1//===-- AArch64ISelDAGToDAG.cpp - A dag to dag inst selector for AArch64 --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the AArch64 target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64.h"
14#include "AArch64ExpandImm.h"
15#include "AArch64MachineFunctionInfo.h"
16#include "AArch64TargetMachine.h"
17#include "MCTargetDesc/AArch64AddressingModes.h"
18#include "llvm/ADT/APSInt.h"
19#include "llvm/CodeGen/ISDOpcodes.h"
20#include "llvm/CodeGen/SDPatternMatch.h"
21#include "llvm/CodeGen/SelectionDAGISel.h"
22#include "llvm/IR/Function.h" // To access function attributes.
23#include "llvm/IR/GlobalValue.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsAArch64.h"
26#include "llvm/Support/Debug.h"
27#include "llvm/Support/ErrorHandling.h"
28#include "llvm/Support/KnownBits.h"
29#include "llvm/Support/MathExtras.h"
30#include "llvm/Support/raw_ostream.h"
31
32using namespace llvm;
33using namespace llvm::SDPatternMatch;
34
35#define DEBUG_TYPE "aarch64-isel"
36#define PASS_NAME "AArch64 Instruction Selection"
37
38// https://github.com/llvm/llvm-project/issues/114425
39#if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG)
40#pragma inline_depth(0)
41#endif
42
43//===--------------------------------------------------------------------===//
44/// AArch64DAGToDAGISel - AArch64 specific code to select AArch64 machine
45/// instructions for SelectionDAG operations.
46///
47namespace {
48
49class AArch64DAGToDAGISel : public SelectionDAGISel {
50
51 /// Subtarget - Keep a pointer to the AArch64Subtarget around so that we can
52 /// make the right decision when generating code for different targets.
53 const AArch64Subtarget *Subtarget;
54
55public:
56 AArch64DAGToDAGISel() = delete;
57
58 explicit AArch64DAGToDAGISel(AArch64TargetMachine &tm,
59 CodeGenOptLevel OptLevel)
60 : SelectionDAGISel(tm, OptLevel), Subtarget(nullptr) {}
61
62 bool runOnMachineFunction(MachineFunction &MF) override {
63 Subtarget = &MF.getSubtarget<AArch64Subtarget>();
64 return SelectionDAGISel::runOnMachineFunction(mf&: MF);
65 }
66
67 void Select(SDNode *Node) override;
68 void PreprocessISelDAG() override;
69
70 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
71 /// inline asm expressions.
72 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
73 InlineAsm::ConstraintCode ConstraintID,
74 std::vector<SDValue> &OutOps) override;
75
76 template <signed Low, signed High, signed Scale>
77 bool SelectRDVLImm(SDValue N, SDValue &Imm);
78
79 template <signed Low, signed High>
80 bool SelectRDSVLShiftImm(SDValue N, SDValue &Imm);
81
82 bool SelectArithExtendedRegister(SDValue N, SDValue &Reg, SDValue &Shift);
83 bool SelectArithUXTXRegister(SDValue N, SDValue &Reg, SDValue &Shift);
84 bool SelectArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
85 bool SelectNegArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
86 bool SelectArithShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
87 return SelectShiftedRegister(N, AllowROR: false, Reg, Shift);
88 }
89 bool SelectLogicalShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
90 return SelectShiftedRegister(N, AllowROR: true, Reg, Shift);
91 }
92 bool SelectAddrModeIndexed7S8(SDValue N, SDValue &Base, SDValue &OffImm) {
93 return SelectAddrModeIndexed7S(N, Size: 1, Base, OffImm);
94 }
95 bool SelectAddrModeIndexed7S16(SDValue N, SDValue &Base, SDValue &OffImm) {
96 return SelectAddrModeIndexed7S(N, Size: 2, Base, OffImm);
97 }
98 bool SelectAddrModeIndexed7S32(SDValue N, SDValue &Base, SDValue &OffImm) {
99 return SelectAddrModeIndexed7S(N, Size: 4, Base, OffImm);
100 }
101 bool SelectAddrModeIndexed7S64(SDValue N, SDValue &Base, SDValue &OffImm) {
102 return SelectAddrModeIndexed7S(N, Size: 8, Base, OffImm);
103 }
104 bool SelectAddrModeIndexed7S128(SDValue N, SDValue &Base, SDValue &OffImm) {
105 return SelectAddrModeIndexed7S(N, Size: 16, Base, OffImm);
106 }
107 bool SelectAddrModeIndexedS9S128(SDValue N, SDValue &Base, SDValue &OffImm) {
108 return SelectAddrModeIndexedBitWidth(N, IsSignedImm: true, BW: 9, Size: 16, Base, OffImm);
109 }
110 bool SelectAddrModeIndexedU6S128(SDValue N, SDValue &Base, SDValue &OffImm) {
111 return SelectAddrModeIndexedBitWidth(N, IsSignedImm: false, BW: 6, Size: 16, Base, OffImm);
112 }
113 bool SelectAddrModeIndexed8(SDValue N, SDValue &Base, SDValue &OffImm) {
114 return SelectAddrModeIndexed(N, Size: 1, Base, OffImm);
115 }
116 bool SelectAddrModeIndexed16(SDValue N, SDValue &Base, SDValue &OffImm) {
117 return SelectAddrModeIndexed(N, Size: 2, Base, OffImm);
118 }
119 bool SelectAddrModeIndexed32(SDValue N, SDValue &Base, SDValue &OffImm) {
120 return SelectAddrModeIndexed(N, Size: 4, Base, OffImm);
121 }
122 bool SelectAddrModeIndexed64(SDValue N, SDValue &Base, SDValue &OffImm) {
123 return SelectAddrModeIndexed(N, Size: 8, Base, OffImm);
124 }
125 bool SelectAddrModeIndexed128(SDValue N, SDValue &Base, SDValue &OffImm) {
126 return SelectAddrModeIndexed(N, Size: 16, Base, OffImm);
127 }
128 bool SelectAddrModeUnscaled8(SDValue N, SDValue &Base, SDValue &OffImm) {
129 return SelectAddrModeUnscaled(N, Size: 1, Base, OffImm);
130 }
131 bool SelectAddrModeUnscaled16(SDValue N, SDValue &Base, SDValue &OffImm) {
132 return SelectAddrModeUnscaled(N, Size: 2, Base, OffImm);
133 }
134 bool SelectAddrModeUnscaled32(SDValue N, SDValue &Base, SDValue &OffImm) {
135 return SelectAddrModeUnscaled(N, Size: 4, Base, OffImm);
136 }
137 bool SelectAddrModeUnscaled64(SDValue N, SDValue &Base, SDValue &OffImm) {
138 return SelectAddrModeUnscaled(N, Size: 8, Base, OffImm);
139 }
140 bool SelectAddrModeUnscaled128(SDValue N, SDValue &Base, SDValue &OffImm) {
141 return SelectAddrModeUnscaled(N, Size: 16, Base, OffImm);
142 }
143 template <unsigned Size, unsigned Max>
144 bool SelectAddrModeIndexedUImm(SDValue N, SDValue &Base, SDValue &OffImm) {
145 // Test if there is an appropriate addressing mode and check if the
146 // immediate fits.
147 bool Found = SelectAddrModeIndexed(N, Size, Base, OffImm);
148 if (Found) {
149 if (auto *CI = dyn_cast<ConstantSDNode>(Val&: OffImm)) {
150 int64_t C = CI->getSExtValue();
151 if (C <= Max)
152 return true;
153 }
154 }
155
156 // Otherwise, base only, materialize address in register.
157 Base = N;
158 OffImm = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i64);
159 return true;
160 }
161
162 template<int Width>
163 bool SelectAddrModeWRO(SDValue N, SDValue &Base, SDValue &Offset,
164 SDValue &SignExtend, SDValue &DoShift) {
165 return SelectAddrModeWRO(N, Size: Width / 8, Base, Offset, SignExtend, DoShift);
166 }
167
168 template<int Width>
169 bool SelectAddrModeXRO(SDValue N, SDValue &Base, SDValue &Offset,
170 SDValue &SignExtend, SDValue &DoShift) {
171 return SelectAddrModeXRO(N, Size: Width / 8, Base, Offset, SignExtend, DoShift);
172 }
173
174 bool SelectExtractHigh(SDValue N, SDValue &Res) {
175 if (Subtarget->isLittleEndian() && N->getOpcode() == ISD::BITCAST)
176 N = N->getOperand(Num: 0);
177 if (N->getOpcode() != ISD::EXTRACT_SUBVECTOR ||
178 !isa<ConstantSDNode>(Val: N->getOperand(Num: 1)))
179 return false;
180 EVT VT = N->getValueType(ResNo: 0);
181 EVT LVT = N->getOperand(Num: 0).getValueType();
182 unsigned Index = N->getConstantOperandVal(Num: 1);
183 if (!VT.is64BitVector() || !LVT.is128BitVector() ||
184 Index != VT.getVectorNumElements())
185 return false;
186 Res = N->getOperand(Num: 0);
187 return true;
188 }
189
190 bool SelectRoundingVLShr(SDValue N, SDValue &Res1, SDValue &Res2) {
191 if (N.getOpcode() != AArch64ISD::VLSHR)
192 return false;
193 SDValue Op = N->getOperand(Num: 0);
194 EVT VT = Op.getValueType();
195 unsigned ShtAmt = N->getConstantOperandVal(Num: 1);
196 if (ShtAmt > VT.getScalarSizeInBits() / 2 || Op.getOpcode() != ISD::ADD)
197 return false;
198
199 APInt Imm;
200 if (Op.getOperand(i: 1).getOpcode() == AArch64ISD::MOVIshift)
201 Imm = APInt(VT.getScalarSizeInBits(),
202 Op.getOperand(i: 1).getConstantOperandVal(i: 0)
203 << Op.getOperand(i: 1).getConstantOperandVal(i: 1));
204 else if (Op.getOperand(i: 1).getOpcode() == AArch64ISD::DUP &&
205 isa<ConstantSDNode>(Val: Op.getOperand(i: 1).getOperand(i: 0)))
206 Imm = APInt(VT.getScalarSizeInBits(),
207 Op.getOperand(i: 1).getConstantOperandVal(i: 0));
208 else
209 return false;
210
211 if (Imm != 1ULL << (ShtAmt - 1))
212 return false;
213
214 Res1 = Op.getOperand(i: 0);
215 Res2 = CurDAG->getTargetConstant(Val: ShtAmt, DL: SDLoc(N), VT: MVT::i32);
216 return true;
217 }
218
219 bool SelectDupZeroOrUndef(SDValue N) {
220 switch(N->getOpcode()) {
221 case ISD::UNDEF:
222 case ISD::POISON:
223 return true;
224 case AArch64ISD::DUP:
225 case ISD::SPLAT_VECTOR: {
226 auto Opnd0 = N->getOperand(Num: 0);
227 if (isNullConstant(V: Opnd0))
228 return true;
229 if (isNullFPConstant(V: Opnd0))
230 return true;
231 break;
232 }
233 default:
234 break;
235 }
236
237 return false;
238 }
239
240 bool SelectAny(SDValue) { return true; }
241
242 bool SelectDupZero(SDValue N) {
243 switch(N->getOpcode()) {
244 case AArch64ISD::DUP:
245 case ISD::SPLAT_VECTOR: {
246 auto Opnd0 = N->getOperand(Num: 0);
247 if (isNullConstant(V: Opnd0))
248 return true;
249 if (isNullFPConstant(V: Opnd0))
250 return true;
251 break;
252 }
253 }
254
255 return false;
256 }
257
258 template <MVT::SimpleValueType VT, bool Negate>
259 bool SelectSVEAddSubImm(SDValue N, SDValue &Imm, SDValue &Shift) {
260 return SelectSVEAddSubImm(N, VT, Imm, Shift, Negate);
261 }
262
263 template <MVT::SimpleValueType VT, bool Negate>
264 bool SelectSVEAddSubSSatImm(SDValue N, SDValue &Imm, SDValue &Shift) {
265 return SelectSVEAddSubSSatImm(N, VT, Imm, Shift, Negate);
266 }
267
268 template <MVT::SimpleValueType VT>
269 bool SelectSVECpyDupImm(SDValue N, SDValue &Imm, SDValue &Shift) {
270 return SelectSVECpyDupImm(N, VT, Imm, Shift);
271 }
272
273 template <MVT::SimpleValueType VT, bool Invert = false>
274 bool SelectSVELogicalImm(SDValue N, SDValue &Imm) {
275 return SelectSVELogicalImm(N, VT, Imm, Invert);
276 }
277
278 template <MVT::SimpleValueType VT>
279 bool SelectSVEArithImm(SDValue N, SDValue &Imm) {
280 return SelectSVEArithImm(N, VT, Imm);
281 }
282
283 template <unsigned Low, unsigned High, bool AllowSaturation = false>
284 bool SelectSVEShiftImm(SDValue N, SDValue &Imm) {
285 return SelectSVEShiftImm(N, Low, High, AllowSaturation, Imm);
286 }
287
288 bool SelectSVEShiftSplatImmR(SDValue N, SDValue &Imm) {
289 if (N->getOpcode() != ISD::SPLAT_VECTOR)
290 return false;
291
292 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
293 return SelectSVEShiftImm(N: N->getOperand(Num: 0), /* Low */ 1,
294 /* High */ EltVT.getFixedSizeInBits(),
295 /* AllowSaturation */ true, Imm);
296 }
297
298 // Returns a suitable CNT/INC/DEC/RDVL multiplier to calculate VSCALE*N.
299 template<signed Min, signed Max, signed Scale, bool Shift>
300 bool SelectCntImm(SDValue N, SDValue &Imm) {
301 if (!isa<ConstantSDNode>(Val: N))
302 return false;
303
304 int64_t MulImm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
305 if (Shift)
306 MulImm = 1LL << MulImm;
307
308 if ((MulImm % std::abs(x: Scale)) != 0)
309 return false;
310
311 MulImm /= Scale;
312 if ((MulImm >= Min) && (MulImm <= Max)) {
313 Imm = CurDAG->getTargetConstant(Val: MulImm, DL: SDLoc(N), VT: MVT::i32);
314 return true;
315 }
316
317 return false;
318 }
319
320 template <signed Max, signed Scale>
321 bool SelectEXTImm(SDValue N, SDValue &Imm) {
322 if (!isa<ConstantSDNode>(Val: N))
323 return false;
324
325 int64_t MulImm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
326
327 if (MulImm >= 0 && MulImm <= Max) {
328 MulImm *= Scale;
329 Imm = CurDAG->getTargetConstant(Val: MulImm, DL: SDLoc(N), VT: MVT::i32);
330 return true;
331 }
332
333 return false;
334 }
335
336 template <unsigned BaseReg, unsigned Max>
337 bool ImmToReg(SDValue N, SDValue &Imm) {
338 if (auto *CI = dyn_cast<ConstantSDNode>(Val&: N)) {
339 uint64_t C = CI->getZExtValue();
340
341 if (C > Max)
342 return false;
343
344 Imm = CurDAG->getRegister(Reg: BaseReg + C, VT: MVT::Other);
345 return true;
346 }
347 return false;
348 }
349
350 /// Form sequences of consecutive 64/128-bit registers for use in NEON
351 /// instructions making use of a vector-list (e.g. ldN, tbl). Vecs must have
352 /// between 1 and 4 elements. If it contains a single element that is returned
353 /// unchanged; otherwise a REG_SEQUENCE value is returned.
354 SDValue createDTuple(ArrayRef<SDValue> Vecs);
355 SDValue createQTuple(ArrayRef<SDValue> Vecs);
356 // Form a sequence of SVE registers for instructions using list of vectors,
357 // e.g. structured loads and stores (ldN, stN).
358 SDValue createZTuple(ArrayRef<SDValue> Vecs);
359
360 // Similar to above, except the register must start at a multiple of the
361 // tuple, e.g. z2 for a 2-tuple, or z8 for a 4-tuple.
362 SDValue createZMulTuple(ArrayRef<SDValue> Regs);
363
364 /// Generic helper for the createDTuple/createQTuple
365 /// functions. Those should almost always be called instead.
366 SDValue createTuple(ArrayRef<SDValue> Vecs, const unsigned RegClassIDs[],
367 const unsigned SubRegs[]);
368
369 void SelectTable(SDNode *N, unsigned NumVecs, unsigned Opc, bool isExt);
370
371 bool tryIndexedLoad(SDNode *N);
372
373 void SelectPtrauthAuth(SDNode *N);
374 void SelectPtrauthResign(SDNode *N);
375 void SelectPtrauthResignWithPC(SDNode *N);
376
377 bool trySelectStackSlotTagP(SDNode *N);
378 void SelectTagP(SDNode *N);
379
380 void SelectLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
381 unsigned SubRegIdx);
382 void SelectPostLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
383 unsigned SubRegIdx);
384 void SelectLoadLane(SDNode *N, unsigned NumVecs, unsigned Opc);
385 void SelectPostLoadLane(SDNode *N, unsigned NumVecs, unsigned Opc);
386 void SelectPredicatedLoad(SDNode *N, unsigned NumVecs, unsigned Scale,
387 unsigned Opc_rr, unsigned Opc_ri,
388 bool IsIntr = false);
389 void SelectContiguousMultiVectorLoad(SDNode *N, unsigned NumVecs,
390 unsigned Scale, unsigned Opc_ri,
391 unsigned Opc_rr);
392 void SelectDestructiveMultiIntrinsic(SDNode *N, unsigned NumVecs,
393 bool IsZmMulti, unsigned Opcode,
394 bool HasPred = false);
395 void SelectPExtPair(SDNode *N, unsigned Opc);
396 void SelectWhilePair(SDNode *N, unsigned Opc);
397 void SelectCVTIntrinsic(SDNode *N, unsigned NumVecs, unsigned Opcode);
398 void SelectCVTIntrinsicFP8(SDNode *N, unsigned NumVecs, unsigned Opcode);
399 void SelectClamp(SDNode *N, unsigned NumVecs, unsigned Opcode);
400 void SelectUnaryMultiIntrinsic(SDNode *N, unsigned NumOutVecs,
401 bool IsTupleInput, unsigned Opc);
402 void SelectFrintFromVT(SDNode *N, unsigned NumVecs, unsigned Opcode);
403
404 template <unsigned MaxIdx, unsigned Scale>
405 void SelectMultiVectorMove(SDNode *N, unsigned NumVecs, unsigned BaseReg,
406 unsigned Op);
407 void SelectMultiVectorMoveZ(SDNode *N, unsigned NumVecs,
408 unsigned Op, unsigned MaxIdx, unsigned Scale,
409 unsigned BaseReg = 0);
410 /// SVE Reg+Imm addressing mode.
411 template <int64_t Min, int64_t Max>
412 bool SelectAddrModeIndexedSVE(SDNode *Root, SDValue N, SDValue &Base,
413 SDValue &OffImm);
414 /// SVE Reg+Reg address mode.
415 template <unsigned Scale>
416 bool SelectSVERegRegAddrMode(SDValue N, SDValue &Base, SDValue &Offset) {
417 return SelectSVERegRegAddrMode(N, Scale, Base, Offset);
418 }
419
420 void SelectMultiVectorLutiLane(SDNode *Node, unsigned NumOutVecs,
421 unsigned Opc, uint32_t MaxImm);
422 void SelectMultiVectorLuti6LaneX4(SDNode *Node, unsigned NumIndexVecs);
423
424 void SelectMultiVectorLuti(SDNode *Node, unsigned NumOutVecs, unsigned Opc,
425 unsigned NumInVecs);
426
427 template <unsigned MaxIdx, unsigned Scale>
428 bool SelectSMETileSlice(SDValue N, SDValue &Vector, SDValue &Offset) {
429 return SelectSMETileSlice(N, MaxSize: MaxIdx, Vector, Offset, Scale);
430 }
431
432 void SelectStore(SDNode *N, unsigned NumVecs, unsigned Opc);
433 void SelectPostStore(SDNode *N, unsigned NumVecs, unsigned Opc);
434 void SelectStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
435 void SelectPostStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
436 void SelectPredicatedStore(SDNode *N, unsigned NumVecs, unsigned Scale,
437 unsigned Opc_rr, unsigned Opc_ri);
438 std::tuple<unsigned, SDValue, SDValue>
439 findAddrModeSVELoadStore(SDNode *N, unsigned Opc_rr, unsigned Opc_ri,
440 const SDValue &OldBase, const SDValue &OldOffset,
441 unsigned Scale);
442
443 bool tryBitfieldExtractOp(SDNode *N);
444 bool tryBitfieldExtractOpFromSExt(SDNode *N);
445 bool tryBitfieldInsertOp(SDNode *N);
446 bool tryBitfieldInsertInZeroOp(SDNode *N);
447 bool tryShiftAmountMod(SDNode *N);
448
449 bool tryReadRegister(SDNode *N);
450 bool tryWriteRegister(SDNode *N);
451
452 bool trySelectCastFixedLengthToScalableVector(SDNode *N);
453 bool trySelectCastScalableToFixedLengthVector(SDNode *N);
454
455 bool trySelectXAR(SDNode *N);
456
457 bool tryFoldCselToFMaxMin(SDNode *N);
458
459// Include the pieces autogenerated from the target description.
460#include "AArch64GenDAGISel.inc"
461
462private:
463 bool SelectShiftedRegister(SDValue N, bool AllowROR, SDValue &Reg,
464 SDValue &Shift);
465 bool SelectShiftedRegisterFromAnd(SDValue N, SDValue &Reg, SDValue &Shift);
466 bool SelectAddrModeIndexed7S(SDValue N, unsigned Size, SDValue &Base,
467 SDValue &OffImm) {
468 return SelectAddrModeIndexedBitWidth(N, IsSignedImm: true, BW: 7, Size, Base, OffImm);
469 }
470 bool SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm, unsigned BW,
471 unsigned Size, SDValue &Base,
472 SDValue &OffImm);
473 bool SelectAddrModeIndexed(SDValue N, unsigned Size, SDValue &Base,
474 SDValue &OffImm);
475 bool SelectAddrModeUnscaled(SDValue N, unsigned Size, SDValue &Base,
476 SDValue &OffImm);
477 bool SelectAddrModeWRO(SDValue N, unsigned Size, SDValue &Base,
478 SDValue &Offset, SDValue &SignExtend,
479 SDValue &DoShift);
480 bool SelectAddrModeXRO(SDValue N, unsigned Size, SDValue &Base,
481 SDValue &Offset, SDValue &SignExtend,
482 SDValue &DoShift);
483 bool isWorthNegatingImm(SDValue V) const;
484 bool isWorthFoldingALU(SDValue V, bool LSL = false) const;
485 bool isWorthFoldingAddr(SDValue V, unsigned Size) const;
486 bool SelectExtendedSHL(SDValue N, unsigned Size, bool WantExtend,
487 SDValue &Offset, SDValue &SignExtend);
488
489 template<unsigned RegWidth>
490 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos) {
491 return SelectCVTFixedPosOperand(N, FixedPos, Width: RegWidth);
492 }
493 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, unsigned Width);
494
495 template <unsigned RegWidth>
496 bool SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos) {
497 return SelectCVTFixedPointVec(N, FixedPos, Width: RegWidth);
498 }
499 bool SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos, unsigned Width);
500
501 template<unsigned RegWidth>
502 bool SelectCVTFixedPosRecipOperand(SDValue N, SDValue &FixedPos) {
503 return SelectCVTFixedPosRecipOperand(N, FixedPos, Width: RegWidth);
504 }
505
506 bool SelectCVTFixedPosRecipOperand(SDValue N, SDValue &FixedPos,
507 unsigned Width);
508
509 template <unsigned FloatWidth>
510 bool SelectCVTFixedPosRecipOperandVec(SDValue N, SDValue &FixedPos) {
511 return SelectCVTFixedPosRecipOperandVec(N, FixedPos, Width: FloatWidth);
512 }
513
514 bool SelectCVTFixedPosRecipOperandVec(SDValue N, SDValue &FixedPos,
515 unsigned Width);
516
517 bool SelectCMP_SWAP(SDNode *N);
518
519 bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
520 bool Negate);
521 bool SelectSVEAddSubImm(SDLoc DL, APInt Value, MVT VT, SDValue &Imm,
522 SDValue &Shift, bool Negate);
523 bool SelectSVEAddSubSSatImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
524 bool Negate);
525 bool SelectSVECpyDupImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift);
526 bool SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm, bool Invert);
527
528 // Match `<NEON Splat> SVEImm` (where <NEON Splat> could be fmov, movi, etc).
529 bool SelectNEONSplatOfSVELogicalImm(SDValue N, SDValue &Imm);
530 bool SelectNEONSplatOfSVEAddSubImm(SDValue N, SDValue &Imm, SDValue &Shift);
531 bool SelectNEONSplatOfSVEArithSImm(SDValue N, SDValue &Imm);
532 bool SelectNEONSplatOfSImm8(SDValue N, SDValue &Imm);
533 bool SelectNEONSplatOfUImm8(SDValue N, SDValue &Imm);
534
535 bool SelectSVESignedArithImm(SDLoc DL, APInt Value, SDValue &Imm);
536 bool SelectSVESignedArithImm(SDValue N, SDValue &Imm);
537 bool SelectSVEShiftImm(SDValue N, uint64_t Low, uint64_t High,
538 bool AllowSaturation, SDValue &Imm);
539
540 bool SelectSVEArithImm(SDValue N, MVT VT, SDValue &Imm);
541 bool SelectSVERegRegAddrMode(SDValue N, unsigned Scale, SDValue &Base,
542 SDValue &Offset);
543 bool SelectSMETileSlice(SDValue N, unsigned MaxSize, SDValue &Vector,
544 SDValue &Offset, unsigned Scale = 1);
545
546 bool SelectAllActivePredicate(SDValue N);
547 bool SelectAnyPredicate(SDValue N);
548
549 bool SelectCmpBranchUImm6Operand(SDNode *P, SDValue N, SDValue &Imm);
550
551 template <bool MatchCBB>
552 bool SelectCmpBranchExtOperand(SDValue N, SDValue &Reg, SDValue &ExtType);
553};
554
555class AArch64DAGToDAGISelLegacy : public SelectionDAGISelLegacy {
556public:
557 static char ID;
558 explicit AArch64DAGToDAGISelLegacy(AArch64TargetMachine &tm,
559 CodeGenOptLevel OptLevel)
560 : SelectionDAGISelLegacy(
561 ID, std::make_unique<AArch64DAGToDAGISel>(args&: tm, args&: OptLevel)) {}
562};
563} // end anonymous namespace
564
565char AArch64DAGToDAGISelLegacy::ID = 0;
566
567INITIALIZE_PASS(AArch64DAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false, false)
568
569AArch64DAGToDAGISelPass::AArch64DAGToDAGISelPass(AArch64TargetMachine &TM)
570 : SelectionDAGISelPass(
571 std::make_unique<AArch64DAGToDAGISel>(args&: TM, args: TM.getOptLevel())) {}
572
573/// addBitcastHints - This method adds bitcast hints to the operands of a node
574/// to help instruction selector determine which operands are in Neon registers.
575static SDValue addBitcastHints(SelectionDAG &DAG, SDNode &N) {
576 SDLoc DL(&N);
577 auto getFloatVT = [&](EVT VT) {
578 EVT ScalarVT = VT.getScalarType();
579 assert((ScalarVT == MVT::i32 || ScalarVT == MVT::i64) && "Unexpected VT");
580 return VT.changeElementType(Context&: *(DAG.getContext()),
581 EltVT: ScalarVT == MVT::i32 ? MVT::f32 : MVT::f64);
582 };
583 SmallVector<SDValue, 2> NewOps;
584 NewOps.reserve(N: N.getNumOperands());
585
586 for (unsigned I = 0, E = N.getNumOperands(); I < E; ++I) {
587 auto bitcasted = DAG.getBitcast(VT: getFloatVT(N.getOperand(Num: I).getValueType()),
588 V: N.getOperand(Num: I));
589 NewOps.push_back(Elt: bitcasted);
590 }
591 EVT OrigVT = N.getValueType(ResNo: 0);
592 SDValue OpNode = DAG.getNode(Opcode: N.getOpcode(), DL, VT: getFloatVT(OrigVT), Ops: NewOps);
593 return DAG.getBitcast(VT: OrigVT, V: OpNode);
594}
595
596/// isIntImmediate - This method tests to see if the node is a constant
597/// operand. If so Imm will receive the 64-bit value.
598static bool isIntImmediate(const SDNode *N, uint64_t &Imm) {
599 if (const ConstantSDNode *C = dyn_cast<const ConstantSDNode>(Val: N)) {
600 Imm = C->getZExtValue();
601 return true;
602 }
603 return false;
604}
605
606// isIntImmediate - This method tests to see if a constant operand.
607// If so Imm will receive the value.
608static bool isIntImmediate(SDValue N, uint64_t &Imm) {
609 return isIntImmediate(N: N.getNode(), Imm);
610}
611
612// isOpcWithIntImmediate - This method tests to see if the node is a specific
613// opcode and that it has a immediate integer right operand.
614// If so Imm will receive the 32 bit value.
615static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc,
616 uint64_t &Imm) {
617 return N->getOpcode() == Opc &&
618 isIntImmediate(N: N->getOperand(Num: 1).getNode(), Imm);
619}
620
621// isIntImmediateEq - This method tests to see if N is a constant operand that
622// is equivalent to 'ImmExpected'.
623#ifndef NDEBUG
624static bool isIntImmediateEq(SDValue N, const uint64_t ImmExpected) {
625 uint64_t Imm;
626 if (!isIntImmediate(N.getNode(), Imm))
627 return false;
628 return Imm == ImmExpected;
629}
630#endif
631
632static APInt DecodeFMOVImm(uint64_t Imm, unsigned RegWidth) {
633 assert(RegWidth == 32 || RegWidth == 64);
634 if (RegWidth == 32)
635 return APInt(RegWidth,
636 uint32_t(AArch64_AM::decodeAdvSIMDModImmType11(Imm)));
637 return APInt(RegWidth, AArch64_AM::decodeAdvSIMDModImmType12(Imm));
638}
639
640// Decodes the raw integer splat value from a NEON splat operation.
641static std::optional<APInt> DecodeNEONSplat(SDValue N,
642 const AArch64Subtarget *Subtarget) {
643 assert(N.getValueType().isInteger() && "Only integers are supported");
644 if (N->getOpcode() == AArch64ISD::NVCAST ||
645 (N->getOpcode() == ISD::BITCAST && Subtarget->isLittleEndian()))
646 N = N->getOperand(Num: 0);
647 unsigned SplatWidth = N.getScalarValueSizeInBits();
648 if (N.getOpcode() == AArch64ISD::FMOV)
649 return DecodeFMOVImm(Imm: N.getConstantOperandVal(i: 0), RegWidth: SplatWidth);
650 if (N->getOpcode() == AArch64ISD::MOVI)
651 return APInt(SplatWidth, N.getConstantOperandVal(i: 0));
652 if (N->getOpcode() == AArch64ISD::MOVIshift)
653 return APInt(SplatWidth, N.getConstantOperandVal(i: 0)
654 << N.getConstantOperandVal(i: 1));
655 if (N->getOpcode() == AArch64ISD::MVNIshift)
656 return ~APInt(SplatWidth, N.getConstantOperandVal(i: 0)
657 << N.getConstantOperandVal(i: 1));
658 if (N->getOpcode() == AArch64ISD::MOVIedit)
659 return APInt(SplatWidth, AArch64_AM::decodeAdvSIMDModImmType10(
660 Imm: N.getConstantOperandVal(i: 0)));
661 if (N->getOpcode() == AArch64ISD::DUP)
662 if (auto *Const = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0)))
663 return Const->getAPIntValue().trunc(width: SplatWidth);
664 APInt SplatVal;
665 if (ISD::isConstantSplatVector(N: N.getNode(), SplatValue&: SplatVal))
666 return SplatVal.trunc(width: SplatWidth);
667 // TODO: Recognize more splat-like NEON operations. See ConstantBuildVector
668 // in AArch64ISelLowering.
669 return std::nullopt;
670}
671
672// If \p N is a NEON splat operation (movi, fmov, etc), return the splat value
673// matching the element size of N.
674static std::optional<APInt>
675GetNEONSplatValue(SDValue N, const AArch64Subtarget *Subtarget) {
676 unsigned SplatWidth = N.getScalarValueSizeInBits();
677 if (std::optional<APInt> SplatVal = DecodeNEONSplat(N, Subtarget)) {
678 if (SplatVal->getBitWidth() <= SplatWidth)
679 return APInt::getSplat(NewLen: SplatWidth, V: *SplatVal);
680 if (SplatVal->isSplat(SplatSizeInBits: SplatWidth))
681 return SplatVal->trunc(width: SplatWidth);
682 }
683 return std::nullopt;
684}
685
686bool AArch64DAGToDAGISel::SelectNEONSplatOfSVELogicalImm(SDValue N,
687 SDValue &Imm) {
688 std::optional<APInt> ImmVal = GetNEONSplatValue(N, Subtarget);
689 if (!ImmVal)
690 return false;
691 uint64_t Encoding;
692 if (!AArch64_AM::isSVELogicalImm(SizeInBits: N.getScalarValueSizeInBits(),
693 ImmVal: ImmVal->getZExtValue(), Encoding))
694 return false;
695
696 Imm = CurDAG->getTargetConstant(Val: Encoding, DL: SDLoc(N), VT: MVT::i64);
697 return true;
698}
699
700bool AArch64DAGToDAGISel::SelectNEONSplatOfSVEAddSubImm(SDValue N, SDValue &Imm,
701 SDValue &Shift) {
702 if (std::optional<APInt> ImmVal = GetNEONSplatValue(N, Subtarget))
703 return SelectSVEAddSubImm(DL: SDLoc(N), Value: *ImmVal,
704 VT: N.getValueType().getScalarType().getSimpleVT(),
705 Imm, Shift,
706 /*Negate=*/false);
707 return false;
708}
709
710bool AArch64DAGToDAGISel::SelectNEONSplatOfSVEArithSImm(SDValue N,
711 SDValue &Imm) {
712 if (std::optional<APInt> ImmVal = GetNEONSplatValue(N, Subtarget))
713 return SelectSVESignedArithImm(DL: SDLoc(N), Value: *ImmVal, Imm);
714 return false;
715}
716
717bool AArch64DAGToDAGISel::SelectNEONSplatOfSImm8(SDValue N, SDValue &Imm) {
718 std::optional<APInt> ImmAPIntVal = GetNEONSplatValue(N, Subtarget);
719 if (!ImmAPIntVal)
720 return false;
721
722 int64_t ImmVal = ImmAPIntVal->getSExtValue();
723 if (ImmVal < -128 || ImmVal > 127)
724 return false;
725
726 Imm = CurDAG->getSignedTargetConstant(Val: ImmVal, DL: SDLoc(N), VT: MVT::i32);
727 return true;
728}
729
730bool AArch64DAGToDAGISel::SelectNEONSplatOfUImm8(SDValue N, SDValue &Imm) {
731 std::optional<APInt> ImmAPIntVal = GetNEONSplatValue(N, Subtarget);
732 if (!ImmAPIntVal)
733 return false;
734
735 uint64_t ImmVal = ImmAPIntVal->getZExtValue();
736 if (ImmVal > 255)
737 return false;
738
739 Imm = CurDAG->getTargetConstant(Val: ImmVal, DL: SDLoc(N), VT: MVT::i32);
740 return true;
741}
742
743bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
744 const SDValue &Op, const InlineAsm::ConstraintCode ConstraintID,
745 std::vector<SDValue> &OutOps) {
746 switch(ConstraintID) {
747 default:
748 llvm_unreachable("Unexpected asm memory constraint");
749 case InlineAsm::ConstraintCode::m:
750 case InlineAsm::ConstraintCode::o:
751 case InlineAsm::ConstraintCode::Q:
752 // We need to make sure that this one operand does not end up in XZR, thus
753 // require the address to be in a pointer register.
754 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
755 const TargetRegisterClass *TRC =
756 TII->getInlineAsmMemoryOperandRegClass(C: ConstraintID);
757 SDLoc dl(Op);
758 SDValue RC = CurDAG->getTargetConstant(Val: TRC->getID(), DL: dl, VT: MVT::i64);
759 SDValue NewOp =
760 SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS,
761 dl, VT: Op.getValueType(),
762 Op1: Op, Op2: RC), 0);
763 OutOps.push_back(x: NewOp);
764 return false;
765 }
766 return true;
767}
768
769/// SelectArithImmed - Select an immediate value that can be represented as
770/// a 12-bit value shifted left by either 0 or 12. If so, return true with
771/// Val set to the 12-bit value and Shift set to the shifter operand.
772bool AArch64DAGToDAGISel::SelectArithImmed(SDValue N, SDValue &Val,
773 SDValue &Shift) {
774 // This function is called from the addsub_shifted_imm ComplexPattern,
775 // which lists [imm] as the list of opcode it's interested in, however
776 // we still need to check whether the operand is actually an immediate
777 // here because the ComplexPattern opcode list is only used in
778 // root-level opcode matching.
779 if (!isa<ConstantSDNode>(Val: N.getNode()))
780 return false;
781
782 uint64_t Immed = N.getNode()->getAsZExtVal();
783
784 if (!AArch64_AM::isLegalArithImmed(C: Immed))
785 return false;
786
787 unsigned ShiftAmt = AArch64_AM::getArithImmedShift(C: Immed);
788 Immed >>= ShiftAmt;
789
790 unsigned ShVal = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShiftAmt);
791 SDLoc dl(N);
792 Val = CurDAG->getTargetConstant(Val: Immed, DL: dl, VT: MVT::i32);
793 Shift = CurDAG->getTargetConstant(Val: ShVal, DL: dl, VT: MVT::i32);
794 return true;
795}
796
797/// SelectNegArithImmed - As above, but negates the value before trying to
798/// select it.
799bool AArch64DAGToDAGISel::SelectNegArithImmed(SDValue N, SDValue &Val,
800 SDValue &Shift) {
801 // This function is called from the addsub_shifted_imm ComplexPattern,
802 // which lists [imm] as the list of opcode it's interested in, however
803 // we still need to check whether the operand is actually an immediate
804 // here because the ComplexPattern opcode list is only used in
805 // root-level opcode matching.
806 if (!isa<ConstantSDNode>(Val: N.getNode()))
807 return false;
808
809 // The immediate operand must be a 24-bit zero-extended immediate.
810 uint64_t Immed = N.getNode()->getAsZExtVal();
811
812 // This negation is almost always valid, but "cmp wN, #0" and "cmn wN, #0"
813 // have the opposite effect on the C flag, so this pattern mustn't match under
814 // those circumstances.
815 if (Immed == 0)
816 return false;
817
818 if (N.getValueType() == MVT::i32)
819 Immed = ~((uint32_t)Immed) + 1;
820 else
821 Immed = ~Immed + 1ULL;
822 if (Immed & 0xFFFFFFFFFF000000ULL)
823 return false;
824
825 Immed &= 0xFFFFFFULL;
826 return SelectArithImmed(N: CurDAG->getConstant(Val: Immed, DL: SDLoc(N), VT: MVT::i32), Val,
827 Shift);
828}
829
830/// getShiftTypeForNode - Translate a shift node to the corresponding
831/// ShiftType value.
832static AArch64_AM::ShiftExtendType getShiftTypeForNode(SDValue N) {
833 switch (N.getOpcode()) {
834 default:
835 return AArch64_AM::InvalidShiftExtend;
836 case ISD::SHL:
837 return AArch64_AM::LSL;
838 case ISD::SRL:
839 return AArch64_AM::LSR;
840 case ISD::SRA:
841 return AArch64_AM::ASR;
842 case ISD::ROTR:
843 return AArch64_AM::ROR;
844 }
845}
846
847static bool isMemOpOrPrefetch(SDNode *N) {
848 return isa<MemSDNode>(Val: *N) || N->getOpcode() == AArch64ISD::PREFETCH;
849}
850
851/// Determine whether it is worth it to fold SHL into the addressing
852/// mode.
853static bool isWorthFoldingSHL(SDValue V) {
854 assert(V.getOpcode() == ISD::SHL && "invalid opcode");
855 // It is worth folding logical shift of up to three places.
856 auto *CSD = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1));
857 if (!CSD)
858 return false;
859 unsigned ShiftVal = CSD->getZExtValue();
860 if (ShiftVal > 3)
861 return false;
862
863 // Check if this particular node is reused in any non-memory related
864 // operation. If yes, do not try to fold this node into the address
865 // computation, since the computation will be kept.
866 const SDNode *Node = V.getNode();
867 for (SDNode *UI : Node->users())
868 if (!isMemOpOrPrefetch(N: UI))
869 for (SDNode *UII : UI->users())
870 if (!isMemOpOrPrefetch(N: UII))
871 return false;
872 return true;
873}
874
875/// Determine whether it is worth to fold V into an extended register addressing
876/// mode.
877bool AArch64DAGToDAGISel::isWorthFoldingAddr(SDValue V, unsigned Size) const {
878 // Trivial if we are optimizing for code size or if there is only
879 // one use of the value.
880 if (CurDAG->shouldOptForSize() || V.hasOneUse())
881 return true;
882
883 // If a subtarget has a slow shift, folding a shift into multiple loads
884 // costs additional micro-ops.
885 if (Subtarget->hasAddrLSLSlow14() && (Size == 2 || Size == 16))
886 return false;
887
888 // Check whether we're going to emit the address arithmetic anyway because
889 // it's used by a non-address operation.
890 if (V.getOpcode() == ISD::SHL && isWorthFoldingSHL(V))
891 return true;
892 if (V.getOpcode() == ISD::ADD) {
893 const SDValue LHS = V.getOperand(i: 0);
894 const SDValue RHS = V.getOperand(i: 1);
895 if (LHS.getOpcode() == ISD::SHL && isWorthFoldingSHL(V: LHS))
896 return true;
897 if (RHS.getOpcode() == ISD::SHL && isWorthFoldingSHL(V: RHS))
898 return true;
899 }
900
901 // It hurts otherwise, since the value will be reused.
902 return false;
903}
904
905/// and (shl/srl/sra, x, c), mask --> shl (srl/sra, x, c1), c2
906/// to select more shifted register
907bool AArch64DAGToDAGISel::SelectShiftedRegisterFromAnd(SDValue N, SDValue &Reg,
908 SDValue &Shift) {
909 EVT VT = N.getValueType();
910 if (VT != MVT::i32 && VT != MVT::i64)
911 return false;
912
913 if (N->getOpcode() != ISD::AND || !N->hasOneUse())
914 return false;
915 SDValue LHS = N.getOperand(i: 0);
916 if (!LHS->hasOneUse())
917 return false;
918
919 unsigned LHSOpcode = LHS->getOpcode();
920 if (LHSOpcode != ISD::SHL && LHSOpcode != ISD::SRL && LHSOpcode != ISD::SRA)
921 return false;
922
923 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1));
924 if (!ShiftAmtNode)
925 return false;
926
927 uint64_t ShiftAmtC = ShiftAmtNode->getZExtValue();
928 ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
929 if (!RHSC)
930 return false;
931
932 APInt AndMask = RHSC->getAPIntValue();
933 unsigned LowZBits, MaskLen;
934 if (!AndMask.isShiftedMask(MaskIdx&: LowZBits, MaskLen))
935 return false;
936
937 unsigned BitWidth = N.getValueSizeInBits();
938 SDLoc DL(LHS);
939 uint64_t NewShiftC;
940 unsigned NewShiftOp;
941 if (LHSOpcode == ISD::SHL) {
942 // LowZBits <= ShiftAmtC will fall into isBitfieldPositioningOp
943 // BitWidth != LowZBits + MaskLen doesn't match the pattern
944 if (LowZBits <= ShiftAmtC || (BitWidth != LowZBits + MaskLen))
945 return false;
946
947 NewShiftC = LowZBits - ShiftAmtC;
948 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
949 } else {
950 if (LowZBits == 0)
951 return false;
952
953 // NewShiftC >= BitWidth will fall into isBitfieldExtractOp
954 NewShiftC = LowZBits + ShiftAmtC;
955 if (NewShiftC >= BitWidth)
956 return false;
957
958 // SRA need all high bits
959 if (LHSOpcode == ISD::SRA && (BitWidth != (LowZBits + MaskLen)))
960 return false;
961
962 // SRL high bits can be 0 or 1
963 if (LHSOpcode == ISD::SRL && (BitWidth > (NewShiftC + MaskLen)))
964 return false;
965
966 if (LHSOpcode == ISD::SRL)
967 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
968 else
969 NewShiftOp = VT == MVT::i64 ? AArch64::SBFMXri : AArch64::SBFMWri;
970 }
971
972 assert(NewShiftC < BitWidth && "Invalid shift amount");
973 SDValue NewShiftAmt = CurDAG->getTargetConstant(Val: NewShiftC, DL, VT);
974 SDValue BitWidthMinus1 = CurDAG->getTargetConstant(Val: BitWidth - 1, DL, VT);
975 Reg = SDValue(CurDAG->getMachineNode(Opcode: NewShiftOp, dl: DL, VT, Op1: LHS->getOperand(Num: 0),
976 Op2: NewShiftAmt, Op3: BitWidthMinus1),
977 0);
978 unsigned ShVal = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: LowZBits);
979 Shift = CurDAG->getTargetConstant(Val: ShVal, DL, VT: MVT::i32);
980 return true;
981}
982
983/// getExtendTypeForNode - Translate an extend node to the corresponding
984/// ExtendType value.
985static AArch64_AM::ShiftExtendType
986getExtendTypeForNode(SDValue N, bool IsLoadStore = false) {
987 if (N.getOpcode() == ISD::SIGN_EXTEND ||
988 N.getOpcode() == ISD::SIGN_EXTEND_INREG) {
989 EVT SrcVT;
990 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG)
991 SrcVT = cast<VTSDNode>(Val: N.getOperand(i: 1))->getVT();
992 else
993 SrcVT = N.getOperand(i: 0).getValueType();
994
995 if (!IsLoadStore && SrcVT == MVT::i8)
996 return AArch64_AM::SXTB;
997 else if (!IsLoadStore && SrcVT == MVT::i16)
998 return AArch64_AM::SXTH;
999 else if (SrcVT == MVT::i32)
1000 return AArch64_AM::SXTW;
1001 assert(SrcVT != MVT::i64 && "extend from 64-bits?");
1002
1003 return AArch64_AM::InvalidShiftExtend;
1004 } else if (N.getOpcode() == ISD::ZERO_EXTEND ||
1005 N.getOpcode() == ISD::ANY_EXTEND) {
1006 EVT SrcVT = N.getOperand(i: 0).getValueType();
1007 if (!IsLoadStore && SrcVT == MVT::i8)
1008 return AArch64_AM::UXTB;
1009 else if (!IsLoadStore && SrcVT == MVT::i16)
1010 return AArch64_AM::UXTH;
1011 else if (SrcVT == MVT::i32)
1012 return AArch64_AM::UXTW;
1013 assert(SrcVT != MVT::i64 && "extend from 64-bits?");
1014
1015 return AArch64_AM::InvalidShiftExtend;
1016 } else if (N.getOpcode() == ISD::AND) {
1017 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1018 if (!CSD)
1019 return AArch64_AM::InvalidShiftExtend;
1020 uint64_t AndMask = CSD->getZExtValue();
1021
1022 switch (AndMask) {
1023 default:
1024 return AArch64_AM::InvalidShiftExtend;
1025 case 0xFF:
1026 return !IsLoadStore ? AArch64_AM::UXTB : AArch64_AM::InvalidShiftExtend;
1027 case 0xFFFF:
1028 return !IsLoadStore ? AArch64_AM::UXTH : AArch64_AM::InvalidShiftExtend;
1029 case 0xFFFFFFFF:
1030 return AArch64_AM::UXTW;
1031 }
1032 }
1033
1034 return AArch64_AM::InvalidShiftExtend;
1035}
1036
1037/// Determine whether constant -V is cheaper to materialise than V.
1038bool AArch64DAGToDAGISel::isWorthNegatingImm(SDValue V) const {
1039 assert(isa<ConstantSDNode>(V) && "invalid node");
1040
1041 EVT VT = V.getValueType();
1042 assert((VT == MVT::i32 || VT == MVT::i64) && "invalid type");
1043
1044 // It's only worth negating the constant if it doesn't have other uses.
1045 if (!V.hasOneUse())
1046 return false;
1047
1048 uint64_t Imm = cast<ConstantSDNode>(Val&: V)->getZExtValue();
1049 unsigned BitSize = VT.getSizeInBits();
1050 SmallVector<AArch64_IMM::ImmInsnModel, 4> OrigCost, NewCost;
1051 AArch64_IMM::expandMOVImm(Imm, BitSize, Insn&: OrigCost);
1052 AArch64_IMM::expandMOVImm(Imm: -Imm, BitSize, Insn&: NewCost);
1053 return NewCost.size() < OrigCost.size();
1054}
1055
1056/// Determine whether it is worth to fold V into an extended register of an
1057/// Add/Sub. LSL means we are folding into an `add w0, w1, w2, lsl #N`
1058/// instruction, and the shift should be treated as worth folding even if has
1059/// multiple uses.
1060bool AArch64DAGToDAGISel::isWorthFoldingALU(SDValue V, bool LSL) const {
1061 // Trivial if we are optimizing for code size or if there is only
1062 // one use of the value.
1063 if (CurDAG->shouldOptForSize() || V.hasOneUse())
1064 return true;
1065
1066 // If a subtarget has a fastpath LSL we can fold a logical shift into
1067 // the add/sub and save a cycle.
1068 if (LSL && Subtarget->hasALULSLFast() && V.getOpcode() == ISD::SHL &&
1069 V.getConstantOperandVal(i: 1) <= 4 &&
1070 getExtendTypeForNode(N: V.getOperand(i: 0)) == AArch64_AM::InvalidShiftExtend)
1071 return true;
1072
1073 // It hurts otherwise, since the value will be reused.
1074 return false;
1075}
1076
1077/// SelectShiftedRegister - Select a "shifted register" operand. If the value
1078/// is not shifted, set the Shift operand to default of "LSL 0". The logical
1079/// instructions allow the shifted register to be rotated, but the arithmetic
1080/// instructions do not. The AllowROR parameter specifies whether ROR is
1081/// supported.
1082bool AArch64DAGToDAGISel::SelectShiftedRegister(SDValue N, bool AllowROR,
1083 SDValue &Reg, SDValue &Shift) {
1084 if (SelectShiftedRegisterFromAnd(N, Reg, Shift))
1085 return true;
1086
1087 AArch64_AM::ShiftExtendType ShType = getShiftTypeForNode(N);
1088 if (ShType == AArch64_AM::InvalidShiftExtend)
1089 return false;
1090 if (!AllowROR && ShType == AArch64_AM::ROR)
1091 return false;
1092
1093 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1094 unsigned BitSize = N.getValueSizeInBits();
1095 unsigned Val = RHS->getZExtValue() & (BitSize - 1);
1096 unsigned ShVal = AArch64_AM::getShifterImm(ST: ShType, Imm: Val);
1097
1098 Reg = N.getOperand(i: 0);
1099 Shift = CurDAG->getTargetConstant(Val: ShVal, DL: SDLoc(N), VT: MVT::i32);
1100 return isWorthFoldingALU(V: N, LSL: true);
1101 }
1102
1103 return false;
1104}
1105
1106/// Instructions that accept extend modifiers like UXTW expect the register
1107/// being extended to be a GPR32, but the incoming DAG might be acting on a
1108/// GPR64 (either via SEXT_INREG or AND). Extract the appropriate low bits if
1109/// this is the case.
1110static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N) {
1111 if (N.getValueType() == MVT::i32)
1112 return N;
1113
1114 SDLoc dl(N);
1115 return CurDAG->getTargetExtractSubreg(SRIdx: AArch64::sub_32, DL: dl, VT: MVT::i32, Operand: N);
1116}
1117
1118// Returns a suitable CNT/INC/DEC/RDVL multiplier to calculate VSCALE*N.
1119template<signed Low, signed High, signed Scale>
1120bool AArch64DAGToDAGISel::SelectRDVLImm(SDValue N, SDValue &Imm) {
1121 if (!isa<ConstantSDNode>(Val: N))
1122 return false;
1123
1124 int64_t MulImm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
1125 if ((MulImm % std::abs(x: Scale)) == 0) {
1126 int64_t RDVLImm = MulImm / Scale;
1127 if ((RDVLImm >= Low) && (RDVLImm <= High)) {
1128 Imm = CurDAG->getSignedTargetConstant(Val: RDVLImm, DL: SDLoc(N), VT: MVT::i32);
1129 return true;
1130 }
1131 }
1132
1133 return false;
1134}
1135
1136// Returns a suitable RDSVL multiplier from a left shift.
1137template <signed Low, signed High>
1138bool AArch64DAGToDAGISel::SelectRDSVLShiftImm(SDValue N, SDValue &Imm) {
1139 if (!isa<ConstantSDNode>(Val: N))
1140 return false;
1141
1142 int64_t MulImm = 1LL << cast<ConstantSDNode>(Val&: N)->getSExtValue();
1143 if (MulImm >= Low && MulImm <= High) {
1144 Imm = CurDAG->getSignedTargetConstant(Val: MulImm, DL: SDLoc(N), VT: MVT::i32);
1145 return true;
1146 }
1147
1148 return false;
1149}
1150
1151/// SelectArithExtendedRegister - Select a "extended register" operand. This
1152/// operand folds in an extend followed by an optional left shift.
1153bool AArch64DAGToDAGISel::SelectArithExtendedRegister(SDValue N, SDValue &Reg,
1154 SDValue &Shift) {
1155 unsigned ShiftVal = 0;
1156 AArch64_AM::ShiftExtendType Ext;
1157
1158 if (N.getOpcode() == ISD::SHL) {
1159 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1160 if (!CSD)
1161 return false;
1162 ShiftVal = CSD->getZExtValue();
1163 if (ShiftVal > 4)
1164 return false;
1165
1166 Ext = getExtendTypeForNode(N: N.getOperand(i: 0));
1167 if (Ext == AArch64_AM::InvalidShiftExtend)
1168 return false;
1169
1170 Reg = N.getOperand(i: 0).getOperand(i: 0);
1171 } else {
1172 Ext = getExtendTypeForNode(N);
1173 if (Ext == AArch64_AM::InvalidShiftExtend)
1174 return false;
1175
1176 // Don't match sext of vector extracts. These can use SMOV, but if we match
1177 // this as an extended register, we'll always fold the extend into an ALU op
1178 // user of the extend (which results in a UMOV).
1179 if (AArch64_AM::isSignExtendShiftType(Type: Ext)) {
1180 SDValue Op = N.getOperand(i: 0);
1181 if (Op->getOpcode() == ISD::ANY_EXTEND)
1182 Op = Op->getOperand(Num: 0);
1183 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
1184 Op.getOperand(i: 0).getValueType().isFixedLengthVector())
1185 return false;
1186 }
1187
1188 Reg = N.getOperand(i: 0);
1189
1190 // Don't match if free 32-bit -> 64-bit zext can be used instead. Use the
1191 // isDef32 as a heuristic for when the operand is likely to be a 32bit def.
1192 auto isDef32 = [](SDValue N) {
1193 unsigned Opc = N.getOpcode();
1194 return Opc != ISD::TRUNCATE && Opc != TargetOpcode::EXTRACT_SUBREG &&
1195 Opc != ISD::CopyFromReg && Opc != ISD::AssertSext &&
1196 Opc != ISD::AssertZext && Opc != ISD::AssertAlign &&
1197 Opc != ISD::FREEZE;
1198 };
1199 if (Ext == AArch64_AM::UXTW && Reg->getValueType(ResNo: 0).getSizeInBits() == 32 &&
1200 isDef32(Reg))
1201 return false;
1202 }
1203
1204 // AArch64 mandates that the RHS of the operation must use the smallest
1205 // register class that could contain the size being extended from. Thus,
1206 // if we're folding a (sext i8), we need the RHS to be a GPR32, even though
1207 // there might not be an actual 32-bit value in the program. We can
1208 // (harmlessly) synthesize one by injected an EXTRACT_SUBREG here.
1209 assert(Ext != AArch64_AM::UXTX && Ext != AArch64_AM::SXTX);
1210 Reg = narrowIfNeeded(CurDAG, N: Reg);
1211 Shift = CurDAG->getTargetConstant(Val: getArithExtendImm(ET: Ext, Imm: ShiftVal), DL: SDLoc(N),
1212 VT: MVT::i32);
1213 return isWorthFoldingALU(V: N);
1214}
1215
1216/// SelectArithUXTXRegister - Select a "UXTX register" operand. This
1217/// operand is referred by the instructions have SP operand
1218bool AArch64DAGToDAGISel::SelectArithUXTXRegister(SDValue N, SDValue &Reg,
1219 SDValue &Shift) {
1220 unsigned ShiftVal = 0;
1221 AArch64_AM::ShiftExtendType Ext;
1222
1223 if (N.getOpcode() != ISD::SHL)
1224 return false;
1225
1226 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1227 if (!CSD)
1228 return false;
1229 ShiftVal = CSD->getZExtValue();
1230 if (ShiftVal > 4)
1231 return false;
1232
1233 Ext = AArch64_AM::UXTX;
1234 Reg = N.getOperand(i: 0);
1235 Shift = CurDAG->getTargetConstant(Val: getArithExtendImm(ET: Ext, Imm: ShiftVal), DL: SDLoc(N),
1236 VT: MVT::i32);
1237 return isWorthFoldingALU(V: N);
1238}
1239
1240/// If there's a use of this ADDlow that's not itself a load/store then we'll
1241/// need to create a real ADD instruction from it anyway and there's no point in
1242/// folding it into the mem op. Theoretically, it shouldn't matter, but there's
1243/// a single pseudo-instruction for an ADRP/ADD pair so over-aggressive folding
1244/// leads to duplicated ADRP instructions.
1245static bool isWorthFoldingADDlow(SDValue N) {
1246 for (auto *User : N->users()) {
1247 if (User->getOpcode() != ISD::LOAD && User->getOpcode() != ISD::STORE &&
1248 User->getOpcode() != ISD::ATOMIC_LOAD &&
1249 User->getOpcode() != ISD::ATOMIC_STORE)
1250 return false;
1251
1252 // ldar and stlr have much more restrictive addressing modes (just a
1253 // register).
1254 if (isStrongerThanMonotonic(AO: cast<MemSDNode>(Val: User)->getSuccessOrdering()))
1255 return false;
1256 }
1257
1258 return true;
1259}
1260
1261/// Check if the immediate offset is valid as a scaled immediate.
1262static bool isValidAsScaledImmediate(int64_t Offset, unsigned Range,
1263 unsigned Size) {
1264 if ((Offset & (Size - 1)) == 0 && Offset >= 0 &&
1265 Offset < (Range << Log2_32(Value: Size)))
1266 return true;
1267 return false;
1268}
1269
1270/// SelectAddrModeIndexedBitWidth - Select a "register plus scaled (un)signed BW-bit
1271/// immediate" address. The "Size" argument is the size in bytes of the memory
1272/// reference, which determines the scale.
1273bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm,
1274 unsigned BW, unsigned Size,
1275 SDValue &Base,
1276 SDValue &OffImm) {
1277 SDLoc dl(N);
1278 const DataLayout &DL = CurDAG->getDataLayout();
1279 const TargetLowering *TLI = getTargetLowering();
1280 if (N.getOpcode() == ISD::FrameIndex) {
1281 int FI = cast<FrameIndexSDNode>(Val&: N)->getIndex();
1282 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1283 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1284 return true;
1285 }
1286
1287 // As opposed to the (12-bit) Indexed addressing mode below, the 7/9-bit signed
1288 // selected here doesn't support labels/immediates, only base+offset.
1289 if (CurDAG->isBaseWithConstantOffset(Op: N)) {
1290 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1291 if (IsSignedImm) {
1292 int64_t RHSC = RHS->getSExtValue();
1293 unsigned Scale = Log2_32(Value: Size);
1294 int64_t Range = 0x1LL << (BW - 1);
1295
1296 if ((RHSC & (Size - 1)) == 0 && RHSC >= -(Range << Scale) &&
1297 RHSC < (Range << Scale)) {
1298 Base = N.getOperand(i: 0);
1299 if (Base.getOpcode() == ISD::FrameIndex) {
1300 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1301 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1302 }
1303 OffImm = CurDAG->getTargetConstant(Val: RHSC >> Scale, DL: dl, VT: MVT::i64);
1304 return true;
1305 }
1306 } else {
1307 // unsigned Immediate
1308 uint64_t RHSC = RHS->getZExtValue();
1309 unsigned Scale = Log2_32(Value: Size);
1310 uint64_t Range = 0x1ULL << BW;
1311
1312 if ((RHSC & (Size - 1)) == 0 && RHSC < (Range << Scale)) {
1313 Base = N.getOperand(i: 0);
1314 if (Base.getOpcode() == ISD::FrameIndex) {
1315 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1316 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1317 }
1318 OffImm = CurDAG->getTargetConstant(Val: RHSC >> Scale, DL: dl, VT: MVT::i64);
1319 return true;
1320 }
1321 }
1322 }
1323 }
1324 // Base only. The address will be materialized into a register before
1325 // the memory is accessed.
1326 // add x0, Xbase, #offset
1327 // stp x1, x2, [x0]
1328 Base = N;
1329 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1330 return true;
1331}
1332
1333/// SelectAddrModeIndexed - Select a "register plus scaled unsigned 12-bit
1334/// immediate" address. The "Size" argument is the size in bytes of the memory
1335/// reference, which determines the scale.
1336bool AArch64DAGToDAGISel::SelectAddrModeIndexed(SDValue N, unsigned Size,
1337 SDValue &Base, SDValue &OffImm) {
1338 SDLoc dl(N);
1339 const DataLayout &DL = CurDAG->getDataLayout();
1340 const TargetLowering *TLI = getTargetLowering();
1341 if (N.getOpcode() == ISD::FrameIndex) {
1342 int FI = cast<FrameIndexSDNode>(Val&: N)->getIndex();
1343 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1344 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1345 return true;
1346 }
1347
1348 if (N.getOpcode() == AArch64ISD::ADDlow && isWorthFoldingADDlow(N)) {
1349 GlobalAddressSDNode *GAN =
1350 dyn_cast<GlobalAddressSDNode>(Val: N.getOperand(i: 1).getNode());
1351 Base = N.getOperand(i: 0);
1352 OffImm = N.getOperand(i: 1);
1353 if (!GAN)
1354 return true;
1355
1356 if (GAN->getOffset() % Size == 0 &&
1357 GAN->getGlobal()->getPointerAlignment(DL) >= Size)
1358 return true;
1359 }
1360
1361 if (CurDAG->isBaseWithConstantOffset(Op: N)) {
1362 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1363 int64_t RHSC = (int64_t)RHS->getZExtValue();
1364 unsigned Scale = Log2_32(Value: Size);
1365 if (isValidAsScaledImmediate(Offset: RHSC, Range: 0x1000, Size)) {
1366 Base = N.getOperand(i: 0);
1367 if (Base.getOpcode() == ISD::FrameIndex) {
1368 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1369 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1370 }
1371 OffImm = CurDAG->getTargetConstant(Val: RHSC >> Scale, DL: dl, VT: MVT::i64);
1372 return true;
1373 }
1374 }
1375 }
1376
1377 // Before falling back to our general case, check if the unscaled
1378 // instructions can handle this. If so, that's preferable.
1379 if (SelectAddrModeUnscaled(N, Size, Base, OffImm))
1380 return false;
1381
1382 // Base only. The address will be materialized into a register before
1383 // the memory is accessed.
1384 // add x0, Xbase, #offset
1385 // ldr x0, [x0]
1386 Base = N;
1387 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1388 return true;
1389}
1390
1391/// SelectAddrModeUnscaled - Select a "register plus unscaled signed 9-bit
1392/// immediate" address. This should only match when there is an offset that
1393/// is not valid for a scaled immediate addressing mode. The "Size" argument
1394/// is the size in bytes of the memory reference, which is needed here to know
1395/// what is valid for a scaled immediate.
1396bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(SDValue N, unsigned Size,
1397 SDValue &Base,
1398 SDValue &OffImm) {
1399 if (!CurDAG->isBaseWithConstantOffset(Op: N))
1400 return false;
1401 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1402 int64_t RHSC = RHS->getSExtValue();
1403 if (RHSC >= -256 && RHSC < 256) {
1404 Base = N.getOperand(i: 0);
1405 if (Base.getOpcode() == ISD::FrameIndex) {
1406 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1407 const TargetLowering *TLI = getTargetLowering();
1408 Base = CurDAG->getTargetFrameIndex(
1409 FI, VT: TLI->getPointerTy(DL: CurDAG->getDataLayout()));
1410 }
1411 OffImm = CurDAG->getTargetConstant(Val: RHSC, DL: SDLoc(N), VT: MVT::i64);
1412 return true;
1413 }
1414 }
1415 return false;
1416}
1417
1418static SDValue Widen(SelectionDAG *CurDAG, SDValue N) {
1419 SDLoc dl(N);
1420 SDValue ImpDef = SDValue(
1421 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl, VT: MVT::i64), 0);
1422 return CurDAG->getTargetInsertSubreg(SRIdx: AArch64::sub_32, DL: dl, VT: MVT::i64, Operand: ImpDef,
1423 Subreg: N);
1424}
1425
1426/// Check if the given SHL node (\p N), can be used to form an
1427/// extended register for an addressing mode.
1428bool AArch64DAGToDAGISel::SelectExtendedSHL(SDValue N, unsigned Size,
1429 bool WantExtend, SDValue &Offset,
1430 SDValue &SignExtend) {
1431 assert(N.getOpcode() == ISD::SHL && "Invalid opcode.");
1432 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1433 if (!CSD || (CSD->getZExtValue() & 0x7) != CSD->getZExtValue())
1434 return false;
1435
1436 SDLoc dl(N);
1437 if (WantExtend) {
1438 AArch64_AM::ShiftExtendType Ext =
1439 getExtendTypeForNode(N: N.getOperand(i: 0), IsLoadStore: true);
1440 if (Ext == AArch64_AM::InvalidShiftExtend)
1441 return false;
1442
1443 Offset = narrowIfNeeded(CurDAG, N: N.getOperand(i: 0).getOperand(i: 0));
1444 SignExtend = CurDAG->getTargetConstant(Val: Ext == AArch64_AM::SXTW, DL: dl,
1445 VT: MVT::i32);
1446 } else {
1447 Offset = N.getOperand(i: 0);
1448 SignExtend = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i32);
1449 }
1450
1451 unsigned LegalShiftVal = Log2_32(Value: Size);
1452 unsigned ShiftVal = CSD->getZExtValue();
1453
1454 if (ShiftVal != 0 && ShiftVal != LegalShiftVal)
1455 return false;
1456
1457 return isWorthFoldingAddr(V: N, Size);
1458}
1459
1460bool AArch64DAGToDAGISel::SelectAddrModeWRO(SDValue N, unsigned Size,
1461 SDValue &Base, SDValue &Offset,
1462 SDValue &SignExtend,
1463 SDValue &DoShift) {
1464 if (N.getOpcode() != ISD::ADD)
1465 return false;
1466 SDValue LHS = N.getOperand(i: 0);
1467 SDValue RHS = N.getOperand(i: 1);
1468 SDLoc dl(N);
1469
1470 // We don't want to match immediate adds here, because they are better lowered
1471 // to the register-immediate addressing modes.
1472 if (isa<ConstantSDNode>(Val: LHS) || isa<ConstantSDNode>(Val: RHS))
1473 return false;
1474
1475 // Check if this particular node is reused in any non-memory related
1476 // operation. If yes, do not try to fold this node into the address
1477 // computation, since the computation will be kept.
1478 const SDNode *Node = N.getNode();
1479 for (SDNode *UI : Node->users()) {
1480 if (!isMemOpOrPrefetch(N: UI))
1481 return false;
1482 }
1483
1484 // Remember if it is worth folding N when it produces extended register.
1485 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(V: N, Size);
1486
1487 // Try to match a shifted extend on the RHS.
1488 if (IsExtendedRegisterWorthFolding && RHS.getOpcode() == ISD::SHL &&
1489 SelectExtendedSHL(N: RHS, Size, WantExtend: true, Offset, SignExtend)) {
1490 Base = LHS;
1491 DoShift = CurDAG->getTargetConstant(Val: true, DL: dl, VT: MVT::i32);
1492 return true;
1493 }
1494
1495 // Try to match a shifted extend on the LHS.
1496 if (IsExtendedRegisterWorthFolding && LHS.getOpcode() == ISD::SHL &&
1497 SelectExtendedSHL(N: LHS, Size, WantExtend: true, Offset, SignExtend)) {
1498 Base = RHS;
1499 DoShift = CurDAG->getTargetConstant(Val: true, DL: dl, VT: MVT::i32);
1500 return true;
1501 }
1502
1503 // There was no shift, whatever else we find.
1504 DoShift = CurDAG->getTargetConstant(Val: false, DL: dl, VT: MVT::i32);
1505
1506 AArch64_AM::ShiftExtendType Ext = AArch64_AM::InvalidShiftExtend;
1507 // Try to match an unshifted extend on the LHS.
1508 if (IsExtendedRegisterWorthFolding &&
1509 (Ext = getExtendTypeForNode(N: LHS, IsLoadStore: true)) !=
1510 AArch64_AM::InvalidShiftExtend) {
1511 Base = RHS;
1512 Offset = narrowIfNeeded(CurDAG, N: LHS.getOperand(i: 0));
1513 SignExtend = CurDAG->getTargetConstant(Val: Ext == AArch64_AM::SXTW, DL: dl,
1514 VT: MVT::i32);
1515 if (isWorthFoldingAddr(V: LHS, Size))
1516 return true;
1517 }
1518
1519 // Try to match an unshifted extend on the RHS.
1520 if (IsExtendedRegisterWorthFolding &&
1521 (Ext = getExtendTypeForNode(N: RHS, IsLoadStore: true)) !=
1522 AArch64_AM::InvalidShiftExtend) {
1523 Base = LHS;
1524 Offset = narrowIfNeeded(CurDAG, N: RHS.getOperand(i: 0));
1525 SignExtend = CurDAG->getTargetConstant(Val: Ext == AArch64_AM::SXTW, DL: dl,
1526 VT: MVT::i32);
1527 if (isWorthFoldingAddr(V: RHS, Size))
1528 return true;
1529 }
1530
1531 return false;
1532}
1533
1534// Check if the given immediate is preferred by ADD. If an immediate can be
1535// encoded in an ADD, or it can be encoded in an "ADD LSL #12" and can not be
1536// encoded by one MOVZ, return true.
1537static bool isPreferredADD(int64_t ImmOff) {
1538 // Constant in [0x0, 0xfff] can be encoded in ADD.
1539 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
1540 return true;
1541 // Check if it can be encoded in an "ADD LSL #12".
1542 if ((ImmOff & 0xffffffffff000fffLL) == 0x0LL)
1543 // As a single MOVZ is faster than a "ADD of LSL #12", ignore such constant.
1544 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
1545 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
1546 return false;
1547}
1548
1549bool AArch64DAGToDAGISel::SelectAddrModeXRO(SDValue N, unsigned Size,
1550 SDValue &Base, SDValue &Offset,
1551 SDValue &SignExtend,
1552 SDValue &DoShift) {
1553 if (N.getOpcode() != ISD::ADD)
1554 return false;
1555 SDValue LHS = N.getOperand(i: 0);
1556 SDValue RHS = N.getOperand(i: 1);
1557 SDLoc DL(N);
1558
1559 // Check if this particular node is reused in any non-memory related
1560 // operation. If yes, do not try to fold this node into the address
1561 // computation, since the computation will be kept.
1562 const SDNode *Node = N.getNode();
1563 for (SDNode *UI : Node->users()) {
1564 if (!isMemOpOrPrefetch(N: UI))
1565 return false;
1566 }
1567
1568 // Watch out if RHS is a wide immediate, it can not be selected into
1569 // [BaseReg+Imm] addressing mode. Also it may not be able to be encoded into
1570 // ADD/SUB. Instead it will use [BaseReg + 0] address mode and generate
1571 // instructions like:
1572 // MOV X0, WideImmediate
1573 // ADD X1, BaseReg, X0
1574 // LDR X2, [X1, 0]
1575 // For such situation, using [BaseReg, XReg] addressing mode can save one
1576 // ADD/SUB:
1577 // MOV X0, WideImmediate
1578 // LDR X2, [BaseReg, X0]
1579 if (isa<ConstantSDNode>(Val: RHS)) {
1580 int64_t ImmOff = (int64_t)RHS->getAsZExtVal();
1581 // Skip the immediate can be selected by load/store addressing mode.
1582 // Also skip the immediate can be encoded by a single ADD (SUB is also
1583 // checked by using -ImmOff).
1584 if (isValidAsScaledImmediate(Offset: ImmOff, Range: 0x1000, Size) ||
1585 isPreferredADD(ImmOff) || isPreferredADD(ImmOff: -ImmOff))
1586 return false;
1587
1588 SDValue Ops[] = { RHS };
1589 SDNode *MOVI =
1590 CurDAG->getMachineNode(Opcode: AArch64::MOVi64imm, dl: DL, VT: MVT::i64, Ops);
1591 SDValue MOVIV = SDValue(MOVI, 0);
1592 // This ADD of two X register will be selected into [Reg+Reg] mode.
1593 N = CurDAG->getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: LHS, N2: MOVIV);
1594 }
1595
1596 // Remember if it is worth folding N when it produces extended register.
1597 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(V: N, Size);
1598
1599 // Try to match a shifted extend on the RHS.
1600 if (IsExtendedRegisterWorthFolding && RHS.getOpcode() == ISD::SHL &&
1601 SelectExtendedSHL(N: RHS, Size, WantExtend: false, Offset, SignExtend)) {
1602 Base = LHS;
1603 DoShift = CurDAG->getTargetConstant(Val: true, DL, VT: MVT::i32);
1604 return true;
1605 }
1606
1607 // Try to match a shifted extend on the LHS.
1608 if (IsExtendedRegisterWorthFolding && LHS.getOpcode() == ISD::SHL &&
1609 SelectExtendedSHL(N: LHS, Size, WantExtend: false, Offset, SignExtend)) {
1610 Base = RHS;
1611 DoShift = CurDAG->getTargetConstant(Val: true, DL, VT: MVT::i32);
1612 return true;
1613 }
1614
1615 // Match any non-shifted, non-extend, non-immediate add expression.
1616 Base = LHS;
1617 Offset = RHS;
1618 SignExtend = CurDAG->getTargetConstant(Val: false, DL, VT: MVT::i32);
1619 DoShift = CurDAG->getTargetConstant(Val: false, DL, VT: MVT::i32);
1620 // Reg1 + Reg2 is free: no check needed.
1621 return true;
1622}
1623
1624SDValue AArch64DAGToDAGISel::createDTuple(ArrayRef<SDValue> Regs) {
1625 static const unsigned RegClassIDs[] = {
1626 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
1627 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
1628 AArch64::dsub2, AArch64::dsub3};
1629
1630 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1631}
1632
1633SDValue AArch64DAGToDAGISel::createQTuple(ArrayRef<SDValue> Regs) {
1634 static const unsigned RegClassIDs[] = {
1635 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
1636 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
1637 AArch64::qsub2, AArch64::qsub3};
1638
1639 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1640}
1641
1642SDValue AArch64DAGToDAGISel::createZTuple(ArrayRef<SDValue> Regs) {
1643 static const unsigned RegClassIDs[] = {AArch64::ZPR2RegClassID,
1644 AArch64::ZPR3RegClassID,
1645 AArch64::ZPR4RegClassID};
1646 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1647 AArch64::zsub2, AArch64::zsub3};
1648
1649 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1650}
1651
1652SDValue AArch64DAGToDAGISel::createZMulTuple(ArrayRef<SDValue> Regs) {
1653 assert(Regs.size() == 2 || Regs.size() == 4);
1654
1655 // The createTuple interface requires 3 RegClassIDs for each possible
1656 // tuple type even though we only have them for ZPR2 and ZPR4.
1657 static const unsigned RegClassIDs[] = {AArch64::ZPR2Mul2RegClassID, 0,
1658 AArch64::ZPR4Mul4RegClassID};
1659 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1660 AArch64::zsub2, AArch64::zsub3};
1661 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1662}
1663
1664SDValue AArch64DAGToDAGISel::createTuple(ArrayRef<SDValue> Regs,
1665 const unsigned RegClassIDs[],
1666 const unsigned SubRegs[]) {
1667 // There's no special register-class for a vector-list of 1 element: it's just
1668 // a vector.
1669 if (Regs.size() == 1)
1670 return Regs[0];
1671
1672 assert(Regs.size() >= 2 && Regs.size() <= 4);
1673
1674 SDLoc DL(Regs[0]);
1675
1676 SmallVector<SDValue, 4> Ops;
1677
1678 // First operand of REG_SEQUENCE is the desired RegClass.
1679 Ops.push_back(
1680 Elt: CurDAG->getTargetConstant(Val: RegClassIDs[Regs.size() - 2], DL, VT: MVT::i32));
1681
1682 // Then we get pairs of source & subregister-position for the components.
1683 for (unsigned i = 0; i < Regs.size(); ++i) {
1684 Ops.push_back(Elt: Regs[i]);
1685 Ops.push_back(Elt: CurDAG->getTargetConstant(Val: SubRegs[i], DL, VT: MVT::i32));
1686 }
1687
1688 SDNode *N =
1689 CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL, VT: MVT::Untyped, Ops);
1690 return SDValue(N, 0);
1691}
1692
1693void AArch64DAGToDAGISel::SelectTable(SDNode *N, unsigned NumVecs, unsigned Opc,
1694 bool isExt) {
1695 SDLoc dl(N);
1696 EVT VT = N->getValueType(ResNo: 0);
1697
1698 unsigned ExtOff = isExt;
1699
1700 // Form a REG_SEQUENCE to force register allocation.
1701 unsigned Vec0Off = ExtOff + 1;
1702 SmallVector<SDValue, 4> Regs(N->ops().slice(N: Vec0Off, M: NumVecs));
1703 SDValue RegSeq = createQTuple(Regs);
1704
1705 SmallVector<SDValue, 6> Ops;
1706 if (isExt)
1707 Ops.push_back(Elt: N->getOperand(Num: 1));
1708 Ops.push_back(Elt: RegSeq);
1709 Ops.push_back(Elt: N->getOperand(Num: NumVecs + ExtOff + 1));
1710 ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: Opc, dl, VT, Ops));
1711}
1712
1713static std::tuple<SDValue, SDValue>
1714extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG) {
1715 SDLoc DL(Disc);
1716 SDValue AddrDisc;
1717 SDValue ConstDisc;
1718
1719 // If this is a blend, remember the constant and address discriminators.
1720 // Otherwise, it's either a constant discriminator, or a non-blended
1721 // address discriminator.
1722 if (Disc->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
1723 Disc->getConstantOperandVal(Num: 0) == Intrinsic::ptrauth_blend) {
1724 AddrDisc = Disc->getOperand(Num: 1);
1725 ConstDisc = Disc->getOperand(Num: 2);
1726 } else {
1727 ConstDisc = Disc;
1728 }
1729
1730 // If the constant discriminator (either the blend RHS, or the entire
1731 // discriminator value) isn't a 16-bit constant, bail out, and let the
1732 // discriminator be computed separately.
1733 auto *ConstDiscN = dyn_cast<ConstantSDNode>(Val&: ConstDisc);
1734 if (!ConstDiscN || !isUInt<16>(x: ConstDiscN->getZExtValue()))
1735 return std::make_tuple(args: DAG->getTargetConstant(Val: 0, DL, VT: MVT::i64), args&: Disc);
1736
1737 // If there's no address discriminator, use XZR directly.
1738 if (!AddrDisc)
1739 AddrDisc = DAG->getRegister(Reg: AArch64::XZR, VT: MVT::i64);
1740
1741 return std::make_tuple(
1742 args: DAG->getTargetConstant(Val: ConstDiscN->getZExtValue(), DL, VT: MVT::i64),
1743 args&: AddrDisc);
1744}
1745
1746void AArch64DAGToDAGISel::SelectPtrauthAuth(SDNode *N) {
1747 SDLoc DL(N);
1748 // IntrinsicID is operand #0
1749 SDValue Val = N->getOperand(Num: 1);
1750 SDValue AUTKey = N->getOperand(Num: 2);
1751 SDValue AUTDisc = N->getOperand(Num: 3);
1752
1753 unsigned AUTKeyC = cast<ConstantSDNode>(Val&: AUTKey)->getZExtValue();
1754 AUTKey = CurDAG->getTargetConstant(Val: AUTKeyC, DL, VT: MVT::i64);
1755
1756 SDValue AUTAddrDisc, AUTConstDisc;
1757 std::tie(args&: AUTConstDisc, args&: AUTAddrDisc) =
1758 extractPtrauthBlendDiscriminators(Disc: AUTDisc, DAG: CurDAG);
1759
1760 if (!Subtarget->isX16X17Safer()) {
1761 std::vector<SDValue> Ops = {Val, AUTKey, AUTConstDisc, AUTAddrDisc};
1762 // Copy deactivation symbol if present.
1763 if (N->getNumOperands() > 4)
1764 Ops.push_back(x: N->getOperand(Num: 4));
1765
1766 SDNode *AUT =
1767 CurDAG->getMachineNode(Opcode: AArch64::AUTxMxN, dl: DL, VT1: MVT::i64, VT2: MVT::i64, Ops);
1768 ReplaceNode(F: N, T: AUT);
1769 } else {
1770 SDValue X16Copy = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl: DL,
1771 Reg: AArch64::X16, N: Val, Glue: SDValue());
1772 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, X16Copy.getValue(R: 1)};
1773
1774 SDNode *AUT = CurDAG->getMachineNode(Opcode: AArch64::AUTx16x17, dl: DL, VT: MVT::i64, Ops);
1775 ReplaceNode(F: N, T: AUT);
1776 }
1777}
1778
1779void AArch64DAGToDAGISel::SelectPtrauthResign(SDNode *N) {
1780 SDLoc DL(N);
1781 // IntrinsicID is operand #0, if W_CHAIN it is #1
1782 int OffsetBase = N->getOpcode() == ISD::INTRINSIC_W_CHAIN ? 1 : 0;
1783 SDValue Val = N->getOperand(Num: OffsetBase + 1);
1784 SDValue AUTKey = N->getOperand(Num: OffsetBase + 2);
1785 SDValue AUTDisc = N->getOperand(Num: OffsetBase + 3);
1786 SDValue PACKey = N->getOperand(Num: OffsetBase + 4);
1787 SDValue PACDisc = N->getOperand(Num: OffsetBase + 5);
1788 uint32_t IntNum = N->getConstantOperandVal(Num: OffsetBase + 0);
1789 bool HasLoad = IntNum == Intrinsic::ptrauth_resign_load_relative;
1790
1791 unsigned AUTKeyC = cast<ConstantSDNode>(Val&: AUTKey)->getZExtValue();
1792 unsigned PACKeyC = cast<ConstantSDNode>(Val&: PACKey)->getZExtValue();
1793
1794 AUTKey = CurDAG->getTargetConstant(Val: AUTKeyC, DL, VT: MVT::i64);
1795 PACKey = CurDAG->getTargetConstant(Val: PACKeyC, DL, VT: MVT::i64);
1796
1797 SDValue AUTAddrDisc, AUTConstDisc;
1798 std::tie(args&: AUTConstDisc, args&: AUTAddrDisc) =
1799 extractPtrauthBlendDiscriminators(Disc: AUTDisc, DAG: CurDAG);
1800
1801 SDValue PACAddrDisc, PACConstDisc;
1802 std::tie(args&: PACConstDisc, args&: PACAddrDisc) =
1803 extractPtrauthBlendDiscriminators(Disc: PACDisc, DAG: CurDAG);
1804
1805 SDValue X16Copy = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl: DL,
1806 Reg: AArch64::X16, N: Val, Glue: SDValue());
1807
1808 if (HasLoad) {
1809 SDValue Addend = N->getOperand(Num: OffsetBase + 6);
1810 SDValue IncomingChain = N->getOperand(Num: 0);
1811 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc,
1812 PACKey, PACConstDisc, PACAddrDisc,
1813 Addend, IncomingChain, X16Copy.getValue(R: 1)};
1814
1815 SDNode *AUTRELLOADPAC = CurDAG->getMachineNode(Opcode: AArch64::AUTRELLOADPAC, dl: DL,
1816 VT1: MVT::i64, VT2: MVT::Other, Ops);
1817 ReplaceNode(F: N, T: AUTRELLOADPAC);
1818 } else {
1819 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, PACKey,
1820 PACConstDisc, PACAddrDisc, X16Copy.getValue(R: 1)};
1821
1822 SDNode *AUTPAC = CurDAG->getMachineNode(Opcode: AArch64::AUTPAC, dl: DL, VT: MVT::i64, Ops);
1823 ReplaceNode(F: N, T: AUTPAC);
1824 }
1825}
1826
1827void AArch64DAGToDAGISel::SelectPtrauthResignWithPC(SDNode *N) {
1828 SDLoc DL(N);
1829 SDValue Val = N->getOperand(Num: 1);
1830 SDValue AUTKey = N->getOperand(Num: 2);
1831 SDValue AUTDisc = N->getOperand(Num: 3);
1832 SDValue AUTPC = N->getOperand(Num: 4);
1833 SDValue PACKey = N->getOperand(Num: 5);
1834 SDValue PACDisc = N->getOperand(Num: 6);
1835
1836 unsigned AUTKeyC = cast<ConstantSDNode>(Val&: AUTKey)->getZExtValue();
1837 unsigned PACKeyC = cast<ConstantSDNode>(Val&: PACKey)->getZExtValue();
1838
1839 AUTKey = CurDAG->getTargetConstant(Val: AUTKeyC, DL, VT: MVT::i64);
1840 PACKey = CurDAG->getTargetConstant(Val: PACKeyC, DL, VT: MVT::i64);
1841
1842 SDValue PACAddrDisc, PACConstDisc;
1843 std::tie(args&: PACConstDisc, args&: PACAddrDisc) =
1844 extractPtrauthBlendDiscriminators(Disc: PACDisc, DAG: CurDAG);
1845
1846 SDValue X17Copy = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl: DL,
1847 Reg: AArch64::X17, N: Val, Glue: SDValue());
1848 SDValue X16Copy = CurDAG->getCopyToReg(
1849 Chain: CurDAG->getEntryNode(), dl: DL, Reg: AArch64::X16, N: AUTDisc, Glue: X17Copy.getValue(R: 1));
1850 SDValue X15Copy = CurDAG->getCopyToReg(
1851 Chain: CurDAG->getEntryNode(), dl: DL, Reg: AArch64::X15, N: AUTPC, Glue: X16Copy.getValue(R: 1));
1852
1853 SDValue Ops[] = {AUTKey, PACKey, PACConstDisc, PACAddrDisc,
1854 X15Copy.getValue(R: 1)};
1855 SDNode *AUTPCPAC =
1856 CurDAG->getMachineNode(Opcode: AArch64::AUTPCPAC, dl: DL, VT: MVT::i64, Ops);
1857 ReplaceNode(F: N, T: AUTPCPAC);
1858}
1859
1860bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *N) {
1861 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
1862 if (LD->isUnindexed())
1863 return false;
1864 EVT VT = LD->getMemoryVT();
1865 EVT DstVT = N->getValueType(ResNo: 0);
1866 ISD::MemIndexedMode AM = LD->getAddressingMode();
1867 bool IsPre = AM == ISD::PRE_INC || AM == ISD::PRE_DEC;
1868 ConstantSDNode *OffsetOp = cast<ConstantSDNode>(Val: LD->getOffset());
1869 int OffsetVal = (int)OffsetOp->getZExtValue();
1870
1871 // We're not doing validity checking here. That was done when checking
1872 // if we should mark the load as indexed or not. We're just selecting
1873 // the right instruction.
1874 unsigned Opcode = 0;
1875
1876 ISD::LoadExtType ExtType = LD->getExtensionType();
1877 bool InsertTo64 = false;
1878 bool UseLd1 =
1879 (VT.is64BitVector() || VT.is128BitVector()) &&
1880 (!Subtarget->isLittleEndian() || (Subtarget->requiresStrictAlign() &&
1881 LD->getAlign() < VT.getStoreSize()));
1882 if (VT == MVT::i64)
1883 Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
1884 else if (VT == MVT::i32) {
1885 if (ExtType == ISD::NON_EXTLOAD)
1886 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1887 else if (ExtType == ISD::SEXTLOAD)
1888 Opcode = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
1889 else {
1890 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1891 InsertTo64 = true;
1892 // The result of the load is only i32. It's the subreg_to_reg that makes
1893 // it into an i64.
1894 DstVT = MVT::i32;
1895 }
1896 } else if (VT == MVT::i16) {
1897 if (ExtType == ISD::SEXTLOAD) {
1898 if (DstVT == MVT::i64)
1899 Opcode = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
1900 else
1901 Opcode = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
1902 } else {
1903 Opcode = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
1904 InsertTo64 = DstVT == MVT::i64;
1905 // The result of the load is only i32. It's the subreg_to_reg that makes
1906 // it into an i64.
1907 DstVT = MVT::i32;
1908 }
1909 } else if (VT == MVT::i8) {
1910 if (ExtType == ISD::SEXTLOAD) {
1911 if (DstVT == MVT::i64)
1912 Opcode = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
1913 else
1914 Opcode = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
1915 } else {
1916 Opcode = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
1917 InsertTo64 = DstVT == MVT::i64;
1918 // The result of the load is only i32. It's the subreg_to_reg that makes
1919 // it into an i64.
1920 DstVT = MVT::i32;
1921 }
1922 } else if (VT == MVT::f16) {
1923 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1924 } else if (VT == MVT::bf16) {
1925 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1926 } else if (VT == MVT::f32) {
1927 Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
1928 } else if (VT == MVT::f64 || (VT.is64BitVector() && !UseLd1)) {
1929 Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
1930 } else if (VT.is128BitVector() && !UseLd1) {
1931 Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
1932 } else if (VT.is64BitVector() && UseLd1) {
1933 if (IsPre || OffsetVal != 8)
1934 return false;
1935 switch (VT.getScalarSizeInBits()) {
1936 case 8:
1937 Opcode = AArch64::LD1Onev8b_POST;
1938 break;
1939 case 16:
1940 Opcode = AArch64::LD1Onev4h_POST;
1941 break;
1942 case 32:
1943 Opcode = AArch64::LD1Onev2s_POST;
1944 break;
1945 case 64:
1946 Opcode = AArch64::LD1Onev1d_POST;
1947 break;
1948 default:
1949 llvm_unreachable("Expected vector element to be a power of 2");
1950 }
1951 } else if (VT.is128BitVector() && UseLd1) {
1952 if (IsPre || OffsetVal != 16)
1953 return false;
1954 switch (VT.getScalarSizeInBits()) {
1955 case 8:
1956 Opcode = AArch64::LD1Onev16b_POST;
1957 break;
1958 case 16:
1959 Opcode = AArch64::LD1Onev8h_POST;
1960 break;
1961 case 32:
1962 Opcode = AArch64::LD1Onev4s_POST;
1963 break;
1964 case 64:
1965 Opcode = AArch64::LD1Onev2d_POST;
1966 break;
1967 default:
1968 llvm_unreachable("Expected vector element to be a power of 2");
1969 }
1970 } else
1971 return false;
1972 SDValue Chain = LD->getChain();
1973 SDValue Base = LD->getBasePtr();
1974 SDLoc dl(N);
1975 // LD1 encodes an immediate offset by using XZR as the offset register.
1976 SDValue Offset = UseLd1 ? CurDAG->getRegister(Reg: AArch64::XZR, VT: MVT::i64)
1977 : CurDAG->getTargetConstant(Val: OffsetVal, DL: dl, VT: MVT::i64);
1978 SDValue Ops[] = { Base, Offset, Chain };
1979 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, VT1: MVT::i64, VT2: DstVT,
1980 VT3: MVT::Other, Ops);
1981
1982 // Transfer memoperands.
1983 MachineMemOperand *MemOp = cast<MemSDNode>(Val: N)->getMemOperand();
1984 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Res), NewMemRefs: {MemOp});
1985
1986 // Either way, we're replacing the node, so tell the caller that.
1987 SDValue LoadedVal = SDValue(Res, 1);
1988 if (InsertTo64) {
1989 SDValue SubReg = CurDAG->getTargetConstant(Val: AArch64::sub_32, DL: dl, VT: MVT::i32);
1990 LoadedVal = SDValue(CurDAG->getMachineNode(Opcode: AArch64::SUBREG_TO_REG, dl,
1991 VT: MVT::i64, Op1: LoadedVal, Op2: SubReg),
1992 0);
1993 }
1994
1995 ReplaceUses(F: SDValue(N, 0), T: LoadedVal);
1996 ReplaceUses(F: SDValue(N, 1), T: SDValue(Res, 0));
1997 ReplaceUses(F: SDValue(N, 2), T: SDValue(Res, 2));
1998 CurDAG->RemoveDeadNode(N);
1999 return true;
2000}
2001
2002void AArch64DAGToDAGISel::SelectLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
2003 unsigned SubRegIdx) {
2004 SDLoc dl(N);
2005 EVT VT = N->getValueType(ResNo: 0);
2006 SDValue Chain = N->getOperand(Num: 0);
2007
2008 SDValue Ops[] = {N->getOperand(Num: 2), // Mem operand;
2009 Chain};
2010
2011 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2012
2013 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2014 SDValue SuperReg = SDValue(Ld, 0);
2015 for (unsigned i = 0; i < NumVecs; ++i)
2016 ReplaceUses(F: SDValue(N, i),
2017 T: CurDAG->getTargetExtractSubreg(SRIdx: SubRegIdx + i, DL: dl, VT, Operand: SuperReg));
2018
2019 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 1));
2020
2021 // Transfer memoperands. In the case of AArch64::LD64B, there won't be one,
2022 // because it's too simple to have needed special treatment during lowering.
2023 if (auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(Val: N)) {
2024 MachineMemOperand *MemOp = MemIntr->getMemOperand();
2025 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Ld), NewMemRefs: {MemOp});
2026 }
2027
2028 CurDAG->RemoveDeadNode(N);
2029}
2030
2031void AArch64DAGToDAGISel::SelectPostLoad(SDNode *N, unsigned NumVecs,
2032 unsigned Opc, unsigned SubRegIdx) {
2033 SDLoc dl(N);
2034 EVT VT = N->getValueType(ResNo: 0);
2035 SDValue Chain = N->getOperand(Num: 0);
2036
2037 SDValue Ops[] = {N->getOperand(Num: 1), // Mem operand
2038 N->getOperand(Num: 2), // Incremental
2039 Chain};
2040
2041 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2042 MVT::Untyped, MVT::Other};
2043
2044 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2045
2046 // Update uses of write back register
2047 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 0));
2048
2049 // Update uses of vector list
2050 SDValue SuperReg = SDValue(Ld, 1);
2051 if (NumVecs == 1)
2052 ReplaceUses(F: SDValue(N, 0), T: SuperReg);
2053 else
2054 for (unsigned i = 0; i < NumVecs; ++i)
2055 ReplaceUses(F: SDValue(N, i),
2056 T: CurDAG->getTargetExtractSubreg(SRIdx: SubRegIdx + i, DL: dl, VT, Operand: SuperReg));
2057
2058 // Transfer memoperands.
2059 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2060 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Ld), NewMemRefs: {MemOp});
2061
2062 // Update the chain
2063 ReplaceUses(F: SDValue(N, NumVecs + 1), T: SDValue(Ld, 2));
2064 CurDAG->RemoveDeadNode(N);
2065}
2066
2067/// Optimize \param OldBase and \param OldOffset selecting the best addressing
2068/// mode. Returns a tuple consisting of an Opcode, an SDValue representing the
2069/// new Base and an SDValue representing the new offset.
2070std::tuple<unsigned, SDValue, SDValue>
2071AArch64DAGToDAGISel::findAddrModeSVELoadStore(SDNode *N, unsigned Opc_rr,
2072 unsigned Opc_ri,
2073 const SDValue &OldBase,
2074 const SDValue &OldOffset,
2075 unsigned Scale) {
2076 SDValue NewBase = OldBase;
2077 SDValue NewOffset = OldOffset;
2078 // Detect a possible Reg+Imm addressing mode.
2079 const bool IsRegImm = SelectAddrModeIndexedSVE</*Min=*/-8, /*Max=*/7>(
2080 Root: N, N: OldBase, Base&: NewBase, OffImm&: NewOffset);
2081
2082 // Detect a possible reg+reg addressing mode, but only if we haven't already
2083 // detected a Reg+Imm one.
2084 const bool IsRegReg =
2085 !IsRegImm && SelectSVERegRegAddrMode(N: OldBase, Scale, Base&: NewBase, Offset&: NewOffset);
2086
2087 // Select the instruction.
2088 return std::make_tuple(args&: IsRegReg ? Opc_rr : Opc_ri, args&: NewBase, args&: NewOffset);
2089}
2090
2091enum class SelectTypeKind {
2092 Int1 = 0,
2093 Int = 1,
2094 FP = 2,
2095 AnyType = 3,
2096};
2097
2098/// This function selects an opcode from a list of opcodes, which is
2099/// expected to be the opcode for { 8-bit, 16-bit, 32-bit, 64-bit }
2100/// element types, in this order.
2101template <SelectTypeKind Kind>
2102static unsigned SelectOpcodeFromVT(EVT VT, ArrayRef<unsigned> Opcodes) {
2103 // Only match scalable vector VTs
2104 if (!VT.isScalableVector())
2105 return 0;
2106
2107 EVT EltVT = VT.getVectorElementType();
2108 unsigned Key = VT.getVectorMinNumElements();
2109 switch (Kind) {
2110 case SelectTypeKind::AnyType:
2111 break;
2112 case SelectTypeKind::Int:
2113 if (EltVT != MVT::i8 && EltVT != MVT::i16 && EltVT != MVT::i32 &&
2114 EltVT != MVT::i64)
2115 return 0;
2116 break;
2117 case SelectTypeKind::Int1:
2118 if (EltVT != MVT::i1)
2119 return 0;
2120 break;
2121 case SelectTypeKind::FP:
2122 if (EltVT == MVT::bf16)
2123 Key = 16;
2124 else if (EltVT != MVT::bf16 && EltVT != MVT::f16 && EltVT != MVT::f32 &&
2125 EltVT != MVT::f64)
2126 return 0;
2127 break;
2128 }
2129
2130 unsigned Offset;
2131 switch (Key) {
2132 case 16: // 8-bit or bf16
2133 Offset = 0;
2134 break;
2135 case 8: // 16-bit
2136 Offset = 1;
2137 break;
2138 case 4: // 32-bit
2139 Offset = 2;
2140 break;
2141 case 2: // 64-bit
2142 Offset = 3;
2143 break;
2144 default:
2145 return 0;
2146 }
2147
2148 return (Opcodes.size() <= Offset) ? 0 : Opcodes[Offset];
2149}
2150
2151// This function is almost identical to SelectWhilePair, but has an
2152// extra check on the range of the immediate operand.
2153// TODO: Merge these two functions together at some point?
2154void AArch64DAGToDAGISel::SelectPExtPair(SDNode *N, unsigned Opc) {
2155 // Immediate can be either 0 or 1.
2156 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2)))
2157 if (Imm->getZExtValue() > 1)
2158 return;
2159
2160 SDLoc DL(N);
2161 EVT VT = N->getValueType(ResNo: 0);
2162 SDValue Ops[] = {N->getOperand(Num: 1), N->getOperand(Num: 2)};
2163 SDNode *WhilePair = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Ops);
2164 SDValue SuperReg = SDValue(WhilePair, 0);
2165
2166 for (unsigned I = 0; I < 2; ++I)
2167 ReplaceUses(F: SDValue(N, I), T: CurDAG->getTargetExtractSubreg(
2168 SRIdx: AArch64::psub0 + I, DL, VT, Operand: SuperReg));
2169
2170 CurDAG->RemoveDeadNode(N);
2171}
2172
2173void AArch64DAGToDAGISel::SelectWhilePair(SDNode *N, unsigned Opc) {
2174 SDLoc DL(N);
2175 EVT VT = N->getValueType(ResNo: 0);
2176
2177 SDValue Ops[] = {N->getOperand(Num: 1), N->getOperand(Num: 2)};
2178
2179 SDNode *WhilePair = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Ops);
2180 SDValue SuperReg = SDValue(WhilePair, 0);
2181
2182 for (unsigned I = 0; I < 2; ++I)
2183 ReplaceUses(F: SDValue(N, I), T: CurDAG->getTargetExtractSubreg(
2184 SRIdx: AArch64::psub0 + I, DL, VT, Operand: SuperReg));
2185
2186 CurDAG->RemoveDeadNode(N);
2187}
2188
2189void AArch64DAGToDAGISel::SelectCVTIntrinsic(SDNode *N, unsigned NumVecs,
2190 unsigned Opcode) {
2191 EVT VT = N->getValueType(ResNo: 0);
2192 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2193 SDValue Ops = createZTuple(Regs);
2194 SDLoc DL(N);
2195 SDNode *Intrinsic = CurDAG->getMachineNode(Opcode, dl: DL, VT: MVT::Untyped, Op1: Ops);
2196 SDValue SuperReg = SDValue(Intrinsic, 0);
2197 for (unsigned i = 0; i < NumVecs; ++i)
2198 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2199 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2200
2201 CurDAG->RemoveDeadNode(N);
2202}
2203
2204void AArch64DAGToDAGISel::SelectCVTIntrinsicFP8(SDNode *N, unsigned NumVecs,
2205 unsigned Opcode) {
2206 SDLoc DL(N);
2207 EVT VT = N->getValueType(ResNo: 0);
2208 SmallVector<SDValue, 4> Ops(N->op_begin() + 2, N->op_end());
2209 Ops.push_back(/*Chain*/ Elt: N->getOperand(Num: 0));
2210
2211 SDNode *Instruction =
2212 CurDAG->getMachineNode(Opcode, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2213 SDValue SuperReg = SDValue(Instruction, 0);
2214
2215 for (unsigned i = 0; i < NumVecs; ++i)
2216 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2217 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2218
2219 // Copy chain
2220 unsigned ChainIdx = NumVecs;
2221 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Instruction, 1));
2222 CurDAG->RemoveDeadNode(N);
2223}
2224
2225void AArch64DAGToDAGISel::SelectDestructiveMultiIntrinsic(SDNode *N,
2226 unsigned NumVecs,
2227 bool IsZmMulti,
2228 unsigned Opcode,
2229 bool HasPred) {
2230 assert(Opcode != 0 && "Unexpected opcode");
2231
2232 SDLoc DL(N);
2233 EVT VT = N->getValueType(ResNo: 0);
2234 SDUse *OpsIter = N->op_begin() + 1; // Skip intrinsic ID
2235 SmallVector<SDValue, 4> Ops;
2236
2237 auto GetMultiVecOperand = [&]() {
2238 SmallVector<SDValue, 4> Regs(OpsIter, OpsIter + NumVecs);
2239 OpsIter += NumVecs;
2240 return createZMulTuple(Regs);
2241 };
2242
2243 if (HasPred)
2244 Ops.push_back(Elt: *OpsIter++);
2245
2246 Ops.push_back(Elt: GetMultiVecOperand());
2247 if (IsZmMulti)
2248 Ops.push_back(Elt: GetMultiVecOperand());
2249 else
2250 Ops.push_back(Elt: *OpsIter++);
2251
2252 // Append any remaining operands.
2253 Ops.append(in_start: OpsIter, in_end: N->op_end());
2254 SDNode *Intrinsic;
2255 Intrinsic = CurDAG->getMachineNode(Opcode, dl: DL, VT: MVT::Untyped, Ops);
2256 SDValue SuperReg = SDValue(Intrinsic, 0);
2257 for (unsigned i = 0; i < NumVecs; ++i)
2258 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2259 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2260
2261 CurDAG->RemoveDeadNode(N);
2262}
2263
2264void AArch64DAGToDAGISel::SelectPredicatedLoad(SDNode *N, unsigned NumVecs,
2265 unsigned Scale, unsigned Opc_ri,
2266 unsigned Opc_rr, bool IsIntr) {
2267 assert(Scale < 5 && "Invalid scaling value.");
2268 SDLoc DL(N);
2269 EVT VT = N->getValueType(ResNo: 0);
2270 SDValue Chain = N->getOperand(Num: 0);
2271
2272 // Optimize addressing mode.
2273 SDValue Base, Offset;
2274 unsigned Opc;
2275 std::tie(args&: Opc, args&: Base, args&: Offset) = findAddrModeSVELoadStore(
2276 N, Opc_rr, Opc_ri, OldBase: N->getOperand(Num: IsIntr ? 3 : 2),
2277 OldOffset: CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64), Scale);
2278
2279 SDValue Ops[] = {N->getOperand(Num: IsIntr ? 2 : 1), // Predicate
2280 Base, // Memory operand
2281 Offset, Chain};
2282
2283 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2284
2285 SDNode *Load = CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: ResTys, Ops);
2286 SDValue SuperReg = SDValue(Load, 0);
2287 for (unsigned i = 0; i < NumVecs; ++i)
2288 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2289 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2290
2291 // Copy chain
2292 unsigned ChainIdx = NumVecs;
2293 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Load, 1));
2294 CurDAG->RemoveDeadNode(N);
2295}
2296
2297void AArch64DAGToDAGISel::SelectContiguousMultiVectorLoad(SDNode *N,
2298 unsigned NumVecs,
2299 unsigned Scale,
2300 unsigned Opc_ri,
2301 unsigned Opc_rr) {
2302 assert(Scale < 4 && "Invalid scaling value.");
2303 SDLoc DL(N);
2304 EVT VT = N->getValueType(ResNo: 0);
2305 SDValue Chain = N->getOperand(Num: 0);
2306
2307 SDValue PNg = N->getOperand(Num: 2);
2308 SDValue Base = N->getOperand(Num: 3);
2309 SDValue Offset = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64);
2310 unsigned Opc;
2311 std::tie(args&: Opc, args&: Base, args&: Offset) =
2312 findAddrModeSVELoadStore(N, Opc_rr, Opc_ri, OldBase: Base, OldOffset: Offset, Scale);
2313
2314 SDValue Ops[] = {PNg, // Predicate-as-counter
2315 Base, // Memory operand
2316 Offset, Chain};
2317
2318 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2319
2320 SDNode *Load = CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: ResTys, Ops);
2321 SDValue SuperReg = SDValue(Load, 0);
2322 for (unsigned i = 0; i < NumVecs; ++i)
2323 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2324 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2325
2326 // Copy chain
2327 unsigned ChainIdx = NumVecs;
2328 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Load, 1));
2329 CurDAG->RemoveDeadNode(N);
2330}
2331
2332void AArch64DAGToDAGISel::SelectFrintFromVT(SDNode *N, unsigned NumVecs,
2333 unsigned Opcode) {
2334 if (N->getValueType(ResNo: 0) != MVT::nxv4f32)
2335 return;
2336 SelectUnaryMultiIntrinsic(N, NumOutVecs: NumVecs, IsTupleInput: true, Opc: Opcode);
2337}
2338
2339void AArch64DAGToDAGISel::SelectMultiVectorLutiLane(SDNode *Node,
2340 unsigned NumOutVecs,
2341 unsigned Opc,
2342 uint32_t MaxImm) {
2343 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 4)))
2344 if (Imm->getZExtValue() > MaxImm)
2345 return;
2346
2347 SDValue ZtValue;
2348 if (!ImmToReg<AArch64::ZT0, 0>(N: Node->getOperand(Num: 2), Imm&: ZtValue))
2349 return;
2350
2351 SDValue Chain = Node->getOperand(Num: 0);
2352 SDValue Ops[] = {ZtValue, Node->getOperand(Num: 3), Node->getOperand(Num: 4), Chain};
2353 SDLoc DL(Node);
2354 EVT VT = Node->getValueType(ResNo: 0);
2355
2356 SDNode *Instruction =
2357 CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2358 SDValue SuperReg = SDValue(Instruction, 0);
2359
2360 for (unsigned I = 0; I < NumOutVecs; ++I)
2361 ReplaceUses(F: SDValue(Node, I), T: CurDAG->getTargetExtractSubreg(
2362 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2363
2364 // Copy chain
2365 unsigned ChainIdx = NumOutVecs;
2366 ReplaceUses(F: SDValue(Node, ChainIdx), T: SDValue(Instruction, 1));
2367 CurDAG->RemoveDeadNode(N: Node);
2368}
2369
2370void AArch64DAGToDAGISel::SelectMultiVectorLuti6LaneX4(SDNode *Node,
2371 unsigned NumIndexVecs) {
2372 assert((NumIndexVecs == 2 || NumIndexVecs == 3) &&
2373 "unexpected number of index vectors");
2374
2375 constexpr unsigned FirstIndexOp = 3;
2376 unsigned ImmOp = FirstIndexOp + NumIndexVecs;
2377 auto *Imm = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: ImmOp));
2378 if (!Imm || Imm->getZExtValue() > 1)
2379 return;
2380
2381 // The luti6 instruction always takes a 2-register Zm index tuple. The x3
2382 // ACLE form provides three index vectors, so the lane selects which adjacent
2383 // pair to use before forming Zm (op 3/4 or op 4/5, with op6 as imm)
2384 unsigned Lane = Imm->getZExtValue();
2385 unsigned IndexOp = FirstIndexOp;
2386 if (NumIndexVecs == 3)
2387 IndexOp += Lane;
2388
2389 SDValue TableTuple = createZTuple(Regs: {Node->getOperand(Num: 1), Node->getOperand(Num: 2)});
2390 SDValue IndexTuple =
2391 createZTuple(Regs: {Node->getOperand(Num: IndexOp), Node->getOperand(Num: IndexOp + 1)});
2392 SDValue Ops[] = {TableTuple, IndexTuple, Node->getOperand(Num: ImmOp)};
2393
2394 SDLoc DL(Node);
2395 EVT VT = Node->getValueType(ResNo: 0);
2396 SDNode *Instruction =
2397 CurDAG->getMachineNode(Opcode: AArch64::LUTI6_4Z2Z2ZI, dl: DL, VT: MVT::Untyped, Ops);
2398 SDValue SuperReg = SDValue(Instruction, 0);
2399
2400 for (unsigned I = 0; I < 4; ++I)
2401 ReplaceUses(F: SDValue(Node, I), T: CurDAG->getTargetExtractSubreg(
2402 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2403
2404 CurDAG->RemoveDeadNode(N: Node);
2405}
2406
2407void AArch64DAGToDAGISel::SelectMultiVectorLuti(SDNode *Node,
2408 unsigned NumOutVecs,
2409 unsigned Opc,
2410 unsigned NumInVecs) {
2411 assert((NumInVecs == 2 || NumInVecs == 3) &&
2412 "unexpected number of input vectors");
2413
2414 SDValue ZtValue;
2415 if (!ImmToReg<AArch64::ZT0, 0>(N: Node->getOperand(Num: 2), Imm&: ZtValue))
2416 return;
2417
2418 SmallVector<SDValue, 4> Regs(Node->ops().slice(N: 3, M: NumInVecs));
2419 SDValue ZTuple = NumInVecs == 3 ? createZTuple(Regs) : createZMulTuple(Regs);
2420 SDValue Ops[] = {ZtValue, ZTuple, Node->getOperand(Num: 0)};
2421
2422 SDLoc DL(Node);
2423 EVT VT = Node->getValueType(ResNo: 0);
2424
2425 SDNode *Instruction =
2426 CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2427 SDValue SuperReg = SDValue(Instruction, 0);
2428
2429 for (unsigned I = 0; I < NumOutVecs; ++I)
2430 ReplaceUses(F: SDValue(Node, I), T: CurDAG->getTargetExtractSubreg(
2431 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2432
2433 ReplaceUses(F: SDValue(Node, NumOutVecs), T: SDValue(Instruction, 1));
2434 CurDAG->RemoveDeadNode(N: Node);
2435}
2436
2437void AArch64DAGToDAGISel::SelectClamp(SDNode *N, unsigned NumVecs,
2438 unsigned Op) {
2439 SDLoc DL(N);
2440 EVT VT = N->getValueType(ResNo: 0);
2441
2442 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2443 SDValue Zd = createZMulTuple(Regs);
2444 SDValue Zn = N->getOperand(Num: 1 + NumVecs);
2445 SDValue Zm = N->getOperand(Num: 2 + NumVecs);
2446
2447 SDValue Ops[] = {Zd, Zn, Zm};
2448
2449 SDNode *Intrinsic = CurDAG->getMachineNode(Opcode: Op, dl: DL, VT: MVT::Untyped, Ops);
2450 SDValue SuperReg = SDValue(Intrinsic, 0);
2451 for (unsigned i = 0; i < NumVecs; ++i)
2452 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2453 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2454
2455 CurDAG->RemoveDeadNode(N);
2456}
2457
2458bool SelectSMETile(unsigned &BaseReg, unsigned TileNum) {
2459 switch (BaseReg) {
2460 default:
2461 return false;
2462 case AArch64::ZA:
2463 case AArch64::ZAB0:
2464 if (TileNum == 0)
2465 break;
2466 return false;
2467 case AArch64::ZAH0:
2468 if (TileNum <= 1)
2469 break;
2470 return false;
2471 case AArch64::ZAS0:
2472 if (TileNum <= 3)
2473 break;
2474 return false;
2475 case AArch64::ZAD0:
2476 if (TileNum <= 7)
2477 break;
2478 return false;
2479 }
2480
2481 BaseReg += TileNum;
2482 return true;
2483}
2484
2485template <unsigned MaxIdx, unsigned Scale>
2486void AArch64DAGToDAGISel::SelectMultiVectorMove(SDNode *N, unsigned NumVecs,
2487 unsigned BaseReg, unsigned Op) {
2488 unsigned TileNum = 0;
2489 if (BaseReg != AArch64::ZA)
2490 TileNum = N->getConstantOperandVal(Num: 2);
2491
2492 if (!SelectSMETile(BaseReg, TileNum))
2493 return;
2494
2495 SDValue SliceBase, Base, Offset;
2496 if (BaseReg == AArch64::ZA)
2497 SliceBase = N->getOperand(Num: 2);
2498 else
2499 SliceBase = N->getOperand(Num: 3);
2500
2501 if (!SelectSMETileSlice(N: SliceBase, MaxSize: MaxIdx, Vector&: Base, Offset, Scale))
2502 return;
2503
2504 SDLoc DL(N);
2505 SDValue SubReg = CurDAG->getRegister(Reg: BaseReg, VT: MVT::Other);
2506 SDValue Ops[] = {SubReg, Base, Offset, /*Chain*/ N->getOperand(Num: 0)};
2507 SDNode *Mov = CurDAG->getMachineNode(Opcode: Op, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2508
2509 EVT VT = N->getValueType(ResNo: 0);
2510 for (unsigned I = 0; I < NumVecs; ++I)
2511 ReplaceUses(F: SDValue(N, I),
2512 T: CurDAG->getTargetExtractSubreg(SRIdx: AArch64::zsub0 + I, DL, VT,
2513 Operand: SDValue(Mov, 0)));
2514 // Copy chain
2515 unsigned ChainIdx = NumVecs;
2516 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Mov, 1));
2517 CurDAG->RemoveDeadNode(N);
2518}
2519
2520void AArch64DAGToDAGISel::SelectMultiVectorMoveZ(SDNode *N, unsigned NumVecs,
2521 unsigned Op, unsigned MaxIdx,
2522 unsigned Scale, unsigned BaseReg) {
2523 // Slice can be in different positions
2524 // The array to vector: llvm.aarch64.sme.readz.<h/v>.<sz>(slice)
2525 // The tile to vector: llvm.aarch64.sme.readz.<h/v>.<sz>(tile, slice)
2526 SDValue SliceBase = N->getOperand(Num: 2);
2527 if (BaseReg != AArch64::ZA)
2528 SliceBase = N->getOperand(Num: 3);
2529
2530 SDValue Base, Offset;
2531 if (!SelectSMETileSlice(N: SliceBase, MaxSize: MaxIdx, Vector&: Base, Offset, Scale))
2532 return;
2533 // The correct Za tile number is computed in Machine Instruction
2534 // See EmitZAInstr
2535 // DAG cannot select Za tile as an output register with ZReg
2536 SDLoc DL(N);
2537 SmallVector<SDValue, 6> Ops;
2538 if (BaseReg != AArch64::ZA )
2539 Ops.push_back(Elt: N->getOperand(Num: 2));
2540 Ops.push_back(Elt: Base);
2541 Ops.push_back(Elt: Offset);
2542 Ops.push_back(Elt: N->getOperand(Num: 0)); //Chain
2543 SDNode *Mov = CurDAG->getMachineNode(Opcode: Op, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2544
2545 EVT VT = N->getValueType(ResNo: 0);
2546 for (unsigned I = 0; I < NumVecs; ++I)
2547 ReplaceUses(F: SDValue(N, I),
2548 T: CurDAG->getTargetExtractSubreg(SRIdx: AArch64::zsub0 + I, DL, VT,
2549 Operand: SDValue(Mov, 0)));
2550
2551 // Copy chain
2552 unsigned ChainIdx = NumVecs;
2553 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Mov, 1));
2554 CurDAG->RemoveDeadNode(N);
2555}
2556
2557void AArch64DAGToDAGISel::SelectUnaryMultiIntrinsic(SDNode *N,
2558 unsigned NumOutVecs,
2559 bool IsTupleInput,
2560 unsigned Opc) {
2561 SDLoc DL(N);
2562 EVT VT = N->getValueType(ResNo: 0);
2563 unsigned NumInVecs = N->getNumOperands() - 1;
2564
2565 SmallVector<SDValue, 6> Ops;
2566 if (IsTupleInput) {
2567 assert((NumInVecs == 2 || NumInVecs == 4) &&
2568 "Don't know how to handle multi-register input!");
2569 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumInVecs));
2570 Ops.push_back(Elt: createZMulTuple(Regs));
2571 } else {
2572 // All intrinsic nodes have the ID as the first operand, hence the "1 + I".
2573 for (unsigned I = 0; I < NumInVecs; I++)
2574 Ops.push_back(Elt: N->getOperand(Num: 1 + I));
2575 }
2576
2577 SDNode *Res = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Ops);
2578 SDValue SuperReg = SDValue(Res, 0);
2579
2580 for (unsigned I = 0; I < NumOutVecs; I++)
2581 ReplaceUses(F: SDValue(N, I), T: CurDAG->getTargetExtractSubreg(
2582 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2583 CurDAG->RemoveDeadNode(N);
2584}
2585
2586void AArch64DAGToDAGISel::SelectStore(SDNode *N, unsigned NumVecs,
2587 unsigned Opc) {
2588 SDLoc dl(N);
2589 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2590
2591 // Form a REG_SEQUENCE to force register allocation.
2592 bool Is128Bit = VT.getSizeInBits() == 128;
2593 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2594 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
2595
2596 SDValue Ops[] = {RegSeq, N->getOperand(Num: NumVecs + 2), N->getOperand(Num: 0)};
2597 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, VT: N->getValueType(ResNo: 0), Ops);
2598
2599 // Transfer memoperands.
2600 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2601 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2602
2603 ReplaceNode(F: N, T: St);
2604}
2605
2606void AArch64DAGToDAGISel::SelectPredicatedStore(SDNode *N, unsigned NumVecs,
2607 unsigned Scale, unsigned Opc_rr,
2608 unsigned Opc_ri) {
2609 SDLoc dl(N);
2610
2611 // Form a REG_SEQUENCE to force register allocation.
2612 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2613 SDValue RegSeq = createZTuple(Regs);
2614
2615 // Optimize addressing mode.
2616 unsigned Opc;
2617 SDValue Offset, Base;
2618 std::tie(args&: Opc, args&: Base, args&: Offset) = findAddrModeSVELoadStore(
2619 N, Opc_rr, Opc_ri, OldBase: N->getOperand(Num: NumVecs + 3),
2620 OldOffset: CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64), Scale);
2621
2622 SDValue Ops[] = {RegSeq, N->getOperand(Num: NumVecs + 2), // predicate
2623 Base, // address
2624 Offset, // offset
2625 N->getOperand(Num: 0)}; // chain
2626 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, VT: N->getValueType(ResNo: 0), Ops);
2627
2628 // Transfer memoperands.
2629 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2630 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2631
2632 ReplaceNode(F: N, T: St);
2633}
2634
2635void AArch64DAGToDAGISel::SelectPostStore(SDNode *N, unsigned NumVecs,
2636 unsigned Opc) {
2637 SDLoc dl(N);
2638 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2639 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2640 MVT::Other}; // Type for the Chain
2641
2642 // Form a REG_SEQUENCE to force register allocation.
2643 bool Is128Bit = VT.getSizeInBits() == 128;
2644 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2645 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
2646
2647 SDValue Ops[] = {RegSeq,
2648 N->getOperand(Num: NumVecs + 1), // base register
2649 N->getOperand(Num: NumVecs + 2), // Incremental
2650 N->getOperand(Num: 0)}; // Chain
2651 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2652
2653 // Transfer memoperands.
2654 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2655 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2656
2657 ReplaceNode(F: N, T: St);
2658}
2659
2660namespace {
2661/// WidenVector - Given a value in the V64 register class, produce the
2662/// equivalent value in the V128 register class.
2663class WidenVector {
2664 SelectionDAG &DAG;
2665
2666public:
2667 WidenVector(SelectionDAG &DAG) : DAG(DAG) {}
2668
2669 SDValue operator()(SDValue V64Reg) {
2670 EVT VT = V64Reg.getValueType();
2671 unsigned NarrowSize = VT.getVectorNumElements();
2672 MVT EltTy = VT.getVectorElementType().getSimpleVT();
2673 MVT WideTy = MVT::getVectorVT(VT: EltTy, NumElements: 2 * NarrowSize);
2674 SDLoc DL(V64Reg);
2675
2676 SDValue Undef =
2677 SDValue(DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: WideTy), 0);
2678 return DAG.getTargetInsertSubreg(SRIdx: AArch64::dsub, DL, VT: WideTy, Operand: Undef, Subreg: V64Reg);
2679 }
2680};
2681} // namespace
2682
2683/// NarrowVector - Given a value in the V128 register class, produce the
2684/// equivalent value in the V64 register class.
2685static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG) {
2686 EVT VT = V128Reg.getValueType();
2687 unsigned WideSize = VT.getVectorNumElements();
2688 MVT EltTy = VT.getVectorElementType().getSimpleVT();
2689 MVT NarrowTy = MVT::getVectorVT(VT: EltTy, NumElements: WideSize / 2);
2690
2691 return DAG.getTargetExtractSubreg(SRIdx: AArch64::dsub, DL: SDLoc(V128Reg), VT: NarrowTy,
2692 Operand: V128Reg);
2693}
2694
2695void AArch64DAGToDAGISel::SelectLoadLane(SDNode *N, unsigned NumVecs,
2696 unsigned Opc) {
2697 SDLoc dl(N);
2698 EVT VT = N->getValueType(ResNo: 0);
2699 bool Narrow = VT.getSizeInBits() == 64;
2700
2701 // Form a REG_SEQUENCE to force register allocation.
2702 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2703
2704 if (Narrow)
2705 transform(Range&: Regs, d_first: Regs.begin(),
2706 F: WidenVector(*CurDAG));
2707
2708 SDValue RegSeq = createQTuple(Regs);
2709
2710 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2711
2712 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 2);
2713
2714 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(Val: LaneNo, DL: dl, VT: MVT::i64),
2715 N->getOperand(Num: NumVecs + 3), N->getOperand(Num: 0)};
2716 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2717 SDValue SuperReg = SDValue(Ld, 0);
2718
2719 EVT WideVT = RegSeq.getOperand(i: 1)->getValueType(ResNo: 0);
2720 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2721 AArch64::qsub2, AArch64::qsub3 };
2722 for (unsigned i = 0; i < NumVecs; ++i) {
2723 SDValue NV = CurDAG->getTargetExtractSubreg(SRIdx: QSubs[i], DL: dl, VT: WideVT, Operand: SuperReg);
2724 if (Narrow)
2725 NV = NarrowVector(V128Reg: NV, DAG&: *CurDAG);
2726 ReplaceUses(F: SDValue(N, i), T: NV);
2727 }
2728
2729 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 1));
2730 CurDAG->RemoveDeadNode(N);
2731}
2732
2733void AArch64DAGToDAGISel::SelectPostLoadLane(SDNode *N, unsigned NumVecs,
2734 unsigned Opc) {
2735 SDLoc dl(N);
2736 EVT VT = N->getValueType(ResNo: 0);
2737 bool Narrow = VT.getSizeInBits() == 64;
2738
2739 // Form a REG_SEQUENCE to force register allocation.
2740 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2741
2742 if (Narrow)
2743 transform(Range&: Regs, d_first: Regs.begin(),
2744 F: WidenVector(*CurDAG));
2745
2746 SDValue RegSeq = createQTuple(Regs);
2747
2748 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2749 RegSeq->getValueType(ResNo: 0), MVT::Other};
2750
2751 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 1);
2752
2753 SDValue Ops[] = {RegSeq,
2754 CurDAG->getTargetConstant(Val: LaneNo, DL: dl,
2755 VT: MVT::i64), // Lane Number
2756 N->getOperand(Num: NumVecs + 2), // Base register
2757 N->getOperand(Num: NumVecs + 3), // Incremental
2758 N->getOperand(Num: 0)};
2759 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2760
2761 // Update uses of the write back register
2762 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 0));
2763
2764 // Update uses of the vector list
2765 SDValue SuperReg = SDValue(Ld, 1);
2766 if (NumVecs == 1) {
2767 ReplaceUses(F: SDValue(N, 0),
2768 T: Narrow ? NarrowVector(V128Reg: SuperReg, DAG&: *CurDAG) : SuperReg);
2769 } else {
2770 EVT WideVT = RegSeq.getOperand(i: 1)->getValueType(ResNo: 0);
2771 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2772 AArch64::qsub2, AArch64::qsub3 };
2773 for (unsigned i = 0; i < NumVecs; ++i) {
2774 SDValue NV = CurDAG->getTargetExtractSubreg(SRIdx: QSubs[i], DL: dl, VT: WideVT,
2775 Operand: SuperReg);
2776 if (Narrow)
2777 NV = NarrowVector(V128Reg: NV, DAG&: *CurDAG);
2778 ReplaceUses(F: SDValue(N, i), T: NV);
2779 }
2780 }
2781
2782 // Update the Chain
2783 ReplaceUses(F: SDValue(N, NumVecs + 1), T: SDValue(Ld, 2));
2784 CurDAG->RemoveDeadNode(N);
2785}
2786
2787void AArch64DAGToDAGISel::SelectStoreLane(SDNode *N, unsigned NumVecs,
2788 unsigned Opc) {
2789 SDLoc dl(N);
2790 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2791 bool Narrow = VT.getSizeInBits() == 64;
2792
2793 // Form a REG_SEQUENCE to force register allocation.
2794 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2795
2796 if (Narrow)
2797 transform(Range&: Regs, d_first: Regs.begin(),
2798 F: WidenVector(*CurDAG));
2799
2800 SDValue RegSeq = createQTuple(Regs);
2801
2802 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 2);
2803
2804 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(Val: LaneNo, DL: dl, VT: MVT::i64),
2805 N->getOperand(Num: NumVecs + 3), N->getOperand(Num: 0)};
2806 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, VT: MVT::Other, Ops);
2807
2808 // Transfer memoperands.
2809 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2810 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2811
2812 ReplaceNode(F: N, T: St);
2813}
2814
2815void AArch64DAGToDAGISel::SelectPostStoreLane(SDNode *N, unsigned NumVecs,
2816 unsigned Opc) {
2817 SDLoc dl(N);
2818 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2819 bool Narrow = VT.getSizeInBits() == 64;
2820
2821 // Form a REG_SEQUENCE to force register allocation.
2822 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2823
2824 if (Narrow)
2825 transform(Range&: Regs, d_first: Regs.begin(),
2826 F: WidenVector(*CurDAG));
2827
2828 SDValue RegSeq = createQTuple(Regs);
2829
2830 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2831 MVT::Other};
2832
2833 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 1);
2834
2835 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(Val: LaneNo, DL: dl, VT: MVT::i64),
2836 N->getOperand(Num: NumVecs + 2), // Base Register
2837 N->getOperand(Num: NumVecs + 3), // Incremental
2838 N->getOperand(Num: 0)};
2839 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2840
2841 // Transfer memoperands.
2842 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2843 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2844
2845 ReplaceNode(F: N, T: St);
2846}
2847
2848static bool isBitfieldExtractOpFromAnd(SelectionDAG *CurDAG, SDNode *N,
2849 unsigned &Opc, SDValue &Opd0,
2850 unsigned &LSB, unsigned &MSB,
2851 unsigned NumberOfIgnoredLowBits,
2852 bool BiggerPattern) {
2853 assert(N->getOpcode() == ISD::AND &&
2854 "N must be a AND operation to call this function");
2855
2856 EVT VT = N->getValueType(ResNo: 0);
2857
2858 // Here we can test the type of VT and return false when the type does not
2859 // match, but since it is done prior to that call in the current context
2860 // we turned that into an assert to avoid redundant code.
2861 assert((VT == MVT::i32 || VT == MVT::i64) &&
2862 "Type checking must have been done before calling this function");
2863
2864 // FIXME: simplify-demanded-bits in DAGCombine will probably have
2865 // changed the AND node to a 32-bit mask operation. We'll have to
2866 // undo that as part of the transform here if we want to catch all
2867 // the opportunities.
2868 // Currently the NumberOfIgnoredLowBits argument helps to recover
2869 // from these situations when matching bigger pattern (bitfield insert).
2870
2871 // For unsigned extracts, check for a shift right and mask
2872 uint64_t AndImm = 0;
2873 if (!isOpcWithIntImmediate(N, Opc: ISD::AND, Imm&: AndImm))
2874 return false;
2875
2876 const SDNode *Op0 = N->getOperand(Num: 0).getNode();
2877
2878 // Because of simplify-demanded-bits in DAGCombine, the mask may have been
2879 // simplified. Try to undo that
2880 AndImm |= maskTrailingOnes<uint64_t>(N: NumberOfIgnoredLowBits);
2881
2882 // The immediate is a mask of the low bits iff imm & (imm+1) == 0
2883 if (AndImm & (AndImm + 1))
2884 return false;
2885
2886 bool ClampMSB = false;
2887 uint64_t SrlImm = 0;
2888 // Handle the SRL + ANY_EXTEND case.
2889 if (VT == MVT::i64 && Op0->getOpcode() == ISD::ANY_EXTEND &&
2890 isOpcWithIntImmediate(N: Op0->getOperand(Num: 0).getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
2891 // Extend the incoming operand of the SRL to 64-bit.
2892 Opd0 = Widen(CurDAG, N: Op0->getOperand(Num: 0).getOperand(i: 0));
2893 // Make sure to clamp the MSB so that we preserve the semantics of the
2894 // original operations.
2895 ClampMSB = true;
2896 } else if (VT == MVT::i32 && Op0->getOpcode() == ISD::TRUNCATE &&
2897 isOpcWithIntImmediate(N: Op0->getOperand(Num: 0).getNode(), Opc: ISD::SRL,
2898 Imm&: SrlImm)) {
2899 // If the shift result was truncated, we can still combine them.
2900 Opd0 = Op0->getOperand(Num: 0).getOperand(i: 0);
2901
2902 // Use the type of SRL node.
2903 VT = Opd0->getValueType(ResNo: 0);
2904 } else if (isOpcWithIntImmediate(N: Op0, Opc: ISD::SRL, Imm&: SrlImm)) {
2905 Opd0 = Op0->getOperand(Num: 0);
2906 ClampMSB = (VT == MVT::i32);
2907 } else if (BiggerPattern) {
2908 // Let's pretend a 0 shift right has been performed.
2909 // The resulting code will be at least as good as the original one
2910 // plus it may expose more opportunities for bitfield insert pattern.
2911 // FIXME: Currently we limit this to the bigger pattern, because
2912 // some optimizations expect AND and not UBFM.
2913 Opd0 = N->getOperand(Num: 0);
2914 } else
2915 return false;
2916
2917 // Bail out on large immediates. This happens when no proper
2918 // combining/constant folding was performed.
2919 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.getSizeInBits())) {
2920 LLVM_DEBUG(
2921 (dbgs() << N
2922 << ": Found large shift immediate, this should not happen\n"));
2923 return false;
2924 }
2925
2926 LSB = SrlImm;
2927 MSB = SrlImm +
2928 (VT == MVT::i32 ? llvm::countr_one<uint32_t>(Value: AndImm)
2929 : llvm::countr_one<uint64_t>(Value: AndImm)) -
2930 1;
2931 if (ClampMSB)
2932 // Since we're moving the extend before the right shift operation, we need
2933 // to clamp the MSB to make sure we don't shift in undefined bits instead of
2934 // the zeros which would get shifted in with the original right shift
2935 // operation.
2936 MSB = MSB > 31 ? 31 : MSB;
2937
2938 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2939 return true;
2940}
2941
2942static bool isBitfieldExtractOpFromSExtInReg(SDNode *N, unsigned &Opc,
2943 SDValue &Opd0, unsigned &Immr,
2944 unsigned &Imms) {
2945 assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG);
2946
2947 EVT VT = N->getValueType(ResNo: 0);
2948 unsigned BitWidth = VT.getSizeInBits();
2949 assert((VT == MVT::i32 || VT == MVT::i64) &&
2950 "Type checking must have been done before calling this function");
2951
2952 SDValue Op = N->getOperand(Num: 0);
2953 if (Op->getOpcode() == ISD::TRUNCATE) {
2954 Op = Op->getOperand(Num: 0);
2955 VT = Op->getValueType(ResNo: 0);
2956 BitWidth = VT.getSizeInBits();
2957 }
2958
2959 uint64_t ShiftImm;
2960 if (!isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::SRL, Imm&: ShiftImm) &&
2961 !isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::SRA, Imm&: ShiftImm))
2962 return false;
2963
2964 unsigned Width = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getSizeInBits();
2965 if (ShiftImm + Width > BitWidth)
2966 return false;
2967
2968 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri;
2969 Opd0 = Op.getOperand(i: 0);
2970 Immr = ShiftImm;
2971 Imms = ShiftImm + Width - 1;
2972 return true;
2973}
2974
2975static bool isSeveralBitsExtractOpFromShr(SDNode *N, unsigned &Opc,
2976 SDValue &Opd0, unsigned &LSB,
2977 unsigned &MSB) {
2978 // We are looking for the following pattern which basically extracts several
2979 // continuous bits from the source value and places it from the LSB of the
2980 // destination value, all other bits of the destination value or set to zero:
2981 //
2982 // Value2 = AND Value, MaskImm
2983 // SRL Value2, ShiftImm
2984 //
2985 // with MaskImm >> ShiftImm to search for the bit width.
2986 //
2987 // This gets selected into a single UBFM:
2988 //
2989 // UBFM Value, ShiftImm, Log2_64(MaskImm)
2990 //
2991
2992 if (N->getOpcode() != ISD::SRL)
2993 return false;
2994
2995 uint64_t AndMask = 0;
2996 if (!isOpcWithIntImmediate(N: N->getOperand(Num: 0).getNode(), Opc: ISD::AND, Imm&: AndMask))
2997 return false;
2998
2999 Opd0 = N->getOperand(Num: 0).getOperand(i: 0);
3000
3001 uint64_t SrlImm = 0;
3002 if (!isIntImmediate(N: N->getOperand(Num: 1), Imm&: SrlImm))
3003 return false;
3004
3005 // Check whether we really have several bits extract here.
3006 if (!isMask_64(Value: AndMask >> SrlImm))
3007 return false;
3008
3009 Opc = N->getValueType(ResNo: 0) == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
3010 LSB = SrlImm;
3011 MSB = llvm::Log2_64(Value: AndMask);
3012 return true;
3013}
3014
3015static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0,
3016 unsigned &Immr, unsigned &Imms,
3017 bool BiggerPattern) {
3018 assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) &&
3019 "N must be a SHR/SRA operation to call this function");
3020
3021 EVT VT = N->getValueType(ResNo: 0);
3022
3023 // Here we can test the type of VT and return false when the type does not
3024 // match, but since it is done prior to that call in the current context
3025 // we turned that into an assert to avoid redundant code.
3026 assert((VT == MVT::i32 || VT == MVT::i64) &&
3027 "Type checking must have been done before calling this function");
3028
3029 // Check for AND + SRL doing several bits extract.
3030 if (isSeveralBitsExtractOpFromShr(N, Opc, Opd0, LSB&: Immr, MSB&: Imms))
3031 return true;
3032
3033 // We're looking for a shift of a shift.
3034 uint64_t ShlImm = 0;
3035 uint64_t TruncBits = 0;
3036 if (isOpcWithIntImmediate(N: N->getOperand(Num: 0).getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
3037 Opd0 = N->getOperand(Num: 0).getOperand(i: 0);
3038 } else if (VT == MVT::i32 && N->getOpcode() == ISD::SRL &&
3039 N->getOperand(Num: 0).getNode()->getOpcode() == ISD::TRUNCATE) {
3040 // We are looking for a shift of truncate. Truncate from i64 to i32 could
3041 // be considered as setting high 32 bits as zero. Our strategy here is to
3042 // always generate 64bit UBFM. This consistency will help the CSE pass
3043 // later find more redundancy.
3044 Opd0 = N->getOperand(Num: 0).getOperand(i: 0);
3045 TruncBits = Opd0->getValueType(ResNo: 0).getSizeInBits() - VT.getSizeInBits();
3046 VT = Opd0.getValueType();
3047 assert(VT == MVT::i64 && "the promoted type should be i64");
3048 } else if (BiggerPattern) {
3049 // Let's pretend a 0 shift left has been performed.
3050 // FIXME: Currently we limit this to the bigger pattern case,
3051 // because some optimizations expect AND and not UBFM
3052 Opd0 = N->getOperand(Num: 0);
3053 } else
3054 return false;
3055
3056 // Missing combines/constant folding may have left us with strange
3057 // constants.
3058 if (ShlImm >= VT.getSizeInBits()) {
3059 LLVM_DEBUG(
3060 (dbgs() << N
3061 << ": Found large shift immediate, this should not happen\n"));
3062 return false;
3063 }
3064
3065 uint64_t SrlImm = 0;
3066 if (!isIntImmediate(N: N->getOperand(Num: 1), Imm&: SrlImm))
3067 return false;
3068
3069 assert(SrlImm > 0 && SrlImm < VT.getSizeInBits() &&
3070 "bad amount in shift node!");
3071 int immr = SrlImm - ShlImm;
3072 Immr = immr < 0 ? immr + VT.getSizeInBits() : immr;
3073 Imms = VT.getSizeInBits() - ShlImm - TruncBits - 1;
3074 // SRA requires a signed extraction
3075 if (VT == MVT::i32)
3076 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
3077 else
3078 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri;
3079 return true;
3080}
3081
3082bool AArch64DAGToDAGISel::tryBitfieldExtractOpFromSExt(SDNode *N) {
3083 assert(N->getOpcode() == ISD::SIGN_EXTEND);
3084
3085 EVT VT = N->getValueType(ResNo: 0);
3086 EVT NarrowVT = N->getOperand(Num: 0)->getValueType(ResNo: 0);
3087 if (VT != MVT::i64 || NarrowVT != MVT::i32)
3088 return false;
3089
3090 uint64_t ShiftImm;
3091 SDValue Op = N->getOperand(Num: 0);
3092 if (!isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::SRA, Imm&: ShiftImm))
3093 return false;
3094
3095 SDLoc dl(N);
3096 // Extend the incoming operand of the shift to 64-bits.
3097 SDValue Opd0 = Widen(CurDAG, N: Op.getOperand(i: 0));
3098 unsigned Immr = ShiftImm;
3099 unsigned Imms = NarrowVT.getSizeInBits() - 1;
3100 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Val: Immr, DL: dl, VT),
3101 CurDAG->getTargetConstant(Val: Imms, DL: dl, VT)};
3102 CurDAG->SelectNodeTo(N, MachineOpc: AArch64::SBFMXri, VT, Ops);
3103 return true;
3104}
3105
3106static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc,
3107 SDValue &Opd0, unsigned &Immr, unsigned &Imms,
3108 unsigned NumberOfIgnoredLowBits = 0,
3109 bool BiggerPattern = false) {
3110 if (N->getValueType(ResNo: 0) != MVT::i32 && N->getValueType(ResNo: 0) != MVT::i64)
3111 return false;
3112
3113 switch (N->getOpcode()) {
3114 default:
3115 if (!N->isMachineOpcode())
3116 return false;
3117 break;
3118 case ISD::AND:
3119 return isBitfieldExtractOpFromAnd(CurDAG, N, Opc, Opd0, LSB&: Immr, MSB&: Imms,
3120 NumberOfIgnoredLowBits, BiggerPattern);
3121 case ISD::SRL:
3122 case ISD::SRA:
3123 return isBitfieldExtractOpFromShr(N, Opc, Opd0, Immr, Imms, BiggerPattern);
3124
3125 case ISD::SIGN_EXTEND_INREG:
3126 return isBitfieldExtractOpFromSExtInReg(N, Opc, Opd0, Immr, Imms);
3127 }
3128
3129 unsigned NOpc = N->getMachineOpcode();
3130 switch (NOpc) {
3131 default:
3132 return false;
3133 case AArch64::SBFMWri:
3134 case AArch64::UBFMWri:
3135 case AArch64::SBFMXri:
3136 case AArch64::UBFMXri:
3137 Opc = NOpc;
3138 Opd0 = N->getOperand(Num: 0);
3139 Immr = N->getConstantOperandVal(Num: 1);
3140 Imms = N->getConstantOperandVal(Num: 2);
3141 return true;
3142 }
3143 // Unreachable
3144 return false;
3145}
3146
3147bool AArch64DAGToDAGISel::tryBitfieldExtractOp(SDNode *N) {
3148 unsigned Opc, Immr, Imms;
3149 SDValue Opd0;
3150 if (!isBitfieldExtractOp(CurDAG, N, Opc, Opd0, Immr, Imms))
3151 return false;
3152
3153 EVT VT = N->getValueType(ResNo: 0);
3154 SDLoc dl(N);
3155
3156 // If the bit extract operation is 64bit but the original type is 32bit, we
3157 // need to add one EXTRACT_SUBREG.
3158 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) {
3159 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Val: Immr, DL: dl, VT: MVT::i64),
3160 CurDAG->getTargetConstant(Val: Imms, DL: dl, VT: MVT::i64)};
3161
3162 SDNode *BFM = CurDAG->getMachineNode(Opcode: Opc, dl, VT: MVT::i64, Ops: Ops64);
3163 SDValue Inner = CurDAG->getTargetExtractSubreg(SRIdx: AArch64::sub_32, DL: dl,
3164 VT: MVT::i32, Operand: SDValue(BFM, 0));
3165 ReplaceNode(F: N, T: Inner.getNode());
3166 return true;
3167 }
3168
3169 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Val: Immr, DL: dl, VT),
3170 CurDAG->getTargetConstant(Val: Imms, DL: dl, VT)};
3171 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
3172 return true;
3173}
3174
3175/// Does DstMask form a complementary pair with the mask provided by
3176/// BitsToBeInserted, suitable for use in a BFI instruction. Roughly speaking,
3177/// this asks whether DstMask zeroes precisely those bits that will be set by
3178/// the other half.
3179static bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted,
3180 unsigned NumberOfIgnoredHighBits, EVT VT) {
3181 assert((VT == MVT::i32 || VT == MVT::i64) &&
3182 "i32 or i64 mask type expected!");
3183 unsigned BitWidth = VT.getSizeInBits() - NumberOfIgnoredHighBits;
3184
3185 // Enable implicitTrunc as we're intentionally ignoring high bits.
3186 APInt SignificantDstMask =
3187 APInt(BitWidth, DstMask, /*isSigned=*/false, /*implicitTrunc=*/true);
3188 APInt SignificantBitsToBeInserted = BitsToBeInserted.zextOrTrunc(width: BitWidth);
3189
3190 return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
3191 (SignificantDstMask | SignificantBitsToBeInserted).isAllOnes();
3192}
3193
3194// Look for bits that will be useful for later uses.
3195// A bit is consider useless as soon as it is dropped and never used
3196// before it as been dropped.
3197// E.g., looking for useful bit of x
3198// 1. y = x & 0x7
3199// 2. z = y >> 2
3200// After #1, x useful bits are 0x7, then the useful bits of x, live through
3201// y.
3202// After #2, the useful bits of x are 0x4.
3203// However, if x is used on an unpredictable instruction, then all its bits
3204// are useful.
3205// E.g.
3206// 1. y = x & 0x7
3207// 2. z = y >> 2
3208// 3. str x, [@x]
3209static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth = 0);
3210
3211static void getUsefulBitsFromAndWithImmediate(SDValue Op, APInt &UsefulBits,
3212 unsigned Depth) {
3213 uint64_t Imm =
3214 cast<const ConstantSDNode>(Val: Op.getOperand(i: 1).getNode())->getZExtValue();
3215 Imm = AArch64_AM::decodeLogicalImmediate(val: Imm, regSize: UsefulBits.getBitWidth());
3216 UsefulBits &= APInt(UsefulBits.getBitWidth(), Imm);
3217 getUsefulBits(Op, UsefulBits, Depth: Depth + 1);
3218}
3219
3220static void getUsefulBitsFromBitfieldMoveOpd(SDValue Op, APInt &UsefulBits,
3221 uint64_t Imm, uint64_t MSB,
3222 unsigned Depth) {
3223 // inherit the bitwidth value
3224 APInt OpUsefulBits(UsefulBits);
3225 OpUsefulBits = 1;
3226
3227 if (MSB >= Imm) {
3228 OpUsefulBits <<= MSB - Imm + 1;
3229 --OpUsefulBits;
3230 // The interesting part will be in the lower part of the result
3231 getUsefulBits(Op, UsefulBits&: OpUsefulBits, Depth: Depth + 1);
3232 // The interesting part was starting at Imm in the argument
3233 OpUsefulBits <<= Imm;
3234 } else {
3235 OpUsefulBits <<= MSB + 1;
3236 --OpUsefulBits;
3237 // The interesting part will be shifted in the result
3238 OpUsefulBits <<= OpUsefulBits.getBitWidth() - Imm;
3239 getUsefulBits(Op, UsefulBits&: OpUsefulBits, Depth: Depth + 1);
3240 // The interesting part was at zero in the argument
3241 OpUsefulBits.lshrInPlace(ShiftAmt: OpUsefulBits.getBitWidth() - Imm);
3242 }
3243
3244 UsefulBits &= OpUsefulBits;
3245}
3246
3247static void getUsefulBitsFromUBFM(SDValue Op, APInt &UsefulBits,
3248 unsigned Depth) {
3249 uint64_t Imm =
3250 cast<const ConstantSDNode>(Val: Op.getOperand(i: 1).getNode())->getZExtValue();
3251 uint64_t MSB =
3252 cast<const ConstantSDNode>(Val: Op.getOperand(i: 2).getNode())->getZExtValue();
3253
3254 getUsefulBitsFromBitfieldMoveOpd(Op, UsefulBits, Imm, MSB, Depth);
3255}
3256
3257static void getUsefulBitsFromOrWithShiftedReg(SDValue Op, APInt &UsefulBits,
3258 unsigned Depth) {
3259 uint64_t ShiftTypeAndValue =
3260 cast<const ConstantSDNode>(Val: Op.getOperand(i: 2).getNode())->getZExtValue();
3261 APInt Mask(UsefulBits);
3262 Mask.clearAllBits();
3263 Mask.flipAllBits();
3264
3265 if (AArch64_AM::getShiftType(Imm: ShiftTypeAndValue) == AArch64_AM::LSL) {
3266 // Shift Left
3267 uint64_t ShiftAmt = AArch64_AM::getShiftValue(Imm: ShiftTypeAndValue);
3268 Mask <<= ShiftAmt;
3269 getUsefulBits(Op, UsefulBits&: Mask, Depth: Depth + 1);
3270 Mask.lshrInPlace(ShiftAmt);
3271 } else if (AArch64_AM::getShiftType(Imm: ShiftTypeAndValue) == AArch64_AM::LSR) {
3272 // Shift Right
3273 // We do not handle AArch64_AM::ASR, because the sign will change the
3274 // number of useful bits
3275 uint64_t ShiftAmt = AArch64_AM::getShiftValue(Imm: ShiftTypeAndValue);
3276 Mask.lshrInPlace(ShiftAmt);
3277 getUsefulBits(Op, UsefulBits&: Mask, Depth: Depth + 1);
3278 Mask <<= ShiftAmt;
3279 } else
3280 return;
3281
3282 UsefulBits &= Mask;
3283}
3284
3285static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits,
3286 unsigned Depth) {
3287 uint64_t Imm =
3288 cast<const ConstantSDNode>(Val: Op.getOperand(i: 2).getNode())->getZExtValue();
3289 uint64_t MSB =
3290 cast<const ConstantSDNode>(Val: Op.getOperand(i: 3).getNode())->getZExtValue();
3291
3292 APInt OpUsefulBits(UsefulBits);
3293 OpUsefulBits = 1;
3294
3295 APInt ResultUsefulBits(UsefulBits.getBitWidth(), 0);
3296 ResultUsefulBits.flipAllBits();
3297 APInt Mask(UsefulBits.getBitWidth(), 0);
3298
3299 getUsefulBits(Op, UsefulBits&: ResultUsefulBits, Depth: Depth + 1);
3300
3301 if (MSB >= Imm) {
3302 // The instruction is a BFXIL.
3303 uint64_t Width = MSB - Imm + 1;
3304 uint64_t LSB = Imm;
3305
3306 OpUsefulBits <<= Width;
3307 --OpUsefulBits;
3308
3309 if (Op.getOperand(i: 1) == Orig) {
3310 // Copy the low bits from the result to bits starting from LSB.
3311 Mask = ResultUsefulBits & OpUsefulBits;
3312 Mask <<= LSB;
3313 }
3314
3315 if (Op.getOperand(i: 0) == Orig)
3316 // Bits starting from LSB in the input contribute to the result.
3317 Mask |= (ResultUsefulBits & ~OpUsefulBits);
3318 } else {
3319 // The instruction is a BFI.
3320 uint64_t Width = MSB + 1;
3321 uint64_t LSB = UsefulBits.getBitWidth() - Imm;
3322
3323 OpUsefulBits <<= Width;
3324 --OpUsefulBits;
3325 OpUsefulBits <<= LSB;
3326
3327 if (Op.getOperand(i: 1) == Orig) {
3328 // Copy the bits from the result to the zero bits.
3329 Mask = ResultUsefulBits & OpUsefulBits;
3330 Mask.lshrInPlace(ShiftAmt: LSB);
3331 }
3332
3333 if (Op.getOperand(i: 0) == Orig)
3334 Mask |= (ResultUsefulBits & ~OpUsefulBits);
3335 }
3336
3337 UsefulBits &= Mask;
3338}
3339
3340static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits,
3341 SDValue Orig, unsigned Depth) {
3342
3343 // Users of this node should have already been instruction selected
3344 // FIXME: Can we turn that into an assert?
3345 if (!UserNode->isMachineOpcode())
3346 return;
3347
3348 switch (UserNode->getMachineOpcode()) {
3349 default:
3350 return;
3351 case AArch64::ANDSWri:
3352 case AArch64::ANDSXri:
3353 case AArch64::ANDWri:
3354 case AArch64::ANDXri:
3355 // We increment Depth only when we call the getUsefulBits
3356 return getUsefulBitsFromAndWithImmediate(Op: SDValue(UserNode, 0), UsefulBits,
3357 Depth);
3358 case AArch64::UBFMWri:
3359 case AArch64::UBFMXri:
3360 return getUsefulBitsFromUBFM(Op: SDValue(UserNode, 0), UsefulBits, Depth);
3361
3362 case AArch64::ORRWrs:
3363 case AArch64::ORRXrs:
3364 if (UserNode->getOperand(Num: 0) != Orig && UserNode->getOperand(Num: 1) == Orig)
3365 getUsefulBitsFromOrWithShiftedReg(Op: SDValue(UserNode, 0), UsefulBits,
3366 Depth);
3367 return;
3368 case AArch64::BFMWri:
3369 case AArch64::BFMXri:
3370 return getUsefulBitsFromBFM(Op: SDValue(UserNode, 0), Orig, UsefulBits, Depth);
3371
3372 case AArch64::STRBBui:
3373 case AArch64::STURBBi:
3374 if (UserNode->getOperand(Num: 0) != Orig)
3375 return;
3376 UsefulBits &= APInt(UsefulBits.getBitWidth(), 0xff);
3377 return;
3378
3379 case AArch64::STRHHui:
3380 case AArch64::STURHHi:
3381 if (UserNode->getOperand(Num: 0) != Orig)
3382 return;
3383 UsefulBits &= APInt(UsefulBits.getBitWidth(), 0xffff);
3384 return;
3385 }
3386}
3387
3388static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth) {
3389 if (Depth >= SelectionDAG::MaxRecursionDepth)
3390 return;
3391 // Initialize UsefulBits
3392 if (!Depth) {
3393 unsigned Bitwidth = Op.getScalarValueSizeInBits();
3394 // At the beginning, assume every produced bits is useful
3395 UsefulBits = APInt(Bitwidth, 0);
3396 UsefulBits.flipAllBits();
3397 }
3398 APInt UsersUsefulBits(UsefulBits.getBitWidth(), 0);
3399
3400 for (SDNode *Node : Op.getNode()->users()) {
3401 // A use cannot produce useful bits
3402 APInt UsefulBitsForUse = APInt(UsefulBits);
3403 getUsefulBitsForUse(UserNode: Node, UsefulBits&: UsefulBitsForUse, Orig: Op, Depth);
3404 UsersUsefulBits |= UsefulBitsForUse;
3405 }
3406 // UsefulBits contains the produced bits that are meaningful for the
3407 // current definition, thus a user cannot make a bit meaningful at
3408 // this point
3409 UsefulBits &= UsersUsefulBits;
3410}
3411
3412/// Create a machine node performing a notional SHL of Op by ShlAmount. If
3413/// ShlAmount is negative, do a (logical) right-shift instead. If ShlAmount is
3414/// 0, return Op unchanged.
3415static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount) {
3416 if (ShlAmount == 0)
3417 return Op;
3418
3419 EVT VT = Op.getValueType();
3420 SDLoc dl(Op);
3421 unsigned BitWidth = VT.getSizeInBits();
3422 unsigned UBFMOpc = BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
3423
3424 SDNode *ShiftNode;
3425 if (ShlAmount > 0) {
3426 // LSL wD, wN, #Amt == UBFM wD, wN, #32-Amt, #31-Amt
3427 ShiftNode = CurDAG->getMachineNode(
3428 Opcode: UBFMOpc, dl, VT, Op1: Op,
3429 Op2: CurDAG->getTargetConstant(Val: BitWidth - ShlAmount, DL: dl, VT),
3430 Op3: CurDAG->getTargetConstant(Val: BitWidth - 1 - ShlAmount, DL: dl, VT));
3431 } else {
3432 // LSR wD, wN, #Amt == UBFM wD, wN, #Amt, #32-1
3433 assert(ShlAmount < 0 && "expected right shift");
3434 int ShrAmount = -ShlAmount;
3435 ShiftNode = CurDAG->getMachineNode(
3436 Opcode: UBFMOpc, dl, VT, Op1: Op, Op2: CurDAG->getTargetConstant(Val: ShrAmount, DL: dl, VT),
3437 Op3: CurDAG->getTargetConstant(Val: BitWidth - 1, DL: dl, VT));
3438 }
3439
3440 return SDValue(ShiftNode, 0);
3441}
3442
3443// For bit-field-positioning pattern "(and (shl VAL, N), ShiftedMask)".
3444static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op,
3445 bool BiggerPattern,
3446 const uint64_t NonZeroBits,
3447 SDValue &Src, int &DstLSB,
3448 int &Width);
3449
3450// For bit-field-positioning pattern "shl VAL, N)".
3451static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op,
3452 bool BiggerPattern,
3453 const uint64_t NonZeroBits,
3454 SDValue &Src, int &DstLSB,
3455 int &Width);
3456
3457/// Does this tree qualify as an attempt to move a bitfield into position,
3458/// essentially "(and (shl VAL, N), Mask)" or (shl VAL, N).
3459static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op,
3460 bool BiggerPattern, SDValue &Src,
3461 int &DstLSB, int &Width) {
3462 EVT VT = Op.getValueType();
3463 unsigned BitWidth = VT.getSizeInBits();
3464 (void)BitWidth;
3465 assert(BitWidth == 32 || BitWidth == 64);
3466
3467 KnownBits Known = CurDAG->computeKnownBits(Op);
3468
3469 // Non-zero in the sense that they're not provably zero, which is the key
3470 // point if we want to use this value
3471 const uint64_t NonZeroBits = (~Known.Zero).getZExtValue();
3472 if (!isShiftedMask_64(Value: NonZeroBits))
3473 return false;
3474
3475 switch (Op.getOpcode()) {
3476 default:
3477 break;
3478 case ISD::AND:
3479 return isBitfieldPositioningOpFromAnd(CurDAG, Op, BiggerPattern,
3480 NonZeroBits, Src, DstLSB, Width);
3481 case ISD::SHL:
3482 return isBitfieldPositioningOpFromShl(CurDAG, Op, BiggerPattern,
3483 NonZeroBits, Src, DstLSB, Width);
3484 }
3485
3486 return false;
3487}
3488
3489static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op,
3490 bool BiggerPattern,
3491 const uint64_t NonZeroBits,
3492 SDValue &Src, int &DstLSB,
3493 int &Width) {
3494 assert(isShiftedMask_64(NonZeroBits) && "Caller guaranteed");
3495
3496 EVT VT = Op.getValueType();
3497 assert((VT == MVT::i32 || VT == MVT::i64) &&
3498 "Caller guarantees VT is one of i32 or i64");
3499 (void)VT;
3500
3501 uint64_t AndImm;
3502 if (!isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::AND, Imm&: AndImm))
3503 return false;
3504
3505 // If (~AndImm & NonZeroBits) is not zero at POS, we know that
3506 // 1) (AndImm & (1 << POS) == 0)
3507 // 2) the result of AND is not zero at POS bit (according to NonZeroBits)
3508 //
3509 // 1) and 2) don't agree so something must be wrong (e.g., in
3510 // 'SelectionDAG::computeKnownBits')
3511 assert((~AndImm & NonZeroBits) == 0 &&
3512 "Something must be wrong (e.g., in SelectionDAG::computeKnownBits)");
3513
3514 SDValue AndOp0 = Op.getOperand(i: 0);
3515
3516 uint64_t ShlImm;
3517 SDValue ShlOp0;
3518 if (isOpcWithIntImmediate(N: AndOp0.getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
3519 // For pattern "and(shl(val, N), shifted-mask)", 'ShlOp0' is set to 'val'.
3520 ShlOp0 = AndOp0.getOperand(i: 0);
3521 } else if (VT == MVT::i64 && AndOp0.getOpcode() == ISD::ANY_EXTEND &&
3522 isOpcWithIntImmediate(N: AndOp0.getOperand(i: 0).getNode(), Opc: ISD::SHL,
3523 Imm&: ShlImm)) {
3524 // For pattern "and(any_extend(shl(val, N)), shifted-mask)"
3525
3526 // ShlVal == shl(val, N), which is a left shift on a smaller type.
3527 SDValue ShlVal = AndOp0.getOperand(i: 0);
3528
3529 // Since this is after type legalization and ShlVal is extended to MVT::i64,
3530 // expect VT to be MVT::i32.
3531 assert((ShlVal.getValueType() == MVT::i32) && "Expect VT to be MVT::i32.");
3532
3533 // Widens 'val' to MVT::i64 as the source of bit field positioning.
3534 ShlOp0 = Widen(CurDAG, N: ShlVal.getOperand(i: 0));
3535 } else
3536 return false;
3537
3538 // For !BiggerPattern, bail out if the AndOp0 has more than one use, since
3539 // then we'll end up generating AndOp0+UBFIZ instead of just keeping
3540 // AndOp0+AND.
3541 if (!BiggerPattern && !AndOp0.hasOneUse())
3542 return false;
3543
3544 DstLSB = llvm::countr_zero(Val: NonZeroBits);
3545 Width = llvm::countr_one(Value: NonZeroBits >> DstLSB);
3546
3547 // Bail out on large Width. This happens when no proper combining / constant
3548 // folding was performed.
3549 if (Width >= (int)VT.getSizeInBits()) {
3550 // If VT is i64, Width > 64 is insensible since NonZeroBits is uint64_t, and
3551 // Width == 64 indicates a missed dag-combine from "(and val, AllOnes)" to
3552 // "val".
3553 // If VT is i32, what Width >= 32 means:
3554 // - For "(and (any_extend(shl val, N)), shifted-mask)", the`and` Op
3555 // demands at least 'Width' bits (after dag-combiner). This together with
3556 // `any_extend` Op (undefined higher bits) indicates missed combination
3557 // when lowering the 'and' IR instruction to an machine IR instruction.
3558 LLVM_DEBUG(
3559 dbgs()
3560 << "Found large Width in bit-field-positioning -- this indicates no "
3561 "proper combining / constant folding was performed\n");
3562 return false;
3563 }
3564
3565 // BFI encompasses sufficiently many nodes that it's worth inserting an extra
3566 // LSL/LSR if the mask in NonZeroBits doesn't quite match up with the ISD::SHL
3567 // amount. BiggerPattern is true when this pattern is being matched for BFI,
3568 // BiggerPattern is false when this pattern is being matched for UBFIZ, in
3569 // which case it is not profitable to insert an extra shift.
3570 if (ShlImm != uint64_t(DstLSB) && !BiggerPattern)
3571 return false;
3572
3573 Src = getLeftShift(CurDAG, Op: ShlOp0, ShlAmount: ShlImm - DstLSB);
3574 return true;
3575}
3576
3577// For node (shl (and val, mask), N)), returns true if the node is equivalent to
3578// UBFIZ.
3579static bool isSeveralBitsPositioningOpFromShl(const uint64_t ShlImm, SDValue Op,
3580 SDValue &Src, int &DstLSB,
3581 int &Width) {
3582 // Caller should have verified that N is a left shift with constant shift
3583 // amount; asserts that.
3584 assert(Op.getOpcode() == ISD::SHL &&
3585 "Op.getNode() should be a SHL node to call this function");
3586 assert(isIntImmediateEq(Op.getOperand(1), ShlImm) &&
3587 "Op.getNode() should shift ShlImm to call this function");
3588
3589 uint64_t AndImm = 0;
3590 SDValue Op0 = Op.getOperand(i: 0);
3591 if (!isOpcWithIntImmediate(N: Op0.getNode(), Opc: ISD::AND, Imm&: AndImm))
3592 return false;
3593
3594 const uint64_t ShiftedAndImm = ((AndImm << ShlImm) >> ShlImm);
3595 if (isMask_64(Value: ShiftedAndImm)) {
3596 // AndImm is a superset of (AllOnes >> ShlImm); in other words, AndImm
3597 // should end with Mask, and could be prefixed with random bits if those
3598 // bits are shifted out.
3599 //
3600 // For example, xyz11111 (with {x,y,z} being 0 or 1) is fine if ShlImm >= 3;
3601 // the AND result corresponding to those bits are shifted out, so it's fine
3602 // to not extract them.
3603 Width = llvm::countr_one(Value: ShiftedAndImm);
3604 DstLSB = ShlImm;
3605 Src = Op0.getOperand(i: 0);
3606 return true;
3607 }
3608 return false;
3609}
3610
3611static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op,
3612 bool BiggerPattern,
3613 const uint64_t NonZeroBits,
3614 SDValue &Src, int &DstLSB,
3615 int &Width) {
3616 assert(isShiftedMask_64(NonZeroBits) && "Caller guaranteed");
3617
3618 EVT VT = Op.getValueType();
3619 assert((VT == MVT::i32 || VT == MVT::i64) &&
3620 "Caller guarantees that type is i32 or i64");
3621 (void)VT;
3622
3623 uint64_t ShlImm;
3624 if (!isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::SHL, Imm&: ShlImm))
3625 return false;
3626
3627 if (!BiggerPattern && !Op.hasOneUse())
3628 return false;
3629
3630 if (isSeveralBitsPositioningOpFromShl(ShlImm, Op, Src, DstLSB, Width))
3631 return true;
3632
3633 DstLSB = llvm::countr_zero(Val: NonZeroBits);
3634 Width = llvm::countr_one(Value: NonZeroBits >> DstLSB);
3635
3636 if (ShlImm != uint64_t(DstLSB) && !BiggerPattern)
3637 return false;
3638
3639 Src = getLeftShift(CurDAG, Op: Op.getOperand(i: 0), ShlAmount: ShlImm - DstLSB);
3640 return true;
3641}
3642
3643static bool isShiftedMask(uint64_t Mask, EVT VT) {
3644 assert(VT == MVT::i32 || VT == MVT::i64);
3645 if (VT == MVT::i32)
3646 return isShiftedMask_32(Value: Mask);
3647 return isShiftedMask_64(Value: Mask);
3648}
3649
3650// Generate a BFI/BFXIL from 'or (and X, MaskImm), OrImm' iff the value being
3651// inserted only sets known zero bits.
3652static bool tryBitfieldInsertOpFromOrAndImm(SDNode *N, SelectionDAG *CurDAG) {
3653 assert(N->getOpcode() == ISD::OR && "Expect a OR operation");
3654
3655 EVT VT = N->getValueType(ResNo: 0);
3656 if (VT != MVT::i32 && VT != MVT::i64)
3657 return false;
3658
3659 unsigned BitWidth = VT.getSizeInBits();
3660
3661 uint64_t OrImm;
3662 if (!isOpcWithIntImmediate(N, Opc: ISD::OR, Imm&: OrImm))
3663 return false;
3664
3665 // Skip this transformation if the ORR immediate can be encoded in the ORR.
3666 // Otherwise, we'll trade an AND+ORR for ORR+BFI/BFXIL, which is most likely
3667 // performance neutral.
3668 if (AArch64_AM::isLogicalImmediate(imm: OrImm, regSize: BitWidth))
3669 return false;
3670
3671 uint64_t MaskImm;
3672 SDValue And = N->getOperand(Num: 0);
3673 // Must be a single use AND with an immediate operand.
3674 if (!And.hasOneUse() ||
3675 !isOpcWithIntImmediate(N: And.getNode(), Opc: ISD::AND, Imm&: MaskImm))
3676 return false;
3677
3678 // Compute the Known Zero for the AND as this allows us to catch more general
3679 // cases than just looking for AND with imm.
3680 KnownBits Known = CurDAG->computeKnownBits(Op: And);
3681
3682 // Non-zero in the sense that they're not provably zero, which is the key
3683 // point if we want to use this value.
3684 uint64_t NotKnownZero = (~Known.Zero).getZExtValue();
3685
3686 // The KnownZero mask must be a shifted mask (e.g., 1110..011, 11100..00).
3687 if (!isShiftedMask(Mask: Known.Zero.getZExtValue(), VT))
3688 return false;
3689
3690 // The bits being inserted must only set those bits that are known to be zero.
3691 if ((OrImm & NotKnownZero) != 0) {
3692 // FIXME: It's okay if the OrImm sets NotKnownZero bits to 1, but we don't
3693 // currently handle this case.
3694 return false;
3695 }
3696
3697 // BFI/BFXIL dst, src, #lsb, #width.
3698 int LSB = llvm::countr_one(Value: NotKnownZero);
3699 int Width = BitWidth - APInt(BitWidth, NotKnownZero).popcount();
3700
3701 // BFI/BFXIL is an alias of BFM, so translate to BFM operands.
3702 unsigned ImmR = (BitWidth - LSB) % BitWidth;
3703 unsigned ImmS = Width - 1;
3704
3705 // If we're creating a BFI instruction avoid cases where we need more
3706 // instructions to materialize the BFI constant as compared to the original
3707 // ORR. A BFXIL will use the same constant as the original ORR, so the code
3708 // should be no worse in this case.
3709 bool IsBFI = LSB != 0;
3710 uint64_t BFIImm = OrImm >> LSB;
3711 if (IsBFI && !AArch64_AM::isLogicalImmediate(imm: BFIImm, regSize: BitWidth)) {
3712 // We have a BFI instruction and we know the constant can't be materialized
3713 // with a ORR-immediate with the zero register.
3714 unsigned OrChunks = 0, BFIChunks = 0;
3715 for (unsigned Shift = 0; Shift < BitWidth; Shift += 16) {
3716 if (((OrImm >> Shift) & 0xFFFF) != 0)
3717 ++OrChunks;
3718 if (((BFIImm >> Shift) & 0xFFFF) != 0)
3719 ++BFIChunks;
3720 }
3721 if (BFIChunks > OrChunks)
3722 return false;
3723 }
3724
3725 // Materialize the constant to be inserted.
3726 SDLoc DL(N);
3727 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
3728 SDNode *MOVI = CurDAG->getMachineNode(
3729 Opcode: MOVIOpc, dl: DL, VT, Op1: CurDAG->getTargetConstant(Val: BFIImm, DL, VT));
3730
3731 // Create the BFI/BFXIL instruction.
3732 SDValue Ops[] = {And.getOperand(i: 0), SDValue(MOVI, 0),
3733 CurDAG->getTargetConstant(Val: ImmR, DL, VT),
3734 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
3735 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3736 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
3737 return true;
3738}
3739
3740static bool isWorthFoldingIntoOrrWithShift(SDValue Dst, SelectionDAG *CurDAG,
3741 SDValue &ShiftedOperand,
3742 uint64_t &EncodedShiftImm) {
3743 // Avoid folding Dst into ORR-with-shift if Dst has other uses than ORR.
3744 if (!Dst.hasOneUse())
3745 return false;
3746
3747 EVT VT = Dst.getValueType();
3748 assert((VT == MVT::i32 || VT == MVT::i64) &&
3749 "Caller should guarantee that VT is one of i32 or i64");
3750 const unsigned SizeInBits = VT.getSizeInBits();
3751
3752 SDLoc DL(Dst.getNode());
3753 uint64_t AndImm, ShlImm;
3754 if (isOpcWithIntImmediate(N: Dst.getNode(), Opc: ISD::AND, Imm&: AndImm) &&
3755 isShiftedMask_64(Value: AndImm)) {
3756 // Avoid transforming 'DstOp0' if it has other uses than the AND node.
3757 SDValue DstOp0 = Dst.getOperand(i: 0);
3758 if (!DstOp0.hasOneUse())
3759 return false;
3760
3761 // An example to illustrate the transformation
3762 // From:
3763 // lsr x8, x1, #1
3764 // and x8, x8, #0x3f80
3765 // bfxil x8, x1, #0, #7
3766 // To:
3767 // and x8, x23, #0x7f
3768 // ubfx x9, x23, #8, #7
3769 // orr x23, x8, x9, lsl #7
3770 //
3771 // The number of instructions remains the same, but ORR is faster than BFXIL
3772 // on many AArch64 processors (or as good as BFXIL if not faster). Besides,
3773 // the dependency chain is improved after the transformation.
3774 uint64_t SrlImm;
3775 if (isOpcWithIntImmediate(N: DstOp0.getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
3776 uint64_t NumTrailingZeroInShiftedMask = llvm::countr_zero(Val: AndImm);
3777 if ((SrlImm + NumTrailingZeroInShiftedMask) < SizeInBits) {
3778 unsigned MaskWidth =
3779 llvm::countr_one(Value: AndImm >> NumTrailingZeroInShiftedMask);
3780 unsigned UBFMOpc =
3781 (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3782 SDNode *UBFMNode = CurDAG->getMachineNode(
3783 Opcode: UBFMOpc, dl: DL, VT, Op1: DstOp0.getOperand(i: 0),
3784 Op2: CurDAG->getTargetConstant(Val: SrlImm + NumTrailingZeroInShiftedMask, DL,
3785 VT),
3786 Op3: CurDAG->getTargetConstant(
3787 Val: SrlImm + NumTrailingZeroInShiftedMask + MaskWidth - 1, DL, VT));
3788 ShiftedOperand = SDValue(UBFMNode, 0);
3789 EncodedShiftImm = AArch64_AM::getShifterImm(
3790 ST: AArch64_AM::LSL, Imm: NumTrailingZeroInShiftedMask);
3791 return true;
3792 }
3793 }
3794 return false;
3795 }
3796
3797 if (isOpcWithIntImmediate(N: Dst.getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
3798 ShiftedOperand = Dst.getOperand(i: 0);
3799 EncodedShiftImm = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShlImm);
3800 return true;
3801 }
3802
3803 uint64_t SrlImm;
3804 if (isOpcWithIntImmediate(N: Dst.getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
3805 ShiftedOperand = Dst.getOperand(i: 0);
3806 EncodedShiftImm = AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: SrlImm);
3807 return true;
3808 }
3809 return false;
3810}
3811
3812// Given an 'ISD::OR' node that is going to be selected as BFM, analyze
3813// the operands and select it to AArch64::ORR with shifted registers if
3814// that's more efficient. Returns true iff selection to AArch64::ORR happens.
3815static bool tryOrrWithShift(SDNode *N, SDValue OrOpd0, SDValue OrOpd1,
3816 SDValue Src, SDValue Dst, SelectionDAG *CurDAG,
3817 const bool BiggerPattern) {
3818 EVT VT = N->getValueType(ResNo: 0);
3819 assert(N->getOpcode() == ISD::OR && "Expect N to be an OR node");
3820 assert(((N->getOperand(0) == OrOpd0 && N->getOperand(1) == OrOpd1) ||
3821 (N->getOperand(1) == OrOpd0 && N->getOperand(0) == OrOpd1)) &&
3822 "Expect OrOpd0 and OrOpd1 to be operands of ISD::OR");
3823 assert((VT == MVT::i32 || VT == MVT::i64) &&
3824 "Expect result type to be i32 or i64 since N is combinable to BFM");
3825 SDLoc DL(N);
3826
3827 // Bail out if BFM simplifies away one node in BFM Dst.
3828 if (OrOpd1 != Dst)
3829 return false;
3830
3831 const unsigned OrrOpc = (VT == MVT::i32) ? AArch64::ORRWrs : AArch64::ORRXrs;
3832 // For "BFM Rd, Rn, #immr, #imms", it's known that BFM simplifies away fewer
3833 // nodes from Rn (or inserts additional shift node) if BiggerPattern is true.
3834 if (BiggerPattern) {
3835 uint64_t SrcAndImm;
3836 if (isOpcWithIntImmediate(N: OrOpd0.getNode(), Opc: ISD::AND, Imm&: SrcAndImm) &&
3837 isMask_64(Value: SrcAndImm) && OrOpd0.getOperand(i: 0) == Src) {
3838 // OrOpd0 = AND Src, #Mask
3839 // So BFM simplifies away one AND node from Src and doesn't simplify away
3840 // nodes from Dst. If ORR with left-shifted operand also simplifies away
3841 // one node (from Rd), ORR is better since it has higher throughput and
3842 // smaller latency than BFM on many AArch64 processors (and for the rest
3843 // ORR is at least as good as BFM).
3844 SDValue ShiftedOperand;
3845 uint64_t EncodedShiftImm;
3846 if (isWorthFoldingIntoOrrWithShift(Dst, CurDAG, ShiftedOperand,
3847 EncodedShiftImm)) {
3848 SDValue Ops[] = {OrOpd0, ShiftedOperand,
3849 CurDAG->getTargetConstant(Val: EncodedShiftImm, DL, VT)};
3850 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
3851 return true;
3852 }
3853 }
3854 return false;
3855 }
3856
3857 assert((!BiggerPattern) && "BiggerPattern should be handled above");
3858
3859 uint64_t ShlImm;
3860 if (isOpcWithIntImmediate(N: OrOpd0.getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
3861 if (OrOpd0.getOperand(i: 0) == Src && OrOpd0.hasOneUse()) {
3862 SDValue Ops[] = {
3863 Dst, Src,
3864 CurDAG->getTargetConstant(
3865 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShlImm), DL, VT)};
3866 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
3867 return true;
3868 }
3869
3870 // Select the following pattern to left-shifted operand rather than BFI.
3871 // %val1 = op ..
3872 // %val2 = shl %val1, #imm
3873 // %res = or %val1, %val2
3874 //
3875 // If N is selected to be BFI, we know that
3876 // 1) OrOpd0 would be the operand from which extract bits (i.e., folded into
3877 // BFI) 2) OrOpd1 would be the destination operand (i.e., preserved)
3878 //
3879 // Instead of selecting N to BFI, fold OrOpd0 as a left shift directly.
3880 if (OrOpd0.getOperand(i: 0) == OrOpd1) {
3881 SDValue Ops[] = {
3882 OrOpd1, OrOpd1,
3883 CurDAG->getTargetConstant(
3884 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShlImm), DL, VT)};
3885 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
3886 return true;
3887 }
3888 }
3889
3890 uint64_t SrlImm;
3891 if (isOpcWithIntImmediate(N: OrOpd0.getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
3892 // Select the following pattern to right-shifted operand rather than BFXIL.
3893 // %val1 = op ..
3894 // %val2 = lshr %val1, #imm
3895 // %res = or %val1, %val2
3896 //
3897 // If N is selected to be BFXIL, we know that
3898 // 1) OrOpd0 would be the operand from which extract bits (i.e., folded into
3899 // BFXIL) 2) OrOpd1 would be the destination operand (i.e., preserved)
3900 //
3901 // Instead of selecting N to BFXIL, fold OrOpd0 as a right shift directly.
3902 if (OrOpd0.getOperand(i: 0) == OrOpd1) {
3903 SDValue Ops[] = {
3904 OrOpd1, OrOpd1,
3905 CurDAG->getTargetConstant(
3906 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: SrlImm), DL, VT)};
3907 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
3908 return true;
3909 }
3910 }
3911
3912 return false;
3913}
3914
3915static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits,
3916 SelectionDAG *CurDAG) {
3917 assert(N->getOpcode() == ISD::OR && "Expect a OR operation");
3918
3919 EVT VT = N->getValueType(ResNo: 0);
3920 if (VT != MVT::i32 && VT != MVT::i64)
3921 return false;
3922
3923 unsigned BitWidth = VT.getSizeInBits();
3924
3925 // Because of simplify-demanded-bits in DAGCombine, involved masks may not
3926 // have the expected shape. Try to undo that.
3927
3928 unsigned NumberOfIgnoredLowBits = UsefulBits.countr_zero();
3929 unsigned NumberOfIgnoredHighBits = UsefulBits.countl_zero();
3930
3931 // Given a OR operation, check if we have the following pattern
3932 // ubfm c, b, imm, imm2 (or something that does the same jobs, see
3933 // isBitfieldExtractOp)
3934 // d = e & mask2 ; where mask is a binary sequence of 1..10..0 and
3935 // countTrailingZeros(mask2) == imm2 - imm + 1
3936 // f = d | c
3937 // if yes, replace the OR instruction with:
3938 // f = BFM Opd0, Opd1, LSB, MSB ; where LSB = imm, and MSB = imm2
3939
3940 // OR is commutative, check all combinations of operand order and values of
3941 // BiggerPattern, i.e.
3942 // Opd0, Opd1, BiggerPattern=false
3943 // Opd1, Opd0, BiggerPattern=false
3944 // Opd0, Opd1, BiggerPattern=true
3945 // Opd1, Opd0, BiggerPattern=true
3946 // Several of these combinations may match, so check with BiggerPattern=false
3947 // first since that will produce better results by matching more instructions
3948 // and/or inserting fewer extra instructions.
3949 for (int I = 0; I < 4; ++I) {
3950
3951 SDValue Dst, Src;
3952 unsigned ImmR, ImmS;
3953 bool BiggerPattern = I / 2;
3954 SDValue OrOpd0Val = N->getOperand(Num: I % 2);
3955 SDNode *OrOpd0 = OrOpd0Val.getNode();
3956 SDValue OrOpd1Val = N->getOperand(Num: (I + 1) % 2);
3957 SDNode *OrOpd1 = OrOpd1Val.getNode();
3958
3959 unsigned BFXOpc;
3960 int DstLSB, Width;
3961 if (isBitfieldExtractOp(CurDAG, N: OrOpd0, Opc&: BFXOpc, Opd0&: Src, Immr&: ImmR, Imms&: ImmS,
3962 NumberOfIgnoredLowBits, BiggerPattern)) {
3963 // Check that the returned opcode is compatible with the pattern,
3964 // i.e., same type and zero extended (U and not S)
3965 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) ||
3966 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32))
3967 continue;
3968
3969 // Compute the width of the bitfield insertion
3970 DstLSB = 0;
3971 Width = ImmS - ImmR + 1;
3972 // FIXME: This constraint is to catch bitfield insertion we may
3973 // want to widen the pattern if we want to grab general bitfield
3974 // move case
3975 if (Width <= 0)
3976 continue;
3977
3978 // If the mask on the insertee is correct, we have a BFXIL operation. We
3979 // can share the ImmR and ImmS values from the already-computed UBFM.
3980 } else if (isBitfieldPositioningOp(CurDAG, Op: OrOpd0Val,
3981 BiggerPattern,
3982 Src, DstLSB, Width)) {
3983 ImmR = (BitWidth - DstLSB) % BitWidth;
3984 ImmS = Width - 1;
3985 } else
3986 continue;
3987
3988 // Check the second part of the pattern
3989 EVT VT = OrOpd1Val.getValueType();
3990 assert((VT == MVT::i32 || VT == MVT::i64) && "unexpected OR operand");
3991
3992 // Compute the Known Zero for the candidate of the first operand.
3993 // This allows to catch more general case than just looking for
3994 // AND with imm. Indeed, simplify-demanded-bits may have removed
3995 // the AND instruction because it proves it was useless.
3996 KnownBits Known = CurDAG->computeKnownBits(Op: OrOpd1Val);
3997
3998 // Check if there is enough room for the second operand to appear
3999 // in the first one
4000 APInt BitsToBeInserted =
4001 APInt::getBitsSet(numBits: Known.getBitWidth(), loBit: DstLSB, hiBit: DstLSB + Width);
4002
4003 if ((BitsToBeInserted & ~Known.Zero) != 0)
4004 continue;
4005
4006 // Set the first operand
4007 uint64_t Imm;
4008 if (isOpcWithIntImmediate(N: OrOpd1, Opc: ISD::AND, Imm) &&
4009 isBitfieldDstMask(DstMask: Imm, BitsToBeInserted, NumberOfIgnoredHighBits, VT))
4010 // In that case, we can eliminate the AND
4011 Dst = OrOpd1->getOperand(Num: 0);
4012 else
4013 // Maybe the AND has been removed by simplify-demanded-bits
4014 // or is useful because it discards more bits
4015 Dst = OrOpd1Val;
4016
4017 // Before selecting ISD::OR node to AArch64::BFM, see if an AArch64::ORR
4018 // with shifted operand is more efficient.
4019 if (tryOrrWithShift(N, OrOpd0: OrOpd0Val, OrOpd1: OrOpd1Val, Src, Dst, CurDAG,
4020 BiggerPattern))
4021 return true;
4022
4023 // both parts match
4024 SDLoc DL(N);
4025 SDValue Ops[] = {Dst, Src, CurDAG->getTargetConstant(Val: ImmR, DL, VT),
4026 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
4027 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
4028 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
4029 return true;
4030 }
4031
4032 // Generate a BFXIL from 'or (and X, Mask0Imm), (and Y, Mask1Imm)' iff
4033 // Mask0Imm and ~Mask1Imm are equivalent and one of the MaskImms is a shifted
4034 // mask (e.g., 0x000ffff0).
4035 uint64_t Mask0Imm, Mask1Imm;
4036 SDValue And0 = N->getOperand(Num: 0);
4037 SDValue And1 = N->getOperand(Num: 1);
4038 if (And0.hasOneUse() && And1.hasOneUse() &&
4039 isOpcWithIntImmediate(N: And0.getNode(), Opc: ISD::AND, Imm&: Mask0Imm) &&
4040 isOpcWithIntImmediate(N: And1.getNode(), Opc: ISD::AND, Imm&: Mask1Imm) &&
4041 APInt(BitWidth, Mask0Imm) == ~APInt(BitWidth, Mask1Imm) &&
4042 (isShiftedMask(Mask: Mask0Imm, VT) || isShiftedMask(Mask: Mask1Imm, VT))) {
4043
4044 // ORR is commutative, so canonicalize to the form 'or (and X, Mask0Imm),
4045 // (and Y, Mask1Imm)' where Mask1Imm is the shifted mask masking off the
4046 // bits to be inserted.
4047 if (isShiftedMask(Mask: Mask0Imm, VT)) {
4048 std::swap(a&: And0, b&: And1);
4049 std::swap(a&: Mask0Imm, b&: Mask1Imm);
4050 }
4051
4052 SDValue Src = And1->getOperand(Num: 0);
4053 SDValue Dst = And0->getOperand(Num: 0);
4054 unsigned LSB = llvm::countr_zero(Val: Mask1Imm);
4055 int Width = BitWidth - APInt(BitWidth, Mask0Imm).popcount();
4056
4057 // The BFXIL inserts the low-order bits from a source register, so right
4058 // shift the needed bits into place.
4059 SDLoc DL(N);
4060 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
4061 uint64_t LsrImm = LSB;
4062 if (Src->hasOneUse() &&
4063 isOpcWithIntImmediate(N: Src.getNode(), Opc: ISD::SRL, Imm&: LsrImm) &&
4064 (LsrImm + LSB) < BitWidth) {
4065 Src = Src->getOperand(Num: 0);
4066 LsrImm += LSB;
4067 }
4068
4069 SDNode *LSR = CurDAG->getMachineNode(
4070 Opcode: ShiftOpc, dl: DL, VT, Op1: Src, Op2: CurDAG->getTargetConstant(Val: LsrImm, DL, VT),
4071 Op3: CurDAG->getTargetConstant(Val: BitWidth - 1, DL, VT));
4072
4073 // BFXIL is an alias of BFM, so translate to BFM operands.
4074 unsigned ImmR = (BitWidth - LSB) % BitWidth;
4075 unsigned ImmS = Width - 1;
4076
4077 // Create the BFXIL instruction.
4078 SDValue Ops[] = {Dst, SDValue(LSR, 0),
4079 CurDAG->getTargetConstant(Val: ImmR, DL, VT),
4080 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
4081 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
4082 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
4083 return true;
4084 }
4085
4086 return false;
4087}
4088
4089bool AArch64DAGToDAGISel::tryBitfieldInsertOp(SDNode *N) {
4090 if (N->getOpcode() != ISD::OR)
4091 return false;
4092
4093 APInt NUsefulBits;
4094 getUsefulBits(Op: SDValue(N, 0), UsefulBits&: NUsefulBits);
4095
4096 // If all bits are not useful, just return UNDEF.
4097 if (!NUsefulBits) {
4098 CurDAG->SelectNodeTo(N, MachineOpc: TargetOpcode::IMPLICIT_DEF, VT: N->getValueType(ResNo: 0));
4099 return true;
4100 }
4101
4102 if (tryBitfieldInsertOpFromOr(N, UsefulBits: NUsefulBits, CurDAG))
4103 return true;
4104
4105 return tryBitfieldInsertOpFromOrAndImm(N, CurDAG);
4106}
4107
4108/// SelectBitfieldInsertInZeroOp - Match a UBFIZ instruction that is the
4109/// equivalent of a left shift by a constant amount followed by an and masking
4110/// out a contiguous set of bits.
4111bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(SDNode *N) {
4112 if (N->getOpcode() != ISD::AND)
4113 return false;
4114
4115 EVT VT = N->getValueType(ResNo: 0);
4116 if (VT != MVT::i32 && VT != MVT::i64)
4117 return false;
4118
4119 SDValue Op0;
4120 int DstLSB, Width;
4121 if (!isBitfieldPositioningOp(CurDAG, Op: SDValue(N, 0), /*BiggerPattern=*/false,
4122 Src&: Op0, DstLSB, Width))
4123 return false;
4124
4125 // ImmR is the rotate right amount.
4126 unsigned ImmR = (VT.getSizeInBits() - DstLSB) % VT.getSizeInBits();
4127 // ImmS is the most significant bit of the source to be moved.
4128 unsigned ImmS = Width - 1;
4129
4130 SDLoc DL(N);
4131 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(Val: ImmR, DL, VT),
4132 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
4133 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
4134 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
4135 return true;
4136}
4137
4138/// tryShiftAmountMod - Take advantage of built-in mod of shift amount in
4139/// variable shift/rotate instructions.
4140bool AArch64DAGToDAGISel::tryShiftAmountMod(SDNode *N) {
4141 EVT VT = N->getValueType(ResNo: 0);
4142
4143 unsigned Opc;
4144 switch (N->getOpcode()) {
4145 case ISD::ROTR:
4146 Opc = (VT == MVT::i32) ? AArch64::RORVWr : AArch64::RORVXr;
4147 break;
4148 case ISD::SHL:
4149 Opc = (VT == MVT::i32) ? AArch64::LSLVWr : AArch64::LSLVXr;
4150 break;
4151 case ISD::SRL:
4152 Opc = (VT == MVT::i32) ? AArch64::LSRVWr : AArch64::LSRVXr;
4153 break;
4154 case ISD::SRA:
4155 Opc = (VT == MVT::i32) ? AArch64::ASRVWr : AArch64::ASRVXr;
4156 break;
4157 default:
4158 return false;
4159 }
4160
4161 uint64_t Size;
4162 uint64_t Bits;
4163 if (VT == MVT::i32) {
4164 Bits = 5;
4165 Size = 32;
4166 } else if (VT == MVT::i64) {
4167 Bits = 6;
4168 Size = 64;
4169 } else
4170 return false;
4171
4172 SDValue ShiftAmt = N->getOperand(Num: 1);
4173 SDLoc DL(N);
4174 SDValue NewShiftAmt;
4175
4176 // Skip over an extend of the shift amount.
4177 if (ShiftAmt->getOpcode() == ISD::ZERO_EXTEND ||
4178 ShiftAmt->getOpcode() == ISD::ANY_EXTEND)
4179 ShiftAmt = ShiftAmt->getOperand(Num: 0);
4180
4181 if (ShiftAmt->getOpcode() == ISD::ADD || ShiftAmt->getOpcode() == ISD::SUB) {
4182 SDValue Add0 = ShiftAmt->getOperand(Num: 0);
4183 SDValue Add1 = ShiftAmt->getOperand(Num: 1);
4184 uint64_t Add0Imm;
4185 uint64_t Add1Imm;
4186 if (isIntImmediate(N: Add1, Imm&: Add1Imm) && (Add1Imm % Size == 0)) {
4187 // If we are shifting by X+/-N where N == 0 mod Size, then just shift by X
4188 // to avoid the ADD/SUB.
4189 NewShiftAmt = Add0;
4190 } else if (ShiftAmt->getOpcode() == ISD::SUB &&
4191 isIntImmediate(N: Add0, Imm&: Add0Imm) && Add0Imm != 0 &&
4192 (Add0Imm % Size == 0)) {
4193 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
4194 // to generate a NEG instead of a SUB from a constant.
4195 unsigned NegOpc;
4196 unsigned ZeroReg;
4197 EVT SubVT = ShiftAmt->getValueType(ResNo: 0);
4198 if (SubVT == MVT::i32) {
4199 NegOpc = AArch64::SUBWrr;
4200 ZeroReg = AArch64::WZR;
4201 } else {
4202 assert(SubVT == MVT::i64);
4203 NegOpc = AArch64::SUBXrr;
4204 ZeroReg = AArch64::XZR;
4205 }
4206 SDValue Zero =
4207 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: ZeroReg, VT: SubVT);
4208 MachineSDNode *Neg =
4209 CurDAG->getMachineNode(Opcode: NegOpc, dl: DL, VT: SubVT, Op1: Zero, Op2: Add1);
4210 NewShiftAmt = SDValue(Neg, 0);
4211 } else if (ShiftAmt->getOpcode() == ISD::SUB &&
4212 isIntImmediate(N: Add0, Imm&: Add0Imm) && (Add0Imm % Size == Size - 1)) {
4213 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
4214 // to generate a NOT instead of a SUB from a constant.
4215 unsigned NotOpc;
4216 unsigned ZeroReg;
4217 EVT SubVT = ShiftAmt->getValueType(ResNo: 0);
4218 if (SubVT == MVT::i32) {
4219 NotOpc = AArch64::ORNWrr;
4220 ZeroReg = AArch64::WZR;
4221 } else {
4222 assert(SubVT == MVT::i64);
4223 NotOpc = AArch64::ORNXrr;
4224 ZeroReg = AArch64::XZR;
4225 }
4226 SDValue Zero =
4227 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: ZeroReg, VT: SubVT);
4228 MachineSDNode *Not =
4229 CurDAG->getMachineNode(Opcode: NotOpc, dl: DL, VT: SubVT, Op1: Zero, Op2: Add1);
4230 NewShiftAmt = SDValue(Not, 0);
4231 } else
4232 return false;
4233 } else {
4234 // If the shift amount is masked with an AND, check that the mask covers the
4235 // bits that are implicitly ANDed off by the above opcodes and if so, skip
4236 // the AND.
4237 uint64_t MaskImm;
4238 if (!isOpcWithIntImmediate(N: ShiftAmt.getNode(), Opc: ISD::AND, Imm&: MaskImm) &&
4239 !isOpcWithIntImmediate(N: ShiftAmt.getNode(), Opc: AArch64ISD::ANDS, Imm&: MaskImm))
4240 return false;
4241
4242 if ((unsigned)llvm::countr_one(Value: MaskImm) < Bits)
4243 return false;
4244
4245 NewShiftAmt = ShiftAmt->getOperand(Num: 0);
4246 }
4247
4248 // Narrow/widen the shift amount to match the size of the shift operation.
4249 if (VT == MVT::i32)
4250 NewShiftAmt = narrowIfNeeded(CurDAG, N: NewShiftAmt);
4251 else if (VT == MVT::i64 && NewShiftAmt->getValueType(ResNo: 0) == MVT::i32) {
4252 SDValue SubReg = CurDAG->getTargetConstant(Val: AArch64::sub_32, DL, VT: MVT::i32);
4253 MachineSDNode *Ext = CurDAG->getMachineNode(Opcode: AArch64::SUBREG_TO_REG, dl: DL, VT,
4254 Op1: NewShiftAmt, Op2: SubReg);
4255 NewShiftAmt = SDValue(Ext, 0);
4256 }
4257
4258 SDValue Ops[] = {N->getOperand(Num: 0), NewShiftAmt};
4259 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
4260 return true;
4261}
4262
4263static bool checkCVTFixedPointOperandWithFBits(SelectionDAG *CurDAG, SDValue N,
4264 SDValue &FixedPos,
4265 unsigned RegWidth,
4266 bool isReciprocal) {
4267 APFloat FVal(0.0);
4268 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: N))
4269 FVal = CN->getValueAPF();
4270 else if (LoadSDNode *LN = dyn_cast<LoadSDNode>(Val&: N)) {
4271 // Some otherwise illegal constants are allowed in this case.
4272 if (LN->getOperand(Num: 1).getOpcode() != AArch64ISD::ADDlow ||
4273 !isa<ConstantPoolSDNode>(Val: LN->getOperand(Num: 1)->getOperand(Num: 1)))
4274 return false;
4275
4276 ConstantPoolSDNode *CN =
4277 dyn_cast<ConstantPoolSDNode>(Val: LN->getOperand(Num: 1)->getOperand(Num: 1));
4278 FVal = cast<ConstantFP>(Val: CN->getConstVal())->getValueAPF();
4279 } else
4280 return false;
4281
4282 if (unsigned FBits =
4283 CheckFixedPointOperandConstant(FVal, RegWidth, isReciprocal)) {
4284 FixedPos = CurDAG->getTargetConstant(Val: FBits, DL: SDLoc(N), VT: MVT::i32);
4285 return true;
4286 }
4287
4288 return false;
4289}
4290
4291static bool checkCVTFixedPointOperandWithFBitsForVectors(SelectionDAG *CurDAG,
4292 SDValue N,
4293 SDValue &FixedPos,
4294 unsigned RegWidth,
4295 bool isReciprocal) {
4296 if ((N.getOpcode() == AArch64ISD::NVCAST || N.getOpcode() == ISD::BITCAST) &&
4297 N.getValueType().getScalarSizeInBits() ==
4298 N.getOperand(i: 0).getValueType().getScalarSizeInBits())
4299 N = N.getOperand(i: 0);
4300
4301 auto ImmToFloat = [RegWidth](APInt Imm) {
4302 switch (RegWidth) {
4303 case 16:
4304 return APFloat(APFloat::IEEEhalf(), Imm);
4305 case 32:
4306 return APFloat(APFloat::IEEEsingle(), Imm);
4307 case 64:
4308 return APFloat(APFloat::IEEEdouble(), Imm);
4309 default:
4310 llvm_unreachable("Unexpected RegWidth!");
4311 };
4312 };
4313
4314 APFloat FVal(0.0);
4315 switch (N->getOpcode()) {
4316 case AArch64ISD::MOVIshift:
4317 FVal = ImmToFloat(APInt(RegWidth, N.getConstantOperandVal(i: 0)
4318 << N.getConstantOperandVal(i: 1)));
4319 break;
4320 case AArch64ISD::FMOV:
4321 FVal = ImmToFloat(DecodeFMOVImm(Imm: N.getConstantOperandVal(i: 0), RegWidth));
4322 break;
4323 case AArch64ISD::DUP:
4324 if (isa<ConstantSDNode>(Val: N.getOperand(i: 0)))
4325 FVal = ImmToFloat(N.getConstantOperandAPInt(i: 0).trunc(width: RegWidth));
4326 else
4327 return false;
4328 break;
4329 default:
4330 return false;
4331 }
4332
4333 if (unsigned FBits =
4334 CheckFixedPointOperandConstant(FVal, RegWidth, isReciprocal)) {
4335 FixedPos = CurDAG->getTargetConstant(Val: FBits, DL: SDLoc(N), VT: MVT::i32);
4336 return true;
4337 }
4338
4339 return false;
4340}
4341
4342bool AArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos,
4343 unsigned RegWidth) {
4344 return checkCVTFixedPointOperandWithFBits(CurDAG, N, FixedPos, RegWidth,
4345 /*isReciprocal*/ false);
4346}
4347
4348bool AArch64DAGToDAGISel::SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos,
4349 unsigned RegWidth) {
4350 return checkCVTFixedPointOperandWithFBitsForVectors(
4351 CurDAG, N, FixedPos, RegWidth, /*isReciprocal*/ false);
4352}
4353
4354bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperandVec(SDValue N,
4355 SDValue &FixedPos,
4356 unsigned RegWidth) {
4357 return checkCVTFixedPointOperandWithFBitsForVectors(
4358 CurDAG, N, FixedPos, RegWidth, /*isReciprocal*/ true);
4359}
4360
4361bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperand(SDValue N,
4362 SDValue &FixedPos,
4363 unsigned RegWidth) {
4364 return checkCVTFixedPointOperandWithFBits(CurDAG, N, FixedPos, RegWidth,
4365 /*isReciprocal*/ true);
4366}
4367
4368// Inspects a register string of the form o0:op1:CRn:CRm:op2 gets the fields
4369// of the string and obtains the integer values from them and combines these
4370// into a single value to be used in the MRS/MSR instruction.
4371static int getIntOperandFromRegisterString(StringRef RegString) {
4372 SmallVector<StringRef, 5> Fields;
4373 RegString.split(A&: Fields, Separator: ':');
4374
4375 if (Fields.size() == 1)
4376 return -1;
4377
4378 assert(Fields.size() == 5
4379 && "Invalid number of fields in read register string");
4380
4381 SmallVector<int, 5> Ops;
4382 bool AllIntFields = true;
4383
4384 for (StringRef Field : Fields) {
4385 unsigned IntField;
4386 AllIntFields &= !Field.getAsInteger(Radix: 10, Result&: IntField);
4387 Ops.push_back(Elt: IntField);
4388 }
4389
4390 assert(AllIntFields &&
4391 "Unexpected non-integer value in special register string.");
4392 (void)AllIntFields;
4393
4394 // Need to combine the integer fields of the string into a single value
4395 // based on the bit encoding of MRS/MSR instruction.
4396 return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) | (Ops[3] << 3) |
4397 (Ops[4]);
4398}
4399
4400// Lower the read_register intrinsic to an MRS instruction node if the special
4401// register string argument is either of the form detailed in the ALCE (the
4402// form described in getIntOperandsFromRegisterString) or is a named register
4403// known by the MRS SysReg mapper.
4404bool AArch64DAGToDAGISel::tryReadRegister(SDNode *N) {
4405 const auto *MD = cast<MDNodeSDNode>(Val: N->getOperand(Num: 1));
4406 const auto *RegString = cast<MDString>(Val: MD->getMD()->getOperand(I: 0));
4407 SDLoc DL(N);
4408
4409 bool ReadIs128Bit = N->getOpcode() == AArch64ISD::MRRS;
4410
4411 unsigned Opcode64Bit = AArch64::MRS;
4412 int Imm = getIntOperandFromRegisterString(RegString: RegString->getString());
4413 if (Imm == -1) {
4414 // No match, Use the sysreg mapper to map the remaining possible strings to
4415 // the value for the register to be used for the instruction operand.
4416 const auto *TheReg =
4417 AArch64SysReg::lookupSysRegByName(Name: RegString->getString());
4418 if (TheReg && TheReg->Readable &&
4419 TheReg->haveFeatures(ActiveFeatures: Subtarget->getFeatureBits()))
4420 Imm = TheReg->Encoding;
4421 else
4422 Imm = AArch64SysReg::parseGenericRegister(Name: RegString->getString());
4423
4424 if (Imm == -1) {
4425 // Still no match, see if this is "pc" or give up.
4426 if (!ReadIs128Bit && RegString->getString() == "pc") {
4427 Opcode64Bit = AArch64::ADR;
4428 Imm = 0;
4429 } else {
4430 // Not a system register. It may name an allocatable 64-bit GPR/FPR read
4431 // by the MSVC __getReg/__getRegFp intrinsics. Emit a pseudo that
4432 // carries the source register as an immediate so the read does not
4433 // reference an undefined physical register (which the machine verifier
4434 // rejects); the AsmPrinter materializes the real mov/fmov.
4435 Register PReg = Subtarget->getTargetLowering()->matchRegisterName(
4436 RegName: RegString->getString());
4437 unsigned PseudoOp = 0;
4438 if (AArch64::GPR64RegClass.contains(Reg: PReg))
4439 PseudoOp = AArch64::READ_REGISTER_GPR64;
4440 else if (AArch64::FPR64RegClass.contains(Reg: PReg))
4441 PseudoOp = AArch64::READ_REGISTER_FPR64;
4442 if (!ReadIs128Bit && PseudoOp && N->getValueType(ResNo: 0) == MVT::i64) {
4443 CurDAG->SelectNodeTo(N, MachineOpc: PseudoOp, VT1: MVT::i64, VT2: MVT::Other,
4444 Ops: {CurDAG->getTargetConstant(Val: PReg, DL, VT: MVT::i32),
4445 N->getOperand(Num: 0)});
4446 return true;
4447 }
4448 return false;
4449 }
4450 }
4451 }
4452
4453 SDValue InChain = N->getOperand(Num: 0);
4454 SDValue SysRegImm = CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i32);
4455 if (!ReadIs128Bit) {
4456 CurDAG->SelectNodeTo(N, MachineOpc: Opcode64Bit, VT1: MVT::i64, VT2: MVT::Other /* Chain */,
4457 Ops: {SysRegImm, InChain});
4458 } else {
4459 SDNode *MRRS = CurDAG->getMachineNode(
4460 Opcode: AArch64::MRRS, dl: DL,
4461 ResultTys: {MVT::Untyped /* XSeqPair */, MVT::Other /* Chain */},
4462 Ops: {SysRegImm, InChain});
4463
4464 // Sysregs are not endian. The even register always contains the low half
4465 // of the register.
4466 SDValue Lo = CurDAG->getTargetExtractSubreg(SRIdx: AArch64::sube64, DL, VT: MVT::i64,
4467 Operand: SDValue(MRRS, 0));
4468 SDValue Hi = CurDAG->getTargetExtractSubreg(SRIdx: AArch64::subo64, DL, VT: MVT::i64,
4469 Operand: SDValue(MRRS, 0));
4470 SDValue OutChain = SDValue(MRRS, 1);
4471
4472 ReplaceUses(F: SDValue(N, 0), T: Lo);
4473 ReplaceUses(F: SDValue(N, 1), T: Hi);
4474 ReplaceUses(F: SDValue(N, 2), T: OutChain);
4475 };
4476 return true;
4477}
4478
4479// Lower the write_register intrinsic to an MSR instruction node if the special
4480// register string argument is either of the form detailed in the ALCE (the
4481// form described in getIntOperandsFromRegisterString) or is a named register
4482// known by the MSR SysReg mapper.
4483bool AArch64DAGToDAGISel::tryWriteRegister(SDNode *N) {
4484 const auto *MD = cast<MDNodeSDNode>(Val: N->getOperand(Num: 1));
4485 const auto *RegString = cast<MDString>(Val: MD->getMD()->getOperand(I: 0));
4486 SDLoc DL(N);
4487
4488 bool WriteIs128Bit = N->getOpcode() == AArch64ISD::MSRR;
4489
4490 if (!WriteIs128Bit) {
4491 // Check if the register was one of those allowed as the pstatefield value
4492 // in the MSR (immediate) instruction. To accept the values allowed in the
4493 // pstatefield for the MSR (immediate) instruction, we also require that an
4494 // immediate value has been provided as an argument, we know that this is
4495 // the case as it has been ensured by semantic checking.
4496 auto trySelectPState = [&](auto PMapper, unsigned State) {
4497 if (PMapper) {
4498 assert(isa<ConstantSDNode>(N->getOperand(2)) &&
4499 "Expected a constant integer expression.");
4500 unsigned Reg = PMapper->Encoding;
4501 uint64_t Immed = N->getConstantOperandVal(Num: 2);
4502 CurDAG->SelectNodeTo(
4503 N, MachineOpc: State, VT: MVT::Other, Op1: CurDAG->getTargetConstant(Val: Reg, DL, VT: MVT::i32),
4504 Op2: CurDAG->getTargetConstant(Val: Immed, DL, VT: MVT::i16), Op3: N->getOperand(Num: 0));
4505 return true;
4506 }
4507 return false;
4508 };
4509
4510 if (trySelectPState(
4511 AArch64PState::lookupPStateImm0_15ByName(Name: RegString->getString()),
4512 AArch64::MSRpstateImm4))
4513 return true;
4514 if (trySelectPState(
4515 AArch64PState::lookupPStateImm0_1ByName(Name: RegString->getString()),
4516 AArch64::MSRpstateImm1))
4517 return true;
4518 }
4519
4520 int Imm = getIntOperandFromRegisterString(RegString: RegString->getString());
4521 if (Imm == -1) {
4522 // Use the sysreg mapper to attempt to map the remaining possible strings
4523 // to the value for the register to be used for the MSR (register)
4524 // instruction operand.
4525 auto TheReg = AArch64SysReg::lookupSysRegByName(Name: RegString->getString());
4526 if (TheReg && TheReg->Writeable &&
4527 TheReg->haveFeatures(ActiveFeatures: Subtarget->getFeatureBits()))
4528 Imm = TheReg->Encoding;
4529 else
4530 Imm = AArch64SysReg::parseGenericRegister(Name: RegString->getString());
4531
4532 if (Imm == -1) {
4533 // Used by the MSVC __setReg/__setRegFp intrinsics. Copy the value into
4534 // the physical register and keep it live with a FAKE_USE so the write is
4535 // not dead-eliminated. (getRegisterByName rejects allocatable registers,
4536 // so the generic write path cannot handle these.)
4537 Register PReg = Subtarget->getTargetLowering()->matchRegisterName(
4538 RegName: RegString->getString());
4539 bool IsGPR = AArch64::GPR64RegClass.contains(Reg: PReg);
4540 bool IsFPR = AArch64::FPR64RegClass.contains(Reg: PReg);
4541 if (!WriteIs128Bit && (IsGPR || IsFPR) &&
4542 N->getOperand(Num: 2).getValueType() == MVT::i64) {
4543 SDValue Copy =
4544 CurDAG->getCopyToReg(Chain: N->getOperand(Num: 0), dl: DL, Reg: PReg, N: N->getOperand(Num: 2));
4545 SDValue RegOp = CurDAG->getRegister(Reg: PReg, VT: MVT::i64);
4546 SDNode *FakeUse = CurDAG->getMachineNode(Opcode: TargetOpcode::FAKE_USE, dl: DL,
4547 VT: MVT::Other, Ops: {RegOp, Copy});
4548 ReplaceUses(F: SDValue(N, 0), T: SDValue(FakeUse, 0));
4549 CurDAG->RemoveDeadNode(N);
4550 return true;
4551 }
4552 return false;
4553 }
4554 }
4555
4556 SDValue InChain = N->getOperand(Num: 0);
4557 if (!WriteIs128Bit) {
4558 CurDAG->SelectNodeTo(N, MachineOpc: AArch64::MSR, VT: MVT::Other,
4559 Op1: CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i32),
4560 Op2: N->getOperand(Num: 2), Op3: InChain);
4561 } else {
4562 // No endian swap. The lower half always goes into the even subreg, and the
4563 // higher half always into the odd supreg.
4564 SDNode *Pair = CurDAG->getMachineNode(
4565 Opcode: TargetOpcode::REG_SEQUENCE, dl: DL, VT: MVT::Untyped /* XSeqPair */,
4566 Ops: {CurDAG->getTargetConstant(Val: AArch64::XSeqPairsClassRegClass.getID(), DL,
4567 VT: MVT::i32),
4568 N->getOperand(Num: 2),
4569 CurDAG->getTargetConstant(Val: AArch64::sube64, DL, VT: MVT::i32),
4570 N->getOperand(Num: 3),
4571 CurDAG->getTargetConstant(Val: AArch64::subo64, DL, VT: MVT::i32)});
4572
4573 CurDAG->SelectNodeTo(N, MachineOpc: AArch64::MSRR, VT: MVT::Other,
4574 Op1: CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i32),
4575 Op2: SDValue(Pair, 0), Op3: InChain);
4576 }
4577
4578 return true;
4579}
4580
4581/// We've got special pseudo-instructions for these
4582bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
4583 unsigned Opcode;
4584 EVT MemTy = cast<MemSDNode>(Val: N)->getMemoryVT();
4585
4586 // Leave IR for LSE if subtarget supports it.
4587 if (Subtarget->hasLSE()) return false;
4588
4589 if (MemTy == MVT::i8)
4590 Opcode = AArch64::CMP_SWAP_8;
4591 else if (MemTy == MVT::i16)
4592 Opcode = AArch64::CMP_SWAP_16;
4593 else if (MemTy == MVT::i32)
4594 Opcode = AArch64::CMP_SWAP_32;
4595 else if (MemTy == MVT::i64)
4596 Opcode = AArch64::CMP_SWAP_64;
4597 else
4598 llvm_unreachable("Unknown AtomicCmpSwap type");
4599
4600 MVT RegTy = MemTy == MVT::i64 ? MVT::i64 : MVT::i32;
4601 SDValue Ops[] = {N->getOperand(Num: 1), N->getOperand(Num: 2), N->getOperand(Num: 3),
4602 N->getOperand(Num: 0)};
4603 SDNode *CmpSwap = CurDAG->getMachineNode(
4604 Opcode, dl: SDLoc(N),
4605 VTs: CurDAG->getVTList(VT1: RegTy, VT2: MVT::i32, VT3: MVT::Other), Ops);
4606
4607 MachineMemOperand *MemOp = cast<MemSDNode>(Val: N)->getMemOperand();
4608 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: CmpSwap), NewMemRefs: {MemOp});
4609
4610 ReplaceUses(F: SDValue(N, 0), T: SDValue(CmpSwap, 0));
4611 ReplaceUses(F: SDValue(N, 1), T: SDValue(CmpSwap, 2));
4612 CurDAG->RemoveDeadNode(N);
4613
4614 return true;
4615}
4616
4617bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm,
4618 SDValue &Shift, bool Negate) {
4619 if (!isa<ConstantSDNode>(Val: N))
4620 return false;
4621
4622 APInt Val =
4623 cast<ConstantSDNode>(Val&: N)->getAPIntValue().trunc(width: VT.getFixedSizeInBits());
4624
4625 return SelectSVEAddSubImm(DL: SDLoc(N), Value: Val, VT, Imm, Shift, Negate);
4626}
4627
4628bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDLoc DL, APInt Val, MVT VT,
4629 SDValue &Imm, SDValue &Shift,
4630 bool Negate) {
4631 if (Negate)
4632 Val = -Val;
4633
4634 switch (VT.SimpleTy) {
4635 case MVT::i8:
4636 // All immediates are supported.
4637 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4638 Imm = CurDAG->getTargetConstant(Val: Val.getZExtValue(), DL, VT: MVT::i32);
4639 return true;
4640 case MVT::i16:
4641 case MVT::i32:
4642 case MVT::i64:
4643 // Support 8bit unsigned immediates.
4644 if ((Val & ~0xff) == 0) {
4645 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4646 Imm = CurDAG->getTargetConstant(Val: Val.getZExtValue(), DL, VT: MVT::i32);
4647 return true;
4648 }
4649 // Support 16bit unsigned immediates that are a multiple of 256.
4650 if ((Val & ~0xff00) == 0) {
4651 Shift = CurDAG->getTargetConstant(Val: 8, DL, VT: MVT::i32);
4652 Imm = CurDAG->getTargetConstant(Val: Val.lshr(shiftAmt: 8).getZExtValue(), DL, VT: MVT::i32);
4653 return true;
4654 }
4655 break;
4656 default:
4657 break;
4658 }
4659
4660 return false;
4661}
4662
4663bool AArch64DAGToDAGISel::SelectSVEAddSubSSatImm(SDValue N, MVT VT,
4664 SDValue &Imm, SDValue &Shift,
4665 bool Negate) {
4666 if (!isa<ConstantSDNode>(Val: N))
4667 return false;
4668
4669 SDLoc DL(N);
4670 int64_t Val = cast<ConstantSDNode>(Val&: N)
4671 ->getAPIntValue()
4672 .trunc(width: VT.getFixedSizeInBits())
4673 .getSExtValue();
4674
4675 if (Negate)
4676 Val = -Val;
4677
4678 // Signed saturating instructions treat their immediate operand as unsigned,
4679 // whereas the related intrinsics define their operands to be signed. This
4680 // means we can only use the immediate form when the operand is non-negative.
4681 if (Val < 0)
4682 return false;
4683
4684 switch (VT.SimpleTy) {
4685 case MVT::i8:
4686 // All positive immediates are supported.
4687 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4688 Imm = CurDAG->getTargetConstant(Val, DL, VT: MVT::i32);
4689 return true;
4690 case MVT::i16:
4691 case MVT::i32:
4692 case MVT::i64:
4693 // Support 8bit positive immediates.
4694 if (Val <= 255) {
4695 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4696 Imm = CurDAG->getTargetConstant(Val, DL, VT: MVT::i32);
4697 return true;
4698 }
4699 // Support 16bit positive immediates that are a multiple of 256.
4700 if (Val <= 65280 && Val % 256 == 0) {
4701 Shift = CurDAG->getTargetConstant(Val: 8, DL, VT: MVT::i32);
4702 Imm = CurDAG->getTargetConstant(Val: Val >> 8, DL, VT: MVT::i32);
4703 return true;
4704 }
4705 break;
4706 default:
4707 break;
4708 }
4709
4710 return false;
4711}
4712
4713bool AArch64DAGToDAGISel::SelectSVECpyDupImm(SDValue N, MVT VT, SDValue &Imm,
4714 SDValue &Shift) {
4715 if (!isa<ConstantSDNode>(Val: N))
4716 return false;
4717
4718 SDLoc DL(N);
4719 int64_t Val = cast<ConstantSDNode>(Val&: N)
4720 ->getAPIntValue()
4721 .trunc(width: VT.getFixedSizeInBits())
4722 .getSExtValue();
4723 int32_t ImmVal, ShiftVal;
4724 if (!AArch64_AM::isSVECpyDupImm(SizeInBits: VT.getScalarSizeInBits(), Val, Imm&: ImmVal,
4725 Shift&: ShiftVal))
4726 return false;
4727
4728 Shift = CurDAG->getTargetConstant(Val: ShiftVal, DL, VT: MVT::i32);
4729 Imm = CurDAG->getTargetConstant(Val: ImmVal, DL, VT: MVT::i32);
4730 return true;
4731}
4732
4733bool AArch64DAGToDAGISel::SelectSVESignedArithImm(SDValue N, SDValue &Imm) {
4734 if (auto CNode = dyn_cast<ConstantSDNode>(Val&: N))
4735 return SelectSVESignedArithImm(DL: SDLoc(N), Value: CNode->getAPIntValue(), Imm);
4736 return false;
4737}
4738
4739bool AArch64DAGToDAGISel::SelectSVESignedArithImm(SDLoc DL, APInt Val,
4740 SDValue &Imm) {
4741 int64_t ImmVal = Val.getSExtValue();
4742 if (ImmVal >= -128 && ImmVal < 128) {
4743 Imm = CurDAG->getSignedTargetConstant(Val: ImmVal, DL, VT: MVT::i32);
4744 return true;
4745 }
4746 return false;
4747}
4748
4749bool AArch64DAGToDAGISel::SelectSVEArithImm(SDValue N, MVT VT, SDValue &Imm) {
4750 if (auto CNode = dyn_cast<ConstantSDNode>(Val&: N)) {
4751 uint64_t ImmVal = CNode->getZExtValue();
4752
4753 switch (VT.SimpleTy) {
4754 case MVT::i8:
4755 ImmVal &= 0xFF;
4756 break;
4757 case MVT::i16:
4758 ImmVal &= 0xFFFF;
4759 break;
4760 case MVT::i32:
4761 ImmVal &= 0xFFFFFFFF;
4762 break;
4763 case MVT::i64:
4764 break;
4765 default:
4766 llvm_unreachable("Unexpected type");
4767 }
4768
4769 if (ImmVal < 256) {
4770 Imm = CurDAG->getTargetConstant(Val: ImmVal, DL: SDLoc(N), VT: MVT::i32);
4771 return true;
4772 }
4773 }
4774 return false;
4775}
4776
4777bool AArch64DAGToDAGISel::SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm,
4778 bool Invert) {
4779 uint64_t ImmVal;
4780 if (auto CI = dyn_cast<ConstantSDNode>(Val&: N))
4781 ImmVal = CI->getZExtValue();
4782 else if (auto CFP = dyn_cast<ConstantFPSDNode>(Val&: N))
4783 ImmVal = CFP->getValueAPF().bitcastToAPInt().getZExtValue();
4784 else
4785 return false;
4786
4787 if (Invert)
4788 ImmVal = ~ImmVal;
4789
4790 uint64_t encoding;
4791 if (!AArch64_AM::isSVELogicalImm(SizeInBits: VT.getScalarSizeInBits(), ImmVal, Encoding&: encoding))
4792 return false;
4793
4794 Imm = CurDAG->getTargetConstant(Val: encoding, DL: SDLoc(N), VT: MVT::i64);
4795 return true;
4796}
4797
4798// SVE shift intrinsics allow shift amounts larger than the element's bitwidth.
4799// Rather than attempt to normalise everything we can sometimes saturate the
4800// shift amount during selection. This function also allows for consistent
4801// isel patterns by ensuring the resulting "Imm" node is of the i32 type
4802// required by the instructions.
4803bool AArch64DAGToDAGISel::SelectSVEShiftImm(SDValue N, uint64_t Low,
4804 uint64_t High, bool AllowSaturation,
4805 SDValue &Imm) {
4806 if (auto *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
4807 uint64_t ImmVal = CN->getZExtValue();
4808
4809 // Reject shift amounts that are too small.
4810 if (ImmVal < Low)
4811 return false;
4812
4813 // Reject or saturate shift amounts that are too big.
4814 if (ImmVal > High) {
4815 if (!AllowSaturation)
4816 return false;
4817 ImmVal = High;
4818 }
4819
4820 Imm = CurDAG->getTargetConstant(Val: ImmVal, DL: SDLoc(N), VT: MVT::i32);
4821 return true;
4822 }
4823
4824 return false;
4825}
4826
4827bool AArch64DAGToDAGISel::trySelectStackSlotTagP(SDNode *N) {
4828 // tagp(FrameIndex, IRGstack, tag_offset):
4829 // since the offset between FrameIndex and IRGstack is a compile-time
4830 // constant, this can be lowered to a single ADDG instruction.
4831 if (!(isa<FrameIndexSDNode>(Val: N->getOperand(Num: 1)))) {
4832 return false;
4833 }
4834
4835 SDValue IRG_SP = N->getOperand(Num: 2);
4836 if (IRG_SP->getOpcode() != ISD::INTRINSIC_W_CHAIN ||
4837 IRG_SP->getConstantOperandVal(Num: 1) != Intrinsic::aarch64_irg_sp) {
4838 return false;
4839 }
4840
4841 const TargetLowering *TLI = getTargetLowering();
4842 SDLoc DL(N);
4843 int FI = cast<FrameIndexSDNode>(Val: N->getOperand(Num: 1))->getIndex();
4844 SDValue FiOp = CurDAG->getTargetFrameIndex(
4845 FI, VT: TLI->getPointerTy(DL: CurDAG->getDataLayout()));
4846 int TagOffset = N->getConstantOperandVal(Num: 3);
4847
4848 SDNode *Out = CurDAG->getMachineNode(
4849 Opcode: AArch64::TAGPstack, dl: DL, VT: MVT::i64,
4850 Ops: {FiOp, CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64), N->getOperand(Num: 2),
4851 CurDAG->getTargetConstant(Val: TagOffset, DL, VT: MVT::i64)});
4852 ReplaceNode(F: N, T: Out);
4853 return true;
4854}
4855
4856void AArch64DAGToDAGISel::SelectTagP(SDNode *N) {
4857 assert(isa<ConstantSDNode>(N->getOperand(3)) &&
4858 "llvm.aarch64.tagp third argument must be an immediate");
4859 if (trySelectStackSlotTagP(N))
4860 return;
4861 // FIXME: above applies in any case when offset between Op1 and Op2 is a
4862 // compile-time constant, not just for stack allocations.
4863
4864 // General case for unrelated pointers in Op1 and Op2.
4865 SDLoc DL(N);
4866 int TagOffset = N->getConstantOperandVal(Num: 3);
4867 SDNode *N1 = CurDAG->getMachineNode(Opcode: AArch64::SUBP, dl: DL, VT: MVT::i64,
4868 Ops: {N->getOperand(Num: 1), N->getOperand(Num: 2)});
4869 SDNode *N2 = CurDAG->getMachineNode(Opcode: AArch64::ADDXrr, dl: DL, VT: MVT::i64,
4870 Ops: {SDValue(N1, 0), N->getOperand(Num: 2)});
4871 SDNode *N3 = CurDAG->getMachineNode(
4872 Opcode: AArch64::ADDG, dl: DL, VT: MVT::i64,
4873 Ops: {SDValue(N2, 0), CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64),
4874 CurDAG->getTargetConstant(Val: TagOffset, DL, VT: MVT::i64)});
4875 ReplaceNode(F: N, T: N3);
4876}
4877
4878bool AArch64DAGToDAGISel::trySelectCastFixedLengthToScalableVector(SDNode *N) {
4879 assert(N->getOpcode() == ISD::INSERT_SUBVECTOR && "Invalid Node!");
4880
4881 // Bail when not a "cast" like insert_subvector.
4882 if (N->getConstantOperandVal(Num: 2) != 0)
4883 return false;
4884 if (!N->getOperand(Num: 0).isUndef())
4885 return false;
4886
4887 // Bail when normal isel should do the job.
4888 EVT VT = N->getValueType(ResNo: 0);
4889 EVT InVT = N->getOperand(Num: 1).getValueType();
4890 if (VT.isFixedLengthVector() || InVT.isScalableVector())
4891 return false;
4892 if (InVT.getSizeInBits() <= 128)
4893 return false;
4894
4895 // NOTE: We can only get here when doing fixed length SVE code generation.
4896 // We do manual selection because the types involved are not linked to real
4897 // registers (despite being legal) and must be coerced into SVE registers.
4898
4899 assert(VT.getSizeInBits().getKnownMinValue() == AArch64::SVEBitsPerBlock &&
4900 "Expected to insert into a packed scalable vector!");
4901
4902 SDLoc DL(N);
4903 auto RC = CurDAG->getTargetConstant(Val: AArch64::ZPRRegClassID, DL, VT: MVT::i64);
4904 ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS, dl: DL, VT,
4905 Op1: N->getOperand(Num: 1), Op2: RC));
4906 return true;
4907}
4908
4909bool AArch64DAGToDAGISel::trySelectCastScalableToFixedLengthVector(SDNode *N) {
4910 assert(N->getOpcode() == ISD::EXTRACT_SUBVECTOR && "Invalid Node!");
4911
4912 // Bail when not a "cast" like extract_subvector.
4913 if (N->getConstantOperandVal(Num: 1) != 0)
4914 return false;
4915
4916 // Bail when normal isel can do the job.
4917 EVT VT = N->getValueType(ResNo: 0);
4918 EVT InVT = N->getOperand(Num: 0).getValueType();
4919 if (VT.isScalableVector() || InVT.isFixedLengthVector())
4920 return false;
4921 if (VT.getSizeInBits() <= 128)
4922 return false;
4923
4924 // NOTE: We can only get here when doing fixed length SVE code generation.
4925 // We do manual selection because the types involved are not linked to real
4926 // registers (despite being legal) and must be coerced into SVE registers.
4927
4928 assert(InVT.getSizeInBits().getKnownMinValue() == AArch64::SVEBitsPerBlock &&
4929 "Expected to extract from a packed scalable vector!");
4930
4931 SDLoc DL(N);
4932 auto RC = CurDAG->getTargetConstant(Val: AArch64::ZPRRegClassID, DL, VT: MVT::i64);
4933 ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS, dl: DL, VT,
4934 Op1: N->getOperand(Num: 0), Op2: RC));
4935 return true;
4936}
4937
4938bool AArch64DAGToDAGISel::trySelectXAR(SDNode *N) {
4939 assert(N->getOpcode() == ISD::OR && "Expected OR instruction");
4940
4941 SDValue N0 = N->getOperand(Num: 0);
4942 SDValue N1 = N->getOperand(Num: 1);
4943
4944 EVT VT = N->getValueType(ResNo: 0);
4945 SDLoc DL(N);
4946
4947 // Essentially: rotr (xor(x, y), imm) -> xar (x, y, imm)
4948 // Rotate by a constant is a funnel shift in IR which is expanded to
4949 // an OR with shifted operands.
4950 // We do the following transform:
4951 // OR N0, N1 -> xar (x, y, imm)
4952 // Where:
4953 // N1 = SRL_PRED true, V, splat(imm) --> rotr amount
4954 // N0 = SHL_PRED true, V, splat(bits-imm)
4955 // V = (xor x, y)
4956 if (VT.isScalableVector() &&
4957 (Subtarget->hasSVE2() ||
4958 (Subtarget->hasSME() && Subtarget->isStreaming()))) {
4959 if (N0.getOpcode() != AArch64ISD::SHL_PRED ||
4960 N1.getOpcode() != AArch64ISD::SRL_PRED)
4961 std::swap(a&: N0, b&: N1);
4962 if (N0.getOpcode() != AArch64ISD::SHL_PRED ||
4963 N1.getOpcode() != AArch64ISD::SRL_PRED)
4964 return false;
4965
4966 auto *TLI = static_cast<const AArch64TargetLowering *>(getTargetLowering());
4967 if (!TLI->isAllActivePredicate(DAG: *CurDAG, N: N0.getOperand(i: 0)) ||
4968 !TLI->isAllActivePredicate(DAG: *CurDAG, N: N1.getOperand(i: 0)))
4969 return false;
4970
4971 if (N0.getOperand(i: 1) != N1.getOperand(i: 1))
4972 return false;
4973
4974 SDValue R1, R2;
4975 bool IsXOROperand = true;
4976 if (N0.getOperand(i: 1).getOpcode() != ISD::XOR) {
4977 IsXOROperand = false;
4978 } else {
4979 R1 = N0.getOperand(i: 1).getOperand(i: 0);
4980 R2 = N1.getOperand(i: 1).getOperand(i: 1);
4981 }
4982
4983 APInt ShlAmt, ShrAmt;
4984 if (!ISD::isConstantSplatVector(N: N0.getOperand(i: 2).getNode(), SplatValue&: ShlAmt) ||
4985 !ISD::isConstantSplatVector(N: N1.getOperand(i: 2).getNode(), SplatValue&: ShrAmt))
4986 return false;
4987
4988 if (ShlAmt + ShrAmt != VT.getScalarSizeInBits())
4989 return false;
4990
4991 if (!IsXOROperand) {
4992 SDValue Zero = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64);
4993 SDNode *MOV = CurDAG->getMachineNode(Opcode: AArch64::MOVIv2d_ns, dl: DL, VT, Op1: Zero);
4994 SDValue MOVIV = SDValue(MOV, 0);
4995
4996 SDValue ZSub = CurDAG->getTargetConstant(Val: AArch64::zsub, DL, VT: MVT::i32);
4997 SDNode *SubRegToReg =
4998 CurDAG->getMachineNode(Opcode: AArch64::SUBREG_TO_REG, dl: DL, VT, Op1: MOVIV, Op2: ZSub);
4999
5000 R1 = N1->getOperand(Num: 1);
5001 R2 = SDValue(SubRegToReg, 0);
5002 }
5003
5004 SDValue Imm =
5005 CurDAG->getTargetConstant(Val: ShrAmt.getZExtValue(), DL, VT: MVT::i32);
5006
5007 SDValue Ops[] = {R1, R2, Imm};
5008 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::Int>(
5009 VT, Opcodes: {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
5010 AArch64::XAR_ZZZI_D})) {
5011 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
5012 return true;
5013 }
5014 return false;
5015 }
5016
5017 // We have Neon SHA3 XAR operation for v2i64 but for types
5018 // v4i32, v8i16, v16i8 we can use SVE operations when SVE2-SHA3
5019 // is available.
5020 EVT SVT;
5021 switch (VT.getSimpleVT().SimpleTy) {
5022 case MVT::v4i32:
5023 case MVT::v2i32:
5024 SVT = MVT::nxv4i32;
5025 break;
5026 case MVT::v8i16:
5027 case MVT::v4i16:
5028 SVT = MVT::nxv8i16;
5029 break;
5030 case MVT::v16i8:
5031 case MVT::v8i8:
5032 SVT = MVT::nxv16i8;
5033 break;
5034 case MVT::v2i64:
5035 case MVT::v1i64:
5036 SVT = Subtarget->hasSHA3() ? MVT::v2i64 : MVT::nxv2i64;
5037 break;
5038 default:
5039 return false;
5040 }
5041
5042 if ((!SVT.isScalableVector() && !Subtarget->hasSHA3()) ||
5043 (SVT.isScalableVector() && !Subtarget->hasSVE2()))
5044 return false;
5045
5046 if (N0->getOpcode() != AArch64ISD::VSHL ||
5047 N1->getOpcode() != AArch64ISD::VLSHR)
5048 return false;
5049
5050 if (N0->getOperand(Num: 0) != N1->getOperand(Num: 0))
5051 return false;
5052
5053 SDValue R1, R2;
5054 bool IsXOROperand = true;
5055 if (N1->getOperand(Num: 0)->getOpcode() != ISD::XOR) {
5056 IsXOROperand = false;
5057 } else {
5058 SDValue XOR = N0.getOperand(i: 0);
5059 R1 = XOR.getOperand(i: 0);
5060 R2 = XOR.getOperand(i: 1);
5061 }
5062
5063 unsigned HsAmt = N0.getConstantOperandVal(i: 1);
5064 unsigned ShAmt = N1.getConstantOperandVal(i: 1);
5065
5066 SDValue Imm = CurDAG->getTargetConstant(
5067 Val: ShAmt, DL, VT: N0.getOperand(i: 1).getValueType(), isOpaque: false);
5068
5069 unsigned VTSizeInBits = VT.getScalarSizeInBits();
5070 if (ShAmt + HsAmt != VTSizeInBits)
5071 return false;
5072
5073 if (!IsXOROperand) {
5074 SDValue Zero = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64);
5075 SDNode *MOV =
5076 CurDAG->getMachineNode(Opcode: AArch64::MOVIv2d_ns, dl: DL, VT: MVT::v2i64, Op1: Zero);
5077 SDValue MOVIV = SDValue(MOV, 0);
5078
5079 R1 = N1->getOperand(Num: 0);
5080 R2 = MOVIV;
5081 }
5082
5083 if (SVT != VT) {
5084 SDValue Undef =
5085 SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: SVT), 0);
5086
5087 if (SVT.isScalableVector() && VT.is64BitVector()) {
5088 EVT QVT = VT.getDoubleNumVectorElementsVT(Context&: *CurDAG->getContext());
5089
5090 SDValue UndefQ = SDValue(
5091 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: QVT), 0);
5092 SDValue DSub = CurDAG->getTargetConstant(Val: AArch64::dsub, DL, VT: MVT::i32);
5093
5094 R1 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: QVT,
5095 Op1: UndefQ, Op2: R1, Op3: DSub),
5096 0);
5097 if (R2.getValueType() == VT)
5098 R2 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: QVT,
5099 Op1: UndefQ, Op2: R2, Op3: DSub),
5100 0);
5101 }
5102
5103 SDValue SubReg = CurDAG->getTargetConstant(
5104 Val: (SVT.isScalableVector() ? AArch64::zsub : AArch64::dsub), DL, VT: MVT::i32);
5105
5106 R1 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: SVT, Op1: Undef,
5107 Op2: R1, Op3: SubReg),
5108 0);
5109
5110 if (SVT.isScalableVector() || R2.getValueType() != SVT)
5111 R2 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: SVT,
5112 Op1: Undef, Op2: R2, Op3: SubReg),
5113 0);
5114 }
5115
5116 SDValue Ops[] = {R1, R2, Imm};
5117 SDNode *XAR = nullptr;
5118
5119 if (SVT.isScalableVector()) {
5120 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::Int>(
5121 VT: SVT, Opcodes: {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
5122 AArch64::XAR_ZZZI_D}))
5123 XAR = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: SVT, Ops);
5124 } else {
5125 XAR = CurDAG->getMachineNode(Opcode: AArch64::XAR, dl: DL, VT: SVT, Ops);
5126 }
5127
5128 assert(XAR && "Unexpected NULL value for XAR instruction in DAG");
5129
5130 if (SVT != VT) {
5131 if (VT.is64BitVector() && SVT.isScalableVector()) {
5132 EVT QVT = VT.getDoubleNumVectorElementsVT(Context&: *CurDAG->getContext());
5133
5134 SDValue ZSub = CurDAG->getTargetConstant(Val: AArch64::zsub, DL, VT: MVT::i32);
5135 SDNode *Q = CurDAG->getMachineNode(Opcode: AArch64::EXTRACT_SUBREG, dl: DL, VT: QVT,
5136 Op1: SDValue(XAR, 0), Op2: ZSub);
5137
5138 SDValue DSub = CurDAG->getTargetConstant(Val: AArch64::dsub, DL, VT: MVT::i32);
5139 XAR = CurDAG->getMachineNode(Opcode: AArch64::EXTRACT_SUBREG, dl: DL, VT,
5140 Op1: SDValue(Q, 0), Op2: DSub);
5141 } else {
5142 SDValue SubReg = CurDAG->getTargetConstant(
5143 Val: (SVT.isScalableVector() ? AArch64::zsub : AArch64::dsub), DL,
5144 VT: MVT::i32);
5145 XAR = CurDAG->getMachineNode(Opcode: AArch64::EXTRACT_SUBREG, dl: DL, VT,
5146 Op1: SDValue(XAR, 0), Op2: SubReg);
5147 }
5148 }
5149 ReplaceNode(F: N, T: XAR);
5150 return true;
5151}
5152
5153/// Returns a copy from WZR or XZR. This can be used during instruction
5154/// selection (it does not require any further selection/legalization).
5155static SDValue getZeroRegister(SelectionDAG &DAG, SDLoc DL, EVT VT) {
5156 assert(VT == MVT::i32 || VT == MVT::i64);
5157 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL,
5158 Reg: VT == MVT::i32 ? AArch64::WZR : AArch64::XZR, VT);
5159}
5160
5161void AArch64DAGToDAGISel::Select(SDNode *Node) {
5162 // If we have a custom node, we already have selected!
5163 if (Node->isMachineOpcode()) {
5164 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
5165 Node->setNodeId(-1);
5166 return;
5167 }
5168
5169 // Few custom selection stuff.
5170 EVT VT = Node->getValueType(ResNo: 0);
5171
5172 switch (Node->getOpcode()) {
5173 default:
5174 break;
5175
5176 case ISD::ATOMIC_CMP_SWAP:
5177 if (SelectCMP_SWAP(N: Node))
5178 return;
5179 break;
5180
5181 case ISD::READ_REGISTER:
5182 case AArch64ISD::MRRS:
5183 if (tryReadRegister(N: Node))
5184 return;
5185 break;
5186
5187 case ISD::WRITE_REGISTER:
5188 case AArch64ISD::MSRR:
5189 if (tryWriteRegister(N: Node))
5190 return;
5191 break;
5192
5193 case ISD::LOAD: {
5194 // Try to select as an indexed load. Fall through to normal processing
5195 // if we can't.
5196 if (tryIndexedLoad(N: Node))
5197 return;
5198 break;
5199 }
5200
5201 case ISD::SRL:
5202 case ISD::AND:
5203 case ISD::SRA:
5204 case ISD::SIGN_EXTEND_INREG:
5205 if (tryBitfieldExtractOp(N: Node))
5206 return;
5207 if (tryBitfieldInsertInZeroOp(N: Node))
5208 return;
5209 [[fallthrough]];
5210 case ISD::ROTR:
5211 case ISD::SHL:
5212 if (tryShiftAmountMod(N: Node))
5213 return;
5214 break;
5215
5216 case ISD::SIGN_EXTEND:
5217 if (tryBitfieldExtractOpFromSExt(N: Node))
5218 return;
5219 break;
5220
5221 case ISD::OR:
5222 if (tryBitfieldInsertOp(N: Node))
5223 return;
5224 if (trySelectXAR(N: Node))
5225 return;
5226 break;
5227
5228 case ISD::EXTRACT_SUBVECTOR: {
5229 if (trySelectCastScalableToFixedLengthVector(N: Node))
5230 return;
5231 break;
5232 }
5233
5234 case ISD::INSERT_SUBVECTOR: {
5235 if (trySelectCastFixedLengthToScalableVector(N: Node))
5236 return;
5237 break;
5238 }
5239
5240 case AArch64ISD::CSEL:
5241 if (tryFoldCselToFMaxMin(N: Node))
5242 return;
5243 break;
5244
5245 case ISD::Constant: {
5246 // Materialize zero constants as copies from WZR/XZR. This allows
5247 // the coalescer to propagate these into other instructions.
5248 ConstantSDNode *ConstNode = cast<ConstantSDNode>(Val: Node);
5249 if (ConstNode->isZero() && (VT == MVT::i32 || VT == MVT::i64)) {
5250 ReplaceNode(F: Node, T: getZeroRegister(DAG&: *CurDAG, DL: SDLoc(Node), VT).getNode());
5251 return;
5252 }
5253 break;
5254 }
5255
5256 case ISD::FrameIndex: {
5257 // Selects to ADDXri FI, 0 which in turn will become ADDXri SP, imm.
5258 int FI = cast<FrameIndexSDNode>(Val: Node)->getIndex();
5259 unsigned Shifter = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0);
5260 const TargetLowering *TLI = getTargetLowering();
5261 SDValue TFI = CurDAG->getTargetFrameIndex(
5262 FI, VT: TLI->getPointerTy(DL: CurDAG->getDataLayout()));
5263 SDLoc DL(Node);
5264 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32),
5265 CurDAG->getTargetConstant(Val: Shifter, DL, VT: MVT::i32) };
5266 CurDAG->SelectNodeTo(N: Node, MachineOpc: AArch64::ADDXri, VT: MVT::i64, Ops);
5267 return;
5268 }
5269 case ISD::INTRINSIC_W_CHAIN: {
5270 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
5271 switch (IntNo) {
5272 default:
5273 break;
5274 case Intrinsic::aarch64_gcsss: {
5275 SDLoc DL(Node);
5276 SDValue Chain = Node->getOperand(Num: 0);
5277 SDValue Val = Node->getOperand(Num: 2);
5278 SDValue Zero = CurDAG->getCopyFromReg(Chain, dl: DL, Reg: AArch64::XZR, VT: MVT::i64);
5279 SDNode *SS1 =
5280 CurDAG->getMachineNode(Opcode: AArch64::GCSSS1, dl: DL, VT: MVT::Other, Op1: Val, Op2: Chain);
5281 SDNode *SS2 = CurDAG->getMachineNode(Opcode: AArch64::GCSSS2, dl: DL, VT1: MVT::i64,
5282 VT2: MVT::Other, Op1: Zero, Op2: SDValue(SS1, 0));
5283 ReplaceNode(F: Node, T: SS2);
5284 return;
5285 }
5286 case Intrinsic::aarch64_ldaxp:
5287 case Intrinsic::aarch64_ldxp: {
5288 unsigned Op =
5289 IntNo == Intrinsic::aarch64_ldaxp ? AArch64::LDAXPX : AArch64::LDXPX;
5290 SDValue MemAddr = Node->getOperand(Num: 2);
5291 SDLoc DL(Node);
5292 SDValue Chain = Node->getOperand(Num: 0);
5293
5294 SDNode *Ld = CurDAG->getMachineNode(Opcode: Op, dl: DL, VT1: MVT::i64, VT2: MVT::i64,
5295 VT3: MVT::Other, Op1: MemAddr, Op2: Chain);
5296
5297 // Transfer memoperands.
5298 MachineMemOperand *MemOp =
5299 cast<MemIntrinsicSDNode>(Val: Node)->getMemOperand();
5300 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Ld), NewMemRefs: {MemOp});
5301 ReplaceNode(F: Node, T: Ld);
5302 return;
5303 }
5304 case Intrinsic::aarch64_stlxp:
5305 case Intrinsic::aarch64_stxp: {
5306 unsigned Op =
5307 IntNo == Intrinsic::aarch64_stlxp ? AArch64::STLXPX : AArch64::STXPX;
5308 SDLoc DL(Node);
5309 SDValue Chain = Node->getOperand(Num: 0);
5310 SDValue ValLo = Node->getOperand(Num: 2);
5311 SDValue ValHi = Node->getOperand(Num: 3);
5312 SDValue MemAddr = Node->getOperand(Num: 4);
5313
5314 // Place arguments in the right order.
5315 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
5316
5317 SDNode *St = CurDAG->getMachineNode(Opcode: Op, dl: DL, VT1: MVT::i32, VT2: MVT::Other, Ops);
5318 // Transfer memoperands.
5319 MachineMemOperand *MemOp =
5320 cast<MemIntrinsicSDNode>(Val: Node)->getMemOperand();
5321 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
5322
5323 ReplaceNode(F: Node, T: St);
5324 return;
5325 }
5326 case Intrinsic::aarch64_neon_ld1x2:
5327 if (VT == MVT::v8i8) {
5328 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8b, SubRegIdx: AArch64::dsub0);
5329 return;
5330 } else if (VT == MVT::v16i8) {
5331 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov16b, SubRegIdx: AArch64::qsub0);
5332 return;
5333 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5334 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4h, SubRegIdx: AArch64::dsub0);
5335 return;
5336 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5337 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8h, SubRegIdx: AArch64::qsub0);
5338 return;
5339 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5340 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2s, SubRegIdx: AArch64::dsub0);
5341 return;
5342 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5343 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4s, SubRegIdx: AArch64::qsub0);
5344 return;
5345 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5346 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d, SubRegIdx: AArch64::dsub0);
5347 return;
5348 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5349 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2d, SubRegIdx: AArch64::qsub0);
5350 return;
5351 }
5352 break;
5353 case Intrinsic::aarch64_neon_ld1x3:
5354 if (VT == MVT::v8i8) {
5355 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8b, SubRegIdx: AArch64::dsub0);
5356 return;
5357 } else if (VT == MVT::v16i8) {
5358 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev16b, SubRegIdx: AArch64::qsub0);
5359 return;
5360 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5361 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4h, SubRegIdx: AArch64::dsub0);
5362 return;
5363 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5364 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8h, SubRegIdx: AArch64::qsub0);
5365 return;
5366 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5367 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2s, SubRegIdx: AArch64::dsub0);
5368 return;
5369 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5370 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4s, SubRegIdx: AArch64::qsub0);
5371 return;
5372 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5373 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d, SubRegIdx: AArch64::dsub0);
5374 return;
5375 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5376 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2d, SubRegIdx: AArch64::qsub0);
5377 return;
5378 }
5379 break;
5380 case Intrinsic::aarch64_neon_ld1x4:
5381 if (VT == MVT::v8i8) {
5382 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8b, SubRegIdx: AArch64::dsub0);
5383 return;
5384 } else if (VT == MVT::v16i8) {
5385 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv16b, SubRegIdx: AArch64::qsub0);
5386 return;
5387 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5388 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4h, SubRegIdx: AArch64::dsub0);
5389 return;
5390 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5391 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8h, SubRegIdx: AArch64::qsub0);
5392 return;
5393 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5394 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2s, SubRegIdx: AArch64::dsub0);
5395 return;
5396 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5397 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4s, SubRegIdx: AArch64::qsub0);
5398 return;
5399 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5400 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d, SubRegIdx: AArch64::dsub0);
5401 return;
5402 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5403 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2d, SubRegIdx: AArch64::qsub0);
5404 return;
5405 }
5406 break;
5407 case Intrinsic::aarch64_neon_ld2:
5408 if (VT == MVT::v8i8) {
5409 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8b, SubRegIdx: AArch64::dsub0);
5410 return;
5411 } else if (VT == MVT::v16i8) {
5412 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov16b, SubRegIdx: AArch64::qsub0);
5413 return;
5414 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5415 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4h, SubRegIdx: AArch64::dsub0);
5416 return;
5417 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5418 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8h, SubRegIdx: AArch64::qsub0);
5419 return;
5420 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5421 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2s, SubRegIdx: AArch64::dsub0);
5422 return;
5423 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5424 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4s, SubRegIdx: AArch64::qsub0);
5425 return;
5426 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5427 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d, SubRegIdx: AArch64::dsub0);
5428 return;
5429 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5430 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2d, SubRegIdx: AArch64::qsub0);
5431 return;
5432 }
5433 break;
5434 case Intrinsic::aarch64_neon_ld3:
5435 if (VT == MVT::v8i8) {
5436 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8b, SubRegIdx: AArch64::dsub0);
5437 return;
5438 } else if (VT == MVT::v16i8) {
5439 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev16b, SubRegIdx: AArch64::qsub0);
5440 return;
5441 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5442 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4h, SubRegIdx: AArch64::dsub0);
5443 return;
5444 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5445 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8h, SubRegIdx: AArch64::qsub0);
5446 return;
5447 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5448 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2s, SubRegIdx: AArch64::dsub0);
5449 return;
5450 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5451 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4s, SubRegIdx: AArch64::qsub0);
5452 return;
5453 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5454 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d, SubRegIdx: AArch64::dsub0);
5455 return;
5456 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5457 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2d, SubRegIdx: AArch64::qsub0);
5458 return;
5459 }
5460 break;
5461 case Intrinsic::aarch64_neon_ld4:
5462 if (VT == MVT::v8i8) {
5463 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8b, SubRegIdx: AArch64::dsub0);
5464 return;
5465 } else if (VT == MVT::v16i8) {
5466 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv16b, SubRegIdx: AArch64::qsub0);
5467 return;
5468 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5469 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4h, SubRegIdx: AArch64::dsub0);
5470 return;
5471 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5472 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8h, SubRegIdx: AArch64::qsub0);
5473 return;
5474 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5475 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2s, SubRegIdx: AArch64::dsub0);
5476 return;
5477 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5478 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4s, SubRegIdx: AArch64::qsub0);
5479 return;
5480 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5481 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d, SubRegIdx: AArch64::dsub0);
5482 return;
5483 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5484 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2d, SubRegIdx: AArch64::qsub0);
5485 return;
5486 }
5487 break;
5488 case Intrinsic::aarch64_neon_ld2r:
5489 if (VT == MVT::v8i8) {
5490 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8b, SubRegIdx: AArch64::dsub0);
5491 return;
5492 } else if (VT == MVT::v16i8) {
5493 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv16b, SubRegIdx: AArch64::qsub0);
5494 return;
5495 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5496 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4h, SubRegIdx: AArch64::dsub0);
5497 return;
5498 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5499 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8h, SubRegIdx: AArch64::qsub0);
5500 return;
5501 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5502 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2s, SubRegIdx: AArch64::dsub0);
5503 return;
5504 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5505 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4s, SubRegIdx: AArch64::qsub0);
5506 return;
5507 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5508 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv1d, SubRegIdx: AArch64::dsub0);
5509 return;
5510 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5511 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2d, SubRegIdx: AArch64::qsub0);
5512 return;
5513 }
5514 break;
5515 case Intrinsic::aarch64_neon_ld3r:
5516 if (VT == MVT::v8i8) {
5517 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8b, SubRegIdx: AArch64::dsub0);
5518 return;
5519 } else if (VT == MVT::v16i8) {
5520 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv16b, SubRegIdx: AArch64::qsub0);
5521 return;
5522 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5523 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4h, SubRegIdx: AArch64::dsub0);
5524 return;
5525 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5526 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8h, SubRegIdx: AArch64::qsub0);
5527 return;
5528 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5529 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2s, SubRegIdx: AArch64::dsub0);
5530 return;
5531 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5532 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4s, SubRegIdx: AArch64::qsub0);
5533 return;
5534 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5535 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv1d, SubRegIdx: AArch64::dsub0);
5536 return;
5537 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5538 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2d, SubRegIdx: AArch64::qsub0);
5539 return;
5540 }
5541 break;
5542 case Intrinsic::aarch64_neon_ld4r:
5543 if (VT == MVT::v8i8) {
5544 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8b, SubRegIdx: AArch64::dsub0);
5545 return;
5546 } else if (VT == MVT::v16i8) {
5547 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv16b, SubRegIdx: AArch64::qsub0);
5548 return;
5549 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5550 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4h, SubRegIdx: AArch64::dsub0);
5551 return;
5552 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5553 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8h, SubRegIdx: AArch64::qsub0);
5554 return;
5555 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5556 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2s, SubRegIdx: AArch64::dsub0);
5557 return;
5558 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5559 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4s, SubRegIdx: AArch64::qsub0);
5560 return;
5561 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5562 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv1d, SubRegIdx: AArch64::dsub0);
5563 return;
5564 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5565 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2d, SubRegIdx: AArch64::qsub0);
5566 return;
5567 }
5568 break;
5569 case Intrinsic::aarch64_neon_ld2lane:
5570 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5571 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i8);
5572 return;
5573 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5574 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5575 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i16);
5576 return;
5577 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5578 VT == MVT::v2f32) {
5579 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i32);
5580 return;
5581 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5582 VT == MVT::v1f64) {
5583 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i64);
5584 return;
5585 }
5586 break;
5587 case Intrinsic::aarch64_neon_ld3lane:
5588 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5589 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i8);
5590 return;
5591 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5592 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5593 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i16);
5594 return;
5595 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5596 VT == MVT::v2f32) {
5597 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i32);
5598 return;
5599 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5600 VT == MVT::v1f64) {
5601 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i64);
5602 return;
5603 }
5604 break;
5605 case Intrinsic::aarch64_neon_ld4lane:
5606 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5607 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i8);
5608 return;
5609 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5610 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5611 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i16);
5612 return;
5613 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5614 VT == MVT::v2f32) {
5615 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i32);
5616 return;
5617 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5618 VT == MVT::v1f64) {
5619 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i64);
5620 return;
5621 }
5622 break;
5623 case Intrinsic::aarch64_ld64b:
5624 SelectLoad(N: Node, NumVecs: 8, Opc: AArch64::LD64B, SubRegIdx: AArch64::x8sub_0);
5625 return;
5626 case Intrinsic::aarch64_sve_ld2q_sret: {
5627 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 4, Opc_ri: AArch64::LD2Q_IMM, Opc_rr: AArch64::LD2Q, IsIntr: true);
5628 return;
5629 }
5630 case Intrinsic::aarch64_sve_ld3q_sret: {
5631 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 4, Opc_ri: AArch64::LD3Q_IMM, Opc_rr: AArch64::LD3Q, IsIntr: true);
5632 return;
5633 }
5634 case Intrinsic::aarch64_sve_ld4q_sret: {
5635 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 4, Opc_ri: AArch64::LD4Q_IMM, Opc_rr: AArch64::LD4Q, IsIntr: true);
5636 return;
5637 }
5638 case Intrinsic::aarch64_sve_ld2_sret: {
5639 if (VT == MVT::nxv16i8) {
5640 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LD2B_IMM, Opc_rr: AArch64::LD2B,
5641 IsIntr: true);
5642 return;
5643 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5644 VT == MVT::nxv8bf16) {
5645 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LD2H_IMM, Opc_rr: AArch64::LD2H,
5646 IsIntr: true);
5647 return;
5648 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5649 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LD2W_IMM, Opc_rr: AArch64::LD2W,
5650 IsIntr: true);
5651 return;
5652 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5653 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LD2D_IMM, Opc_rr: AArch64::LD2D,
5654 IsIntr: true);
5655 return;
5656 }
5657 break;
5658 }
5659 case Intrinsic::aarch64_sve_ld1_pn_x2: {
5660 if (VT == MVT::nxv16i8) {
5661 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5662 SelectContiguousMultiVectorLoad(
5663 N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LD1B_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1B_2Z_PSEUDO);
5664 else if (Subtarget->hasSVE2p1())
5665 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LD1B_2Z_IMM,
5666 Opc_rr: AArch64::LD1B_2Z);
5667 else
5668 break;
5669 return;
5670 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5671 VT == MVT::nxv8bf16) {
5672 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5673 SelectContiguousMultiVectorLoad(
5674 N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LD1H_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1H_2Z_PSEUDO);
5675 else if (Subtarget->hasSVE2p1())
5676 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LD1H_2Z_IMM,
5677 Opc_rr: AArch64::LD1H_2Z);
5678 else
5679 break;
5680 return;
5681 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5682 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5683 SelectContiguousMultiVectorLoad(
5684 N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LD1W_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1W_2Z_PSEUDO);
5685 else if (Subtarget->hasSVE2p1())
5686 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LD1W_2Z_IMM,
5687 Opc_rr: AArch64::LD1W_2Z);
5688 else
5689 break;
5690 return;
5691 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5692 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5693 SelectContiguousMultiVectorLoad(
5694 N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LD1D_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1D_2Z_PSEUDO);
5695 else if (Subtarget->hasSVE2p1())
5696 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LD1D_2Z_IMM,
5697 Opc_rr: AArch64::LD1D_2Z);
5698 else
5699 break;
5700 return;
5701 }
5702 break;
5703 }
5704 case Intrinsic::aarch64_sve_ld1_pn_x4: {
5705 if (VT == MVT::nxv16i8) {
5706 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5707 SelectContiguousMultiVectorLoad(
5708 N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LD1B_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1B_4Z_PSEUDO);
5709 else if (Subtarget->hasSVE2p1())
5710 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LD1B_4Z_IMM,
5711 Opc_rr: AArch64::LD1B_4Z);
5712 else
5713 break;
5714 return;
5715 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5716 VT == MVT::nxv8bf16) {
5717 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5718 SelectContiguousMultiVectorLoad(
5719 N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LD1H_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1H_4Z_PSEUDO);
5720 else if (Subtarget->hasSVE2p1())
5721 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LD1H_4Z_IMM,
5722 Opc_rr: AArch64::LD1H_4Z);
5723 else
5724 break;
5725 return;
5726 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5727 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5728 SelectContiguousMultiVectorLoad(
5729 N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LD1W_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1W_4Z_PSEUDO);
5730 else if (Subtarget->hasSVE2p1())
5731 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LD1W_4Z_IMM,
5732 Opc_rr: AArch64::LD1W_4Z);
5733 else
5734 break;
5735 return;
5736 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5737 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5738 SelectContiguousMultiVectorLoad(
5739 N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LD1D_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1D_4Z_PSEUDO);
5740 else if (Subtarget->hasSVE2p1())
5741 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LD1D_4Z_IMM,
5742 Opc_rr: AArch64::LD1D_4Z);
5743 else
5744 break;
5745 return;
5746 }
5747 break;
5748 }
5749 case Intrinsic::aarch64_sve_ldnt1_pn_x2: {
5750 if (VT == MVT::nxv16i8) {
5751 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5752 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 0,
5753 Opc_ri: AArch64::LDNT1B_2Z_IMM_PSEUDO,
5754 Opc_rr: AArch64::LDNT1B_2Z_PSEUDO);
5755 else if (Subtarget->hasSVE2p1())
5756 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LDNT1B_2Z_IMM,
5757 Opc_rr: AArch64::LDNT1B_2Z);
5758 else
5759 break;
5760 return;
5761 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5762 VT == MVT::nxv8bf16) {
5763 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5764 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 1,
5765 Opc_ri: AArch64::LDNT1H_2Z_IMM_PSEUDO,
5766 Opc_rr: AArch64::LDNT1H_2Z_PSEUDO);
5767 else if (Subtarget->hasSVE2p1())
5768 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LDNT1H_2Z_IMM,
5769 Opc_rr: AArch64::LDNT1H_2Z);
5770 else
5771 break;
5772 return;
5773 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5774 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5775 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 2,
5776 Opc_ri: AArch64::LDNT1W_2Z_IMM_PSEUDO,
5777 Opc_rr: AArch64::LDNT1W_2Z_PSEUDO);
5778 else if (Subtarget->hasSVE2p1())
5779 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LDNT1W_2Z_IMM,
5780 Opc_rr: AArch64::LDNT1W_2Z);
5781 else
5782 break;
5783 return;
5784 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5785 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5786 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 3,
5787 Opc_ri: AArch64::LDNT1D_2Z_IMM_PSEUDO,
5788 Opc_rr: AArch64::LDNT1D_2Z_PSEUDO);
5789 else if (Subtarget->hasSVE2p1())
5790 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LDNT1D_2Z_IMM,
5791 Opc_rr: AArch64::LDNT1D_2Z);
5792 else
5793 break;
5794 return;
5795 }
5796 break;
5797 }
5798 case Intrinsic::aarch64_sve_ldnt1_pn_x4: {
5799 if (VT == MVT::nxv16i8) {
5800 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5801 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 0,
5802 Opc_ri: AArch64::LDNT1B_4Z_IMM_PSEUDO,
5803 Opc_rr: AArch64::LDNT1B_4Z_PSEUDO);
5804 else if (Subtarget->hasSVE2p1())
5805 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LDNT1B_4Z_IMM,
5806 Opc_rr: AArch64::LDNT1B_4Z);
5807 else
5808 break;
5809 return;
5810 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5811 VT == MVT::nxv8bf16) {
5812 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5813 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 1,
5814 Opc_ri: AArch64::LDNT1H_4Z_IMM_PSEUDO,
5815 Opc_rr: AArch64::LDNT1H_4Z_PSEUDO);
5816 else if (Subtarget->hasSVE2p1())
5817 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LDNT1H_4Z_IMM,
5818 Opc_rr: AArch64::LDNT1H_4Z);
5819 else
5820 break;
5821 return;
5822 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5823 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5824 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 2,
5825 Opc_ri: AArch64::LDNT1W_4Z_IMM_PSEUDO,
5826 Opc_rr: AArch64::LDNT1W_4Z_PSEUDO);
5827 else if (Subtarget->hasSVE2p1())
5828 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LDNT1W_4Z_IMM,
5829 Opc_rr: AArch64::LDNT1W_4Z);
5830 else
5831 break;
5832 return;
5833 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5834 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5835 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 3,
5836 Opc_ri: AArch64::LDNT1D_4Z_IMM_PSEUDO,
5837 Opc_rr: AArch64::LDNT1D_4Z_PSEUDO);
5838 else if (Subtarget->hasSVE2p1())
5839 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LDNT1D_4Z_IMM,
5840 Opc_rr: AArch64::LDNT1D_4Z);
5841 else
5842 break;
5843 return;
5844 }
5845 break;
5846 }
5847 case Intrinsic::aarch64_sve_ld3_sret: {
5848 if (VT == MVT::nxv16i8) {
5849 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 0, Opc_ri: AArch64::LD3B_IMM, Opc_rr: AArch64::LD3B,
5850 IsIntr: true);
5851 return;
5852 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5853 VT == MVT::nxv8bf16) {
5854 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 1, Opc_ri: AArch64::LD3H_IMM, Opc_rr: AArch64::LD3H,
5855 IsIntr: true);
5856 return;
5857 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5858 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 2, Opc_ri: AArch64::LD3W_IMM, Opc_rr: AArch64::LD3W,
5859 IsIntr: true);
5860 return;
5861 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5862 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 3, Opc_ri: AArch64::LD3D_IMM, Opc_rr: AArch64::LD3D,
5863 IsIntr: true);
5864 return;
5865 }
5866 break;
5867 }
5868 case Intrinsic::aarch64_sve_ld4_sret: {
5869 if (VT == MVT::nxv16i8) {
5870 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LD4B_IMM, Opc_rr: AArch64::LD4B,
5871 IsIntr: true);
5872 return;
5873 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5874 VT == MVT::nxv8bf16) {
5875 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LD4H_IMM, Opc_rr: AArch64::LD4H,
5876 IsIntr: true);
5877 return;
5878 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5879 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LD4W_IMM, Opc_rr: AArch64::LD4W,
5880 IsIntr: true);
5881 return;
5882 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5883 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LD4D_IMM, Opc_rr: AArch64::LD4D,
5884 IsIntr: true);
5885 return;
5886 }
5887 break;
5888 }
5889 case Intrinsic::aarch64_sme_read_hor_vg2: {
5890 if (VT == MVT::nxv16i8) {
5891 SelectMultiVectorMove<14, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAB0,
5892 Op: AArch64::MOVA_2ZMXI_H_B);
5893 return;
5894 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5895 VT == MVT::nxv8bf16) {
5896 SelectMultiVectorMove<6, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAH0,
5897 Op: AArch64::MOVA_2ZMXI_H_H);
5898 return;
5899 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5900 SelectMultiVectorMove<2, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAS0,
5901 Op: AArch64::MOVA_2ZMXI_H_S);
5902 return;
5903 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5904 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAD0,
5905 Op: AArch64::MOVA_2ZMXI_H_D);
5906 return;
5907 }
5908 break;
5909 }
5910 case Intrinsic::aarch64_sme_read_ver_vg2: {
5911 if (VT == MVT::nxv16i8) {
5912 SelectMultiVectorMove<14, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAB0,
5913 Op: AArch64::MOVA_2ZMXI_V_B);
5914 return;
5915 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5916 VT == MVT::nxv8bf16) {
5917 SelectMultiVectorMove<6, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAH0,
5918 Op: AArch64::MOVA_2ZMXI_V_H);
5919 return;
5920 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5921 SelectMultiVectorMove<2, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAS0,
5922 Op: AArch64::MOVA_2ZMXI_V_S);
5923 return;
5924 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5925 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAD0,
5926 Op: AArch64::MOVA_2ZMXI_V_D);
5927 return;
5928 }
5929 break;
5930 }
5931 case Intrinsic::aarch64_sme_read_hor_vg4: {
5932 if (VT == MVT::nxv16i8) {
5933 SelectMultiVectorMove<12, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAB0,
5934 Op: AArch64::MOVA_4ZMXI_H_B);
5935 return;
5936 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5937 VT == MVT::nxv8bf16) {
5938 SelectMultiVectorMove<4, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAH0,
5939 Op: AArch64::MOVA_4ZMXI_H_H);
5940 return;
5941 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5942 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAS0,
5943 Op: AArch64::MOVA_4ZMXI_H_S);
5944 return;
5945 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5946 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAD0,
5947 Op: AArch64::MOVA_4ZMXI_H_D);
5948 return;
5949 }
5950 break;
5951 }
5952 case Intrinsic::aarch64_sme_read_ver_vg4: {
5953 if (VT == MVT::nxv16i8) {
5954 SelectMultiVectorMove<12, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAB0,
5955 Op: AArch64::MOVA_4ZMXI_V_B);
5956 return;
5957 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5958 VT == MVT::nxv8bf16) {
5959 SelectMultiVectorMove<4, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAH0,
5960 Op: AArch64::MOVA_4ZMXI_V_H);
5961 return;
5962 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5963 SelectMultiVectorMove<0, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAS0,
5964 Op: AArch64::MOVA_4ZMXI_V_S);
5965 return;
5966 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5967 SelectMultiVectorMove<0, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAD0,
5968 Op: AArch64::MOVA_4ZMXI_V_D);
5969 return;
5970 }
5971 break;
5972 }
5973 case Intrinsic::aarch64_sme_read_vg1x2: {
5974 SelectMultiVectorMove<7, 1>(N: Node, NumVecs: 2, BaseReg: AArch64::ZA,
5975 Op: AArch64::MOVA_VG2_2ZMXI);
5976 return;
5977 }
5978 case Intrinsic::aarch64_sme_read_vg1x4: {
5979 SelectMultiVectorMove<7, 1>(N: Node, NumVecs: 4, BaseReg: AArch64::ZA,
5980 Op: AArch64::MOVA_VG4_4ZMXI);
5981 return;
5982 }
5983 case Intrinsic::aarch64_sme_readz_horiz_x2: {
5984 if (VT == MVT::nxv16i8) {
5985 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_B_PSEUDO, MaxIdx: 14, Scale: 2);
5986 return;
5987 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5988 VT == MVT::nxv8bf16) {
5989 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_H_PSEUDO, MaxIdx: 6, Scale: 2);
5990 return;
5991 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5992 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_S_PSEUDO, MaxIdx: 2, Scale: 2);
5993 return;
5994 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5995 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_D_PSEUDO, MaxIdx: 0, Scale: 2);
5996 return;
5997 }
5998 break;
5999 }
6000 case Intrinsic::aarch64_sme_readz_vert_x2: {
6001 if (VT == MVT::nxv16i8) {
6002 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_B_PSEUDO, MaxIdx: 14, Scale: 2);
6003 return;
6004 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6005 VT == MVT::nxv8bf16) {
6006 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_H_PSEUDO, MaxIdx: 6, Scale: 2);
6007 return;
6008 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6009 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_S_PSEUDO, MaxIdx: 2, Scale: 2);
6010 return;
6011 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6012 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_D_PSEUDO, MaxIdx: 0, Scale: 2);
6013 return;
6014 }
6015 break;
6016 }
6017 case Intrinsic::aarch64_sme_readz_horiz_x4: {
6018 if (VT == MVT::nxv16i8) {
6019 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_B_PSEUDO, MaxIdx: 12, Scale: 4);
6020 return;
6021 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6022 VT == MVT::nxv8bf16) {
6023 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_H_PSEUDO, MaxIdx: 4, Scale: 4);
6024 return;
6025 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6026 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_S_PSEUDO, MaxIdx: 0, Scale: 4);
6027 return;
6028 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6029 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_D_PSEUDO, MaxIdx: 0, Scale: 4);
6030 return;
6031 }
6032 break;
6033 }
6034 case Intrinsic::aarch64_sme_readz_vert_x4: {
6035 if (VT == MVT::nxv16i8) {
6036 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_B_PSEUDO, MaxIdx: 12, Scale: 4);
6037 return;
6038 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6039 VT == MVT::nxv8bf16) {
6040 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_H_PSEUDO, MaxIdx: 4, Scale: 4);
6041 return;
6042 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6043 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_S_PSEUDO, MaxIdx: 0, Scale: 4);
6044 return;
6045 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6046 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_D_PSEUDO, MaxIdx: 0, Scale: 4);
6047 return;
6048 }
6049 break;
6050 }
6051 case Intrinsic::aarch64_sme_readz_x2: {
6052 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_VG2_2ZMXI_PSEUDO, MaxIdx: 7, Scale: 1,
6053 BaseReg: AArch64::ZA);
6054 return;
6055 }
6056 case Intrinsic::aarch64_sme_readz_x4: {
6057 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_VG4_4ZMXI_PSEUDO, MaxIdx: 7, Scale: 1,
6058 BaseReg: AArch64::ZA);
6059 return;
6060 }
6061 case Intrinsic::swift_async_context_addr: {
6062 SDLoc DL(Node);
6063 SDValue Chain = Node->getOperand(Num: 0);
6064 SDValue CopyFP = CurDAG->getCopyFromReg(Chain, dl: DL, Reg: AArch64::FP, VT: MVT::i64);
6065 SDValue Res = SDValue(
6066 CurDAG->getMachineNode(Opcode: AArch64::SUBXri, dl: DL, VT: MVT::i64, Op1: CopyFP,
6067 Op2: CurDAG->getTargetConstant(Val: 8, DL, VT: MVT::i32),
6068 Op3: CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32)),
6069 0);
6070 ReplaceUses(F: SDValue(Node, 0), T: Res);
6071 ReplaceUses(F: SDValue(Node, 1), T: CopyFP.getValue(R: 1));
6072 CurDAG->RemoveDeadNode(N: Node);
6073
6074 auto &MF = CurDAG->getMachineFunction();
6075 MF.getFrameInfo().setFrameAddressIsTaken(true);
6076 MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
6077 return;
6078 }
6079 case Intrinsic::aarch64_sme_luti2_lane_zt_x4: {
6080 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6081 VT: Node->getValueType(ResNo: 0),
6082 Opcodes: {AArch64::LUTI2_4ZTZI_B, AArch64::LUTI2_4ZTZI_H,
6083 AArch64::LUTI2_4ZTZI_S}))
6084 // Second Immediate must be <= 3:
6085 SelectMultiVectorLutiLane(Node, NumOutVecs: 4, Opc, MaxImm: 3);
6086 return;
6087 }
6088 case Intrinsic::aarch64_sme_luti4_lane_zt_x4: {
6089 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6090 VT: Node->getValueType(ResNo: 0),
6091 Opcodes: {0, AArch64::LUTI4_4ZTZI_H, AArch64::LUTI4_4ZTZI_S}))
6092 // Second Immediate must be <= 1:
6093 SelectMultiVectorLutiLane(Node, NumOutVecs: 4, Opc, MaxImm: 1);
6094 return;
6095 }
6096 case Intrinsic::aarch64_sme_luti2_lane_zt_x2: {
6097 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6098 VT: Node->getValueType(ResNo: 0),
6099 Opcodes: {AArch64::LUTI2_2ZTZI_B, AArch64::LUTI2_2ZTZI_H,
6100 AArch64::LUTI2_2ZTZI_S}))
6101 // Second Immediate must be <= 7:
6102 SelectMultiVectorLutiLane(Node, NumOutVecs: 2, Opc, MaxImm: 7);
6103 return;
6104 }
6105 case Intrinsic::aarch64_sme_luti4_lane_zt_x2: {
6106 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6107 VT: Node->getValueType(ResNo: 0),
6108 Opcodes: {AArch64::LUTI4_2ZTZI_B, AArch64::LUTI4_2ZTZI_H,
6109 AArch64::LUTI4_2ZTZI_S}))
6110 // Second Immediate must be <= 3:
6111 SelectMultiVectorLutiLane(Node, NumOutVecs: 2, Opc, MaxImm: 3);
6112 return;
6113 }
6114 case Intrinsic::aarch64_sme_luti4_zt_x4: {
6115 SelectMultiVectorLuti(Node, NumOutVecs: 4, Opc: AArch64::LUTI4_4ZZT2Z, NumInVecs: 2);
6116 return;
6117 }
6118 case Intrinsic::aarch64_sme_luti6_zt_x4: {
6119 SelectMultiVectorLuti(Node, NumOutVecs: 4, Opc: AArch64::LUTI6_4ZT3Z, NumInVecs: 3);
6120 return;
6121 }
6122 case Intrinsic::aarch64_sve_fp8_cvtl1_x2:
6123 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6124 VT: Node->getValueType(ResNo: 0),
6125 Opcodes: {AArch64::BF1CVTL_2ZZ_BtoH, AArch64::F1CVTL_2ZZ_BtoH}))
6126 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6127 return;
6128 case Intrinsic::aarch64_sve_fp8_cvtl2_x2:
6129 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6130 VT: Node->getValueType(ResNo: 0),
6131 Opcodes: {AArch64::BF2CVTL_2ZZ_BtoH, AArch64::F2CVTL_2ZZ_BtoH}))
6132 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6133 return;
6134 case Intrinsic::aarch64_sve_fp8_cvt1_x2:
6135 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6136 VT: Node->getValueType(ResNo: 0),
6137 Opcodes: {AArch64::BF1CVT_2ZZ_BtoH, AArch64::F1CVT_2ZZ_BtoH}))
6138 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6139 return;
6140 case Intrinsic::aarch64_sve_fp8_cvt2_x2:
6141 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6142 VT: Node->getValueType(ResNo: 0),
6143 Opcodes: {AArch64::BF2CVT_2ZZ_BtoH, AArch64::F2CVT_2ZZ_BtoH}))
6144 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6145 return;
6146 case Intrinsic::ptrauth_resign_load_relative:
6147 SelectPtrauthResign(N: Node);
6148 return;
6149 }
6150 } break;
6151 case ISD::INTRINSIC_WO_CHAIN: {
6152 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
6153 switch (IntNo) {
6154 default:
6155 break;
6156 case Intrinsic::aarch64_tagp:
6157 SelectTagP(N: Node);
6158 return;
6159
6160 case Intrinsic::ptrauth_auth:
6161 SelectPtrauthAuth(N: Node);
6162 return;
6163
6164 case Intrinsic::ptrauth_resign:
6165 SelectPtrauthResign(N: Node);
6166 return;
6167
6168 case Intrinsic::ptrauth_auth_with_pc_and_resign:
6169 SelectPtrauthResignWithPC(N: Node);
6170 return;
6171
6172 case Intrinsic::aarch64_neon_tbl2:
6173 SelectTable(N: Node, NumVecs: 2,
6174 Opc: VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two,
6175 isExt: false);
6176 return;
6177 case Intrinsic::aarch64_neon_tbl3:
6178 SelectTable(N: Node, NumVecs: 3, Opc: VT == MVT::v8i8 ? AArch64::TBLv8i8Three
6179 : AArch64::TBLv16i8Three,
6180 isExt: false);
6181 return;
6182 case Intrinsic::aarch64_neon_tbl4:
6183 SelectTable(N: Node, NumVecs: 4, Opc: VT == MVT::v8i8 ? AArch64::TBLv8i8Four
6184 : AArch64::TBLv16i8Four,
6185 isExt: false);
6186 return;
6187 case Intrinsic::aarch64_neon_tbx2:
6188 SelectTable(N: Node, NumVecs: 2,
6189 Opc: VT == MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two,
6190 isExt: true);
6191 return;
6192 case Intrinsic::aarch64_neon_tbx3:
6193 SelectTable(N: Node, NumVecs: 3, Opc: VT == MVT::v8i8 ? AArch64::TBXv8i8Three
6194 : AArch64::TBXv16i8Three,
6195 isExt: true);
6196 return;
6197 case Intrinsic::aarch64_neon_tbx4:
6198 SelectTable(N: Node, NumVecs: 4, Opc: VT == MVT::v8i8 ? AArch64::TBXv8i8Four
6199 : AArch64::TBXv16i8Four,
6200 isExt: true);
6201 return;
6202 case Intrinsic::aarch64_sve_srshl_single_x2:
6203 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6204 VT: Node->getValueType(ResNo: 0),
6205 Opcodes: {AArch64::SRSHL_VG2_2ZZ_B, AArch64::SRSHL_VG2_2ZZ_H,
6206 AArch64::SRSHL_VG2_2ZZ_S, AArch64::SRSHL_VG2_2ZZ_D}))
6207 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6208 return;
6209 case Intrinsic::aarch64_sve_srshl_single_x4:
6210 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6211 VT: Node->getValueType(ResNo: 0),
6212 Opcodes: {AArch64::SRSHL_VG4_4ZZ_B, AArch64::SRSHL_VG4_4ZZ_H,
6213 AArch64::SRSHL_VG4_4ZZ_S, AArch64::SRSHL_VG4_4ZZ_D}))
6214 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6215 return;
6216 case Intrinsic::aarch64_sme_luti6_lane_x4_x2:
6217 SelectMultiVectorLuti6LaneX4(Node, NumIndexVecs: 2);
6218 return;
6219 case Intrinsic::aarch64_sme_luti6_lane_x4_x3:
6220 SelectMultiVectorLuti6LaneX4(Node, NumIndexVecs: 3);
6221 return;
6222 case Intrinsic::aarch64_sve_urshl_single_x2:
6223 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6224 VT: Node->getValueType(ResNo: 0),
6225 Opcodes: {AArch64::URSHL_VG2_2ZZ_B, AArch64::URSHL_VG2_2ZZ_H,
6226 AArch64::URSHL_VG2_2ZZ_S, AArch64::URSHL_VG2_2ZZ_D}))
6227 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6228 return;
6229 case Intrinsic::aarch64_sve_urshl_single_x4:
6230 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6231 VT: Node->getValueType(ResNo: 0),
6232 Opcodes: {AArch64::URSHL_VG4_4ZZ_B, AArch64::URSHL_VG4_4ZZ_H,
6233 AArch64::URSHL_VG4_4ZZ_S, AArch64::URSHL_VG4_4ZZ_D}))
6234 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6235 return;
6236 case Intrinsic::aarch64_sve_srshl_x2:
6237 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6238 VT: Node->getValueType(ResNo: 0),
6239 Opcodes: {AArch64::SRSHL_VG2_2Z2Z_B, AArch64::SRSHL_VG2_2Z2Z_H,
6240 AArch64::SRSHL_VG2_2Z2Z_S, AArch64::SRSHL_VG2_2Z2Z_D}))
6241 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6242 return;
6243 case Intrinsic::aarch64_sve_srshl_x4:
6244 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6245 VT: Node->getValueType(ResNo: 0),
6246 Opcodes: {AArch64::SRSHL_VG4_4Z4Z_B, AArch64::SRSHL_VG4_4Z4Z_H,
6247 AArch64::SRSHL_VG4_4Z4Z_S, AArch64::SRSHL_VG4_4Z4Z_D}))
6248 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6249 return;
6250 case Intrinsic::aarch64_sve_urshl_x2:
6251 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6252 VT: Node->getValueType(ResNo: 0),
6253 Opcodes: {AArch64::URSHL_VG2_2Z2Z_B, AArch64::URSHL_VG2_2Z2Z_H,
6254 AArch64::URSHL_VG2_2Z2Z_S, AArch64::URSHL_VG2_2Z2Z_D}))
6255 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6256 return;
6257 case Intrinsic::aarch64_sve_urshl_x4:
6258 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6259 VT: Node->getValueType(ResNo: 0),
6260 Opcodes: {AArch64::URSHL_VG4_4Z4Z_B, AArch64::URSHL_VG4_4Z4Z_H,
6261 AArch64::URSHL_VG4_4Z4Z_S, AArch64::URSHL_VG4_4Z4Z_D}))
6262 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6263 return;
6264 case Intrinsic::aarch64_sve_sqdmulh_single_vgx2:
6265 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6266 VT: Node->getValueType(ResNo: 0),
6267 Opcodes: {AArch64::SQDMULH_VG2_2ZZ_B, AArch64::SQDMULH_VG2_2ZZ_H,
6268 AArch64::SQDMULH_VG2_2ZZ_S, AArch64::SQDMULH_VG2_2ZZ_D}))
6269 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6270 return;
6271 case Intrinsic::aarch64_sve_sqdmulh_single_vgx4:
6272 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6273 VT: Node->getValueType(ResNo: 0),
6274 Opcodes: {AArch64::SQDMULH_VG4_4ZZ_B, AArch64::SQDMULH_VG4_4ZZ_H,
6275 AArch64::SQDMULH_VG4_4ZZ_S, AArch64::SQDMULH_VG4_4ZZ_D}))
6276 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6277 return;
6278 case Intrinsic::aarch64_sve_sqdmulh_vgx2:
6279 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6280 VT: Node->getValueType(ResNo: 0),
6281 Opcodes: {AArch64::SQDMULH_VG2_2Z2Z_B, AArch64::SQDMULH_VG2_2Z2Z_H,
6282 AArch64::SQDMULH_VG2_2Z2Z_S, AArch64::SQDMULH_VG2_2Z2Z_D}))
6283 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6284 return;
6285 case Intrinsic::aarch64_sve_sqdmulh_vgx4:
6286 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6287 VT: Node->getValueType(ResNo: 0),
6288 Opcodes: {AArch64::SQDMULH_VG4_4Z4Z_B, AArch64::SQDMULH_VG4_4Z4Z_H,
6289 AArch64::SQDMULH_VG4_4Z4Z_S, AArch64::SQDMULH_VG4_4Z4Z_D}))
6290 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6291 return;
6292 case Intrinsic::aarch64_sme_fp8_scale_single_x2:
6293 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6294 VT: Node->getValueType(ResNo: 0),
6295 Opcodes: {0, AArch64::FSCALE_2ZZ_H, AArch64::FSCALE_2ZZ_S,
6296 AArch64::FSCALE_2ZZ_D}))
6297 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6298 return;
6299 case Intrinsic::aarch64_sme_fp8_scale_single_x4:
6300 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6301 VT: Node->getValueType(ResNo: 0),
6302 Opcodes: {0, AArch64::FSCALE_4ZZ_H, AArch64::FSCALE_4ZZ_S,
6303 AArch64::FSCALE_4ZZ_D}))
6304 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6305 return;
6306 case Intrinsic::aarch64_sme_fp8_scale_x2:
6307 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6308 VT: Node->getValueType(ResNo: 0),
6309 Opcodes: {0, AArch64::FSCALE_2Z2Z_H, AArch64::FSCALE_2Z2Z_S,
6310 AArch64::FSCALE_2Z2Z_D}))
6311 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6312 return;
6313 case Intrinsic::aarch64_sme_fp8_scale_x4:
6314 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6315 VT: Node->getValueType(ResNo: 0),
6316 Opcodes: {0, AArch64::FSCALE_4Z4Z_H, AArch64::FSCALE_4Z4Z_S,
6317 AArch64::FSCALE_4Z4Z_D}))
6318 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6319 return;
6320 case Intrinsic::aarch64_sve_whilege_x2:
6321 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6322 VT: Node->getValueType(ResNo: 0),
6323 Opcodes: {AArch64::WHILEGE_2PXX_B, AArch64::WHILEGE_2PXX_H,
6324 AArch64::WHILEGE_2PXX_S, AArch64::WHILEGE_2PXX_D}))
6325 SelectWhilePair(N: Node, Opc: Op);
6326 return;
6327 case Intrinsic::aarch64_sve_whilegt_x2:
6328 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6329 VT: Node->getValueType(ResNo: 0),
6330 Opcodes: {AArch64::WHILEGT_2PXX_B, AArch64::WHILEGT_2PXX_H,
6331 AArch64::WHILEGT_2PXX_S, AArch64::WHILEGT_2PXX_D}))
6332 SelectWhilePair(N: Node, Opc: Op);
6333 return;
6334 case Intrinsic::aarch64_sve_whilehi_x2:
6335 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6336 VT: Node->getValueType(ResNo: 0),
6337 Opcodes: {AArch64::WHILEHI_2PXX_B, AArch64::WHILEHI_2PXX_H,
6338 AArch64::WHILEHI_2PXX_S, AArch64::WHILEHI_2PXX_D}))
6339 SelectWhilePair(N: Node, Opc: Op);
6340 return;
6341 case Intrinsic::aarch64_sve_whilehs_x2:
6342 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6343 VT: Node->getValueType(ResNo: 0),
6344 Opcodes: {AArch64::WHILEHS_2PXX_B, AArch64::WHILEHS_2PXX_H,
6345 AArch64::WHILEHS_2PXX_S, AArch64::WHILEHS_2PXX_D}))
6346 SelectWhilePair(N: Node, Opc: Op);
6347 return;
6348 case Intrinsic::aarch64_sve_whilele_x2:
6349 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6350 VT: Node->getValueType(ResNo: 0),
6351 Opcodes: {AArch64::WHILELE_2PXX_B, AArch64::WHILELE_2PXX_H,
6352 AArch64::WHILELE_2PXX_S, AArch64::WHILELE_2PXX_D}))
6353 SelectWhilePair(N: Node, Opc: Op);
6354 return;
6355 case Intrinsic::aarch64_sve_whilelo_x2:
6356 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6357 VT: Node->getValueType(ResNo: 0),
6358 Opcodes: {AArch64::WHILELO_2PXX_B, AArch64::WHILELO_2PXX_H,
6359 AArch64::WHILELO_2PXX_S, AArch64::WHILELO_2PXX_D}))
6360 SelectWhilePair(N: Node, Opc: Op);
6361 return;
6362 case Intrinsic::aarch64_sve_whilels_x2:
6363 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6364 VT: Node->getValueType(ResNo: 0),
6365 Opcodes: {AArch64::WHILELS_2PXX_B, AArch64::WHILELS_2PXX_H,
6366 AArch64::WHILELS_2PXX_S, AArch64::WHILELS_2PXX_D}))
6367 SelectWhilePair(N: Node, Opc: Op);
6368 return;
6369 case Intrinsic::aarch64_sve_whilelt_x2:
6370 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6371 VT: Node->getValueType(ResNo: 0),
6372 Opcodes: {AArch64::WHILELT_2PXX_B, AArch64::WHILELT_2PXX_H,
6373 AArch64::WHILELT_2PXX_S, AArch64::WHILELT_2PXX_D}))
6374 SelectWhilePair(N: Node, Opc: Op);
6375 return;
6376 case Intrinsic::aarch64_sve_smax_single_x2:
6377 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6378 VT: Node->getValueType(ResNo: 0),
6379 Opcodes: {AArch64::SMAX_VG2_2ZZ_B, AArch64::SMAX_VG2_2ZZ_H,
6380 AArch64::SMAX_VG2_2ZZ_S, AArch64::SMAX_VG2_2ZZ_D}))
6381 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6382 return;
6383 case Intrinsic::aarch64_sve_umax_single_x2:
6384 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6385 VT: Node->getValueType(ResNo: 0),
6386 Opcodes: {AArch64::UMAX_VG2_2ZZ_B, AArch64::UMAX_VG2_2ZZ_H,
6387 AArch64::UMAX_VG2_2ZZ_S, AArch64::UMAX_VG2_2ZZ_D}))
6388 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6389 return;
6390 case Intrinsic::aarch64_sve_fmax_single_x2:
6391 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6392 VT: Node->getValueType(ResNo: 0),
6393 Opcodes: {AArch64::BFMAX_VG2_2ZZ_H, AArch64::FMAX_VG2_2ZZ_H,
6394 AArch64::FMAX_VG2_2ZZ_S, AArch64::FMAX_VG2_2ZZ_D}))
6395 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6396 return;
6397 case Intrinsic::aarch64_sve_smax_single_x4:
6398 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6399 VT: Node->getValueType(ResNo: 0),
6400 Opcodes: {AArch64::SMAX_VG4_4ZZ_B, AArch64::SMAX_VG4_4ZZ_H,
6401 AArch64::SMAX_VG4_4ZZ_S, AArch64::SMAX_VG4_4ZZ_D}))
6402 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6403 return;
6404 case Intrinsic::aarch64_sve_umax_single_x4:
6405 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6406 VT: Node->getValueType(ResNo: 0),
6407 Opcodes: {AArch64::UMAX_VG4_4ZZ_B, AArch64::UMAX_VG4_4ZZ_H,
6408 AArch64::UMAX_VG4_4ZZ_S, AArch64::UMAX_VG4_4ZZ_D}))
6409 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6410 return;
6411 case Intrinsic::aarch64_sve_fmax_single_x4:
6412 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6413 VT: Node->getValueType(ResNo: 0),
6414 Opcodes: {AArch64::BFMAX_VG4_4ZZ_H, AArch64::FMAX_VG4_4ZZ_H,
6415 AArch64::FMAX_VG4_4ZZ_S, AArch64::FMAX_VG4_4ZZ_D}))
6416 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6417 return;
6418 case Intrinsic::aarch64_sve_smin_single_x2:
6419 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6420 VT: Node->getValueType(ResNo: 0),
6421 Opcodes: {AArch64::SMIN_VG2_2ZZ_B, AArch64::SMIN_VG2_2ZZ_H,
6422 AArch64::SMIN_VG2_2ZZ_S, AArch64::SMIN_VG2_2ZZ_D}))
6423 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6424 return;
6425 case Intrinsic::aarch64_sve_umin_single_x2:
6426 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6427 VT: Node->getValueType(ResNo: 0),
6428 Opcodes: {AArch64::UMIN_VG2_2ZZ_B, AArch64::UMIN_VG2_2ZZ_H,
6429 AArch64::UMIN_VG2_2ZZ_S, AArch64::UMIN_VG2_2ZZ_D}))
6430 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6431 return;
6432 case Intrinsic::aarch64_sve_fmin_single_x2:
6433 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6434 VT: Node->getValueType(ResNo: 0),
6435 Opcodes: {AArch64::BFMIN_VG2_2ZZ_H, AArch64::FMIN_VG2_2ZZ_H,
6436 AArch64::FMIN_VG2_2ZZ_S, AArch64::FMIN_VG2_2ZZ_D}))
6437 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6438 return;
6439 case Intrinsic::aarch64_sve_smin_single_x4:
6440 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6441 VT: Node->getValueType(ResNo: 0),
6442 Opcodes: {AArch64::SMIN_VG4_4ZZ_B, AArch64::SMIN_VG4_4ZZ_H,
6443 AArch64::SMIN_VG4_4ZZ_S, AArch64::SMIN_VG4_4ZZ_D}))
6444 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6445 return;
6446 case Intrinsic::aarch64_sve_umin_single_x4:
6447 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6448 VT: Node->getValueType(ResNo: 0),
6449 Opcodes: {AArch64::UMIN_VG4_4ZZ_B, AArch64::UMIN_VG4_4ZZ_H,
6450 AArch64::UMIN_VG4_4ZZ_S, AArch64::UMIN_VG4_4ZZ_D}))
6451 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6452 return;
6453 case Intrinsic::aarch64_sve_fmin_single_x4:
6454 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6455 VT: Node->getValueType(ResNo: 0),
6456 Opcodes: {AArch64::BFMIN_VG4_4ZZ_H, AArch64::FMIN_VG4_4ZZ_H,
6457 AArch64::FMIN_VG4_4ZZ_S, AArch64::FMIN_VG4_4ZZ_D}))
6458 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6459 return;
6460 case Intrinsic::aarch64_sve_smax_x2:
6461 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6462 VT: Node->getValueType(ResNo: 0),
6463 Opcodes: {AArch64::SMAX_VG2_2Z2Z_B, AArch64::SMAX_VG2_2Z2Z_H,
6464 AArch64::SMAX_VG2_2Z2Z_S, AArch64::SMAX_VG2_2Z2Z_D}))
6465 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6466 return;
6467 case Intrinsic::aarch64_sve_umax_x2:
6468 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6469 VT: Node->getValueType(ResNo: 0),
6470 Opcodes: {AArch64::UMAX_VG2_2Z2Z_B, AArch64::UMAX_VG2_2Z2Z_H,
6471 AArch64::UMAX_VG2_2Z2Z_S, AArch64::UMAX_VG2_2Z2Z_D}))
6472 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6473 return;
6474 case Intrinsic::aarch64_sve_fmax_x2:
6475 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6476 VT: Node->getValueType(ResNo: 0),
6477 Opcodes: {AArch64::BFMAX_VG2_2Z2Z_H, AArch64::FMAX_VG2_2Z2Z_H,
6478 AArch64::FMAX_VG2_2Z2Z_S, AArch64::FMAX_VG2_2Z2Z_D}))
6479 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6480 return;
6481 case Intrinsic::aarch64_sve_smax_x4:
6482 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6483 VT: Node->getValueType(ResNo: 0),
6484 Opcodes: {AArch64::SMAX_VG4_4Z4Z_B, AArch64::SMAX_VG4_4Z4Z_H,
6485 AArch64::SMAX_VG4_4Z4Z_S, AArch64::SMAX_VG4_4Z4Z_D}))
6486 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6487 return;
6488 case Intrinsic::aarch64_sve_umax_x4:
6489 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6490 VT: Node->getValueType(ResNo: 0),
6491 Opcodes: {AArch64::UMAX_VG4_4Z4Z_B, AArch64::UMAX_VG4_4Z4Z_H,
6492 AArch64::UMAX_VG4_4Z4Z_S, AArch64::UMAX_VG4_4Z4Z_D}))
6493 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6494 return;
6495 case Intrinsic::aarch64_sve_fmax_x4:
6496 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6497 VT: Node->getValueType(ResNo: 0),
6498 Opcodes: {AArch64::BFMAX_VG4_4Z2Z_H, AArch64::FMAX_VG4_4Z4Z_H,
6499 AArch64::FMAX_VG4_4Z4Z_S, AArch64::FMAX_VG4_4Z4Z_D}))
6500 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6501 return;
6502 case Intrinsic::aarch64_sme_famax_x2:
6503 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6504 VT: Node->getValueType(ResNo: 0),
6505 Opcodes: {0, AArch64::FAMAX_2Z2Z_H, AArch64::FAMAX_2Z2Z_S,
6506 AArch64::FAMAX_2Z2Z_D}))
6507 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6508 return;
6509 case Intrinsic::aarch64_sme_famax_x4:
6510 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6511 VT: Node->getValueType(ResNo: 0),
6512 Opcodes: {0, AArch64::FAMAX_4Z4Z_H, AArch64::FAMAX_4Z4Z_S,
6513 AArch64::FAMAX_4Z4Z_D}))
6514 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6515 return;
6516 case Intrinsic::aarch64_sme_famin_x2:
6517 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6518 VT: Node->getValueType(ResNo: 0),
6519 Opcodes: {0, AArch64::FAMIN_2Z2Z_H, AArch64::FAMIN_2Z2Z_S,
6520 AArch64::FAMIN_2Z2Z_D}))
6521 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6522 return;
6523 case Intrinsic::aarch64_sme_famin_x4:
6524 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6525 VT: Node->getValueType(ResNo: 0),
6526 Opcodes: {0, AArch64::FAMIN_4Z4Z_H, AArch64::FAMIN_4Z4Z_S,
6527 AArch64::FAMIN_4Z4Z_D}))
6528 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6529 return;
6530 case Intrinsic::aarch64_sve_smin_x2:
6531 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6532 VT: Node->getValueType(ResNo: 0),
6533 Opcodes: {AArch64::SMIN_VG2_2Z2Z_B, AArch64::SMIN_VG2_2Z2Z_H,
6534 AArch64::SMIN_VG2_2Z2Z_S, AArch64::SMIN_VG2_2Z2Z_D}))
6535 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6536 return;
6537 case Intrinsic::aarch64_sve_umin_x2:
6538 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6539 VT: Node->getValueType(ResNo: 0),
6540 Opcodes: {AArch64::UMIN_VG2_2Z2Z_B, AArch64::UMIN_VG2_2Z2Z_H,
6541 AArch64::UMIN_VG2_2Z2Z_S, AArch64::UMIN_VG2_2Z2Z_D}))
6542 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6543 return;
6544 case Intrinsic::aarch64_sve_fmin_x2:
6545 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6546 VT: Node->getValueType(ResNo: 0),
6547 Opcodes: {AArch64::BFMIN_VG2_2Z2Z_H, AArch64::FMIN_VG2_2Z2Z_H,
6548 AArch64::FMIN_VG2_2Z2Z_S, AArch64::FMIN_VG2_2Z2Z_D}))
6549 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6550 return;
6551 case Intrinsic::aarch64_sve_smin_x4:
6552 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6553 VT: Node->getValueType(ResNo: 0),
6554 Opcodes: {AArch64::SMIN_VG4_4Z4Z_B, AArch64::SMIN_VG4_4Z4Z_H,
6555 AArch64::SMIN_VG4_4Z4Z_S, AArch64::SMIN_VG4_4Z4Z_D}))
6556 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6557 return;
6558 case Intrinsic::aarch64_sve_umin_x4:
6559 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6560 VT: Node->getValueType(ResNo: 0),
6561 Opcodes: {AArch64::UMIN_VG4_4Z4Z_B, AArch64::UMIN_VG4_4Z4Z_H,
6562 AArch64::UMIN_VG4_4Z4Z_S, AArch64::UMIN_VG4_4Z4Z_D}))
6563 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6564 return;
6565 case Intrinsic::aarch64_sve_fmin_x4:
6566 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6567 VT: Node->getValueType(ResNo: 0),
6568 Opcodes: {AArch64::BFMIN_VG4_4Z2Z_H, AArch64::FMIN_VG4_4Z4Z_H,
6569 AArch64::FMIN_VG4_4Z4Z_S, AArch64::FMIN_VG4_4Z4Z_D}))
6570 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6571 return;
6572 case Intrinsic::aarch64_sve_fmaxnm_single_x2 :
6573 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6574 VT: Node->getValueType(ResNo: 0),
6575 Opcodes: {AArch64::BFMAXNM_VG2_2ZZ_H, AArch64::FMAXNM_VG2_2ZZ_H,
6576 AArch64::FMAXNM_VG2_2ZZ_S, AArch64::FMAXNM_VG2_2ZZ_D}))
6577 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6578 return;
6579 case Intrinsic::aarch64_sve_fmaxnm_single_x4 :
6580 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6581 VT: Node->getValueType(ResNo: 0),
6582 Opcodes: {AArch64::BFMAXNM_VG4_4ZZ_H, AArch64::FMAXNM_VG4_4ZZ_H,
6583 AArch64::FMAXNM_VG4_4ZZ_S, AArch64::FMAXNM_VG4_4ZZ_D}))
6584 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6585 return;
6586 case Intrinsic::aarch64_sve_fminnm_single_x2:
6587 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6588 VT: Node->getValueType(ResNo: 0),
6589 Opcodes: {AArch64::BFMINNM_VG2_2ZZ_H, AArch64::FMINNM_VG2_2ZZ_H,
6590 AArch64::FMINNM_VG2_2ZZ_S, AArch64::FMINNM_VG2_2ZZ_D}))
6591 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6592 return;
6593 case Intrinsic::aarch64_sve_fminnm_single_x4:
6594 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6595 VT: Node->getValueType(ResNo: 0),
6596 Opcodes: {AArch64::BFMINNM_VG4_4ZZ_H, AArch64::FMINNM_VG4_4ZZ_H,
6597 AArch64::FMINNM_VG4_4ZZ_S, AArch64::FMINNM_VG4_4ZZ_D}))
6598 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6599 return;
6600 case Intrinsic::aarch64_sve_fscale_single_x4:
6601 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::BFSCALE_4ZZ);
6602 return;
6603 case Intrinsic::aarch64_sve_fscale_single_x2:
6604 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::BFSCALE_2ZZ);
6605 return;
6606 case Intrinsic::aarch64_sve_fmul_single_x4:
6607 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6608 VT: Node->getValueType(ResNo: 0),
6609 Opcodes: {AArch64::BFMUL_4ZZ, AArch64::FMUL_4ZZ_H, AArch64::FMUL_4ZZ_S,
6610 AArch64::FMUL_4ZZ_D}))
6611 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6612 return;
6613 case Intrinsic::aarch64_sve_fmul_single_x2:
6614 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6615 VT: Node->getValueType(ResNo: 0),
6616 Opcodes: {AArch64::BFMUL_2ZZ, AArch64::FMUL_2ZZ_H, AArch64::FMUL_2ZZ_S,
6617 AArch64::FMUL_2ZZ_D}))
6618 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6619 return;
6620 case Intrinsic::aarch64_sve_fmaxnm_x2:
6621 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6622 VT: Node->getValueType(ResNo: 0),
6623 Opcodes: {AArch64::BFMAXNM_VG2_2Z2Z_H, AArch64::FMAXNM_VG2_2Z2Z_H,
6624 AArch64::FMAXNM_VG2_2Z2Z_S, AArch64::FMAXNM_VG2_2Z2Z_D}))
6625 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6626 return;
6627 case Intrinsic::aarch64_sve_fmaxnm_x4:
6628 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6629 VT: Node->getValueType(ResNo: 0),
6630 Opcodes: {AArch64::BFMAXNM_VG4_4Z2Z_H, AArch64::FMAXNM_VG4_4Z4Z_H,
6631 AArch64::FMAXNM_VG4_4Z4Z_S, AArch64::FMAXNM_VG4_4Z4Z_D}))
6632 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6633 return;
6634 case Intrinsic::aarch64_sve_fminnm_x2:
6635 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6636 VT: Node->getValueType(ResNo: 0),
6637 Opcodes: {AArch64::BFMINNM_VG2_2Z2Z_H, AArch64::FMINNM_VG2_2Z2Z_H,
6638 AArch64::FMINNM_VG2_2Z2Z_S, AArch64::FMINNM_VG2_2Z2Z_D}))
6639 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6640 return;
6641 case Intrinsic::aarch64_sve_fminnm_x4:
6642 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6643 VT: Node->getValueType(ResNo: 0),
6644 Opcodes: {AArch64::BFMINNM_VG4_4Z2Z_H, AArch64::FMINNM_VG4_4Z4Z_H,
6645 AArch64::FMINNM_VG4_4Z4Z_S, AArch64::FMINNM_VG4_4Z4Z_D}))
6646 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6647 return;
6648 case Intrinsic::aarch64_sve_aese_lane_x2:
6649 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESE_2ZZI_B);
6650 return;
6651 case Intrinsic::aarch64_sve_aesd_lane_x2:
6652 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESD_2ZZI_B);
6653 return;
6654 case Intrinsic::aarch64_sve_aesemc_lane_x2:
6655 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESEMC_2ZZI_B);
6656 return;
6657 case Intrinsic::aarch64_sve_aesdimc_lane_x2:
6658 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESDIMC_2ZZI_B);
6659 return;
6660 case Intrinsic::aarch64_sve_aese_lane_x4:
6661 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESE_4ZZI_B);
6662 return;
6663 case Intrinsic::aarch64_sve_aesd_lane_x4:
6664 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESD_4ZZI_B);
6665 return;
6666 case Intrinsic::aarch64_sve_aesemc_lane_x4:
6667 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESEMC_4ZZI_B);
6668 return;
6669 case Intrinsic::aarch64_sve_aesdimc_lane_x4:
6670 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESDIMC_4ZZI_B);
6671 return;
6672 case Intrinsic::aarch64_sve_pmlal_pair_x2:
6673 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::PMLAL_2ZZZ_Q);
6674 return;
6675 case Intrinsic::aarch64_sve_pmull_pair_x2: {
6676 SDLoc DL(Node);
6677 SmallVector<SDValue, 4> Regs(Node->ops().slice(N: 1, M: 2));
6678 SDNode *Res =
6679 CurDAG->getMachineNode(Opcode: AArch64::PMULL_2ZZZ_Q, dl: DL, VT: MVT::Untyped, Ops: Regs);
6680 SDValue SuperReg = SDValue(Res, 0);
6681 for (unsigned I = 0; I < 2; I++)
6682 ReplaceUses(F: SDValue(Node, I),
6683 T: CurDAG->getTargetExtractSubreg(SRIdx: AArch64::zsub0 + I, DL, VT,
6684 Operand: SuperReg));
6685 CurDAG->RemoveDeadNode(N: Node);
6686 return;
6687 }
6688 case Intrinsic::aarch64_sve_fscale_x4:
6689 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: AArch64::BFSCALE_4Z4Z);
6690 return;
6691 case Intrinsic::aarch64_sve_fscale_x2:
6692 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: AArch64::BFSCALE_2Z2Z);
6693 return;
6694 case Intrinsic::aarch64_sve_fmul_x4:
6695 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6696 VT: Node->getValueType(ResNo: 0),
6697 Opcodes: {AArch64::BFMUL_4Z4Z, AArch64::FMUL_4Z4Z_H, AArch64::FMUL_4Z4Z_S,
6698 AArch64::FMUL_4Z4Z_D}))
6699 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6700 return;
6701 case Intrinsic::aarch64_sve_fmul_x2:
6702 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6703 VT: Node->getValueType(ResNo: 0),
6704 Opcodes: {AArch64::BFMUL_2Z2Z, AArch64::FMUL_2Z2Z_H, AArch64::FMUL_2Z2Z_S,
6705 AArch64::FMUL_2Z2Z_D}))
6706 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6707 return;
6708 case Intrinsic::aarch64_sve_fcvtzs_x2:
6709 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::FCVTZS_2Z2Z_StoS);
6710 return;
6711 case Intrinsic::aarch64_sve_scvtf_x2:
6712 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::SCVTF_2Z2Z_StoS);
6713 return;
6714 case Intrinsic::aarch64_sve_fcvtzu_x2:
6715 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::FCVTZU_2Z2Z_StoS);
6716 return;
6717 case Intrinsic::aarch64_sve_ucvtf_x2:
6718 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::UCVTF_2Z2Z_StoS);
6719 return;
6720 case Intrinsic::aarch64_sve_fcvtzs_x4:
6721 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::FCVTZS_4Z4Z_StoS);
6722 return;
6723 case Intrinsic::aarch64_sve_scvtf_x4:
6724 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::SCVTF_4Z4Z_StoS);
6725 return;
6726 case Intrinsic::aarch64_sve_fcvtzu_x4:
6727 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::FCVTZU_4Z4Z_StoS);
6728 return;
6729 case Intrinsic::aarch64_sve_ucvtf_x4:
6730 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::UCVTF_4Z4Z_StoS);
6731 return;
6732 case Intrinsic::aarch64_sve_fcvt_widen_x2:
6733 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, IsTupleInput: false, Opc: AArch64::FCVT_2ZZ_H_S);
6734 return;
6735 case Intrinsic::aarch64_sve_fcvtl_widen_x2:
6736 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, IsTupleInput: false, Opc: AArch64::FCVTL_2ZZ_H_S);
6737 return;
6738 case Intrinsic::aarch64_sve_sclamp_single_x2:
6739 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6740 VT: Node->getValueType(ResNo: 0),
6741 Opcodes: {AArch64::SCLAMP_VG2_2Z2Z_B, AArch64::SCLAMP_VG2_2Z2Z_H,
6742 AArch64::SCLAMP_VG2_2Z2Z_S, AArch64::SCLAMP_VG2_2Z2Z_D}))
6743 SelectClamp(N: Node, NumVecs: 2, Op);
6744 return;
6745 case Intrinsic::aarch64_sve_uclamp_single_x2:
6746 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6747 VT: Node->getValueType(ResNo: 0),
6748 Opcodes: {AArch64::UCLAMP_VG2_2Z2Z_B, AArch64::UCLAMP_VG2_2Z2Z_H,
6749 AArch64::UCLAMP_VG2_2Z2Z_S, AArch64::UCLAMP_VG2_2Z2Z_D}))
6750 SelectClamp(N: Node, NumVecs: 2, Op);
6751 return;
6752 case Intrinsic::aarch64_sve_fclamp_single_x2:
6753 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6754 VT: Node->getValueType(ResNo: 0),
6755 Opcodes: {0, AArch64::FCLAMP_VG2_2Z2Z_H, AArch64::FCLAMP_VG2_2Z2Z_S,
6756 AArch64::FCLAMP_VG2_2Z2Z_D}))
6757 SelectClamp(N: Node, NumVecs: 2, Op);
6758 return;
6759 case Intrinsic::aarch64_sve_bfclamp_single_x2:
6760 SelectClamp(N: Node, NumVecs: 2, Op: AArch64::BFCLAMP_VG2_2ZZZ_H);
6761 return;
6762 case Intrinsic::aarch64_sve_sclamp_single_x4:
6763 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6764 VT: Node->getValueType(ResNo: 0),
6765 Opcodes: {AArch64::SCLAMP_VG4_4Z4Z_B, AArch64::SCLAMP_VG4_4Z4Z_H,
6766 AArch64::SCLAMP_VG4_4Z4Z_S, AArch64::SCLAMP_VG4_4Z4Z_D}))
6767 SelectClamp(N: Node, NumVecs: 4, Op);
6768 return;
6769 case Intrinsic::aarch64_sve_uclamp_single_x4:
6770 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6771 VT: Node->getValueType(ResNo: 0),
6772 Opcodes: {AArch64::UCLAMP_VG4_4Z4Z_B, AArch64::UCLAMP_VG4_4Z4Z_H,
6773 AArch64::UCLAMP_VG4_4Z4Z_S, AArch64::UCLAMP_VG4_4Z4Z_D}))
6774 SelectClamp(N: Node, NumVecs: 4, Op);
6775 return;
6776 case Intrinsic::aarch64_sve_fclamp_single_x4:
6777 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6778 VT: Node->getValueType(ResNo: 0),
6779 Opcodes: {0, AArch64::FCLAMP_VG4_4Z4Z_H, AArch64::FCLAMP_VG4_4Z4Z_S,
6780 AArch64::FCLAMP_VG4_4Z4Z_D}))
6781 SelectClamp(N: Node, NumVecs: 4, Op);
6782 return;
6783 case Intrinsic::aarch64_sve_bfclamp_single_x4:
6784 SelectClamp(N: Node, NumVecs: 4, Op: AArch64::BFCLAMP_VG4_4ZZZ_H);
6785 return;
6786 case Intrinsic::aarch64_sve_add_single_x2:
6787 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6788 VT: Node->getValueType(ResNo: 0),
6789 Opcodes: {AArch64::ADD_VG2_2ZZ_B, AArch64::ADD_VG2_2ZZ_H,
6790 AArch64::ADD_VG2_2ZZ_S, AArch64::ADD_VG2_2ZZ_D}))
6791 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6792 return;
6793 case Intrinsic::aarch64_sve_add_single_x4:
6794 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6795 VT: Node->getValueType(ResNo: 0),
6796 Opcodes: {AArch64::ADD_VG4_4ZZ_B, AArch64::ADD_VG4_4ZZ_H,
6797 AArch64::ADD_VG4_4ZZ_S, AArch64::ADD_VG4_4ZZ_D}))
6798 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6799 return;
6800 case Intrinsic::aarch64_sve_zip_x2:
6801 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6802 VT: Node->getValueType(ResNo: 0),
6803 Opcodes: {AArch64::ZIP_VG2_2ZZZ_B, AArch64::ZIP_VG2_2ZZZ_H,
6804 AArch64::ZIP_VG2_2ZZZ_S, AArch64::ZIP_VG2_2ZZZ_D}))
6805 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6806 return;
6807 case Intrinsic::aarch64_sve_zipq_x2:
6808 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false,
6809 Opc: AArch64::ZIP_VG2_2ZZZ_Q);
6810 return;
6811 case Intrinsic::aarch64_sve_zip_x4:
6812 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6813 VT: Node->getValueType(ResNo: 0),
6814 Opcodes: {AArch64::ZIP_VG4_4Z4Z_B, AArch64::ZIP_VG4_4Z4Z_H,
6815 AArch64::ZIP_VG4_4Z4Z_S, AArch64::ZIP_VG4_4Z4Z_D}))
6816 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6817 return;
6818 case Intrinsic::aarch64_sve_zipq_x4:
6819 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true,
6820 Opc: AArch64::ZIP_VG4_4Z4Z_Q);
6821 return;
6822 case Intrinsic::aarch64_sve_uzp_x2:
6823 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6824 VT: Node->getValueType(ResNo: 0),
6825 Opcodes: {AArch64::UZP_VG2_2ZZZ_B, AArch64::UZP_VG2_2ZZZ_H,
6826 AArch64::UZP_VG2_2ZZZ_S, AArch64::UZP_VG2_2ZZZ_D}))
6827 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6828 return;
6829 case Intrinsic::aarch64_sve_uzpq_x2:
6830 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false,
6831 Opc: AArch64::UZP_VG2_2ZZZ_Q);
6832 return;
6833 case Intrinsic::aarch64_sve_uzp_x4:
6834 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6835 VT: Node->getValueType(ResNo: 0),
6836 Opcodes: {AArch64::UZP_VG4_4Z4Z_B, AArch64::UZP_VG4_4Z4Z_H,
6837 AArch64::UZP_VG4_4Z4Z_S, AArch64::UZP_VG4_4Z4Z_D}))
6838 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6839 return;
6840 case Intrinsic::aarch64_sve_uzpq_x4:
6841 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true,
6842 Opc: AArch64::UZP_VG4_4Z4Z_Q);
6843 return;
6844 case Intrinsic::aarch64_sve_sel_x2:
6845 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6846 VT: Node->getValueType(ResNo: 0),
6847 Opcodes: {AArch64::SEL_VG2_2ZC2Z2Z_B, AArch64::SEL_VG2_2ZC2Z2Z_H,
6848 AArch64::SEL_VG2_2ZC2Z2Z_S, AArch64::SEL_VG2_2ZC2Z2Z_D}))
6849 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op, /*HasPred=*/true);
6850 return;
6851 case Intrinsic::aarch64_sve_sel_x4:
6852 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6853 VT: Node->getValueType(ResNo: 0),
6854 Opcodes: {AArch64::SEL_VG4_4ZC4Z4Z_B, AArch64::SEL_VG4_4ZC4Z4Z_H,
6855 AArch64::SEL_VG4_4ZC4Z4Z_S, AArch64::SEL_VG4_4ZC4Z4Z_D}))
6856 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op, /*HasPred=*/true);
6857 return;
6858 case Intrinsic::aarch64_sve_frinta_x2:
6859 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTA_2Z2Z_S);
6860 return;
6861 case Intrinsic::aarch64_sve_frinta_x4:
6862 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTA_4Z4Z_S);
6863 return;
6864 case Intrinsic::aarch64_sve_frintm_x2:
6865 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTM_2Z2Z_S);
6866 return;
6867 case Intrinsic::aarch64_sve_frintm_x4:
6868 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTM_4Z4Z_S);
6869 return;
6870 case Intrinsic::aarch64_sve_frintn_x2:
6871 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTN_2Z2Z_S);
6872 return;
6873 case Intrinsic::aarch64_sve_frintn_x4:
6874 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTN_4Z4Z_S);
6875 return;
6876 case Intrinsic::aarch64_sve_frintp_x2:
6877 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTP_2Z2Z_S);
6878 return;
6879 case Intrinsic::aarch64_sve_frintp_x4:
6880 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTP_4Z4Z_S);
6881 return;
6882 case Intrinsic::aarch64_sve_sunpk_x2:
6883 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6884 VT: Node->getValueType(ResNo: 0),
6885 Opcodes: {0, AArch64::SUNPK_VG2_2ZZ_H, AArch64::SUNPK_VG2_2ZZ_S,
6886 AArch64::SUNPK_VG2_2ZZ_D}))
6887 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6888 return;
6889 case Intrinsic::aarch64_sve_uunpk_x2:
6890 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6891 VT: Node->getValueType(ResNo: 0),
6892 Opcodes: {0, AArch64::UUNPK_VG2_2ZZ_H, AArch64::UUNPK_VG2_2ZZ_S,
6893 AArch64::UUNPK_VG2_2ZZ_D}))
6894 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6895 return;
6896 case Intrinsic::aarch64_sve_sunpk_x4:
6897 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6898 VT: Node->getValueType(ResNo: 0),
6899 Opcodes: {0, AArch64::SUNPK_VG4_4Z2Z_H, AArch64::SUNPK_VG4_4Z2Z_S,
6900 AArch64::SUNPK_VG4_4Z2Z_D}))
6901 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6902 return;
6903 case Intrinsic::aarch64_sve_uunpk_x4:
6904 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6905 VT: Node->getValueType(ResNo: 0),
6906 Opcodes: {0, AArch64::UUNPK_VG4_4Z2Z_H, AArch64::UUNPK_VG4_4Z2Z_S,
6907 AArch64::UUNPK_VG4_4Z2Z_D}))
6908 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6909 return;
6910 case Intrinsic::aarch64_sve_pext_x2: {
6911 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6912 VT: Node->getValueType(ResNo: 0),
6913 Opcodes: {AArch64::PEXT_2PCI_B, AArch64::PEXT_2PCI_H, AArch64::PEXT_2PCI_S,
6914 AArch64::PEXT_2PCI_D}))
6915 SelectPExtPair(N: Node, Opc: Op);
6916 return;
6917 }
6918 }
6919 break;
6920 }
6921 case ISD::INTRINSIC_VOID: {
6922 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
6923 if (Node->getNumOperands() >= 3)
6924 VT = Node->getOperand(Num: 2)->getValueType(ResNo: 0);
6925 switch (IntNo) {
6926 default:
6927 break;
6928 case Intrinsic::aarch64_neon_st1x2: {
6929 if (VT == MVT::v8i8) {
6930 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8b);
6931 return;
6932 } else if (VT == MVT::v16i8) {
6933 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov16b);
6934 return;
6935 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6936 VT == MVT::v4bf16) {
6937 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4h);
6938 return;
6939 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6940 VT == MVT::v8bf16) {
6941 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8h);
6942 return;
6943 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6944 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2s);
6945 return;
6946 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6947 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4s);
6948 return;
6949 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6950 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2d);
6951 return;
6952 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6953 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d);
6954 return;
6955 }
6956 break;
6957 }
6958 case Intrinsic::aarch64_neon_st1x3: {
6959 if (VT == MVT::v8i8) {
6960 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8b);
6961 return;
6962 } else if (VT == MVT::v16i8) {
6963 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev16b);
6964 return;
6965 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6966 VT == MVT::v4bf16) {
6967 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4h);
6968 return;
6969 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6970 VT == MVT::v8bf16) {
6971 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8h);
6972 return;
6973 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6974 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2s);
6975 return;
6976 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6977 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4s);
6978 return;
6979 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6980 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2d);
6981 return;
6982 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6983 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d);
6984 return;
6985 }
6986 break;
6987 }
6988 case Intrinsic::aarch64_neon_st1x4: {
6989 if (VT == MVT::v8i8) {
6990 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8b);
6991 return;
6992 } else if (VT == MVT::v16i8) {
6993 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv16b);
6994 return;
6995 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6996 VT == MVT::v4bf16) {
6997 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4h);
6998 return;
6999 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7000 VT == MVT::v8bf16) {
7001 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8h);
7002 return;
7003 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7004 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2s);
7005 return;
7006 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7007 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4s);
7008 return;
7009 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7010 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2d);
7011 return;
7012 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7013 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d);
7014 return;
7015 }
7016 break;
7017 }
7018 case Intrinsic::aarch64_neon_st2: {
7019 if (VT == MVT::v8i8) {
7020 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8b);
7021 return;
7022 } else if (VT == MVT::v16i8) {
7023 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov16b);
7024 return;
7025 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
7026 VT == MVT::v4bf16) {
7027 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4h);
7028 return;
7029 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7030 VT == MVT::v8bf16) {
7031 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8h);
7032 return;
7033 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7034 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2s);
7035 return;
7036 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7037 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4s);
7038 return;
7039 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7040 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2d);
7041 return;
7042 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7043 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d);
7044 return;
7045 }
7046 break;
7047 }
7048 case Intrinsic::aarch64_neon_st3: {
7049 if (VT == MVT::v8i8) {
7050 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8b);
7051 return;
7052 } else if (VT == MVT::v16i8) {
7053 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev16b);
7054 return;
7055 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
7056 VT == MVT::v4bf16) {
7057 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4h);
7058 return;
7059 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7060 VT == MVT::v8bf16) {
7061 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8h);
7062 return;
7063 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7064 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2s);
7065 return;
7066 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7067 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4s);
7068 return;
7069 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7070 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2d);
7071 return;
7072 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7073 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d);
7074 return;
7075 }
7076 break;
7077 }
7078 case Intrinsic::aarch64_neon_st4: {
7079 if (VT == MVT::v8i8) {
7080 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8b);
7081 return;
7082 } else if (VT == MVT::v16i8) {
7083 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv16b);
7084 return;
7085 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
7086 VT == MVT::v4bf16) {
7087 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4h);
7088 return;
7089 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7090 VT == MVT::v8bf16) {
7091 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8h);
7092 return;
7093 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7094 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2s);
7095 return;
7096 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7097 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4s);
7098 return;
7099 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7100 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2d);
7101 return;
7102 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7103 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d);
7104 return;
7105 }
7106 break;
7107 }
7108 case Intrinsic::aarch64_neon_st2lane: {
7109 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7110 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i8);
7111 return;
7112 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7113 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7114 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i16);
7115 return;
7116 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7117 VT == MVT::v2f32) {
7118 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i32);
7119 return;
7120 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7121 VT == MVT::v1f64) {
7122 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i64);
7123 return;
7124 }
7125 break;
7126 }
7127 case Intrinsic::aarch64_neon_st3lane: {
7128 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7129 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i8);
7130 return;
7131 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7132 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7133 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i16);
7134 return;
7135 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7136 VT == MVT::v2f32) {
7137 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i32);
7138 return;
7139 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7140 VT == MVT::v1f64) {
7141 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i64);
7142 return;
7143 }
7144 break;
7145 }
7146 case Intrinsic::aarch64_neon_st4lane: {
7147 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7148 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i8);
7149 return;
7150 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7151 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7152 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i16);
7153 return;
7154 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7155 VT == MVT::v2f32) {
7156 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i32);
7157 return;
7158 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7159 VT == MVT::v1f64) {
7160 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i64);
7161 return;
7162 }
7163 break;
7164 }
7165 case Intrinsic::aarch64_sve_st2q: {
7166 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 4, Opc_rr: AArch64::ST2Q, Opc_ri: AArch64::ST2Q_IMM);
7167 return;
7168 }
7169 case Intrinsic::aarch64_sve_st3q: {
7170 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 4, Opc_rr: AArch64::ST3Q, Opc_ri: AArch64::ST3Q_IMM);
7171 return;
7172 }
7173 case Intrinsic::aarch64_sve_st4q: {
7174 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 4, Opc_rr: AArch64::ST4Q, Opc_ri: AArch64::ST4Q_IMM);
7175 return;
7176 }
7177 case Intrinsic::aarch64_sve_st2: {
7178 if (VT == MVT::nxv16i8) {
7179 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 0, Opc_rr: AArch64::ST2B, Opc_ri: AArch64::ST2B_IMM);
7180 return;
7181 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7182 VT == MVT::nxv8bf16) {
7183 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 1, Opc_rr: AArch64::ST2H, Opc_ri: AArch64::ST2H_IMM);
7184 return;
7185 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7186 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 2, Opc_rr: AArch64::ST2W, Opc_ri: AArch64::ST2W_IMM);
7187 return;
7188 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7189 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 3, Opc_rr: AArch64::ST2D, Opc_ri: AArch64::ST2D_IMM);
7190 return;
7191 }
7192 break;
7193 }
7194 case Intrinsic::aarch64_sve_st3: {
7195 if (VT == MVT::nxv16i8) {
7196 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 0, Opc_rr: AArch64::ST3B, Opc_ri: AArch64::ST3B_IMM);
7197 return;
7198 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7199 VT == MVT::nxv8bf16) {
7200 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 1, Opc_rr: AArch64::ST3H, Opc_ri: AArch64::ST3H_IMM);
7201 return;
7202 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7203 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 2, Opc_rr: AArch64::ST3W, Opc_ri: AArch64::ST3W_IMM);
7204 return;
7205 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7206 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 3, Opc_rr: AArch64::ST3D, Opc_ri: AArch64::ST3D_IMM);
7207 return;
7208 }
7209 break;
7210 }
7211 case Intrinsic::aarch64_sve_st4: {
7212 if (VT == MVT::nxv16i8) {
7213 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 0, Opc_rr: AArch64::ST4B, Opc_ri: AArch64::ST4B_IMM);
7214 return;
7215 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7216 VT == MVT::nxv8bf16) {
7217 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 1, Opc_rr: AArch64::ST4H, Opc_ri: AArch64::ST4H_IMM);
7218 return;
7219 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7220 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 2, Opc_rr: AArch64::ST4W, Opc_ri: AArch64::ST4W_IMM);
7221 return;
7222 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7223 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 3, Opc_rr: AArch64::ST4D, Opc_ri: AArch64::ST4D_IMM);
7224 return;
7225 }
7226 break;
7227 }
7228 }
7229 break;
7230 }
7231 case AArch64ISD::LD2post: {
7232 if (VT == MVT::v8i8) {
7233 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8b_POST, SubRegIdx: AArch64::dsub0);
7234 return;
7235 } else if (VT == MVT::v16i8) {
7236 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov16b_POST, SubRegIdx: AArch64::qsub0);
7237 return;
7238 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7239 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4h_POST, SubRegIdx: AArch64::dsub0);
7240 return;
7241 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7242 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8h_POST, SubRegIdx: AArch64::qsub0);
7243 return;
7244 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7245 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2s_POST, SubRegIdx: AArch64::dsub0);
7246 return;
7247 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7248 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4s_POST, SubRegIdx: AArch64::qsub0);
7249 return;
7250 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7251 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d_POST, SubRegIdx: AArch64::dsub0);
7252 return;
7253 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7254 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2d_POST, SubRegIdx: AArch64::qsub0);
7255 return;
7256 }
7257 break;
7258 }
7259 case AArch64ISD::LD3post: {
7260 if (VT == MVT::v8i8) {
7261 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8b_POST, SubRegIdx: AArch64::dsub0);
7262 return;
7263 } else if (VT == MVT::v16i8) {
7264 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev16b_POST, SubRegIdx: AArch64::qsub0);
7265 return;
7266 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7267 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4h_POST, SubRegIdx: AArch64::dsub0);
7268 return;
7269 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7270 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8h_POST, SubRegIdx: AArch64::qsub0);
7271 return;
7272 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7273 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2s_POST, SubRegIdx: AArch64::dsub0);
7274 return;
7275 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7276 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4s_POST, SubRegIdx: AArch64::qsub0);
7277 return;
7278 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7279 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d_POST, SubRegIdx: AArch64::dsub0);
7280 return;
7281 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7282 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2d_POST, SubRegIdx: AArch64::qsub0);
7283 return;
7284 }
7285 break;
7286 }
7287 case AArch64ISD::LD4post: {
7288 if (VT == MVT::v8i8) {
7289 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8b_POST, SubRegIdx: AArch64::dsub0);
7290 return;
7291 } else if (VT == MVT::v16i8) {
7292 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv16b_POST, SubRegIdx: AArch64::qsub0);
7293 return;
7294 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7295 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4h_POST, SubRegIdx: AArch64::dsub0);
7296 return;
7297 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7298 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8h_POST, SubRegIdx: AArch64::qsub0);
7299 return;
7300 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7301 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2s_POST, SubRegIdx: AArch64::dsub0);
7302 return;
7303 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7304 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4s_POST, SubRegIdx: AArch64::qsub0);
7305 return;
7306 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7307 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d_POST, SubRegIdx: AArch64::dsub0);
7308 return;
7309 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7310 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2d_POST, SubRegIdx: AArch64::qsub0);
7311 return;
7312 }
7313 break;
7314 }
7315 case AArch64ISD::LD1x2post: {
7316 if (VT == MVT::v8i8) {
7317 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8b_POST, SubRegIdx: AArch64::dsub0);
7318 return;
7319 } else if (VT == MVT::v16i8) {
7320 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov16b_POST, SubRegIdx: AArch64::qsub0);
7321 return;
7322 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7323 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4h_POST, SubRegIdx: AArch64::dsub0);
7324 return;
7325 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7326 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8h_POST, SubRegIdx: AArch64::qsub0);
7327 return;
7328 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7329 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2s_POST, SubRegIdx: AArch64::dsub0);
7330 return;
7331 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7332 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4s_POST, SubRegIdx: AArch64::qsub0);
7333 return;
7334 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7335 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d_POST, SubRegIdx: AArch64::dsub0);
7336 return;
7337 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7338 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2d_POST, SubRegIdx: AArch64::qsub0);
7339 return;
7340 }
7341 break;
7342 }
7343 case AArch64ISD::LD1x3post: {
7344 if (VT == MVT::v8i8) {
7345 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8b_POST, SubRegIdx: AArch64::dsub0);
7346 return;
7347 } else if (VT == MVT::v16i8) {
7348 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev16b_POST, SubRegIdx: AArch64::qsub0);
7349 return;
7350 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7351 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4h_POST, SubRegIdx: AArch64::dsub0);
7352 return;
7353 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7354 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8h_POST, SubRegIdx: AArch64::qsub0);
7355 return;
7356 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7357 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2s_POST, SubRegIdx: AArch64::dsub0);
7358 return;
7359 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7360 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4s_POST, SubRegIdx: AArch64::qsub0);
7361 return;
7362 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7363 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d_POST, SubRegIdx: AArch64::dsub0);
7364 return;
7365 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7366 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2d_POST, SubRegIdx: AArch64::qsub0);
7367 return;
7368 }
7369 break;
7370 }
7371 case AArch64ISD::LD1x4post: {
7372 if (VT == MVT::v8i8) {
7373 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8b_POST, SubRegIdx: AArch64::dsub0);
7374 return;
7375 } else if (VT == MVT::v16i8) {
7376 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv16b_POST, SubRegIdx: AArch64::qsub0);
7377 return;
7378 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7379 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4h_POST, SubRegIdx: AArch64::dsub0);
7380 return;
7381 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7382 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8h_POST, SubRegIdx: AArch64::qsub0);
7383 return;
7384 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7385 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2s_POST, SubRegIdx: AArch64::dsub0);
7386 return;
7387 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7388 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4s_POST, SubRegIdx: AArch64::qsub0);
7389 return;
7390 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7391 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d_POST, SubRegIdx: AArch64::dsub0);
7392 return;
7393 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7394 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2d_POST, SubRegIdx: AArch64::qsub0);
7395 return;
7396 }
7397 break;
7398 }
7399 case AArch64ISD::LD1DUPpost: {
7400 if (VT == MVT::v8i8) {
7401 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv8b_POST, SubRegIdx: AArch64::dsub0);
7402 return;
7403 } else if (VT == MVT::v16i8) {
7404 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv16b_POST, SubRegIdx: AArch64::qsub0);
7405 return;
7406 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7407 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv4h_POST, SubRegIdx: AArch64::dsub0);
7408 return;
7409 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7410 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv8h_POST, SubRegIdx: AArch64::qsub0);
7411 return;
7412 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7413 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv2s_POST, SubRegIdx: AArch64::dsub0);
7414 return;
7415 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7416 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv4s_POST, SubRegIdx: AArch64::qsub0);
7417 return;
7418 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7419 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv1d_POST, SubRegIdx: AArch64::dsub0);
7420 return;
7421 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7422 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv2d_POST, SubRegIdx: AArch64::qsub0);
7423 return;
7424 }
7425 break;
7426 }
7427 case AArch64ISD::LD2DUPpost: {
7428 if (VT == MVT::v8i8) {
7429 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8b_POST, SubRegIdx: AArch64::dsub0);
7430 return;
7431 } else if (VT == MVT::v16i8) {
7432 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv16b_POST, SubRegIdx: AArch64::qsub0);
7433 return;
7434 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7435 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4h_POST, SubRegIdx: AArch64::dsub0);
7436 return;
7437 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7438 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8h_POST, SubRegIdx: AArch64::qsub0);
7439 return;
7440 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7441 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2s_POST, SubRegIdx: AArch64::dsub0);
7442 return;
7443 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7444 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4s_POST, SubRegIdx: AArch64::qsub0);
7445 return;
7446 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7447 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv1d_POST, SubRegIdx: AArch64::dsub0);
7448 return;
7449 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7450 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2d_POST, SubRegIdx: AArch64::qsub0);
7451 return;
7452 }
7453 break;
7454 }
7455 case AArch64ISD::LD3DUPpost: {
7456 if (VT == MVT::v8i8) {
7457 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8b_POST, SubRegIdx: AArch64::dsub0);
7458 return;
7459 } else if (VT == MVT::v16i8) {
7460 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv16b_POST, SubRegIdx: AArch64::qsub0);
7461 return;
7462 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7463 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4h_POST, SubRegIdx: AArch64::dsub0);
7464 return;
7465 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7466 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8h_POST, SubRegIdx: AArch64::qsub0);
7467 return;
7468 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7469 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2s_POST, SubRegIdx: AArch64::dsub0);
7470 return;
7471 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7472 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4s_POST, SubRegIdx: AArch64::qsub0);
7473 return;
7474 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7475 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv1d_POST, SubRegIdx: AArch64::dsub0);
7476 return;
7477 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7478 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2d_POST, SubRegIdx: AArch64::qsub0);
7479 return;
7480 }
7481 break;
7482 }
7483 case AArch64ISD::LD4DUPpost: {
7484 if (VT == MVT::v8i8) {
7485 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8b_POST, SubRegIdx: AArch64::dsub0);
7486 return;
7487 } else if (VT == MVT::v16i8) {
7488 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv16b_POST, SubRegIdx: AArch64::qsub0);
7489 return;
7490 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7491 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4h_POST, SubRegIdx: AArch64::dsub0);
7492 return;
7493 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7494 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8h_POST, SubRegIdx: AArch64::qsub0);
7495 return;
7496 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7497 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2s_POST, SubRegIdx: AArch64::dsub0);
7498 return;
7499 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7500 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4s_POST, SubRegIdx: AArch64::qsub0);
7501 return;
7502 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7503 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv1d_POST, SubRegIdx: AArch64::dsub0);
7504 return;
7505 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7506 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2d_POST, SubRegIdx: AArch64::qsub0);
7507 return;
7508 }
7509 break;
7510 }
7511 case AArch64ISD::LD1LANEpost: {
7512 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7513 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i8_POST);
7514 return;
7515 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7516 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7517 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i16_POST);
7518 return;
7519 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7520 VT == MVT::v2f32) {
7521 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i32_POST);
7522 return;
7523 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7524 VT == MVT::v1f64) {
7525 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i64_POST);
7526 return;
7527 }
7528 break;
7529 }
7530 case AArch64ISD::LD2LANEpost: {
7531 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7532 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i8_POST);
7533 return;
7534 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7535 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7536 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i16_POST);
7537 return;
7538 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7539 VT == MVT::v2f32) {
7540 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i32_POST);
7541 return;
7542 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7543 VT == MVT::v1f64) {
7544 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i64_POST);
7545 return;
7546 }
7547 break;
7548 }
7549 case AArch64ISD::LD3LANEpost: {
7550 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7551 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i8_POST);
7552 return;
7553 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7554 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7555 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i16_POST);
7556 return;
7557 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7558 VT == MVT::v2f32) {
7559 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i32_POST);
7560 return;
7561 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7562 VT == MVT::v1f64) {
7563 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i64_POST);
7564 return;
7565 }
7566 break;
7567 }
7568 case AArch64ISD::LD4LANEpost: {
7569 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7570 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i8_POST);
7571 return;
7572 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7573 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7574 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i16_POST);
7575 return;
7576 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7577 VT == MVT::v2f32) {
7578 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i32_POST);
7579 return;
7580 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7581 VT == MVT::v1f64) {
7582 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i64_POST);
7583 return;
7584 }
7585 break;
7586 }
7587 case AArch64ISD::ST2post: {
7588 VT = Node->getOperand(Num: 1).getValueType();
7589 if (VT == MVT::v8i8) {
7590 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8b_POST);
7591 return;
7592 } else if (VT == MVT::v16i8) {
7593 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov16b_POST);
7594 return;
7595 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7596 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4h_POST);
7597 return;
7598 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7599 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8h_POST);
7600 return;
7601 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7602 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2s_POST);
7603 return;
7604 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7605 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4s_POST);
7606 return;
7607 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7608 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2d_POST);
7609 return;
7610 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7611 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d_POST);
7612 return;
7613 }
7614 break;
7615 }
7616 case AArch64ISD::ST3post: {
7617 VT = Node->getOperand(Num: 1).getValueType();
7618 if (VT == MVT::v8i8) {
7619 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8b_POST);
7620 return;
7621 } else if (VT == MVT::v16i8) {
7622 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev16b_POST);
7623 return;
7624 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7625 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4h_POST);
7626 return;
7627 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7628 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8h_POST);
7629 return;
7630 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7631 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2s_POST);
7632 return;
7633 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7634 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4s_POST);
7635 return;
7636 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7637 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2d_POST);
7638 return;
7639 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7640 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d_POST);
7641 return;
7642 }
7643 break;
7644 }
7645 case AArch64ISD::ST4post: {
7646 VT = Node->getOperand(Num: 1).getValueType();
7647 if (VT == MVT::v8i8) {
7648 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8b_POST);
7649 return;
7650 } else if (VT == MVT::v16i8) {
7651 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv16b_POST);
7652 return;
7653 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7654 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4h_POST);
7655 return;
7656 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7657 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8h_POST);
7658 return;
7659 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7660 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2s_POST);
7661 return;
7662 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7663 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4s_POST);
7664 return;
7665 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7666 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2d_POST);
7667 return;
7668 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7669 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d_POST);
7670 return;
7671 }
7672 break;
7673 }
7674 case AArch64ISD::ST1x2post: {
7675 VT = Node->getOperand(Num: 1).getValueType();
7676 if (VT == MVT::v8i8) {
7677 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8b_POST);
7678 return;
7679 } else if (VT == MVT::v16i8) {
7680 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov16b_POST);
7681 return;
7682 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7683 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4h_POST);
7684 return;
7685 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7686 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8h_POST);
7687 return;
7688 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7689 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2s_POST);
7690 return;
7691 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7692 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4s_POST);
7693 return;
7694 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7695 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d_POST);
7696 return;
7697 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7698 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2d_POST);
7699 return;
7700 }
7701 break;
7702 }
7703 case AArch64ISD::ST1x3post: {
7704 VT = Node->getOperand(Num: 1).getValueType();
7705 if (VT == MVT::v8i8) {
7706 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8b_POST);
7707 return;
7708 } else if (VT == MVT::v16i8) {
7709 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev16b_POST);
7710 return;
7711 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7712 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4h_POST);
7713 return;
7714 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16 ) {
7715 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8h_POST);
7716 return;
7717 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7718 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2s_POST);
7719 return;
7720 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7721 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4s_POST);
7722 return;
7723 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7724 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d_POST);
7725 return;
7726 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7727 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2d_POST);
7728 return;
7729 }
7730 break;
7731 }
7732 case AArch64ISD::ST1x4post: {
7733 VT = Node->getOperand(Num: 1).getValueType();
7734 if (VT == MVT::v8i8) {
7735 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8b_POST);
7736 return;
7737 } else if (VT == MVT::v16i8) {
7738 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv16b_POST);
7739 return;
7740 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7741 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4h_POST);
7742 return;
7743 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7744 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8h_POST);
7745 return;
7746 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7747 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2s_POST);
7748 return;
7749 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7750 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4s_POST);
7751 return;
7752 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7753 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d_POST);
7754 return;
7755 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7756 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2d_POST);
7757 return;
7758 }
7759 break;
7760 }
7761 case AArch64ISD::ST2LANEpost: {
7762 VT = Node->getOperand(Num: 1).getValueType();
7763 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7764 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i8_POST);
7765 return;
7766 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7767 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7768 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i16_POST);
7769 return;
7770 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7771 VT == MVT::v2f32) {
7772 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i32_POST);
7773 return;
7774 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7775 VT == MVT::v1f64) {
7776 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i64_POST);
7777 return;
7778 }
7779 break;
7780 }
7781 case AArch64ISD::ST3LANEpost: {
7782 VT = Node->getOperand(Num: 1).getValueType();
7783 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7784 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i8_POST);
7785 return;
7786 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7787 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7788 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i16_POST);
7789 return;
7790 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7791 VT == MVT::v2f32) {
7792 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i32_POST);
7793 return;
7794 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7795 VT == MVT::v1f64) {
7796 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i64_POST);
7797 return;
7798 }
7799 break;
7800 }
7801 case AArch64ISD::ST4LANEpost: {
7802 VT = Node->getOperand(Num: 1).getValueType();
7803 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7804 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i8_POST);
7805 return;
7806 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7807 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7808 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i16_POST);
7809 return;
7810 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7811 VT == MVT::v2f32) {
7812 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i32_POST);
7813 return;
7814 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7815 VT == MVT::v1f64) {
7816 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i64_POST);
7817 return;
7818 }
7819 break;
7820 }
7821 }
7822
7823 // Select the default instruction
7824 SelectCode(N: Node);
7825}
7826
7827/// createAArch64ISelDag - This pass converts a legalized DAG into a
7828/// AArch64-specific DAG, ready for instruction scheduling.
7829FunctionPass *llvm::createAArch64ISelDag(AArch64TargetMachine &TM,
7830 CodeGenOptLevel OptLevel) {
7831 return new AArch64DAGToDAGISelLegacy(TM, OptLevel);
7832}
7833
7834/// When \p PredVT is a scalable vector predicate in the form
7835/// MVT::nx<M>xi1, it builds the correspondent scalable vector of
7836/// integers MVT::nx<M>xi<bits> s.t. M x bits = 128. When targeting
7837/// structured vectors (NumVec >1), the output data type is
7838/// MVT::nx<M*NumVec>xi<bits> s.t. M x bits = 128. If the input
7839/// PredVT is not in the form MVT::nx<M>xi1, it returns an invalid
7840/// EVT.
7841static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT,
7842 unsigned NumVec) {
7843 assert(NumVec > 0 && NumVec < 5 && "Invalid number of vectors.");
7844 if (!PredVT.isScalableVectorOf(EltVT: MVT::i1))
7845 return EVT();
7846
7847 if (PredVT != MVT::nxv16i1 && PredVT != MVT::nxv8i1 &&
7848 PredVT != MVT::nxv4i1 && PredVT != MVT::nxv2i1)
7849 return EVT();
7850
7851 ElementCount EC = PredVT.getVectorElementCount();
7852 EVT ScalarVT =
7853 EVT::getIntegerVT(Context&: Ctx, BitWidth: AArch64::SVEBitsPerBlock / EC.getKnownMinValue());
7854 EVT MemVT = EVT::getVectorVT(Context&: Ctx, VT: ScalarVT, EC: EC * NumVec);
7855
7856 return MemVT;
7857}
7858
7859/// Builds an integer vector type large enough to hold \p NumVec instances
7860/// of \p VecVT.
7861static EVT getMultipleVectorType(LLVMContext &Ctx, EVT VecVT, unsigned NumVec) {
7862 return EVT::getVectorVT(Context&: Ctx, VT: VecVT.getScalarType().changeTypeToInteger(),
7863 EC: VecVT.getVectorElementCount() * NumVec);
7864}
7865
7866/// Return the EVT of the data associated to a memory operation in \p
7867/// Root. If such EVT cannot be retrieved, it returns an invalid EVT.
7868static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root) {
7869 if (auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(Val: Root))
7870 return MemIntr->getMemoryVT();
7871
7872 if (isa<MemSDNode>(Val: Root)) {
7873 EVT MemVT = cast<MemSDNode>(Val: Root)->getMemoryVT();
7874
7875 EVT DataVT;
7876 if (auto *Load = dyn_cast<LoadSDNode>(Val: Root))
7877 DataVT = Load->getValueType(ResNo: 0);
7878 else if (auto *Load = dyn_cast<MaskedLoadSDNode>(Val: Root))
7879 DataVT = Load->getValueType(ResNo: 0);
7880 else if (auto *Store = dyn_cast<StoreSDNode>(Val: Root))
7881 DataVT = Store->getValue().getValueType();
7882 else if (auto *Store = dyn_cast<MaskedStoreSDNode>(Val: Root))
7883 DataVT = Store->getValue().getValueType();
7884 else
7885 llvm_unreachable("Unexpected MemSDNode!");
7886
7887 return DataVT.changeVectorElementType(Context&: Ctx, EltVT: MemVT.getVectorElementType());
7888 }
7889
7890 const unsigned Opcode = Root->getOpcode();
7891 // For custom ISD nodes, we have to look at them individually to extract the
7892 // type of the data moved to/from memory.
7893 switch (Opcode) {
7894 case AArch64ISD::LD1_MERGE_ZERO:
7895 case AArch64ISD::LD1S_MERGE_ZERO:
7896 case AArch64ISD::LDNF1_MERGE_ZERO:
7897 case AArch64ISD::LDNF1S_MERGE_ZERO:
7898 return cast<VTSDNode>(Val: Root->getOperand(Num: 3))->getVT();
7899 case AArch64ISD::ST1_PRED:
7900 return cast<VTSDNode>(Val: Root->getOperand(Num: 4))->getVT();
7901 default:
7902 break;
7903 }
7904
7905 if (Opcode != ISD::INTRINSIC_VOID && Opcode != ISD::INTRINSIC_W_CHAIN)
7906 return EVT();
7907
7908 switch (Root->getConstantOperandVal(Num: 1)) {
7909 default:
7910 return EVT();
7911 case Intrinsic::aarch64_sme_ldr:
7912 case Intrinsic::aarch64_sme_str:
7913 return MVT::nxv16i8;
7914 case Intrinsic::aarch64_sve_prf:
7915 // We are using an SVE prefetch intrinsic. Type must be inferred from the
7916 // width of the predicate.
7917 return getPackedVectorTypeFromPredicateType(
7918 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/1);
7919 case Intrinsic::aarch64_sve_ld2_sret:
7920 case Intrinsic::aarch64_sve_ld2q_sret:
7921 return getPackedVectorTypeFromPredicateType(
7922 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/2);
7923 case Intrinsic::aarch64_sve_st2q:
7924 return getPackedVectorTypeFromPredicateType(
7925 Ctx, PredVT: Root->getOperand(Num: 4)->getValueType(ResNo: 0), /*NumVec=*/2);
7926 case Intrinsic::aarch64_sve_ld3_sret:
7927 case Intrinsic::aarch64_sve_ld3q_sret:
7928 return getPackedVectorTypeFromPredicateType(
7929 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/3);
7930 case Intrinsic::aarch64_sve_st3q:
7931 return getPackedVectorTypeFromPredicateType(
7932 Ctx, PredVT: Root->getOperand(Num: 5)->getValueType(ResNo: 0), /*NumVec=*/3);
7933 case Intrinsic::aarch64_sve_ld4_sret:
7934 case Intrinsic::aarch64_sve_ld4q_sret:
7935 return getPackedVectorTypeFromPredicateType(
7936 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/4);
7937 case Intrinsic::aarch64_sve_st4q:
7938 return getPackedVectorTypeFromPredicateType(
7939 Ctx, PredVT: Root->getOperand(Num: 6)->getValueType(ResNo: 0), /*NumVec=*/4);
7940 case Intrinsic::aarch64_sve_ld1_pn_x2:
7941 case Intrinsic::aarch64_sve_ldnt1_pn_x2:
7942 return getMultipleVectorType(Ctx, VecVT: Root->getValueType(ResNo: 0),
7943 /*NumVec=*/2);
7944 case Intrinsic::aarch64_sve_ld1_pn_x4:
7945 case Intrinsic::aarch64_sve_ldnt1_pn_x4:
7946 return getMultipleVectorType(Ctx, VecVT: Root->getValueType(ResNo: 0),
7947 /*NumVec=*/4);
7948 case Intrinsic::aarch64_sve_st1_pn_x2:
7949 case Intrinsic::aarch64_sve_stnt1_pn_x2:
7950 return getMultipleVectorType(Ctx, VecVT: Root->getOperand(Num: 2).getValueType(),
7951 /*NumVec=*/2);
7952 case Intrinsic::aarch64_sve_st1_pn_x4:
7953 case Intrinsic::aarch64_sve_stnt1_pn_x4:
7954 return getMultipleVectorType(Ctx, VecVT: Root->getOperand(Num: 2).getValueType(),
7955 /*NumVec=*/4);
7956 case Intrinsic::aarch64_sve_ld1udq:
7957 case Intrinsic::aarch64_sve_st1dq:
7958 return EVT(MVT::nxv1i64);
7959 case Intrinsic::aarch64_sve_ld1uwq:
7960 case Intrinsic::aarch64_sve_st1wq:
7961 return EVT(MVT::nxv1i32);
7962 }
7963}
7964
7965/// SelectAddrModeIndexedSVE - Attempt selection of the addressing mode:
7966/// Base + OffImm * sizeof(MemVT) for Min >= OffImm <= Max
7967/// where Root is the memory access using N for its address.
7968template <int64_t Min, int64_t Max>
7969bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(SDNode *Root, SDValue N,
7970 SDValue &Base,
7971 SDValue &OffImm) {
7972 const EVT MemVT = getMemVTFromNode(Ctx&: *(CurDAG->getContext()), Root);
7973 const DataLayout &DL = CurDAG->getDataLayout();
7974 const MachineFrameInfo &MFI = MF->getFrameInfo();
7975
7976 if (N.getOpcode() == ISD::FrameIndex) {
7977 int FI = cast<FrameIndexSDNode>(Val&: N)->getIndex();
7978 // We can only encode VL scaled offsets, so only fold in frame indexes
7979 // referencing SVE objects.
7980 if (MFI.hasScalableStackID(ObjectIdx: FI)) {
7981 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
7982 OffImm = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i64);
7983 return true;
7984 }
7985
7986 return false;
7987 }
7988
7989 if (MemVT == EVT())
7990 return false;
7991
7992 if (N.getOpcode() != ISD::ADD)
7993 return false;
7994
7995 SDValue VScale = N.getOperand(i: 1);
7996 int64_t MulImm = std::numeric_limits<int64_t>::max();
7997 if (VScale.getOpcode() == ISD::VSCALE) {
7998 MulImm = cast<ConstantSDNode>(Val: VScale.getOperand(i: 0))->getSExtValue();
7999 } else if (auto C = dyn_cast<ConstantSDNode>(Val&: VScale)) {
8000 int64_t ByteOffset = C->getSExtValue();
8001 const auto KnownVScale =
8002 Subtarget->getSVEVectorSizeInBits() / AArch64::SVEBitsPerBlock;
8003
8004 if (!KnownVScale || ByteOffset % KnownVScale != 0)
8005 return false;
8006
8007 MulImm = ByteOffset / KnownVScale;
8008 } else
8009 return false;
8010
8011 TypeSize TS = MemVT.getSizeInBits();
8012 int64_t MemWidthBytes = static_cast<int64_t>(TS.getKnownMinValue()) / 8;
8013
8014 if ((MulImm % MemWidthBytes) != 0)
8015 return false;
8016
8017 int64_t Offset = MulImm / MemWidthBytes;
8018 if (Offset < Min || Offset > Max)
8019 return false;
8020
8021 Base = N.getOperand(i: 0);
8022 if (Base.getOpcode() == ISD::FrameIndex) {
8023 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
8024 // We can only encode VL scaled offsets, so only fold in frame indexes
8025 // referencing SVE objects.
8026 if (MFI.hasScalableStackID(ObjectIdx: FI))
8027 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
8028 }
8029
8030 OffImm = CurDAG->getTargetConstant(Val: Offset, DL: SDLoc(N), VT: MVT::i64);
8031 return true;
8032}
8033
8034/// Select register plus register addressing mode for SVE, with scaled
8035/// offset.
8036bool AArch64DAGToDAGISel::SelectSVERegRegAddrMode(SDValue N, unsigned Scale,
8037 SDValue &Base,
8038 SDValue &Offset) {
8039 if (N.getOpcode() != ISD::ADD)
8040 return false;
8041
8042 // Process an ADD node.
8043 const SDValue LHS = N.getOperand(i: 0);
8044 const SDValue RHS = N.getOperand(i: 1);
8045
8046 // 8 bit data does not come with the SHL node, so it is treated
8047 // separately.
8048 if (Scale == 0) {
8049 Base = LHS;
8050 Offset = RHS;
8051 return true;
8052 }
8053
8054 if (auto C = dyn_cast<ConstantSDNode>(Val: RHS)) {
8055 int64_t ImmOff = C->getSExtValue();
8056 unsigned Size = 1 << Scale;
8057
8058 // To use the reg+reg addressing mode, the immediate must be a multiple of
8059 // the vector element's byte size.
8060 if (ImmOff % Size)
8061 return false;
8062
8063 SDLoc DL(N);
8064 Base = LHS;
8065 Offset = CurDAG->getTargetConstant(Val: ImmOff >> Scale, DL, VT: MVT::i64);
8066 SDValue Ops[] = {Offset};
8067 SDNode *MI = CurDAG->getMachineNode(Opcode: AArch64::MOVi64imm, dl: DL, VT: MVT::i64, Ops);
8068 Offset = SDValue(MI, 0);
8069 return true;
8070 }
8071
8072 // Check if the RHS is a shift node with a constant.
8073 if (RHS.getOpcode() != ISD::SHL)
8074 return false;
8075
8076 const SDValue ShiftRHS = RHS.getOperand(i: 1);
8077 if (auto *C = dyn_cast<ConstantSDNode>(Val: ShiftRHS))
8078 if (C->getZExtValue() == Scale) {
8079 Base = LHS;
8080 Offset = RHS.getOperand(i: 0);
8081 return true;
8082 }
8083
8084 return false;
8085}
8086
8087bool AArch64DAGToDAGISel::SelectAllActivePredicate(SDValue N) {
8088 const AArch64TargetLowering *TLI =
8089 static_cast<const AArch64TargetLowering *>(getTargetLowering());
8090
8091 return TLI->isAllActivePredicate(DAG: *CurDAG, N);
8092}
8093
8094bool AArch64DAGToDAGISel::SelectAnyPredicate(SDValue N) {
8095 return N.getValueType().isScalableVectorOf(EltVT: MVT::i1);
8096}
8097
8098bool AArch64DAGToDAGISel::SelectSMETileSlice(SDValue N, unsigned MaxSize,
8099 SDValue &Base, SDValue &Offset,
8100 unsigned Scale) {
8101 auto MatchConstantOffset = [&](SDValue CN) -> SDValue {
8102 if (auto *C = dyn_cast<ConstantSDNode>(Val&: CN)) {
8103 int64_t ImmOff = C->getSExtValue();
8104 if ((ImmOff > 0 && ImmOff <= MaxSize && (ImmOff % Scale == 0)))
8105 return CurDAG->getTargetConstant(Val: ImmOff / Scale, DL: SDLoc(N), VT: MVT::i64);
8106 }
8107 return SDValue();
8108 };
8109
8110 if (SDValue C = MatchConstantOffset(N)) {
8111 Base = getZeroRegister(DAG&: *CurDAG, DL: SDLoc(N), VT: MVT::i32);
8112 Offset = C;
8113 return true;
8114 }
8115
8116 // Try to untangle an ADD node into a 'reg + offset'
8117 if (CurDAG->isBaseWithConstantOffset(Op: N)) {
8118 if (SDValue C = MatchConstantOffset(N.getOperand(i: 1))) {
8119 Base = N.getOperand(i: 0);
8120 Offset = C;
8121 return true;
8122 }
8123 }
8124
8125 // By default, just match reg + 0.
8126 Base = N;
8127 Offset = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i64);
8128 return true;
8129}
8130
8131bool AArch64DAGToDAGISel::SelectCmpBranchUImm6Operand(SDNode *P, SDValue N,
8132 SDValue &Imm) {
8133 AArch64CC::CondCode CC =
8134 static_cast<AArch64CC::CondCode>(P->getConstantOperandVal(Num: 1));
8135 if (auto *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
8136 // Check conservatively if the immediate fits the valid range [0, 64).
8137 // Immediate variants for GE and HS definitely need to be decremented
8138 // when lowering the pseudos later, so an immediate of 1 would become 0.
8139 // For the inverse conditions LT and LO we don't know for sure if they
8140 // will need a decrement but should the decision be made to reverse the
8141 // branch condition, we again end up with the need to decrement.
8142 // The same argument holds for LE, LS, GT and HI and possibly
8143 // incremented immediates. This can lead to slightly less optimal
8144 // codegen, e.g. we never codegen the legal case
8145 // cblt w0, #63, A
8146 // because we could end up with the illegal case
8147 // cbge w0, #64, B
8148 // should the decision to reverse the branch direction be made. For the
8149 // lower bound cases this is no problem since we can express comparisons
8150 // against 0 with either tbz/tnbz or using wzr/xzr.
8151 uint64_t LowerBound = 0, UpperBound = 64;
8152 switch (CC) {
8153 case AArch64CC::GE:
8154 case AArch64CC::HS:
8155 case AArch64CC::LT:
8156 case AArch64CC::LO:
8157 LowerBound = 1;
8158 break;
8159 case AArch64CC::LE:
8160 case AArch64CC::LS:
8161 case AArch64CC::GT:
8162 case AArch64CC::HI:
8163 UpperBound = 63;
8164 break;
8165 default:
8166 break;
8167 }
8168
8169 if (CN->getAPIntValue().uge(RHS: LowerBound) &&
8170 CN->getAPIntValue().ult(RHS: UpperBound)) {
8171 SDLoc DL(N);
8172 Imm = CurDAG->getTargetConstant(Val: CN->getZExtValue(), DL, VT: N.getValueType());
8173 return true;
8174 }
8175 }
8176
8177 return false;
8178}
8179
8180template <bool MatchCBB>
8181bool AArch64DAGToDAGISel::SelectCmpBranchExtOperand(SDValue N, SDValue &Reg,
8182 SDValue &ExtType) {
8183
8184 // Use an invalid shift-extend value to indicate we don't need to extend later
8185 if (N.getOpcode() == ISD::AssertZext || N.getOpcode() == ISD::AssertSext) {
8186 EVT Ty = cast<VTSDNode>(Val: N.getOperand(i: 1))->getVT();
8187 if (Ty != (MatchCBB ? MVT::i8 : MVT::i16))
8188 return false;
8189 Reg = N.getOperand(i: 0);
8190 ExtType = CurDAG->getSignedTargetConstant(Val: AArch64_AM::InvalidShiftExtend,
8191 DL: SDLoc(N), VT: MVT::i32);
8192 return true;
8193 }
8194
8195 AArch64_AM::ShiftExtendType ET = getExtendTypeForNode(N);
8196
8197 if ((MatchCBB && (ET == AArch64_AM::UXTB || ET == AArch64_AM::SXTB)) ||
8198 (!MatchCBB && (ET == AArch64_AM::UXTH || ET == AArch64_AM::SXTH))) {
8199 Reg = N.getOperand(i: 0);
8200 ExtType =
8201 CurDAG->getTargetConstant(Val: getExtendEncoding(ET), DL: SDLoc(N), VT: MVT::i32);
8202 return true;
8203 }
8204
8205 return false;
8206}
8207
8208/// Try to fold AArch64 CSEL/FCMP patterns to FMAXNM/FMINNM.
8209///
8210/// This is intentionally done in PreprocessISelDAG rather than DAGCombine:
8211/// doing this earlier based on the defining operation of X can be invalidated
8212/// by later DAG combines. At this point the DAG is being prepared for
8213/// instruction selection, so the use of isKnownNeverSNaN(X) applies to the
8214/// final SDValue being selected.
8215/// Only handles FCMP(X, C) with scalar FP types, where C is a non-NaN constant.
8216/// The nsz requirement is needed only when C is zero, to avoid signed-zero
8217/// mismatches. The never-sNaN check is required because AArch64 FMAXNM/FMINNM
8218/// differ from fcmp+fcsel for signaling NaN inputs.
8219bool AArch64DAGToDAGISel::tryFoldCselToFMaxMin(SDNode *N) {
8220 EVT VT = N->getValueType(ResNo: 0);
8221
8222 // Scalar FP only.
8223 if (!VT.isFloatingPoint() || VT.isVector())
8224 return false;
8225
8226 SDValue TVal = N->getOperand(Num: 0);
8227 SDValue FVal = N->getOperand(Num: 1);
8228 SDValue CCVal = N->getOperand(Num: 2);
8229 SDValue Cmp = N->getOperand(Num: 3);
8230
8231 if (Cmp.getOpcode() != AArch64ISD::FCMP)
8232 return false;
8233
8234 auto *CC = dyn_cast<ConstantSDNode>(Val&: CCVal);
8235 if (!CC)
8236 return false;
8237
8238 SDValue CmpLHS = Cmp.getOperand(i: 0);
8239 SDValue CmpRHS = Cmp.getOperand(i: 1);
8240 unsigned CondCode = CC->getZExtValue();
8241
8242 // Map VT and operation (max/min) to machine opcode.
8243 auto getOpc = [](EVT VT, bool isMax) -> unsigned {
8244 if (VT == MVT::f16)
8245 return isMax ? AArch64::FMAXNMHrr : AArch64::FMINNMHrr;
8246 else if (VT == MVT::f32)
8247 return isMax ? AArch64::FMAXNMSrr : AArch64::FMINNMSrr;
8248 else if (VT == MVT::f64)
8249 return isMax ? AArch64::FMAXNMDrr : AArch64::FMINNMDrr;
8250 else
8251 return 0; // unsupported
8252 };
8253
8254 // Determine whether to use max or min based on condition code and operands.
8255 bool isMax;
8256 if (CondCode == AArch64CC::GT || CondCode == AArch64CC::GE) {
8257 if (TVal == CmpLHS && FVal == CmpRHS)
8258 isMax = true;
8259 else
8260 return false;
8261 } else if (CondCode == AArch64CC::MI || CondCode == AArch64CC::LS) {
8262 if (TVal == CmpLHS && FVal == CmpRHS)
8263 isMax = false;
8264 else
8265 return false;
8266 } else {
8267 return false;
8268 }
8269
8270 // Get the machine opcode for this VT and operation.
8271 unsigned Opc = getOpc(VT, isMax);
8272 if (!Opc)
8273 return false;
8274
8275 // Constant must be non-NaN.
8276 auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: CmpRHS);
8277 if (!CFP || CFP->getValueAPF().isNaN())
8278 return false;
8279
8280 // nsz flag required only when constant is zero: fmaxnm(+0,-0)=+0 differs from
8281 // fcmp+select's -0. For non-zero constants, semantics are identical.
8282 if (CFP->isZero() && !N->getFlags().hasNoSignedZeros())
8283 return false;
8284
8285 // Only fold if variable operand is never sNaN.
8286 // This runs after DAG combines, so later combines cannot remove a defining
8287 // operation used by isKnownNeverSNaN().
8288 if (!CurDAG->isKnownNeverSNaN(Op: CmpLHS))
8289 return false;
8290
8291 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Op1: CmpLHS, Op2: CmpRHS);
8292 return true;
8293}
8294
8295void AArch64DAGToDAGISel::PreprocessISelDAG() {
8296 bool MadeChange = false;
8297 for (SDNode &N : llvm::make_early_inc_range(Range: CurDAG->allnodes())) {
8298 if (N.use_empty())
8299 continue;
8300
8301 SDValue Result;
8302 switch (N.getOpcode()) {
8303 case ISD::SCALAR_TO_VECTOR: {
8304 EVT ScalarTy = N.getValueType(ResNo: 0).getVectorElementType();
8305 if ((ScalarTy == MVT::i32 || ScalarTy == MVT::i64) &&
8306 ScalarTy == N.getOperand(Num: 0).getValueType())
8307 Result = addBitcastHints(DAG&: *CurDAG, N);
8308
8309 break;
8310 }
8311 case AArch64ISD::VSHL: {
8312 // Undo mul(shl(A,C),B) -> shl(mul(A,B),C) canonicalisation when A is an
8313 // extend that can be folded into the shift.
8314 EVT VT = N.getValueType(ResNo: 0);
8315 SDValue A, B, C = N.getOperand(Num: 1);
8316 if (sd_match(N: N.getOperand(Num: 0),
8317 P: m_OneUse(P: m_Mul(L: m_Value(N&: A, P: m_AnyOf(preds: m_ZExt(Op: m_Value()),
8318 preds: m_SExt(Op: m_Value()))),
8319 R: m_Value(N&: B))))) {
8320 // If both mul operands are extended, preserve the smull/umull idiom.
8321 if (B.getOpcode() == A.getOpcode())
8322 break;
8323 SDLoc DL(&N);
8324 SDValue SHL = CurDAG->getNode(Opcode: AArch64ISD::VSHL, DL, VT, N1: A, N2: C);
8325 Result = CurDAG->getNode(Opcode: ISD::MUL, DL, VT, N1: SHL, N2: B);
8326 }
8327 break;
8328 }
8329 default:
8330 break;
8331 }
8332
8333 if (Result) {
8334 LLVM_DEBUG(dbgs() << "AArch64 DAG preprocessing replacing:\nOld: ");
8335 LLVM_DEBUG(N.dump(CurDAG));
8336 LLVM_DEBUG(dbgs() << "\nNew: ");
8337 LLVM_DEBUG(Result.dump(CurDAG));
8338 LLVM_DEBUG(dbgs() << "\n");
8339
8340 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(&N, 0), To: Result);
8341 MadeChange = true;
8342 }
8343 }
8344
8345 if (MadeChange)
8346 CurDAG->RemoveDeadNodes();
8347
8348 SelectionDAGISel::PreprocessISelDAG();
8349}
8350