1//===-- AArch64ISelDAGToDAG.cpp - A dag to dag inst selector for AArch64 --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the AArch64 target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64.h"
14#include "AArch64MachineFunctionInfo.h"
15#include "AArch64TargetMachine.h"
16#include "MCTargetDesc/AArch64AddressingModes.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/CodeGen/ISDOpcodes.h"
19#include "llvm/CodeGen/SelectionDAGISel.h"
20#include "llvm/IR/Function.h" // To access function attributes.
21#include "llvm/IR/GlobalValue.h"
22#include "llvm/IR/Intrinsics.h"
23#include "llvm/IR/IntrinsicsAArch64.h"
24#include "llvm/Support/Debug.h"
25#include "llvm/Support/ErrorHandling.h"
26#include "llvm/Support/KnownBits.h"
27#include "llvm/Support/MathExtras.h"
28#include "llvm/Support/raw_ostream.h"
29
30using namespace llvm;
31
32#define DEBUG_TYPE "aarch64-isel"
33#define PASS_NAME "AArch64 Instruction Selection"
34
35// https://github.com/llvm/llvm-project/issues/114425
36#if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG)
37#pragma inline_depth(0)
38#endif
39
40//===--------------------------------------------------------------------===//
41/// AArch64DAGToDAGISel - AArch64 specific code to select AArch64 machine
42/// instructions for SelectionDAG operations.
43///
44namespace {
45
46class AArch64DAGToDAGISel : public SelectionDAGISel {
47
48 /// Subtarget - Keep a pointer to the AArch64Subtarget around so that we can
49 /// make the right decision when generating code for different targets.
50 const AArch64Subtarget *Subtarget;
51
52public:
53 AArch64DAGToDAGISel() = delete;
54
55 explicit AArch64DAGToDAGISel(AArch64TargetMachine &tm,
56 CodeGenOptLevel OptLevel)
57 : SelectionDAGISel(tm, OptLevel), Subtarget(nullptr) {}
58
59 bool runOnMachineFunction(MachineFunction &MF) override {
60 Subtarget = &MF.getSubtarget<AArch64Subtarget>();
61 return SelectionDAGISel::runOnMachineFunction(mf&: MF);
62 }
63
64 void Select(SDNode *Node) override;
65 void PreprocessISelDAG() override;
66
67 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
68 /// inline asm expressions.
69 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
70 InlineAsm::ConstraintCode ConstraintID,
71 std::vector<SDValue> &OutOps) override;
72
73 template <signed Low, signed High, signed Scale>
74 bool SelectRDVLImm(SDValue N, SDValue &Imm);
75
76 template <signed Low, signed High>
77 bool SelectRDSVLShiftImm(SDValue N, SDValue &Imm);
78
79 bool SelectArithExtendedRegister(SDValue N, SDValue &Reg, SDValue &Shift);
80 bool SelectArithUXTXRegister(SDValue N, SDValue &Reg, SDValue &Shift);
81 bool SelectArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
82 bool SelectNegArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
83 bool SelectArithShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
84 return SelectShiftedRegister(N, AllowROR: false, Reg, Shift);
85 }
86 bool SelectLogicalShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
87 return SelectShiftedRegister(N, AllowROR: true, Reg, Shift);
88 }
89 bool SelectAddrModeIndexed7S8(SDValue N, SDValue &Base, SDValue &OffImm) {
90 return SelectAddrModeIndexed7S(N, Size: 1, Base, OffImm);
91 }
92 bool SelectAddrModeIndexed7S16(SDValue N, SDValue &Base, SDValue &OffImm) {
93 return SelectAddrModeIndexed7S(N, Size: 2, Base, OffImm);
94 }
95 bool SelectAddrModeIndexed7S32(SDValue N, SDValue &Base, SDValue &OffImm) {
96 return SelectAddrModeIndexed7S(N, Size: 4, Base, OffImm);
97 }
98 bool SelectAddrModeIndexed7S64(SDValue N, SDValue &Base, SDValue &OffImm) {
99 return SelectAddrModeIndexed7S(N, Size: 8, Base, OffImm);
100 }
101 bool SelectAddrModeIndexed7S128(SDValue N, SDValue &Base, SDValue &OffImm) {
102 return SelectAddrModeIndexed7S(N, Size: 16, Base, OffImm);
103 }
104 bool SelectAddrModeIndexedS9S128(SDValue N, SDValue &Base, SDValue &OffImm) {
105 return SelectAddrModeIndexedBitWidth(N, IsSignedImm: true, BW: 9, Size: 16, Base, OffImm);
106 }
107 bool SelectAddrModeIndexedU6S128(SDValue N, SDValue &Base, SDValue &OffImm) {
108 return SelectAddrModeIndexedBitWidth(N, IsSignedImm: false, BW: 6, Size: 16, Base, OffImm);
109 }
110 bool SelectAddrModeIndexed8(SDValue N, SDValue &Base, SDValue &OffImm) {
111 return SelectAddrModeIndexed(N, Size: 1, Base, OffImm);
112 }
113 bool SelectAddrModeIndexed16(SDValue N, SDValue &Base, SDValue &OffImm) {
114 return SelectAddrModeIndexed(N, Size: 2, Base, OffImm);
115 }
116 bool SelectAddrModeIndexed32(SDValue N, SDValue &Base, SDValue &OffImm) {
117 return SelectAddrModeIndexed(N, Size: 4, Base, OffImm);
118 }
119 bool SelectAddrModeIndexed64(SDValue N, SDValue &Base, SDValue &OffImm) {
120 return SelectAddrModeIndexed(N, Size: 8, Base, OffImm);
121 }
122 bool SelectAddrModeIndexed128(SDValue N, SDValue &Base, SDValue &OffImm) {
123 return SelectAddrModeIndexed(N, Size: 16, Base, OffImm);
124 }
125 bool SelectAddrModeUnscaled8(SDValue N, SDValue &Base, SDValue &OffImm) {
126 return SelectAddrModeUnscaled(N, Size: 1, Base, OffImm);
127 }
128 bool SelectAddrModeUnscaled16(SDValue N, SDValue &Base, SDValue &OffImm) {
129 return SelectAddrModeUnscaled(N, Size: 2, Base, OffImm);
130 }
131 bool SelectAddrModeUnscaled32(SDValue N, SDValue &Base, SDValue &OffImm) {
132 return SelectAddrModeUnscaled(N, Size: 4, Base, OffImm);
133 }
134 bool SelectAddrModeUnscaled64(SDValue N, SDValue &Base, SDValue &OffImm) {
135 return SelectAddrModeUnscaled(N, Size: 8, Base, OffImm);
136 }
137 bool SelectAddrModeUnscaled128(SDValue N, SDValue &Base, SDValue &OffImm) {
138 return SelectAddrModeUnscaled(N, Size: 16, Base, OffImm);
139 }
140 template <unsigned Size, unsigned Max>
141 bool SelectAddrModeIndexedUImm(SDValue N, SDValue &Base, SDValue &OffImm) {
142 // Test if there is an appropriate addressing mode and check if the
143 // immediate fits.
144 bool Found = SelectAddrModeIndexed(N, Size, Base, OffImm);
145 if (Found) {
146 if (auto *CI = dyn_cast<ConstantSDNode>(Val&: OffImm)) {
147 int64_t C = CI->getSExtValue();
148 if (C <= Max)
149 return true;
150 }
151 }
152
153 // Otherwise, base only, materialize address in register.
154 Base = N;
155 OffImm = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i64);
156 return true;
157 }
158
159 template<int Width>
160 bool SelectAddrModeWRO(SDValue N, SDValue &Base, SDValue &Offset,
161 SDValue &SignExtend, SDValue &DoShift) {
162 return SelectAddrModeWRO(N, Size: Width / 8, Base, Offset, SignExtend, DoShift);
163 }
164
165 template<int Width>
166 bool SelectAddrModeXRO(SDValue N, SDValue &Base, SDValue &Offset,
167 SDValue &SignExtend, SDValue &DoShift) {
168 return SelectAddrModeXRO(N, Size: Width / 8, Base, Offset, SignExtend, DoShift);
169 }
170
171 bool SelectExtractHigh(SDValue N, SDValue &Res) {
172 if (Subtarget->isLittleEndian() && N->getOpcode() == ISD::BITCAST)
173 N = N->getOperand(Num: 0);
174 if (N->getOpcode() != ISD::EXTRACT_SUBVECTOR ||
175 !isa<ConstantSDNode>(Val: N->getOperand(Num: 1)))
176 return false;
177 EVT VT = N->getValueType(ResNo: 0);
178 EVT LVT = N->getOperand(Num: 0).getValueType();
179 unsigned Index = N->getConstantOperandVal(Num: 1);
180 if (!VT.is64BitVector() || !LVT.is128BitVector() ||
181 Index != VT.getVectorNumElements())
182 return false;
183 Res = N->getOperand(Num: 0);
184 return true;
185 }
186
187 bool SelectRoundingVLShr(SDValue N, SDValue &Res1, SDValue &Res2) {
188 if (N.getOpcode() != AArch64ISD::VLSHR)
189 return false;
190 SDValue Op = N->getOperand(Num: 0);
191 EVT VT = Op.getValueType();
192 unsigned ShtAmt = N->getConstantOperandVal(Num: 1);
193 if (ShtAmt > VT.getScalarSizeInBits() / 2 || Op.getOpcode() != ISD::ADD)
194 return false;
195
196 APInt Imm;
197 if (Op.getOperand(i: 1).getOpcode() == AArch64ISD::MOVIshift)
198 Imm = APInt(VT.getScalarSizeInBits(),
199 Op.getOperand(i: 1).getConstantOperandVal(i: 0)
200 << Op.getOperand(i: 1).getConstantOperandVal(i: 1));
201 else if (Op.getOperand(i: 1).getOpcode() == AArch64ISD::DUP &&
202 isa<ConstantSDNode>(Val: Op.getOperand(i: 1).getOperand(i: 0)))
203 Imm = APInt(VT.getScalarSizeInBits(),
204 Op.getOperand(i: 1).getConstantOperandVal(i: 0));
205 else
206 return false;
207
208 if (Imm != 1ULL << (ShtAmt - 1))
209 return false;
210
211 Res1 = Op.getOperand(i: 0);
212 Res2 = CurDAG->getTargetConstant(Val: ShtAmt, DL: SDLoc(N), VT: MVT::i32);
213 return true;
214 }
215
216 bool SelectDupZeroOrUndef(SDValue N) {
217 switch(N->getOpcode()) {
218 case ISD::UNDEF:
219 return true;
220 case AArch64ISD::DUP:
221 case ISD::SPLAT_VECTOR: {
222 auto Opnd0 = N->getOperand(Num: 0);
223 if (isNullConstant(V: Opnd0))
224 return true;
225 if (isNullFPConstant(V: Opnd0))
226 return true;
227 break;
228 }
229 default:
230 break;
231 }
232
233 return false;
234 }
235
236 bool SelectAny(SDValue) { return true; }
237
238 bool SelectDupZero(SDValue N) {
239 switch(N->getOpcode()) {
240 case AArch64ISD::DUP:
241 case ISD::SPLAT_VECTOR: {
242 auto Opnd0 = N->getOperand(Num: 0);
243 if (isNullConstant(V: Opnd0))
244 return true;
245 if (isNullFPConstant(V: Opnd0))
246 return true;
247 break;
248 }
249 }
250
251 return false;
252 }
253
254 template <MVT::SimpleValueType VT, bool Negate>
255 bool SelectSVEAddSubImm(SDValue N, SDValue &Imm, SDValue &Shift) {
256 return SelectSVEAddSubImm(N, VT, Imm, Shift, Negate);
257 }
258
259 template <MVT::SimpleValueType VT, bool Negate>
260 bool SelectSVEAddSubSSatImm(SDValue N, SDValue &Imm, SDValue &Shift) {
261 return SelectSVEAddSubSSatImm(N, VT, Imm, Shift, Negate);
262 }
263
264 template <MVT::SimpleValueType VT>
265 bool SelectSVECpyDupImm(SDValue N, SDValue &Imm, SDValue &Shift) {
266 return SelectSVECpyDupImm(N, VT, Imm, Shift);
267 }
268
269 template <MVT::SimpleValueType VT, bool Invert = false>
270 bool SelectSVELogicalImm(SDValue N, SDValue &Imm) {
271 return SelectSVELogicalImm(N, VT, Imm, Invert);
272 }
273
274 template <MVT::SimpleValueType VT>
275 bool SelectSVEArithImm(SDValue N, SDValue &Imm) {
276 return SelectSVEArithImm(N, VT, Imm);
277 }
278
279 template <unsigned Low, unsigned High, bool AllowSaturation = false>
280 bool SelectSVEShiftImm(SDValue N, SDValue &Imm) {
281 return SelectSVEShiftImm(N, Low, High, AllowSaturation, Imm);
282 }
283
284 bool SelectSVEShiftSplatImmR(SDValue N, SDValue &Imm) {
285 if (N->getOpcode() != ISD::SPLAT_VECTOR)
286 return false;
287
288 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
289 return SelectSVEShiftImm(N: N->getOperand(Num: 0), /* Low */ 1,
290 /* High */ EltVT.getFixedSizeInBits(),
291 /* AllowSaturation */ true, Imm);
292 }
293
294 // Returns a suitable CNT/INC/DEC/RDVL multiplier to calculate VSCALE*N.
295 template<signed Min, signed Max, signed Scale, bool Shift>
296 bool SelectCntImm(SDValue N, SDValue &Imm) {
297 if (!isa<ConstantSDNode>(Val: N))
298 return false;
299
300 int64_t MulImm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
301 if (Shift)
302 MulImm = 1LL << MulImm;
303
304 if ((MulImm % std::abs(x: Scale)) != 0)
305 return false;
306
307 MulImm /= Scale;
308 if ((MulImm >= Min) && (MulImm <= Max)) {
309 Imm = CurDAG->getTargetConstant(Val: MulImm, DL: SDLoc(N), VT: MVT::i32);
310 return true;
311 }
312
313 return false;
314 }
315
316 template <signed Max, signed Scale>
317 bool SelectEXTImm(SDValue N, SDValue &Imm) {
318 if (!isa<ConstantSDNode>(Val: N))
319 return false;
320
321 int64_t MulImm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
322
323 if (MulImm >= 0 && MulImm <= Max) {
324 MulImm *= Scale;
325 Imm = CurDAG->getTargetConstant(Val: MulImm, DL: SDLoc(N), VT: MVT::i32);
326 return true;
327 }
328
329 return false;
330 }
331
332 template <unsigned BaseReg, unsigned Max>
333 bool ImmToReg(SDValue N, SDValue &Imm) {
334 if (auto *CI = dyn_cast<ConstantSDNode>(Val&: N)) {
335 uint64_t C = CI->getZExtValue();
336
337 if (C > Max)
338 return false;
339
340 Imm = CurDAG->getRegister(Reg: BaseReg + C, VT: MVT::Other);
341 return true;
342 }
343 return false;
344 }
345
346 /// Form sequences of consecutive 64/128-bit registers for use in NEON
347 /// instructions making use of a vector-list (e.g. ldN, tbl). Vecs must have
348 /// between 1 and 4 elements. If it contains a single element that is returned
349 /// unchanged; otherwise a REG_SEQUENCE value is returned.
350 SDValue createDTuple(ArrayRef<SDValue> Vecs);
351 SDValue createQTuple(ArrayRef<SDValue> Vecs);
352 // Form a sequence of SVE registers for instructions using list of vectors,
353 // e.g. structured loads and stores (ldN, stN).
354 SDValue createZTuple(ArrayRef<SDValue> Vecs);
355
356 // Similar to above, except the register must start at a multiple of the
357 // tuple, e.g. z2 for a 2-tuple, or z8 for a 4-tuple.
358 SDValue createZMulTuple(ArrayRef<SDValue> Regs);
359
360 /// Generic helper for the createDTuple/createQTuple
361 /// functions. Those should almost always be called instead.
362 SDValue createTuple(ArrayRef<SDValue> Vecs, const unsigned RegClassIDs[],
363 const unsigned SubRegs[]);
364
365 void SelectTable(SDNode *N, unsigned NumVecs, unsigned Opc, bool isExt);
366
367 bool tryIndexedLoad(SDNode *N);
368
369 void SelectPtrauthAuth(SDNode *N);
370 void SelectPtrauthResign(SDNode *N);
371 void SelectPtrauthResignWithPC(SDNode *N);
372
373 bool trySelectStackSlotTagP(SDNode *N);
374 void SelectTagP(SDNode *N);
375
376 void SelectLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
377 unsigned SubRegIdx);
378 void SelectPostLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
379 unsigned SubRegIdx);
380 void SelectLoadLane(SDNode *N, unsigned NumVecs, unsigned Opc);
381 void SelectPostLoadLane(SDNode *N, unsigned NumVecs, unsigned Opc);
382 void SelectPredicatedLoad(SDNode *N, unsigned NumVecs, unsigned Scale,
383 unsigned Opc_rr, unsigned Opc_ri,
384 bool IsIntr = false);
385 void SelectContiguousMultiVectorLoad(SDNode *N, unsigned NumVecs,
386 unsigned Scale, unsigned Opc_ri,
387 unsigned Opc_rr);
388 void SelectDestructiveMultiIntrinsic(SDNode *N, unsigned NumVecs,
389 bool IsZmMulti, unsigned Opcode,
390 bool HasPred = false);
391 void SelectPExtPair(SDNode *N, unsigned Opc);
392 void SelectWhilePair(SDNode *N, unsigned Opc);
393 void SelectCVTIntrinsic(SDNode *N, unsigned NumVecs, unsigned Opcode);
394 void SelectCVTIntrinsicFP8(SDNode *N, unsigned NumVecs, unsigned Opcode);
395 void SelectClamp(SDNode *N, unsigned NumVecs, unsigned Opcode);
396 void SelectUnaryMultiIntrinsic(SDNode *N, unsigned NumOutVecs,
397 bool IsTupleInput, unsigned Opc);
398 void SelectFrintFromVT(SDNode *N, unsigned NumVecs, unsigned Opcode);
399
400 template <unsigned MaxIdx, unsigned Scale>
401 void SelectMultiVectorMove(SDNode *N, unsigned NumVecs, unsigned BaseReg,
402 unsigned Op);
403 void SelectMultiVectorMoveZ(SDNode *N, unsigned NumVecs,
404 unsigned Op, unsigned MaxIdx, unsigned Scale,
405 unsigned BaseReg = 0);
406 /// SVE Reg+Imm addressing mode.
407 template <int64_t Min, int64_t Max>
408 bool SelectAddrModeIndexedSVE(SDNode *Root, SDValue N, SDValue &Base,
409 SDValue &OffImm);
410 /// SVE Reg+Reg address mode.
411 template <unsigned Scale>
412 bool SelectSVERegRegAddrMode(SDValue N, SDValue &Base, SDValue &Offset) {
413 return SelectSVERegRegAddrMode(N, Scale, Base, Offset);
414 }
415
416 void SelectMultiVectorLutiLane(SDNode *Node, unsigned NumOutVecs,
417 unsigned Opc, uint32_t MaxImm);
418 void SelectMultiVectorLuti6LaneX4(SDNode *Node, unsigned NumIndexVecs);
419
420 void SelectMultiVectorLuti(SDNode *Node, unsigned NumOutVecs, unsigned Opc,
421 unsigned NumInVecs);
422
423 template <unsigned MaxIdx, unsigned Scale>
424 bool SelectSMETileSlice(SDValue N, SDValue &Vector, SDValue &Offset) {
425 return SelectSMETileSlice(N, MaxSize: MaxIdx, Vector, Offset, Scale);
426 }
427
428 void SelectStore(SDNode *N, unsigned NumVecs, unsigned Opc);
429 void SelectPostStore(SDNode *N, unsigned NumVecs, unsigned Opc);
430 void SelectStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
431 void SelectPostStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
432 void SelectPredicatedStore(SDNode *N, unsigned NumVecs, unsigned Scale,
433 unsigned Opc_rr, unsigned Opc_ri);
434 std::tuple<unsigned, SDValue, SDValue>
435 findAddrModeSVELoadStore(SDNode *N, unsigned Opc_rr, unsigned Opc_ri,
436 const SDValue &OldBase, const SDValue &OldOffset,
437 unsigned Scale);
438
439 bool tryBitfieldExtractOp(SDNode *N);
440 bool tryBitfieldExtractOpFromSExt(SDNode *N);
441 bool tryBitfieldInsertOp(SDNode *N);
442 bool tryBitfieldInsertInZeroOp(SDNode *N);
443 bool tryShiftAmountMod(SDNode *N);
444
445 bool tryReadRegister(SDNode *N);
446 bool tryWriteRegister(SDNode *N);
447
448 bool trySelectCastFixedLengthToScalableVector(SDNode *N);
449 bool trySelectCastScalableToFixedLengthVector(SDNode *N);
450
451 bool trySelectXAR(SDNode *N);
452
453 SDValue tryFoldCselToFMaxMin(SDNode &N);
454
455// Include the pieces autogenerated from the target description.
456#include "AArch64GenDAGISel.inc"
457
458private:
459 bool SelectShiftedRegister(SDValue N, bool AllowROR, SDValue &Reg,
460 SDValue &Shift);
461 bool SelectShiftedRegisterFromAnd(SDValue N, SDValue &Reg, SDValue &Shift);
462 bool SelectAddrModeIndexed7S(SDValue N, unsigned Size, SDValue &Base,
463 SDValue &OffImm) {
464 return SelectAddrModeIndexedBitWidth(N, IsSignedImm: true, BW: 7, Size, Base, OffImm);
465 }
466 bool SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm, unsigned BW,
467 unsigned Size, SDValue &Base,
468 SDValue &OffImm);
469 bool SelectAddrModeIndexed(SDValue N, unsigned Size, SDValue &Base,
470 SDValue &OffImm);
471 bool SelectAddrModeUnscaled(SDValue N, unsigned Size, SDValue &Base,
472 SDValue &OffImm);
473 bool SelectAddrModeWRO(SDValue N, unsigned Size, SDValue &Base,
474 SDValue &Offset, SDValue &SignExtend,
475 SDValue &DoShift);
476 bool SelectAddrModeXRO(SDValue N, unsigned Size, SDValue &Base,
477 SDValue &Offset, SDValue &SignExtend,
478 SDValue &DoShift);
479 bool isWorthFoldingALU(SDValue V, bool LSL = false) const;
480 bool isWorthFoldingAddr(SDValue V, unsigned Size) const;
481 bool SelectExtendedSHL(SDValue N, unsigned Size, bool WantExtend,
482 SDValue &Offset, SDValue &SignExtend);
483
484 template<unsigned RegWidth>
485 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos) {
486 return SelectCVTFixedPosOperand(N, FixedPos, Width: RegWidth);
487 }
488 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, unsigned Width);
489
490 template <unsigned RegWidth>
491 bool SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos) {
492 return SelectCVTFixedPointVec(N, FixedPos, Width: RegWidth);
493 }
494 bool SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos, unsigned Width);
495
496 template<unsigned RegWidth>
497 bool SelectCVTFixedPosRecipOperand(SDValue N, SDValue &FixedPos) {
498 return SelectCVTFixedPosRecipOperand(N, FixedPos, Width: RegWidth);
499 }
500
501 bool SelectCVTFixedPosRecipOperand(SDValue N, SDValue &FixedPos,
502 unsigned Width);
503
504 template <unsigned FloatWidth>
505 bool SelectCVTFixedPosRecipOperandVec(SDValue N, SDValue &FixedPos) {
506 return SelectCVTFixedPosRecipOperandVec(N, FixedPos, Width: FloatWidth);
507 }
508
509 bool SelectCVTFixedPosRecipOperandVec(SDValue N, SDValue &FixedPos,
510 unsigned Width);
511
512 bool SelectCMP_SWAP(SDNode *N);
513
514 bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
515 bool Negate);
516 bool SelectSVEAddSubImm(SDLoc DL, APInt Value, MVT VT, SDValue &Imm,
517 SDValue &Shift, bool Negate);
518 bool SelectSVEAddSubSSatImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
519 bool Negate);
520 bool SelectSVECpyDupImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift);
521 bool SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm, bool Invert);
522
523 // Match `<NEON Splat> SVEImm` (where <NEON Splat> could be fmov, movi, etc).
524 bool SelectNEONSplatOfSVELogicalImm(SDValue N, SDValue &Imm);
525 bool SelectNEONSplatOfSVEAddSubImm(SDValue N, SDValue &Imm, SDValue &Shift);
526 bool SelectNEONSplatOfSVEArithSImm(SDValue N, SDValue &Imm);
527
528 bool SelectSVESignedArithImm(SDLoc DL, APInt Value, SDValue &Imm);
529 bool SelectSVESignedArithImm(SDValue N, SDValue &Imm);
530 bool SelectSVEShiftImm(SDValue N, uint64_t Low, uint64_t High,
531 bool AllowSaturation, SDValue &Imm);
532
533 bool SelectSVEArithImm(SDValue N, MVT VT, SDValue &Imm);
534 bool SelectSVERegRegAddrMode(SDValue N, unsigned Scale, SDValue &Base,
535 SDValue &Offset);
536 bool SelectSMETileSlice(SDValue N, unsigned MaxSize, SDValue &Vector,
537 SDValue &Offset, unsigned Scale = 1);
538
539 bool SelectAllActivePredicate(SDValue N);
540 bool SelectAnyPredicate(SDValue N);
541
542 bool SelectCmpBranchUImm6Operand(SDNode *P, SDValue N, SDValue &Imm);
543
544 template <bool MatchCBB>
545 bool SelectCmpBranchExtOperand(SDValue N, SDValue &Reg, SDValue &ExtType);
546};
547
548class AArch64DAGToDAGISelLegacy : public SelectionDAGISelLegacy {
549public:
550 static char ID;
551 explicit AArch64DAGToDAGISelLegacy(AArch64TargetMachine &tm,
552 CodeGenOptLevel OptLevel)
553 : SelectionDAGISelLegacy(
554 ID, std::make_unique<AArch64DAGToDAGISel>(args&: tm, args&: OptLevel)) {}
555};
556} // end anonymous namespace
557
558char AArch64DAGToDAGISelLegacy::ID = 0;
559
560INITIALIZE_PASS(AArch64DAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false, false)
561
562AArch64DAGToDAGISelPass::AArch64DAGToDAGISelPass(AArch64TargetMachine &TM)
563 : SelectionDAGISelPass(
564 std::make_unique<AArch64DAGToDAGISel>(args&: TM, args: TM.getOptLevel())) {}
565
566/// addBitcastHints - This method adds bitcast hints to the operands of a node
567/// to help instruction selector determine which operands are in Neon registers.
568static SDValue addBitcastHints(SelectionDAG &DAG, SDNode &N) {
569 SDLoc DL(&N);
570 auto getFloatVT = [&](EVT VT) {
571 EVT ScalarVT = VT.getScalarType();
572 assert((ScalarVT == MVT::i32 || ScalarVT == MVT::i64) && "Unexpected VT");
573 return VT.changeElementType(Context&: *(DAG.getContext()),
574 EltVT: ScalarVT == MVT::i32 ? MVT::f32 : MVT::f64);
575 };
576 SmallVector<SDValue, 2> NewOps;
577 NewOps.reserve(N: N.getNumOperands());
578
579 for (unsigned I = 0, E = N.getNumOperands(); I < E; ++I) {
580 auto bitcasted = DAG.getBitcast(VT: getFloatVT(N.getOperand(Num: I).getValueType()),
581 V: N.getOperand(Num: I));
582 NewOps.push_back(Elt: bitcasted);
583 }
584 EVT OrigVT = N.getValueType(ResNo: 0);
585 SDValue OpNode = DAG.getNode(Opcode: N.getOpcode(), DL, VT: getFloatVT(OrigVT), Ops: NewOps);
586 return DAG.getBitcast(VT: OrigVT, V: OpNode);
587}
588
589/// isIntImmediate - This method tests to see if the node is a constant
590/// operand. If so Imm will receive the 64-bit value.
591static bool isIntImmediate(const SDNode *N, uint64_t &Imm) {
592 if (const ConstantSDNode *C = dyn_cast<const ConstantSDNode>(Val: N)) {
593 Imm = C->getZExtValue();
594 return true;
595 }
596 return false;
597}
598
599// isIntImmediate - This method tests to see if a constant operand.
600// If so Imm will receive the value.
601static bool isIntImmediate(SDValue N, uint64_t &Imm) {
602 return isIntImmediate(N: N.getNode(), Imm);
603}
604
605// isOpcWithIntImmediate - This method tests to see if the node is a specific
606// opcode and that it has a immediate integer right operand.
607// If so Imm will receive the 32 bit value.
608static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc,
609 uint64_t &Imm) {
610 return N->getOpcode() == Opc &&
611 isIntImmediate(N: N->getOperand(Num: 1).getNode(), Imm);
612}
613
614// isIntImmediateEq - This method tests to see if N is a constant operand that
615// is equivalent to 'ImmExpected'.
616#ifndef NDEBUG
617static bool isIntImmediateEq(SDValue N, const uint64_t ImmExpected) {
618 uint64_t Imm;
619 if (!isIntImmediate(N.getNode(), Imm))
620 return false;
621 return Imm == ImmExpected;
622}
623#endif
624
625static APInt DecodeFMOVImm(uint64_t Imm, unsigned RegWidth) {
626 assert(RegWidth == 32 || RegWidth == 64);
627 if (RegWidth == 32)
628 return APInt(RegWidth,
629 uint32_t(AArch64_AM::decodeAdvSIMDModImmType11(Imm)));
630 return APInt(RegWidth, AArch64_AM::decodeAdvSIMDModImmType12(Imm));
631}
632
633// Decodes the raw integer splat value from a NEON splat operation.
634static std::optional<APInt> DecodeNEONSplat(SDValue N) {
635 assert(N.getValueType().isInteger() && "Only integers are supported");
636 if (N->getOpcode() == AArch64ISD::NVCAST)
637 N = N->getOperand(Num: 0);
638 unsigned SplatWidth = N.getScalarValueSizeInBits();
639 if (N.getOpcode() == AArch64ISD::FMOV)
640 return DecodeFMOVImm(Imm: N.getConstantOperandVal(i: 0), RegWidth: SplatWidth);
641 if (N->getOpcode() == AArch64ISD::MOVI)
642 return APInt(SplatWidth, N.getConstantOperandVal(i: 0));
643 if (N->getOpcode() == AArch64ISD::MOVIshift)
644 return APInt(SplatWidth, N.getConstantOperandVal(i: 0)
645 << N.getConstantOperandVal(i: 1));
646 if (N->getOpcode() == AArch64ISD::MVNIshift)
647 return ~APInt(SplatWidth, N.getConstantOperandVal(i: 0)
648 << N.getConstantOperandVal(i: 1));
649 if (N->getOpcode() == AArch64ISD::MOVIedit)
650 return APInt(SplatWidth, AArch64_AM::decodeAdvSIMDModImmType10(
651 Imm: N.getConstantOperandVal(i: 0)));
652 if (N->getOpcode() == AArch64ISD::DUP)
653 if (auto *Const = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0)))
654 return Const->getAPIntValue().trunc(width: SplatWidth);
655 // TODO: Recognize more splat-like NEON operations. See ConstantBuildVector
656 // in AArch64ISelLowering.
657 return std::nullopt;
658}
659
660// If \p N is a NEON splat operation (movi, fmov, etc), return the splat value
661// matching the element size of N.
662static std::optional<APInt> GetNEONSplatValue(SDValue N) {
663 unsigned SplatWidth = N.getScalarValueSizeInBits();
664 if (std::optional<APInt> SplatVal = DecodeNEONSplat(N)) {
665 if (SplatVal->getBitWidth() <= SplatWidth)
666 return APInt::getSplat(NewLen: SplatWidth, V: *SplatVal);
667 if (SplatVal->isSplat(SplatSizeInBits: SplatWidth))
668 return SplatVal->trunc(width: SplatWidth);
669 }
670 return std::nullopt;
671}
672
673bool AArch64DAGToDAGISel::SelectNEONSplatOfSVELogicalImm(SDValue N,
674 SDValue &Imm) {
675 std::optional<APInt> ImmVal = GetNEONSplatValue(N);
676 if (!ImmVal)
677 return false;
678 uint64_t Encoding;
679 if (!AArch64_AM::isSVELogicalImm(SizeInBits: N.getScalarValueSizeInBits(),
680 ImmVal: ImmVal->getZExtValue(), Encoding))
681 return false;
682
683 Imm = CurDAG->getTargetConstant(Val: Encoding, DL: SDLoc(N), VT: MVT::i64);
684 return true;
685}
686
687bool AArch64DAGToDAGISel::SelectNEONSplatOfSVEAddSubImm(SDValue N, SDValue &Imm,
688 SDValue &Shift) {
689 if (std::optional<APInt> ImmVal = GetNEONSplatValue(N))
690 return SelectSVEAddSubImm(DL: SDLoc(N), Value: *ImmVal,
691 VT: N.getValueType().getScalarType().getSimpleVT(),
692 Imm, Shift,
693 /*Negate=*/false);
694 return false;
695}
696
697bool AArch64DAGToDAGISel::SelectNEONSplatOfSVEArithSImm(SDValue N,
698 SDValue &Imm) {
699 if (std::optional<APInt> ImmVal = GetNEONSplatValue(N))
700 return SelectSVESignedArithImm(DL: SDLoc(N), Value: *ImmVal, Imm);
701 return false;
702}
703
704bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
705 const SDValue &Op, const InlineAsm::ConstraintCode ConstraintID,
706 std::vector<SDValue> &OutOps) {
707 switch(ConstraintID) {
708 default:
709 llvm_unreachable("Unexpected asm memory constraint");
710 case InlineAsm::ConstraintCode::m:
711 case InlineAsm::ConstraintCode::o:
712 case InlineAsm::ConstraintCode::Q:
713 // We need to make sure that this one operand does not end up in XZR, thus
714 // require the address to be in a PointerRegClass register.
715 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
716 const TargetRegisterClass *TRC = TRI->getPointerRegClass();
717 SDLoc dl(Op);
718 SDValue RC = CurDAG->getTargetConstant(Val: TRC->getID(), DL: dl, VT: MVT::i64);
719 SDValue NewOp =
720 SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS,
721 dl, VT: Op.getValueType(),
722 Op1: Op, Op2: RC), 0);
723 OutOps.push_back(x: NewOp);
724 return false;
725 }
726 return true;
727}
728
729/// SelectArithImmed - Select an immediate value that can be represented as
730/// a 12-bit value shifted left by either 0 or 12. If so, return true with
731/// Val set to the 12-bit value and Shift set to the shifter operand.
732bool AArch64DAGToDAGISel::SelectArithImmed(SDValue N, SDValue &Val,
733 SDValue &Shift) {
734 // This function is called from the addsub_shifted_imm ComplexPattern,
735 // which lists [imm] as the list of opcode it's interested in, however
736 // we still need to check whether the operand is actually an immediate
737 // here because the ComplexPattern opcode list is only used in
738 // root-level opcode matching.
739 if (!isa<ConstantSDNode>(Val: N.getNode()))
740 return false;
741
742 uint64_t Immed = N.getNode()->getAsZExtVal();
743
744 if (!AArch64_AM::isLegalArithImmed(C: Immed))
745 return false;
746
747 unsigned ShiftAmt = AArch64_AM::getArithImmedShift(C: Immed);
748 Immed >>= ShiftAmt;
749
750 unsigned ShVal = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShiftAmt);
751 SDLoc dl(N);
752 Val = CurDAG->getTargetConstant(Val: Immed, DL: dl, VT: MVT::i32);
753 Shift = CurDAG->getTargetConstant(Val: ShVal, DL: dl, VT: MVT::i32);
754 return true;
755}
756
757/// SelectNegArithImmed - As above, but negates the value before trying to
758/// select it.
759bool AArch64DAGToDAGISel::SelectNegArithImmed(SDValue N, SDValue &Val,
760 SDValue &Shift) {
761 // This function is called from the addsub_shifted_imm ComplexPattern,
762 // which lists [imm] as the list of opcode it's interested in, however
763 // we still need to check whether the operand is actually an immediate
764 // here because the ComplexPattern opcode list is only used in
765 // root-level opcode matching.
766 if (!isa<ConstantSDNode>(Val: N.getNode()))
767 return false;
768
769 // The immediate operand must be a 24-bit zero-extended immediate.
770 uint64_t Immed = N.getNode()->getAsZExtVal();
771
772 // This negation is almost always valid, but "cmp wN, #0" and "cmn wN, #0"
773 // have the opposite effect on the C flag, so this pattern mustn't match under
774 // those circumstances.
775 if (Immed == 0)
776 return false;
777
778 if (N.getValueType() == MVT::i32)
779 Immed = ~((uint32_t)Immed) + 1;
780 else
781 Immed = ~Immed + 1ULL;
782 if (Immed & 0xFFFFFFFFFF000000ULL)
783 return false;
784
785 Immed &= 0xFFFFFFULL;
786 return SelectArithImmed(N: CurDAG->getConstant(Val: Immed, DL: SDLoc(N), VT: MVT::i32), Val,
787 Shift);
788}
789
790/// getShiftTypeForNode - Translate a shift node to the corresponding
791/// ShiftType value.
792static AArch64_AM::ShiftExtendType getShiftTypeForNode(SDValue N) {
793 switch (N.getOpcode()) {
794 default:
795 return AArch64_AM::InvalidShiftExtend;
796 case ISD::SHL:
797 return AArch64_AM::LSL;
798 case ISD::SRL:
799 return AArch64_AM::LSR;
800 case ISD::SRA:
801 return AArch64_AM::ASR;
802 case ISD::ROTR:
803 return AArch64_AM::ROR;
804 }
805}
806
807static bool isMemOpOrPrefetch(SDNode *N) {
808 return isa<MemSDNode>(Val: *N) || N->getOpcode() == AArch64ISD::PREFETCH;
809}
810
811/// Determine whether it is worth it to fold SHL into the addressing
812/// mode.
813static bool isWorthFoldingSHL(SDValue V) {
814 assert(V.getOpcode() == ISD::SHL && "invalid opcode");
815 // It is worth folding logical shift of up to three places.
816 auto *CSD = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1));
817 if (!CSD)
818 return false;
819 unsigned ShiftVal = CSD->getZExtValue();
820 if (ShiftVal > 3)
821 return false;
822
823 // Check if this particular node is reused in any non-memory related
824 // operation. If yes, do not try to fold this node into the address
825 // computation, since the computation will be kept.
826 const SDNode *Node = V.getNode();
827 for (SDNode *UI : Node->users())
828 if (!isMemOpOrPrefetch(N: UI))
829 for (SDNode *UII : UI->users())
830 if (!isMemOpOrPrefetch(N: UII))
831 return false;
832 return true;
833}
834
835/// Determine whether it is worth to fold V into an extended register addressing
836/// mode.
837bool AArch64DAGToDAGISel::isWorthFoldingAddr(SDValue V, unsigned Size) const {
838 // Trivial if we are optimizing for code size or if there is only
839 // one use of the value.
840 if (CurDAG->shouldOptForSize() || V.hasOneUse())
841 return true;
842
843 // If a subtarget has a slow shift, folding a shift into multiple loads
844 // costs additional micro-ops.
845 if (Subtarget->hasAddrLSLSlow14() && (Size == 2 || Size == 16))
846 return false;
847
848 // Check whether we're going to emit the address arithmetic anyway because
849 // it's used by a non-address operation.
850 if (V.getOpcode() == ISD::SHL && isWorthFoldingSHL(V))
851 return true;
852 if (V.getOpcode() == ISD::ADD) {
853 const SDValue LHS = V.getOperand(i: 0);
854 const SDValue RHS = V.getOperand(i: 1);
855 if (LHS.getOpcode() == ISD::SHL && isWorthFoldingSHL(V: LHS))
856 return true;
857 if (RHS.getOpcode() == ISD::SHL && isWorthFoldingSHL(V: RHS))
858 return true;
859 }
860
861 // It hurts otherwise, since the value will be reused.
862 return false;
863}
864
865/// and (shl/srl/sra, x, c), mask --> shl (srl/sra, x, c1), c2
866/// to select more shifted register
867bool AArch64DAGToDAGISel::SelectShiftedRegisterFromAnd(SDValue N, SDValue &Reg,
868 SDValue &Shift) {
869 EVT VT = N.getValueType();
870 if (VT != MVT::i32 && VT != MVT::i64)
871 return false;
872
873 if (N->getOpcode() != ISD::AND || !N->hasOneUse())
874 return false;
875 SDValue LHS = N.getOperand(i: 0);
876 if (!LHS->hasOneUse())
877 return false;
878
879 unsigned LHSOpcode = LHS->getOpcode();
880 if (LHSOpcode != ISD::SHL && LHSOpcode != ISD::SRL && LHSOpcode != ISD::SRA)
881 return false;
882
883 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1));
884 if (!ShiftAmtNode)
885 return false;
886
887 uint64_t ShiftAmtC = ShiftAmtNode->getZExtValue();
888 ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
889 if (!RHSC)
890 return false;
891
892 APInt AndMask = RHSC->getAPIntValue();
893 unsigned LowZBits, MaskLen;
894 if (!AndMask.isShiftedMask(MaskIdx&: LowZBits, MaskLen))
895 return false;
896
897 unsigned BitWidth = N.getValueSizeInBits();
898 SDLoc DL(LHS);
899 uint64_t NewShiftC;
900 unsigned NewShiftOp;
901 if (LHSOpcode == ISD::SHL) {
902 // LowZBits <= ShiftAmtC will fall into isBitfieldPositioningOp
903 // BitWidth != LowZBits + MaskLen doesn't match the pattern
904 if (LowZBits <= ShiftAmtC || (BitWidth != LowZBits + MaskLen))
905 return false;
906
907 NewShiftC = LowZBits - ShiftAmtC;
908 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
909 } else {
910 if (LowZBits == 0)
911 return false;
912
913 // NewShiftC >= BitWidth will fall into isBitfieldExtractOp
914 NewShiftC = LowZBits + ShiftAmtC;
915 if (NewShiftC >= BitWidth)
916 return false;
917
918 // SRA need all high bits
919 if (LHSOpcode == ISD::SRA && (BitWidth != (LowZBits + MaskLen)))
920 return false;
921
922 // SRL high bits can be 0 or 1
923 if (LHSOpcode == ISD::SRL && (BitWidth > (NewShiftC + MaskLen)))
924 return false;
925
926 if (LHSOpcode == ISD::SRL)
927 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
928 else
929 NewShiftOp = VT == MVT::i64 ? AArch64::SBFMXri : AArch64::SBFMWri;
930 }
931
932 assert(NewShiftC < BitWidth && "Invalid shift amount");
933 SDValue NewShiftAmt = CurDAG->getTargetConstant(Val: NewShiftC, DL, VT);
934 SDValue BitWidthMinus1 = CurDAG->getTargetConstant(Val: BitWidth - 1, DL, VT);
935 Reg = SDValue(CurDAG->getMachineNode(Opcode: NewShiftOp, dl: DL, VT, Op1: LHS->getOperand(Num: 0),
936 Op2: NewShiftAmt, Op3: BitWidthMinus1),
937 0);
938 unsigned ShVal = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: LowZBits);
939 Shift = CurDAG->getTargetConstant(Val: ShVal, DL, VT: MVT::i32);
940 return true;
941}
942
943/// getExtendTypeForNode - Translate an extend node to the corresponding
944/// ExtendType value.
945static AArch64_AM::ShiftExtendType
946getExtendTypeForNode(SDValue N, bool IsLoadStore = false) {
947 if (N.getOpcode() == ISD::SIGN_EXTEND ||
948 N.getOpcode() == ISD::SIGN_EXTEND_INREG) {
949 EVT SrcVT;
950 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG)
951 SrcVT = cast<VTSDNode>(Val: N.getOperand(i: 1))->getVT();
952 else
953 SrcVT = N.getOperand(i: 0).getValueType();
954
955 if (!IsLoadStore && SrcVT == MVT::i8)
956 return AArch64_AM::SXTB;
957 else if (!IsLoadStore && SrcVT == MVT::i16)
958 return AArch64_AM::SXTH;
959 else if (SrcVT == MVT::i32)
960 return AArch64_AM::SXTW;
961 assert(SrcVT != MVT::i64 && "extend from 64-bits?");
962
963 return AArch64_AM::InvalidShiftExtend;
964 } else if (N.getOpcode() == ISD::ZERO_EXTEND ||
965 N.getOpcode() == ISD::ANY_EXTEND) {
966 EVT SrcVT = N.getOperand(i: 0).getValueType();
967 if (!IsLoadStore && SrcVT == MVT::i8)
968 return AArch64_AM::UXTB;
969 else if (!IsLoadStore && SrcVT == MVT::i16)
970 return AArch64_AM::UXTH;
971 else if (SrcVT == MVT::i32)
972 return AArch64_AM::UXTW;
973 assert(SrcVT != MVT::i64 && "extend from 64-bits?");
974
975 return AArch64_AM::InvalidShiftExtend;
976 } else if (N.getOpcode() == ISD::AND) {
977 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
978 if (!CSD)
979 return AArch64_AM::InvalidShiftExtend;
980 uint64_t AndMask = CSD->getZExtValue();
981
982 switch (AndMask) {
983 default:
984 return AArch64_AM::InvalidShiftExtend;
985 case 0xFF:
986 return !IsLoadStore ? AArch64_AM::UXTB : AArch64_AM::InvalidShiftExtend;
987 case 0xFFFF:
988 return !IsLoadStore ? AArch64_AM::UXTH : AArch64_AM::InvalidShiftExtend;
989 case 0xFFFFFFFF:
990 return AArch64_AM::UXTW;
991 }
992 }
993
994 return AArch64_AM::InvalidShiftExtend;
995}
996
997/// Determine whether it is worth to fold V into an extended register of an
998/// Add/Sub. LSL means we are folding into an `add w0, w1, w2, lsl #N`
999/// instruction, and the shift should be treated as worth folding even if has
1000/// multiple uses.
1001bool AArch64DAGToDAGISel::isWorthFoldingALU(SDValue V, bool LSL) const {
1002 // Trivial if we are optimizing for code size or if there is only
1003 // one use of the value.
1004 if (CurDAG->shouldOptForSize() || V.hasOneUse())
1005 return true;
1006
1007 // If a subtarget has a fastpath LSL we can fold a logical shift into
1008 // the add/sub and save a cycle.
1009 if (LSL && Subtarget->hasALULSLFast() && V.getOpcode() == ISD::SHL &&
1010 V.getConstantOperandVal(i: 1) <= 4 &&
1011 getExtendTypeForNode(N: V.getOperand(i: 0)) == AArch64_AM::InvalidShiftExtend)
1012 return true;
1013
1014 // It hurts otherwise, since the value will be reused.
1015 return false;
1016}
1017
1018/// SelectShiftedRegister - Select a "shifted register" operand. If the value
1019/// is not shifted, set the Shift operand to default of "LSL 0". The logical
1020/// instructions allow the shifted register to be rotated, but the arithmetic
1021/// instructions do not. The AllowROR parameter specifies whether ROR is
1022/// supported.
1023bool AArch64DAGToDAGISel::SelectShiftedRegister(SDValue N, bool AllowROR,
1024 SDValue &Reg, SDValue &Shift) {
1025 if (SelectShiftedRegisterFromAnd(N, Reg, Shift))
1026 return true;
1027
1028 AArch64_AM::ShiftExtendType ShType = getShiftTypeForNode(N);
1029 if (ShType == AArch64_AM::InvalidShiftExtend)
1030 return false;
1031 if (!AllowROR && ShType == AArch64_AM::ROR)
1032 return false;
1033
1034 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1035 unsigned BitSize = N.getValueSizeInBits();
1036 unsigned Val = RHS->getZExtValue() & (BitSize - 1);
1037 unsigned ShVal = AArch64_AM::getShifterImm(ST: ShType, Imm: Val);
1038
1039 Reg = N.getOperand(i: 0);
1040 Shift = CurDAG->getTargetConstant(Val: ShVal, DL: SDLoc(N), VT: MVT::i32);
1041 return isWorthFoldingALU(V: N, LSL: true);
1042 }
1043
1044 return false;
1045}
1046
1047/// Instructions that accept extend modifiers like UXTW expect the register
1048/// being extended to be a GPR32, but the incoming DAG might be acting on a
1049/// GPR64 (either via SEXT_INREG or AND). Extract the appropriate low bits if
1050/// this is the case.
1051static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N) {
1052 if (N.getValueType() == MVT::i32)
1053 return N;
1054
1055 SDLoc dl(N);
1056 return CurDAG->getTargetExtractSubreg(SRIdx: AArch64::sub_32, DL: dl, VT: MVT::i32, Operand: N);
1057}
1058
1059// Returns a suitable CNT/INC/DEC/RDVL multiplier to calculate VSCALE*N.
1060template<signed Low, signed High, signed Scale>
1061bool AArch64DAGToDAGISel::SelectRDVLImm(SDValue N, SDValue &Imm) {
1062 if (!isa<ConstantSDNode>(Val: N))
1063 return false;
1064
1065 int64_t MulImm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
1066 if ((MulImm % std::abs(x: Scale)) == 0) {
1067 int64_t RDVLImm = MulImm / Scale;
1068 if ((RDVLImm >= Low) && (RDVLImm <= High)) {
1069 Imm = CurDAG->getSignedTargetConstant(Val: RDVLImm, DL: SDLoc(N), VT: MVT::i32);
1070 return true;
1071 }
1072 }
1073
1074 return false;
1075}
1076
1077// Returns a suitable RDSVL multiplier from a left shift.
1078template <signed Low, signed High>
1079bool AArch64DAGToDAGISel::SelectRDSVLShiftImm(SDValue N, SDValue &Imm) {
1080 if (!isa<ConstantSDNode>(Val: N))
1081 return false;
1082
1083 int64_t MulImm = 1LL << cast<ConstantSDNode>(Val&: N)->getSExtValue();
1084 if (MulImm >= Low && MulImm <= High) {
1085 Imm = CurDAG->getSignedTargetConstant(Val: MulImm, DL: SDLoc(N), VT: MVT::i32);
1086 return true;
1087 }
1088
1089 return false;
1090}
1091
1092/// SelectArithExtendedRegister - Select a "extended register" operand. This
1093/// operand folds in an extend followed by an optional left shift.
1094bool AArch64DAGToDAGISel::SelectArithExtendedRegister(SDValue N, SDValue &Reg,
1095 SDValue &Shift) {
1096 unsigned ShiftVal = 0;
1097 AArch64_AM::ShiftExtendType Ext;
1098
1099 if (N.getOpcode() == ISD::SHL) {
1100 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1101 if (!CSD)
1102 return false;
1103 ShiftVal = CSD->getZExtValue();
1104 if (ShiftVal > 4)
1105 return false;
1106
1107 Ext = getExtendTypeForNode(N: N.getOperand(i: 0));
1108 if (Ext == AArch64_AM::InvalidShiftExtend)
1109 return false;
1110
1111 Reg = N.getOperand(i: 0).getOperand(i: 0);
1112 } else {
1113 Ext = getExtendTypeForNode(N);
1114 if (Ext == AArch64_AM::InvalidShiftExtend)
1115 return false;
1116
1117 // Don't match sext of vector extracts. These can use SMOV, but if we match
1118 // this as an extended register, we'll always fold the extend into an ALU op
1119 // user of the extend (which results in a UMOV).
1120 if (AArch64_AM::isSignExtendShiftType(Type: Ext)) {
1121 SDValue Op = N.getOperand(i: 0);
1122 if (Op->getOpcode() == ISD::ANY_EXTEND)
1123 Op = Op->getOperand(Num: 0);
1124 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
1125 Op.getOperand(i: 0).getValueType().isFixedLengthVector())
1126 return false;
1127 }
1128
1129 Reg = N.getOperand(i: 0);
1130
1131 // Don't match if free 32-bit -> 64-bit zext can be used instead. Use the
1132 // isDef32 as a heuristic for when the operand is likely to be a 32bit def.
1133 auto isDef32 = [](SDValue N) {
1134 unsigned Opc = N.getOpcode();
1135 return Opc != ISD::TRUNCATE && Opc != TargetOpcode::EXTRACT_SUBREG &&
1136 Opc != ISD::CopyFromReg && Opc != ISD::AssertSext &&
1137 Opc != ISD::AssertZext && Opc != ISD::AssertAlign &&
1138 Opc != ISD::FREEZE;
1139 };
1140 if (Ext == AArch64_AM::UXTW && Reg->getValueType(ResNo: 0).getSizeInBits() == 32 &&
1141 isDef32(Reg))
1142 return false;
1143 }
1144
1145 // AArch64 mandates that the RHS of the operation must use the smallest
1146 // register class that could contain the size being extended from. Thus,
1147 // if we're folding a (sext i8), we need the RHS to be a GPR32, even though
1148 // there might not be an actual 32-bit value in the program. We can
1149 // (harmlessly) synthesize one by injected an EXTRACT_SUBREG here.
1150 assert(Ext != AArch64_AM::UXTX && Ext != AArch64_AM::SXTX);
1151 Reg = narrowIfNeeded(CurDAG, N: Reg);
1152 Shift = CurDAG->getTargetConstant(Val: getArithExtendImm(ET: Ext, Imm: ShiftVal), DL: SDLoc(N),
1153 VT: MVT::i32);
1154 return isWorthFoldingALU(V: N);
1155}
1156
1157/// SelectArithUXTXRegister - Select a "UXTX register" operand. This
1158/// operand is referred by the instructions have SP operand
1159bool AArch64DAGToDAGISel::SelectArithUXTXRegister(SDValue N, SDValue &Reg,
1160 SDValue &Shift) {
1161 unsigned ShiftVal = 0;
1162 AArch64_AM::ShiftExtendType Ext;
1163
1164 if (N.getOpcode() != ISD::SHL)
1165 return false;
1166
1167 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1168 if (!CSD)
1169 return false;
1170 ShiftVal = CSD->getZExtValue();
1171 if (ShiftVal > 4)
1172 return false;
1173
1174 Ext = AArch64_AM::UXTX;
1175 Reg = N.getOperand(i: 0);
1176 Shift = CurDAG->getTargetConstant(Val: getArithExtendImm(ET: Ext, Imm: ShiftVal), DL: SDLoc(N),
1177 VT: MVT::i32);
1178 return isWorthFoldingALU(V: N);
1179}
1180
1181/// If there's a use of this ADDlow that's not itself a load/store then we'll
1182/// need to create a real ADD instruction from it anyway and there's no point in
1183/// folding it into the mem op. Theoretically, it shouldn't matter, but there's
1184/// a single pseudo-instruction for an ADRP/ADD pair so over-aggressive folding
1185/// leads to duplicated ADRP instructions.
1186static bool isWorthFoldingADDlow(SDValue N) {
1187 for (auto *User : N->users()) {
1188 if (User->getOpcode() != ISD::LOAD && User->getOpcode() != ISD::STORE &&
1189 User->getOpcode() != ISD::ATOMIC_LOAD &&
1190 User->getOpcode() != ISD::ATOMIC_STORE)
1191 return false;
1192
1193 // ldar and stlr have much more restrictive addressing modes (just a
1194 // register).
1195 if (isStrongerThanMonotonic(AO: cast<MemSDNode>(Val: User)->getSuccessOrdering()))
1196 return false;
1197 }
1198
1199 return true;
1200}
1201
1202/// Check if the immediate offset is valid as a scaled immediate.
1203static bool isValidAsScaledImmediate(int64_t Offset, unsigned Range,
1204 unsigned Size) {
1205 if ((Offset & (Size - 1)) == 0 && Offset >= 0 &&
1206 Offset < (Range << Log2_32(Value: Size)))
1207 return true;
1208 return false;
1209}
1210
1211/// SelectAddrModeIndexedBitWidth - Select a "register plus scaled (un)signed BW-bit
1212/// immediate" address. The "Size" argument is the size in bytes of the memory
1213/// reference, which determines the scale.
1214bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm,
1215 unsigned BW, unsigned Size,
1216 SDValue &Base,
1217 SDValue &OffImm) {
1218 SDLoc dl(N);
1219 const DataLayout &DL = CurDAG->getDataLayout();
1220 const TargetLowering *TLI = getTargetLowering();
1221 if (N.getOpcode() == ISD::FrameIndex) {
1222 int FI = cast<FrameIndexSDNode>(Val&: N)->getIndex();
1223 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1224 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1225 return true;
1226 }
1227
1228 // As opposed to the (12-bit) Indexed addressing mode below, the 7/9-bit signed
1229 // selected here doesn't support labels/immediates, only base+offset.
1230 if (CurDAG->isBaseWithConstantOffset(Op: N)) {
1231 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1232 if (IsSignedImm) {
1233 int64_t RHSC = RHS->getSExtValue();
1234 unsigned Scale = Log2_32(Value: Size);
1235 int64_t Range = 0x1LL << (BW - 1);
1236
1237 if ((RHSC & (Size - 1)) == 0 && RHSC >= -(Range << Scale) &&
1238 RHSC < (Range << Scale)) {
1239 Base = N.getOperand(i: 0);
1240 if (Base.getOpcode() == ISD::FrameIndex) {
1241 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1242 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1243 }
1244 OffImm = CurDAG->getTargetConstant(Val: RHSC >> Scale, DL: dl, VT: MVT::i64);
1245 return true;
1246 }
1247 } else {
1248 // unsigned Immediate
1249 uint64_t RHSC = RHS->getZExtValue();
1250 unsigned Scale = Log2_32(Value: Size);
1251 uint64_t Range = 0x1ULL << BW;
1252
1253 if ((RHSC & (Size - 1)) == 0 && RHSC < (Range << Scale)) {
1254 Base = N.getOperand(i: 0);
1255 if (Base.getOpcode() == ISD::FrameIndex) {
1256 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1257 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1258 }
1259 OffImm = CurDAG->getTargetConstant(Val: RHSC >> Scale, DL: dl, VT: MVT::i64);
1260 return true;
1261 }
1262 }
1263 }
1264 }
1265 // Base only. The address will be materialized into a register before
1266 // the memory is accessed.
1267 // add x0, Xbase, #offset
1268 // stp x1, x2, [x0]
1269 Base = N;
1270 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1271 return true;
1272}
1273
1274/// SelectAddrModeIndexed - Select a "register plus scaled unsigned 12-bit
1275/// immediate" address. The "Size" argument is the size in bytes of the memory
1276/// reference, which determines the scale.
1277bool AArch64DAGToDAGISel::SelectAddrModeIndexed(SDValue N, unsigned Size,
1278 SDValue &Base, SDValue &OffImm) {
1279 SDLoc dl(N);
1280 const DataLayout &DL = CurDAG->getDataLayout();
1281 const TargetLowering *TLI = getTargetLowering();
1282 if (N.getOpcode() == ISD::FrameIndex) {
1283 int FI = cast<FrameIndexSDNode>(Val&: N)->getIndex();
1284 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1285 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1286 return true;
1287 }
1288
1289 if (N.getOpcode() == AArch64ISD::ADDlow && isWorthFoldingADDlow(N)) {
1290 GlobalAddressSDNode *GAN =
1291 dyn_cast<GlobalAddressSDNode>(Val: N.getOperand(i: 1).getNode());
1292 Base = N.getOperand(i: 0);
1293 OffImm = N.getOperand(i: 1);
1294 if (!GAN)
1295 return true;
1296
1297 if (GAN->getOffset() % Size == 0 &&
1298 GAN->getGlobal()->getPointerAlignment(DL) >= Size)
1299 return true;
1300 }
1301
1302 if (CurDAG->isBaseWithConstantOffset(Op: N)) {
1303 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1304 int64_t RHSC = (int64_t)RHS->getZExtValue();
1305 unsigned Scale = Log2_32(Value: Size);
1306 if (isValidAsScaledImmediate(Offset: RHSC, Range: 0x1000, Size)) {
1307 Base = N.getOperand(i: 0);
1308 if (Base.getOpcode() == ISD::FrameIndex) {
1309 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1310 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1311 }
1312 OffImm = CurDAG->getTargetConstant(Val: RHSC >> Scale, DL: dl, VT: MVT::i64);
1313 return true;
1314 }
1315 }
1316 }
1317
1318 // Before falling back to our general case, check if the unscaled
1319 // instructions can handle this. If so, that's preferable.
1320 if (SelectAddrModeUnscaled(N, Size, Base, OffImm))
1321 return false;
1322
1323 // Base only. The address will be materialized into a register before
1324 // the memory is accessed.
1325 // add x0, Xbase, #offset
1326 // ldr x0, [x0]
1327 Base = N;
1328 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1329 return true;
1330}
1331
1332/// SelectAddrModeUnscaled - Select a "register plus unscaled signed 9-bit
1333/// immediate" address. This should only match when there is an offset that
1334/// is not valid for a scaled immediate addressing mode. The "Size" argument
1335/// is the size in bytes of the memory reference, which is needed here to know
1336/// what is valid for a scaled immediate.
1337bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(SDValue N, unsigned Size,
1338 SDValue &Base,
1339 SDValue &OffImm) {
1340 if (!CurDAG->isBaseWithConstantOffset(Op: N))
1341 return false;
1342 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1343 int64_t RHSC = RHS->getSExtValue();
1344 if (RHSC >= -256 && RHSC < 256) {
1345 Base = N.getOperand(i: 0);
1346 if (Base.getOpcode() == ISD::FrameIndex) {
1347 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1348 const TargetLowering *TLI = getTargetLowering();
1349 Base = CurDAG->getTargetFrameIndex(
1350 FI, VT: TLI->getPointerTy(DL: CurDAG->getDataLayout()));
1351 }
1352 OffImm = CurDAG->getTargetConstant(Val: RHSC, DL: SDLoc(N), VT: MVT::i64);
1353 return true;
1354 }
1355 }
1356 return false;
1357}
1358
1359static SDValue Widen(SelectionDAG *CurDAG, SDValue N) {
1360 SDLoc dl(N);
1361 SDValue ImpDef = SDValue(
1362 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl, VT: MVT::i64), 0);
1363 return CurDAG->getTargetInsertSubreg(SRIdx: AArch64::sub_32, DL: dl, VT: MVT::i64, Operand: ImpDef,
1364 Subreg: N);
1365}
1366
1367/// Check if the given SHL node (\p N), can be used to form an
1368/// extended register for an addressing mode.
1369bool AArch64DAGToDAGISel::SelectExtendedSHL(SDValue N, unsigned Size,
1370 bool WantExtend, SDValue &Offset,
1371 SDValue &SignExtend) {
1372 assert(N.getOpcode() == ISD::SHL && "Invalid opcode.");
1373 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1374 if (!CSD || (CSD->getZExtValue() & 0x7) != CSD->getZExtValue())
1375 return false;
1376
1377 SDLoc dl(N);
1378 if (WantExtend) {
1379 AArch64_AM::ShiftExtendType Ext =
1380 getExtendTypeForNode(N: N.getOperand(i: 0), IsLoadStore: true);
1381 if (Ext == AArch64_AM::InvalidShiftExtend)
1382 return false;
1383
1384 Offset = narrowIfNeeded(CurDAG, N: N.getOperand(i: 0).getOperand(i: 0));
1385 SignExtend = CurDAG->getTargetConstant(Val: Ext == AArch64_AM::SXTW, DL: dl,
1386 VT: MVT::i32);
1387 } else {
1388 Offset = N.getOperand(i: 0);
1389 SignExtend = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i32);
1390 }
1391
1392 unsigned LegalShiftVal = Log2_32(Value: Size);
1393 unsigned ShiftVal = CSD->getZExtValue();
1394
1395 if (ShiftVal != 0 && ShiftVal != LegalShiftVal)
1396 return false;
1397
1398 return isWorthFoldingAddr(V: N, Size);
1399}
1400
1401bool AArch64DAGToDAGISel::SelectAddrModeWRO(SDValue N, unsigned Size,
1402 SDValue &Base, SDValue &Offset,
1403 SDValue &SignExtend,
1404 SDValue &DoShift) {
1405 if (N.getOpcode() != ISD::ADD)
1406 return false;
1407 SDValue LHS = N.getOperand(i: 0);
1408 SDValue RHS = N.getOperand(i: 1);
1409 SDLoc dl(N);
1410
1411 // We don't want to match immediate adds here, because they are better lowered
1412 // to the register-immediate addressing modes.
1413 if (isa<ConstantSDNode>(Val: LHS) || isa<ConstantSDNode>(Val: RHS))
1414 return false;
1415
1416 // Check if this particular node is reused in any non-memory related
1417 // operation. If yes, do not try to fold this node into the address
1418 // computation, since the computation will be kept.
1419 const SDNode *Node = N.getNode();
1420 for (SDNode *UI : Node->users()) {
1421 if (!isMemOpOrPrefetch(N: UI))
1422 return false;
1423 }
1424
1425 // Remember if it is worth folding N when it produces extended register.
1426 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(V: N, Size);
1427
1428 // Try to match a shifted extend on the RHS.
1429 if (IsExtendedRegisterWorthFolding && RHS.getOpcode() == ISD::SHL &&
1430 SelectExtendedSHL(N: RHS, Size, WantExtend: true, Offset, SignExtend)) {
1431 Base = LHS;
1432 DoShift = CurDAG->getTargetConstant(Val: true, DL: dl, VT: MVT::i32);
1433 return true;
1434 }
1435
1436 // Try to match a shifted extend on the LHS.
1437 if (IsExtendedRegisterWorthFolding && LHS.getOpcode() == ISD::SHL &&
1438 SelectExtendedSHL(N: LHS, Size, WantExtend: true, Offset, SignExtend)) {
1439 Base = RHS;
1440 DoShift = CurDAG->getTargetConstant(Val: true, DL: dl, VT: MVT::i32);
1441 return true;
1442 }
1443
1444 // There was no shift, whatever else we find.
1445 DoShift = CurDAG->getTargetConstant(Val: false, DL: dl, VT: MVT::i32);
1446
1447 AArch64_AM::ShiftExtendType Ext = AArch64_AM::InvalidShiftExtend;
1448 // Try to match an unshifted extend on the LHS.
1449 if (IsExtendedRegisterWorthFolding &&
1450 (Ext = getExtendTypeForNode(N: LHS, IsLoadStore: true)) !=
1451 AArch64_AM::InvalidShiftExtend) {
1452 Base = RHS;
1453 Offset = narrowIfNeeded(CurDAG, N: LHS.getOperand(i: 0));
1454 SignExtend = CurDAG->getTargetConstant(Val: Ext == AArch64_AM::SXTW, DL: dl,
1455 VT: MVT::i32);
1456 if (isWorthFoldingAddr(V: LHS, Size))
1457 return true;
1458 }
1459
1460 // Try to match an unshifted extend on the RHS.
1461 if (IsExtendedRegisterWorthFolding &&
1462 (Ext = getExtendTypeForNode(N: RHS, IsLoadStore: true)) !=
1463 AArch64_AM::InvalidShiftExtend) {
1464 Base = LHS;
1465 Offset = narrowIfNeeded(CurDAG, N: RHS.getOperand(i: 0));
1466 SignExtend = CurDAG->getTargetConstant(Val: Ext == AArch64_AM::SXTW, DL: dl,
1467 VT: MVT::i32);
1468 if (isWorthFoldingAddr(V: RHS, Size))
1469 return true;
1470 }
1471
1472 return false;
1473}
1474
1475// Check if the given immediate is preferred by ADD. If an immediate can be
1476// encoded in an ADD, or it can be encoded in an "ADD LSL #12" and can not be
1477// encoded by one MOVZ, return true.
1478static bool isPreferredADD(int64_t ImmOff) {
1479 // Constant in [0x0, 0xfff] can be encoded in ADD.
1480 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
1481 return true;
1482 // Check if it can be encoded in an "ADD LSL #12".
1483 if ((ImmOff & 0xffffffffff000fffLL) == 0x0LL)
1484 // As a single MOVZ is faster than a "ADD of LSL #12", ignore such constant.
1485 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
1486 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
1487 return false;
1488}
1489
1490bool AArch64DAGToDAGISel::SelectAddrModeXRO(SDValue N, unsigned Size,
1491 SDValue &Base, SDValue &Offset,
1492 SDValue &SignExtend,
1493 SDValue &DoShift) {
1494 if (N.getOpcode() != ISD::ADD)
1495 return false;
1496 SDValue LHS = N.getOperand(i: 0);
1497 SDValue RHS = N.getOperand(i: 1);
1498 SDLoc DL(N);
1499
1500 // Check if this particular node is reused in any non-memory related
1501 // operation. If yes, do not try to fold this node into the address
1502 // computation, since the computation will be kept.
1503 const SDNode *Node = N.getNode();
1504 for (SDNode *UI : Node->users()) {
1505 if (!isMemOpOrPrefetch(N: UI))
1506 return false;
1507 }
1508
1509 // Watch out if RHS is a wide immediate, it can not be selected into
1510 // [BaseReg+Imm] addressing mode. Also it may not be able to be encoded into
1511 // ADD/SUB. Instead it will use [BaseReg + 0] address mode and generate
1512 // instructions like:
1513 // MOV X0, WideImmediate
1514 // ADD X1, BaseReg, X0
1515 // LDR X2, [X1, 0]
1516 // For such situation, using [BaseReg, XReg] addressing mode can save one
1517 // ADD/SUB:
1518 // MOV X0, WideImmediate
1519 // LDR X2, [BaseReg, X0]
1520 if (isa<ConstantSDNode>(Val: RHS)) {
1521 int64_t ImmOff = (int64_t)RHS->getAsZExtVal();
1522 // Skip the immediate can be selected by load/store addressing mode.
1523 // Also skip the immediate can be encoded by a single ADD (SUB is also
1524 // checked by using -ImmOff).
1525 if (isValidAsScaledImmediate(Offset: ImmOff, Range: 0x1000, Size) ||
1526 isPreferredADD(ImmOff) || isPreferredADD(ImmOff: -ImmOff))
1527 return false;
1528
1529 SDValue Ops[] = { RHS };
1530 SDNode *MOVI =
1531 CurDAG->getMachineNode(Opcode: AArch64::MOVi64imm, dl: DL, VT: MVT::i64, Ops);
1532 SDValue MOVIV = SDValue(MOVI, 0);
1533 // This ADD of two X register will be selected into [Reg+Reg] mode.
1534 N = CurDAG->getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: LHS, N2: MOVIV);
1535 }
1536
1537 // Remember if it is worth folding N when it produces extended register.
1538 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(V: N, Size);
1539
1540 // Try to match a shifted extend on the RHS.
1541 if (IsExtendedRegisterWorthFolding && RHS.getOpcode() == ISD::SHL &&
1542 SelectExtendedSHL(N: RHS, Size, WantExtend: false, Offset, SignExtend)) {
1543 Base = LHS;
1544 DoShift = CurDAG->getTargetConstant(Val: true, DL, VT: MVT::i32);
1545 return true;
1546 }
1547
1548 // Try to match a shifted extend on the LHS.
1549 if (IsExtendedRegisterWorthFolding && LHS.getOpcode() == ISD::SHL &&
1550 SelectExtendedSHL(N: LHS, Size, WantExtend: false, Offset, SignExtend)) {
1551 Base = RHS;
1552 DoShift = CurDAG->getTargetConstant(Val: true, DL, VT: MVT::i32);
1553 return true;
1554 }
1555
1556 // Match any non-shifted, non-extend, non-immediate add expression.
1557 Base = LHS;
1558 Offset = RHS;
1559 SignExtend = CurDAG->getTargetConstant(Val: false, DL, VT: MVT::i32);
1560 DoShift = CurDAG->getTargetConstant(Val: false, DL, VT: MVT::i32);
1561 // Reg1 + Reg2 is free: no check needed.
1562 return true;
1563}
1564
1565SDValue AArch64DAGToDAGISel::createDTuple(ArrayRef<SDValue> Regs) {
1566 static const unsigned RegClassIDs[] = {
1567 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
1568 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
1569 AArch64::dsub2, AArch64::dsub3};
1570
1571 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1572}
1573
1574SDValue AArch64DAGToDAGISel::createQTuple(ArrayRef<SDValue> Regs) {
1575 static const unsigned RegClassIDs[] = {
1576 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
1577 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
1578 AArch64::qsub2, AArch64::qsub3};
1579
1580 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1581}
1582
1583SDValue AArch64DAGToDAGISel::createZTuple(ArrayRef<SDValue> Regs) {
1584 static const unsigned RegClassIDs[] = {AArch64::ZPR2RegClassID,
1585 AArch64::ZPR3RegClassID,
1586 AArch64::ZPR4RegClassID};
1587 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1588 AArch64::zsub2, AArch64::zsub3};
1589
1590 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1591}
1592
1593SDValue AArch64DAGToDAGISel::createZMulTuple(ArrayRef<SDValue> Regs) {
1594 assert(Regs.size() == 2 || Regs.size() == 4);
1595
1596 // The createTuple interface requires 3 RegClassIDs for each possible
1597 // tuple type even though we only have them for ZPR2 and ZPR4.
1598 static const unsigned RegClassIDs[] = {AArch64::ZPR2Mul2RegClassID, 0,
1599 AArch64::ZPR4Mul4RegClassID};
1600 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1601 AArch64::zsub2, AArch64::zsub3};
1602 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1603}
1604
1605SDValue AArch64DAGToDAGISel::createTuple(ArrayRef<SDValue> Regs,
1606 const unsigned RegClassIDs[],
1607 const unsigned SubRegs[]) {
1608 // There's no special register-class for a vector-list of 1 element: it's just
1609 // a vector.
1610 if (Regs.size() == 1)
1611 return Regs[0];
1612
1613 assert(Regs.size() >= 2 && Regs.size() <= 4);
1614
1615 SDLoc DL(Regs[0]);
1616
1617 SmallVector<SDValue, 4> Ops;
1618
1619 // First operand of REG_SEQUENCE is the desired RegClass.
1620 Ops.push_back(
1621 Elt: CurDAG->getTargetConstant(Val: RegClassIDs[Regs.size() - 2], DL, VT: MVT::i32));
1622
1623 // Then we get pairs of source & subregister-position for the components.
1624 for (unsigned i = 0; i < Regs.size(); ++i) {
1625 Ops.push_back(Elt: Regs[i]);
1626 Ops.push_back(Elt: CurDAG->getTargetConstant(Val: SubRegs[i], DL, VT: MVT::i32));
1627 }
1628
1629 SDNode *N =
1630 CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL, VT: MVT::Untyped, Ops);
1631 return SDValue(N, 0);
1632}
1633
1634void AArch64DAGToDAGISel::SelectTable(SDNode *N, unsigned NumVecs, unsigned Opc,
1635 bool isExt) {
1636 SDLoc dl(N);
1637 EVT VT = N->getValueType(ResNo: 0);
1638
1639 unsigned ExtOff = isExt;
1640
1641 // Form a REG_SEQUENCE to force register allocation.
1642 unsigned Vec0Off = ExtOff + 1;
1643 SmallVector<SDValue, 4> Regs(N->ops().slice(N: Vec0Off, M: NumVecs));
1644 SDValue RegSeq = createQTuple(Regs);
1645
1646 SmallVector<SDValue, 6> Ops;
1647 if (isExt)
1648 Ops.push_back(Elt: N->getOperand(Num: 1));
1649 Ops.push_back(Elt: RegSeq);
1650 Ops.push_back(Elt: N->getOperand(Num: NumVecs + ExtOff + 1));
1651 ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: Opc, dl, VT, Ops));
1652}
1653
1654static std::tuple<SDValue, SDValue>
1655extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG) {
1656 SDLoc DL(Disc);
1657 SDValue AddrDisc;
1658 SDValue ConstDisc;
1659
1660 // If this is a blend, remember the constant and address discriminators.
1661 // Otherwise, it's either a constant discriminator, or a non-blended
1662 // address discriminator.
1663 if (Disc->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
1664 Disc->getConstantOperandVal(Num: 0) == Intrinsic::ptrauth_blend) {
1665 AddrDisc = Disc->getOperand(Num: 1);
1666 ConstDisc = Disc->getOperand(Num: 2);
1667 } else {
1668 ConstDisc = Disc;
1669 }
1670
1671 // If the constant discriminator (either the blend RHS, or the entire
1672 // discriminator value) isn't a 16-bit constant, bail out, and let the
1673 // discriminator be computed separately.
1674 auto *ConstDiscN = dyn_cast<ConstantSDNode>(Val&: ConstDisc);
1675 if (!ConstDiscN || !isUInt<16>(x: ConstDiscN->getZExtValue()))
1676 return std::make_tuple(args: DAG->getTargetConstant(Val: 0, DL, VT: MVT::i64), args&: Disc);
1677
1678 // If there's no address discriminator, use XZR directly.
1679 if (!AddrDisc)
1680 AddrDisc = DAG->getRegister(Reg: AArch64::XZR, VT: MVT::i64);
1681
1682 return std::make_tuple(
1683 args: DAG->getTargetConstant(Val: ConstDiscN->getZExtValue(), DL, VT: MVT::i64),
1684 args&: AddrDisc);
1685}
1686
1687void AArch64DAGToDAGISel::SelectPtrauthAuth(SDNode *N) {
1688 SDLoc DL(N);
1689 // IntrinsicID is operand #0
1690 SDValue Val = N->getOperand(Num: 1);
1691 SDValue AUTKey = N->getOperand(Num: 2);
1692 SDValue AUTDisc = N->getOperand(Num: 3);
1693
1694 unsigned AUTKeyC = cast<ConstantSDNode>(Val&: AUTKey)->getZExtValue();
1695 AUTKey = CurDAG->getTargetConstant(Val: AUTKeyC, DL, VT: MVT::i64);
1696
1697 SDValue AUTAddrDisc, AUTConstDisc;
1698 std::tie(args&: AUTConstDisc, args&: AUTAddrDisc) =
1699 extractPtrauthBlendDiscriminators(Disc: AUTDisc, DAG: CurDAG);
1700
1701 if (!Subtarget->isX16X17Safer()) {
1702 std::vector<SDValue> Ops = {Val, AUTKey, AUTConstDisc, AUTAddrDisc};
1703 // Copy deactivation symbol if present.
1704 if (N->getNumOperands() > 4)
1705 Ops.push_back(x: N->getOperand(Num: 4));
1706
1707 SDNode *AUT =
1708 CurDAG->getMachineNode(Opcode: AArch64::AUTxMxN, dl: DL, VT1: MVT::i64, VT2: MVT::i64, Ops);
1709 ReplaceNode(F: N, T: AUT);
1710 } else {
1711 SDValue X16Copy = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl: DL,
1712 Reg: AArch64::X16, N: Val, Glue: SDValue());
1713 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, X16Copy.getValue(R: 1)};
1714
1715 SDNode *AUT = CurDAG->getMachineNode(Opcode: AArch64::AUTx16x17, dl: DL, VT: MVT::i64, Ops);
1716 ReplaceNode(F: N, T: AUT);
1717 }
1718}
1719
1720void AArch64DAGToDAGISel::SelectPtrauthResign(SDNode *N) {
1721 SDLoc DL(N);
1722 // IntrinsicID is operand #0, if W_CHAIN it is #1
1723 int OffsetBase = N->getOpcode() == ISD::INTRINSIC_W_CHAIN ? 1 : 0;
1724 SDValue Val = N->getOperand(Num: OffsetBase + 1);
1725 SDValue AUTKey = N->getOperand(Num: OffsetBase + 2);
1726 SDValue AUTDisc = N->getOperand(Num: OffsetBase + 3);
1727 SDValue PACKey = N->getOperand(Num: OffsetBase + 4);
1728 SDValue PACDisc = N->getOperand(Num: OffsetBase + 5);
1729 uint32_t IntNum = N->getConstantOperandVal(Num: OffsetBase + 0);
1730 bool HasLoad = IntNum == Intrinsic::ptrauth_resign_load_relative;
1731
1732 unsigned AUTKeyC = cast<ConstantSDNode>(Val&: AUTKey)->getZExtValue();
1733 unsigned PACKeyC = cast<ConstantSDNode>(Val&: PACKey)->getZExtValue();
1734
1735 AUTKey = CurDAG->getTargetConstant(Val: AUTKeyC, DL, VT: MVT::i64);
1736 PACKey = CurDAG->getTargetConstant(Val: PACKeyC, DL, VT: MVT::i64);
1737
1738 SDValue AUTAddrDisc, AUTConstDisc;
1739 std::tie(args&: AUTConstDisc, args&: AUTAddrDisc) =
1740 extractPtrauthBlendDiscriminators(Disc: AUTDisc, DAG: CurDAG);
1741
1742 SDValue PACAddrDisc, PACConstDisc;
1743 std::tie(args&: PACConstDisc, args&: PACAddrDisc) =
1744 extractPtrauthBlendDiscriminators(Disc: PACDisc, DAG: CurDAG);
1745
1746 SDValue X16Copy = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl: DL,
1747 Reg: AArch64::X16, N: Val, Glue: SDValue());
1748
1749 if (HasLoad) {
1750 SDValue Addend = N->getOperand(Num: OffsetBase + 6);
1751 SDValue IncomingChain = N->getOperand(Num: 0);
1752 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc,
1753 PACKey, PACConstDisc, PACAddrDisc,
1754 Addend, IncomingChain, X16Copy.getValue(R: 1)};
1755
1756 SDNode *AUTRELLOADPAC = CurDAG->getMachineNode(Opcode: AArch64::AUTRELLOADPAC, dl: DL,
1757 VT1: MVT::i64, VT2: MVT::Other, Ops);
1758 ReplaceNode(F: N, T: AUTRELLOADPAC);
1759 } else {
1760 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, PACKey,
1761 PACConstDisc, PACAddrDisc, X16Copy.getValue(R: 1)};
1762
1763 SDNode *AUTPAC = CurDAG->getMachineNode(Opcode: AArch64::AUTPAC, dl: DL, VT: MVT::i64, Ops);
1764 ReplaceNode(F: N, T: AUTPAC);
1765 }
1766}
1767
1768void AArch64DAGToDAGISel::SelectPtrauthResignWithPC(SDNode *N) {
1769 SDLoc DL(N);
1770 SDValue Val = N->getOperand(Num: 1);
1771 SDValue AUTKey = N->getOperand(Num: 2);
1772 SDValue AUTDisc = N->getOperand(Num: 3);
1773 SDValue AUTPC = N->getOperand(Num: 4);
1774 SDValue PACKey = N->getOperand(Num: 5);
1775 SDValue PACDisc = N->getOperand(Num: 6);
1776
1777 unsigned AUTKeyC = cast<ConstantSDNode>(Val&: AUTKey)->getZExtValue();
1778 unsigned PACKeyC = cast<ConstantSDNode>(Val&: PACKey)->getZExtValue();
1779
1780 AUTKey = CurDAG->getTargetConstant(Val: AUTKeyC, DL, VT: MVT::i64);
1781 PACKey = CurDAG->getTargetConstant(Val: PACKeyC, DL, VT: MVT::i64);
1782
1783 SDValue PACAddrDisc, PACConstDisc;
1784 std::tie(args&: PACConstDisc, args&: PACAddrDisc) =
1785 extractPtrauthBlendDiscriminators(Disc: PACDisc, DAG: CurDAG);
1786
1787 SDValue X17Copy = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl: DL,
1788 Reg: AArch64::X17, N: Val, Glue: SDValue());
1789 SDValue X16Copy = CurDAG->getCopyToReg(
1790 Chain: CurDAG->getEntryNode(), dl: DL, Reg: AArch64::X16, N: AUTDisc, Glue: X17Copy.getValue(R: 1));
1791 SDValue X15Copy = CurDAG->getCopyToReg(
1792 Chain: CurDAG->getEntryNode(), dl: DL, Reg: AArch64::X15, N: AUTPC, Glue: X16Copy.getValue(R: 1));
1793
1794 SDValue Ops[] = {AUTKey, PACKey, PACConstDisc, PACAddrDisc,
1795 X15Copy.getValue(R: 1)};
1796 SDNode *AUTPCPAC =
1797 CurDAG->getMachineNode(Opcode: AArch64::AUTPCPAC, dl: DL, VT: MVT::i64, Ops);
1798 ReplaceNode(F: N, T: AUTPCPAC);
1799}
1800
1801bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *N) {
1802 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
1803 if (LD->isUnindexed())
1804 return false;
1805 EVT VT = LD->getMemoryVT();
1806 EVT DstVT = N->getValueType(ResNo: 0);
1807 ISD::MemIndexedMode AM = LD->getAddressingMode();
1808 bool IsPre = AM == ISD::PRE_INC || AM == ISD::PRE_DEC;
1809 ConstantSDNode *OffsetOp = cast<ConstantSDNode>(Val: LD->getOffset());
1810 int OffsetVal = (int)OffsetOp->getZExtValue();
1811
1812 // We're not doing validity checking here. That was done when checking
1813 // if we should mark the load as indexed or not. We're just selecting
1814 // the right instruction.
1815 unsigned Opcode = 0;
1816
1817 ISD::LoadExtType ExtType = LD->getExtensionType();
1818 bool InsertTo64 = false;
1819 if (VT == MVT::i64)
1820 Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
1821 else if (VT == MVT::i32) {
1822 if (ExtType == ISD::NON_EXTLOAD)
1823 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1824 else if (ExtType == ISD::SEXTLOAD)
1825 Opcode = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
1826 else {
1827 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1828 InsertTo64 = true;
1829 // The result of the load is only i32. It's the subreg_to_reg that makes
1830 // it into an i64.
1831 DstVT = MVT::i32;
1832 }
1833 } else if (VT == MVT::i16) {
1834 if (ExtType == ISD::SEXTLOAD) {
1835 if (DstVT == MVT::i64)
1836 Opcode = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
1837 else
1838 Opcode = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
1839 } else {
1840 Opcode = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
1841 InsertTo64 = DstVT == MVT::i64;
1842 // The result of the load is only i32. It's the subreg_to_reg that makes
1843 // it into an i64.
1844 DstVT = MVT::i32;
1845 }
1846 } else if (VT == MVT::i8) {
1847 if (ExtType == ISD::SEXTLOAD) {
1848 if (DstVT == MVT::i64)
1849 Opcode = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
1850 else
1851 Opcode = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
1852 } else {
1853 Opcode = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
1854 InsertTo64 = DstVT == MVT::i64;
1855 // The result of the load is only i32. It's the subreg_to_reg that makes
1856 // it into an i64.
1857 DstVT = MVT::i32;
1858 }
1859 } else if (VT == MVT::f16) {
1860 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1861 } else if (VT == MVT::bf16) {
1862 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1863 } else if (VT == MVT::f32) {
1864 Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
1865 } else if (VT == MVT::f64 ||
1866 (VT.is64BitVector() && Subtarget->isLittleEndian())) {
1867 Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
1868 } else if (VT.is128BitVector() && Subtarget->isLittleEndian()) {
1869 Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
1870 } else if (VT.is64BitVector()) {
1871 if (IsPre || OffsetVal != 8)
1872 return false;
1873 switch (VT.getScalarSizeInBits()) {
1874 case 8:
1875 Opcode = AArch64::LD1Onev8b_POST;
1876 break;
1877 case 16:
1878 Opcode = AArch64::LD1Onev4h_POST;
1879 break;
1880 case 32:
1881 Opcode = AArch64::LD1Onev2s_POST;
1882 break;
1883 case 64:
1884 Opcode = AArch64::LD1Onev1d_POST;
1885 break;
1886 default:
1887 llvm_unreachable("Expected vector element to be a power of 2");
1888 }
1889 } else if (VT.is128BitVector()) {
1890 if (IsPre || OffsetVal != 16)
1891 return false;
1892 switch (VT.getScalarSizeInBits()) {
1893 case 8:
1894 Opcode = AArch64::LD1Onev16b_POST;
1895 break;
1896 case 16:
1897 Opcode = AArch64::LD1Onev8h_POST;
1898 break;
1899 case 32:
1900 Opcode = AArch64::LD1Onev4s_POST;
1901 break;
1902 case 64:
1903 Opcode = AArch64::LD1Onev2d_POST;
1904 break;
1905 default:
1906 llvm_unreachable("Expected vector element to be a power of 2");
1907 }
1908 } else
1909 return false;
1910 SDValue Chain = LD->getChain();
1911 SDValue Base = LD->getBasePtr();
1912 SDLoc dl(N);
1913 // LD1 encodes an immediate offset by using XZR as the offset register.
1914 SDValue Offset = (VT.isVector() && !Subtarget->isLittleEndian())
1915 ? CurDAG->getRegister(Reg: AArch64::XZR, VT: MVT::i64)
1916 : CurDAG->getTargetConstant(Val: OffsetVal, DL: dl, VT: MVT::i64);
1917 SDValue Ops[] = { Base, Offset, Chain };
1918 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, VT1: MVT::i64, VT2: DstVT,
1919 VT3: MVT::Other, Ops);
1920
1921 // Transfer memoperands.
1922 MachineMemOperand *MemOp = cast<MemSDNode>(Val: N)->getMemOperand();
1923 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Res), NewMemRefs: {MemOp});
1924
1925 // Either way, we're replacing the node, so tell the caller that.
1926 SDValue LoadedVal = SDValue(Res, 1);
1927 if (InsertTo64) {
1928 SDValue SubReg = CurDAG->getTargetConstant(Val: AArch64::sub_32, DL: dl, VT: MVT::i32);
1929 LoadedVal = SDValue(CurDAG->getMachineNode(Opcode: AArch64::SUBREG_TO_REG, dl,
1930 VT: MVT::i64, Op1: LoadedVal, Op2: SubReg),
1931 0);
1932 }
1933
1934 ReplaceUses(F: SDValue(N, 0), T: LoadedVal);
1935 ReplaceUses(F: SDValue(N, 1), T: SDValue(Res, 0));
1936 ReplaceUses(F: SDValue(N, 2), T: SDValue(Res, 2));
1937 CurDAG->RemoveDeadNode(N);
1938 return true;
1939}
1940
1941void AArch64DAGToDAGISel::SelectLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
1942 unsigned SubRegIdx) {
1943 SDLoc dl(N);
1944 EVT VT = N->getValueType(ResNo: 0);
1945 SDValue Chain = N->getOperand(Num: 0);
1946
1947 SDValue Ops[] = {N->getOperand(Num: 2), // Mem operand;
1948 Chain};
1949
1950 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1951
1952 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
1953 SDValue SuperReg = SDValue(Ld, 0);
1954 for (unsigned i = 0; i < NumVecs; ++i)
1955 ReplaceUses(F: SDValue(N, i),
1956 T: CurDAG->getTargetExtractSubreg(SRIdx: SubRegIdx + i, DL: dl, VT, Operand: SuperReg));
1957
1958 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 1));
1959
1960 // Transfer memoperands. In the case of AArch64::LD64B, there won't be one,
1961 // because it's too simple to have needed special treatment during lowering.
1962 if (auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(Val: N)) {
1963 MachineMemOperand *MemOp = MemIntr->getMemOperand();
1964 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Ld), NewMemRefs: {MemOp});
1965 }
1966
1967 CurDAG->RemoveDeadNode(N);
1968}
1969
1970void AArch64DAGToDAGISel::SelectPostLoad(SDNode *N, unsigned NumVecs,
1971 unsigned Opc, unsigned SubRegIdx) {
1972 SDLoc dl(N);
1973 EVT VT = N->getValueType(ResNo: 0);
1974 SDValue Chain = N->getOperand(Num: 0);
1975
1976 SDValue Ops[] = {N->getOperand(Num: 1), // Mem operand
1977 N->getOperand(Num: 2), // Incremental
1978 Chain};
1979
1980 const EVT ResTys[] = {MVT::i64, // Type of the write back register
1981 MVT::Untyped, MVT::Other};
1982
1983 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
1984
1985 // Update uses of write back register
1986 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 0));
1987
1988 // Update uses of vector list
1989 SDValue SuperReg = SDValue(Ld, 1);
1990 if (NumVecs == 1)
1991 ReplaceUses(F: SDValue(N, 0), T: SuperReg);
1992 else
1993 for (unsigned i = 0; i < NumVecs; ++i)
1994 ReplaceUses(F: SDValue(N, i),
1995 T: CurDAG->getTargetExtractSubreg(SRIdx: SubRegIdx + i, DL: dl, VT, Operand: SuperReg));
1996
1997 // Transfer memoperands.
1998 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
1999 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Ld), NewMemRefs: {MemOp});
2000
2001 // Update the chain
2002 ReplaceUses(F: SDValue(N, NumVecs + 1), T: SDValue(Ld, 2));
2003 CurDAG->RemoveDeadNode(N);
2004}
2005
2006/// Optimize \param OldBase and \param OldOffset selecting the best addressing
2007/// mode. Returns a tuple consisting of an Opcode, an SDValue representing the
2008/// new Base and an SDValue representing the new offset.
2009std::tuple<unsigned, SDValue, SDValue>
2010AArch64DAGToDAGISel::findAddrModeSVELoadStore(SDNode *N, unsigned Opc_rr,
2011 unsigned Opc_ri,
2012 const SDValue &OldBase,
2013 const SDValue &OldOffset,
2014 unsigned Scale) {
2015 SDValue NewBase = OldBase;
2016 SDValue NewOffset = OldOffset;
2017 // Detect a possible Reg+Imm addressing mode.
2018 const bool IsRegImm = SelectAddrModeIndexedSVE</*Min=*/-8, /*Max=*/7>(
2019 Root: N, N: OldBase, Base&: NewBase, OffImm&: NewOffset);
2020
2021 // Detect a possible reg+reg addressing mode, but only if we haven't already
2022 // detected a Reg+Imm one.
2023 const bool IsRegReg =
2024 !IsRegImm && SelectSVERegRegAddrMode(N: OldBase, Scale, Base&: NewBase, Offset&: NewOffset);
2025
2026 // Select the instruction.
2027 return std::make_tuple(args&: IsRegReg ? Opc_rr : Opc_ri, args&: NewBase, args&: NewOffset);
2028}
2029
2030enum class SelectTypeKind {
2031 Int1 = 0,
2032 Int = 1,
2033 FP = 2,
2034 AnyType = 3,
2035};
2036
2037/// This function selects an opcode from a list of opcodes, which is
2038/// expected to be the opcode for { 8-bit, 16-bit, 32-bit, 64-bit }
2039/// element types, in this order.
2040template <SelectTypeKind Kind>
2041static unsigned SelectOpcodeFromVT(EVT VT, ArrayRef<unsigned> Opcodes) {
2042 // Only match scalable vector VTs
2043 if (!VT.isScalableVector())
2044 return 0;
2045
2046 EVT EltVT = VT.getVectorElementType();
2047 unsigned Key = VT.getVectorMinNumElements();
2048 switch (Kind) {
2049 case SelectTypeKind::AnyType:
2050 break;
2051 case SelectTypeKind::Int:
2052 if (EltVT != MVT::i8 && EltVT != MVT::i16 && EltVT != MVT::i32 &&
2053 EltVT != MVT::i64)
2054 return 0;
2055 break;
2056 case SelectTypeKind::Int1:
2057 if (EltVT != MVT::i1)
2058 return 0;
2059 break;
2060 case SelectTypeKind::FP:
2061 if (EltVT == MVT::bf16)
2062 Key = 16;
2063 else if (EltVT != MVT::bf16 && EltVT != MVT::f16 && EltVT != MVT::f32 &&
2064 EltVT != MVT::f64)
2065 return 0;
2066 break;
2067 }
2068
2069 unsigned Offset;
2070 switch (Key) {
2071 case 16: // 8-bit or bf16
2072 Offset = 0;
2073 break;
2074 case 8: // 16-bit
2075 Offset = 1;
2076 break;
2077 case 4: // 32-bit
2078 Offset = 2;
2079 break;
2080 case 2: // 64-bit
2081 Offset = 3;
2082 break;
2083 default:
2084 return 0;
2085 }
2086
2087 return (Opcodes.size() <= Offset) ? 0 : Opcodes[Offset];
2088}
2089
2090// This function is almost identical to SelectWhilePair, but has an
2091// extra check on the range of the immediate operand.
2092// TODO: Merge these two functions together at some point?
2093void AArch64DAGToDAGISel::SelectPExtPair(SDNode *N, unsigned Opc) {
2094 // Immediate can be either 0 or 1.
2095 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2)))
2096 if (Imm->getZExtValue() > 1)
2097 return;
2098
2099 SDLoc DL(N);
2100 EVT VT = N->getValueType(ResNo: 0);
2101 SDValue Ops[] = {N->getOperand(Num: 1), N->getOperand(Num: 2)};
2102 SDNode *WhilePair = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Ops);
2103 SDValue SuperReg = SDValue(WhilePair, 0);
2104
2105 for (unsigned I = 0; I < 2; ++I)
2106 ReplaceUses(F: SDValue(N, I), T: CurDAG->getTargetExtractSubreg(
2107 SRIdx: AArch64::psub0 + I, DL, VT, Operand: SuperReg));
2108
2109 CurDAG->RemoveDeadNode(N);
2110}
2111
2112void AArch64DAGToDAGISel::SelectWhilePair(SDNode *N, unsigned Opc) {
2113 SDLoc DL(N);
2114 EVT VT = N->getValueType(ResNo: 0);
2115
2116 SDValue Ops[] = {N->getOperand(Num: 1), N->getOperand(Num: 2)};
2117
2118 SDNode *WhilePair = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Ops);
2119 SDValue SuperReg = SDValue(WhilePair, 0);
2120
2121 for (unsigned I = 0; I < 2; ++I)
2122 ReplaceUses(F: SDValue(N, I), T: CurDAG->getTargetExtractSubreg(
2123 SRIdx: AArch64::psub0 + I, DL, VT, Operand: SuperReg));
2124
2125 CurDAG->RemoveDeadNode(N);
2126}
2127
2128void AArch64DAGToDAGISel::SelectCVTIntrinsic(SDNode *N, unsigned NumVecs,
2129 unsigned Opcode) {
2130 EVT VT = N->getValueType(ResNo: 0);
2131 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2132 SDValue Ops = createZTuple(Regs);
2133 SDLoc DL(N);
2134 SDNode *Intrinsic = CurDAG->getMachineNode(Opcode, dl: DL, VT: MVT::Untyped, Op1: Ops);
2135 SDValue SuperReg = SDValue(Intrinsic, 0);
2136 for (unsigned i = 0; i < NumVecs; ++i)
2137 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2138 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2139
2140 CurDAG->RemoveDeadNode(N);
2141}
2142
2143void AArch64DAGToDAGISel::SelectCVTIntrinsicFP8(SDNode *N, unsigned NumVecs,
2144 unsigned Opcode) {
2145 SDLoc DL(N);
2146 EVT VT = N->getValueType(ResNo: 0);
2147 SmallVector<SDValue, 4> Ops(N->op_begin() + 2, N->op_end());
2148 Ops.push_back(/*Chain*/ Elt: N->getOperand(Num: 0));
2149
2150 SDNode *Instruction =
2151 CurDAG->getMachineNode(Opcode, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2152 SDValue SuperReg = SDValue(Instruction, 0);
2153
2154 for (unsigned i = 0; i < NumVecs; ++i)
2155 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2156 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2157
2158 // Copy chain
2159 unsigned ChainIdx = NumVecs;
2160 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Instruction, 1));
2161 CurDAG->RemoveDeadNode(N);
2162}
2163
2164void AArch64DAGToDAGISel::SelectDestructiveMultiIntrinsic(SDNode *N,
2165 unsigned NumVecs,
2166 bool IsZmMulti,
2167 unsigned Opcode,
2168 bool HasPred) {
2169 assert(Opcode != 0 && "Unexpected opcode");
2170
2171 SDLoc DL(N);
2172 EVT VT = N->getValueType(ResNo: 0);
2173 SDUse *OpsIter = N->op_begin() + 1; // Skip intrinsic ID
2174 SmallVector<SDValue, 4> Ops;
2175
2176 auto GetMultiVecOperand = [&]() {
2177 SmallVector<SDValue, 4> Regs(OpsIter, OpsIter + NumVecs);
2178 OpsIter += NumVecs;
2179 return createZMulTuple(Regs);
2180 };
2181
2182 if (HasPred)
2183 Ops.push_back(Elt: *OpsIter++);
2184
2185 Ops.push_back(Elt: GetMultiVecOperand());
2186 if (IsZmMulti)
2187 Ops.push_back(Elt: GetMultiVecOperand());
2188 else
2189 Ops.push_back(Elt: *OpsIter++);
2190
2191 // Append any remaining operands.
2192 Ops.append(in_start: OpsIter, in_end: N->op_end());
2193 SDNode *Intrinsic;
2194 Intrinsic = CurDAG->getMachineNode(Opcode, dl: DL, VT: MVT::Untyped, Ops);
2195 SDValue SuperReg = SDValue(Intrinsic, 0);
2196 for (unsigned i = 0; i < NumVecs; ++i)
2197 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2198 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2199
2200 CurDAG->RemoveDeadNode(N);
2201}
2202
2203void AArch64DAGToDAGISel::SelectPredicatedLoad(SDNode *N, unsigned NumVecs,
2204 unsigned Scale, unsigned Opc_ri,
2205 unsigned Opc_rr, bool IsIntr) {
2206 assert(Scale < 5 && "Invalid scaling value.");
2207 SDLoc DL(N);
2208 EVT VT = N->getValueType(ResNo: 0);
2209 SDValue Chain = N->getOperand(Num: 0);
2210
2211 // Optimize addressing mode.
2212 SDValue Base, Offset;
2213 unsigned Opc;
2214 std::tie(args&: Opc, args&: Base, args&: Offset) = findAddrModeSVELoadStore(
2215 N, Opc_rr, Opc_ri, OldBase: N->getOperand(Num: IsIntr ? 3 : 2),
2216 OldOffset: CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64), Scale);
2217
2218 SDValue Ops[] = {N->getOperand(Num: IsIntr ? 2 : 1), // Predicate
2219 Base, // Memory operand
2220 Offset, Chain};
2221
2222 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2223
2224 SDNode *Load = CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: ResTys, Ops);
2225 SDValue SuperReg = SDValue(Load, 0);
2226 for (unsigned i = 0; i < NumVecs; ++i)
2227 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2228 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2229
2230 // Copy chain
2231 unsigned ChainIdx = NumVecs;
2232 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Load, 1));
2233 CurDAG->RemoveDeadNode(N);
2234}
2235
2236void AArch64DAGToDAGISel::SelectContiguousMultiVectorLoad(SDNode *N,
2237 unsigned NumVecs,
2238 unsigned Scale,
2239 unsigned Opc_ri,
2240 unsigned Opc_rr) {
2241 assert(Scale < 4 && "Invalid scaling value.");
2242 SDLoc DL(N);
2243 EVT VT = N->getValueType(ResNo: 0);
2244 SDValue Chain = N->getOperand(Num: 0);
2245
2246 SDValue PNg = N->getOperand(Num: 2);
2247 SDValue Base = N->getOperand(Num: 3);
2248 SDValue Offset = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64);
2249 unsigned Opc;
2250 std::tie(args&: Opc, args&: Base, args&: Offset) =
2251 findAddrModeSVELoadStore(N, Opc_rr, Opc_ri, OldBase: Base, OldOffset: Offset, Scale);
2252
2253 SDValue Ops[] = {PNg, // Predicate-as-counter
2254 Base, // Memory operand
2255 Offset, Chain};
2256
2257 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2258
2259 SDNode *Load = CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: ResTys, Ops);
2260 SDValue SuperReg = SDValue(Load, 0);
2261 for (unsigned i = 0; i < NumVecs; ++i)
2262 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2263 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2264
2265 // Copy chain
2266 unsigned ChainIdx = NumVecs;
2267 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Load, 1));
2268 CurDAG->RemoveDeadNode(N);
2269}
2270
2271void AArch64DAGToDAGISel::SelectFrintFromVT(SDNode *N, unsigned NumVecs,
2272 unsigned Opcode) {
2273 if (N->getValueType(ResNo: 0) != MVT::nxv4f32)
2274 return;
2275 SelectUnaryMultiIntrinsic(N, NumOutVecs: NumVecs, IsTupleInput: true, Opc: Opcode);
2276}
2277
2278void AArch64DAGToDAGISel::SelectMultiVectorLutiLane(SDNode *Node,
2279 unsigned NumOutVecs,
2280 unsigned Opc,
2281 uint32_t MaxImm) {
2282 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 4)))
2283 if (Imm->getZExtValue() > MaxImm)
2284 return;
2285
2286 SDValue ZtValue;
2287 if (!ImmToReg<AArch64::ZT0, 0>(N: Node->getOperand(Num: 2), Imm&: ZtValue))
2288 return;
2289
2290 SDValue Chain = Node->getOperand(Num: 0);
2291 SDValue Ops[] = {ZtValue, Node->getOperand(Num: 3), Node->getOperand(Num: 4), Chain};
2292 SDLoc DL(Node);
2293 EVT VT = Node->getValueType(ResNo: 0);
2294
2295 SDNode *Instruction =
2296 CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2297 SDValue SuperReg = SDValue(Instruction, 0);
2298
2299 for (unsigned I = 0; I < NumOutVecs; ++I)
2300 ReplaceUses(F: SDValue(Node, I), T: CurDAG->getTargetExtractSubreg(
2301 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2302
2303 // Copy chain
2304 unsigned ChainIdx = NumOutVecs;
2305 ReplaceUses(F: SDValue(Node, ChainIdx), T: SDValue(Instruction, 1));
2306 CurDAG->RemoveDeadNode(N: Node);
2307}
2308
2309void AArch64DAGToDAGISel::SelectMultiVectorLuti6LaneX4(SDNode *Node,
2310 unsigned NumIndexVecs) {
2311 assert((NumIndexVecs == 2 || NumIndexVecs == 3) &&
2312 "unexpected number of index vectors");
2313
2314 constexpr unsigned FirstIndexOp = 3;
2315 unsigned ImmOp = FirstIndexOp + NumIndexVecs;
2316 auto *Imm = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: ImmOp));
2317 if (!Imm || Imm->getZExtValue() > 1)
2318 return;
2319
2320 // The luti6 instruction always takes a 2-register Zm index tuple. The x3
2321 // ACLE form provides three index vectors, so the lane selects which adjacent
2322 // pair to use before forming Zm (op 3/4 or op 4/5, with op6 as imm)
2323 unsigned Lane = Imm->getZExtValue();
2324 unsigned IndexOp = FirstIndexOp;
2325 if (NumIndexVecs == 3)
2326 IndexOp += Lane;
2327
2328 SDValue TableTuple = createZTuple(Regs: {Node->getOperand(Num: 1), Node->getOperand(Num: 2)});
2329 SDValue IndexTuple =
2330 createZTuple(Regs: {Node->getOperand(Num: IndexOp), Node->getOperand(Num: IndexOp + 1)});
2331 SDValue Ops[] = {TableTuple, IndexTuple, Node->getOperand(Num: ImmOp)};
2332
2333 SDLoc DL(Node);
2334 EVT VT = Node->getValueType(ResNo: 0);
2335 SDNode *Instruction =
2336 CurDAG->getMachineNode(Opcode: AArch64::LUTI6_4Z2Z2ZI, dl: DL, VT: MVT::Untyped, Ops);
2337 SDValue SuperReg = SDValue(Instruction, 0);
2338
2339 for (unsigned I = 0; I < 4; ++I)
2340 ReplaceUses(F: SDValue(Node, I), T: CurDAG->getTargetExtractSubreg(
2341 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2342
2343 CurDAG->RemoveDeadNode(N: Node);
2344}
2345
2346void AArch64DAGToDAGISel::SelectMultiVectorLuti(SDNode *Node,
2347 unsigned NumOutVecs,
2348 unsigned Opc,
2349 unsigned NumInVecs) {
2350 assert((NumInVecs == 2 || NumInVecs == 3) &&
2351 "unexpected number of input vectors");
2352
2353 SDValue ZtValue;
2354 if (!ImmToReg<AArch64::ZT0, 0>(N: Node->getOperand(Num: 2), Imm&: ZtValue))
2355 return;
2356
2357 SmallVector<SDValue, 4> Regs(Node->ops().slice(N: 3, M: NumInVecs));
2358 SDValue ZTuple = NumInVecs == 3 ? createZTuple(Regs) : createZMulTuple(Regs);
2359 SDValue Ops[] = {ZtValue, ZTuple, Node->getOperand(Num: 0)};
2360
2361 SDLoc DL(Node);
2362 EVT VT = Node->getValueType(ResNo: 0);
2363
2364 SDNode *Instruction =
2365 CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2366 SDValue SuperReg = SDValue(Instruction, 0);
2367
2368 for (unsigned I = 0; I < NumOutVecs; ++I)
2369 ReplaceUses(F: SDValue(Node, I), T: CurDAG->getTargetExtractSubreg(
2370 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2371
2372 ReplaceUses(F: SDValue(Node, NumOutVecs), T: SDValue(Instruction, 1));
2373 CurDAG->RemoveDeadNode(N: Node);
2374}
2375
2376void AArch64DAGToDAGISel::SelectClamp(SDNode *N, unsigned NumVecs,
2377 unsigned Op) {
2378 SDLoc DL(N);
2379 EVT VT = N->getValueType(ResNo: 0);
2380
2381 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2382 SDValue Zd = createZMulTuple(Regs);
2383 SDValue Zn = N->getOperand(Num: 1 + NumVecs);
2384 SDValue Zm = N->getOperand(Num: 2 + NumVecs);
2385
2386 SDValue Ops[] = {Zd, Zn, Zm};
2387
2388 SDNode *Intrinsic = CurDAG->getMachineNode(Opcode: Op, dl: DL, VT: MVT::Untyped, Ops);
2389 SDValue SuperReg = SDValue(Intrinsic, 0);
2390 for (unsigned i = 0; i < NumVecs; ++i)
2391 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2392 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2393
2394 CurDAG->RemoveDeadNode(N);
2395}
2396
2397bool SelectSMETile(unsigned &BaseReg, unsigned TileNum) {
2398 switch (BaseReg) {
2399 default:
2400 return false;
2401 case AArch64::ZA:
2402 case AArch64::ZAB0:
2403 if (TileNum == 0)
2404 break;
2405 return false;
2406 case AArch64::ZAH0:
2407 if (TileNum <= 1)
2408 break;
2409 return false;
2410 case AArch64::ZAS0:
2411 if (TileNum <= 3)
2412 break;
2413 return false;
2414 case AArch64::ZAD0:
2415 if (TileNum <= 7)
2416 break;
2417 return false;
2418 }
2419
2420 BaseReg += TileNum;
2421 return true;
2422}
2423
2424template <unsigned MaxIdx, unsigned Scale>
2425void AArch64DAGToDAGISel::SelectMultiVectorMove(SDNode *N, unsigned NumVecs,
2426 unsigned BaseReg, unsigned Op) {
2427 unsigned TileNum = 0;
2428 if (BaseReg != AArch64::ZA)
2429 TileNum = N->getConstantOperandVal(Num: 2);
2430
2431 if (!SelectSMETile(BaseReg, TileNum))
2432 return;
2433
2434 SDValue SliceBase, Base, Offset;
2435 if (BaseReg == AArch64::ZA)
2436 SliceBase = N->getOperand(Num: 2);
2437 else
2438 SliceBase = N->getOperand(Num: 3);
2439
2440 if (!SelectSMETileSlice(N: SliceBase, MaxSize: MaxIdx, Vector&: Base, Offset, Scale))
2441 return;
2442
2443 SDLoc DL(N);
2444 SDValue SubReg = CurDAG->getRegister(Reg: BaseReg, VT: MVT::Other);
2445 SDValue Ops[] = {SubReg, Base, Offset, /*Chain*/ N->getOperand(Num: 0)};
2446 SDNode *Mov = CurDAG->getMachineNode(Opcode: Op, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2447
2448 EVT VT = N->getValueType(ResNo: 0);
2449 for (unsigned I = 0; I < NumVecs; ++I)
2450 ReplaceUses(F: SDValue(N, I),
2451 T: CurDAG->getTargetExtractSubreg(SRIdx: AArch64::zsub0 + I, DL, VT,
2452 Operand: SDValue(Mov, 0)));
2453 // Copy chain
2454 unsigned ChainIdx = NumVecs;
2455 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Mov, 1));
2456 CurDAG->RemoveDeadNode(N);
2457}
2458
2459void AArch64DAGToDAGISel::SelectMultiVectorMoveZ(SDNode *N, unsigned NumVecs,
2460 unsigned Op, unsigned MaxIdx,
2461 unsigned Scale, unsigned BaseReg) {
2462 // Slice can be in different positions
2463 // The array to vector: llvm.aarch64.sme.readz.<h/v>.<sz>(slice)
2464 // The tile to vector: llvm.aarch64.sme.readz.<h/v>.<sz>(tile, slice)
2465 SDValue SliceBase = N->getOperand(Num: 2);
2466 if (BaseReg != AArch64::ZA)
2467 SliceBase = N->getOperand(Num: 3);
2468
2469 SDValue Base, Offset;
2470 if (!SelectSMETileSlice(N: SliceBase, MaxSize: MaxIdx, Vector&: Base, Offset, Scale))
2471 return;
2472 // The correct Za tile number is computed in Machine Instruction
2473 // See EmitZAInstr
2474 // DAG cannot select Za tile as an output register with ZReg
2475 SDLoc DL(N);
2476 SmallVector<SDValue, 6> Ops;
2477 if (BaseReg != AArch64::ZA )
2478 Ops.push_back(Elt: N->getOperand(Num: 2));
2479 Ops.push_back(Elt: Base);
2480 Ops.push_back(Elt: Offset);
2481 Ops.push_back(Elt: N->getOperand(Num: 0)); //Chain
2482 SDNode *Mov = CurDAG->getMachineNode(Opcode: Op, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2483
2484 EVT VT = N->getValueType(ResNo: 0);
2485 for (unsigned I = 0; I < NumVecs; ++I)
2486 ReplaceUses(F: SDValue(N, I),
2487 T: CurDAG->getTargetExtractSubreg(SRIdx: AArch64::zsub0 + I, DL, VT,
2488 Operand: SDValue(Mov, 0)));
2489
2490 // Copy chain
2491 unsigned ChainIdx = NumVecs;
2492 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Mov, 1));
2493 CurDAG->RemoveDeadNode(N);
2494}
2495
2496void AArch64DAGToDAGISel::SelectUnaryMultiIntrinsic(SDNode *N,
2497 unsigned NumOutVecs,
2498 bool IsTupleInput,
2499 unsigned Opc) {
2500 SDLoc DL(N);
2501 EVT VT = N->getValueType(ResNo: 0);
2502 unsigned NumInVecs = N->getNumOperands() - 1;
2503
2504 SmallVector<SDValue, 6> Ops;
2505 if (IsTupleInput) {
2506 assert((NumInVecs == 2 || NumInVecs == 4) &&
2507 "Don't know how to handle multi-register input!");
2508 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumInVecs));
2509 Ops.push_back(Elt: createZMulTuple(Regs));
2510 } else {
2511 // All intrinsic nodes have the ID as the first operand, hence the "1 + I".
2512 for (unsigned I = 0; I < NumInVecs; I++)
2513 Ops.push_back(Elt: N->getOperand(Num: 1 + I));
2514 }
2515
2516 SDNode *Res = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Ops);
2517 SDValue SuperReg = SDValue(Res, 0);
2518
2519 for (unsigned I = 0; I < NumOutVecs; I++)
2520 ReplaceUses(F: SDValue(N, I), T: CurDAG->getTargetExtractSubreg(
2521 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2522 CurDAG->RemoveDeadNode(N);
2523}
2524
2525void AArch64DAGToDAGISel::SelectStore(SDNode *N, unsigned NumVecs,
2526 unsigned Opc) {
2527 SDLoc dl(N);
2528 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2529
2530 // Form a REG_SEQUENCE to force register allocation.
2531 bool Is128Bit = VT.getSizeInBits() == 128;
2532 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2533 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
2534
2535 SDValue Ops[] = {RegSeq, N->getOperand(Num: NumVecs + 2), N->getOperand(Num: 0)};
2536 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, VT: N->getValueType(ResNo: 0), Ops);
2537
2538 // Transfer memoperands.
2539 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2540 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2541
2542 ReplaceNode(F: N, T: St);
2543}
2544
2545void AArch64DAGToDAGISel::SelectPredicatedStore(SDNode *N, unsigned NumVecs,
2546 unsigned Scale, unsigned Opc_rr,
2547 unsigned Opc_ri) {
2548 SDLoc dl(N);
2549
2550 // Form a REG_SEQUENCE to force register allocation.
2551 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2552 SDValue RegSeq = createZTuple(Regs);
2553
2554 // Optimize addressing mode.
2555 unsigned Opc;
2556 SDValue Offset, Base;
2557 std::tie(args&: Opc, args&: Base, args&: Offset) = findAddrModeSVELoadStore(
2558 N, Opc_rr, Opc_ri, OldBase: N->getOperand(Num: NumVecs + 3),
2559 OldOffset: CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64), Scale);
2560
2561 SDValue Ops[] = {RegSeq, N->getOperand(Num: NumVecs + 2), // predicate
2562 Base, // address
2563 Offset, // offset
2564 N->getOperand(Num: 0)}; // chain
2565 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, VT: N->getValueType(ResNo: 0), Ops);
2566
2567 ReplaceNode(F: N, T: St);
2568}
2569
2570void AArch64DAGToDAGISel::SelectPostStore(SDNode *N, unsigned NumVecs,
2571 unsigned Opc) {
2572 SDLoc dl(N);
2573 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2574 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2575 MVT::Other}; // Type for the Chain
2576
2577 // Form a REG_SEQUENCE to force register allocation.
2578 bool Is128Bit = VT.getSizeInBits() == 128;
2579 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2580 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
2581
2582 SDValue Ops[] = {RegSeq,
2583 N->getOperand(Num: NumVecs + 1), // base register
2584 N->getOperand(Num: NumVecs + 2), // Incremental
2585 N->getOperand(Num: 0)}; // Chain
2586 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2587
2588 // Transfer memoperands.
2589 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2590 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2591
2592 ReplaceNode(F: N, T: St);
2593}
2594
2595namespace {
2596/// WidenVector - Given a value in the V64 register class, produce the
2597/// equivalent value in the V128 register class.
2598class WidenVector {
2599 SelectionDAG &DAG;
2600
2601public:
2602 WidenVector(SelectionDAG &DAG) : DAG(DAG) {}
2603
2604 SDValue operator()(SDValue V64Reg) {
2605 EVT VT = V64Reg.getValueType();
2606 unsigned NarrowSize = VT.getVectorNumElements();
2607 MVT EltTy = VT.getVectorElementType().getSimpleVT();
2608 MVT WideTy = MVT::getVectorVT(VT: EltTy, NumElements: 2 * NarrowSize);
2609 SDLoc DL(V64Reg);
2610
2611 SDValue Undef =
2612 SDValue(DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: WideTy), 0);
2613 return DAG.getTargetInsertSubreg(SRIdx: AArch64::dsub, DL, VT: WideTy, Operand: Undef, Subreg: V64Reg);
2614 }
2615};
2616} // namespace
2617
2618/// NarrowVector - Given a value in the V128 register class, produce the
2619/// equivalent value in the V64 register class.
2620static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG) {
2621 EVT VT = V128Reg.getValueType();
2622 unsigned WideSize = VT.getVectorNumElements();
2623 MVT EltTy = VT.getVectorElementType().getSimpleVT();
2624 MVT NarrowTy = MVT::getVectorVT(VT: EltTy, NumElements: WideSize / 2);
2625
2626 return DAG.getTargetExtractSubreg(SRIdx: AArch64::dsub, DL: SDLoc(V128Reg), VT: NarrowTy,
2627 Operand: V128Reg);
2628}
2629
2630void AArch64DAGToDAGISel::SelectLoadLane(SDNode *N, unsigned NumVecs,
2631 unsigned Opc) {
2632 SDLoc dl(N);
2633 EVT VT = N->getValueType(ResNo: 0);
2634 bool Narrow = VT.getSizeInBits() == 64;
2635
2636 // Form a REG_SEQUENCE to force register allocation.
2637 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2638
2639 if (Narrow)
2640 transform(Range&: Regs, d_first: Regs.begin(),
2641 F: WidenVector(*CurDAG));
2642
2643 SDValue RegSeq = createQTuple(Regs);
2644
2645 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2646
2647 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 2);
2648
2649 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(Val: LaneNo, DL: dl, VT: MVT::i64),
2650 N->getOperand(Num: NumVecs + 3), N->getOperand(Num: 0)};
2651 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2652 SDValue SuperReg = SDValue(Ld, 0);
2653
2654 EVT WideVT = RegSeq.getOperand(i: 1)->getValueType(ResNo: 0);
2655 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2656 AArch64::qsub2, AArch64::qsub3 };
2657 for (unsigned i = 0; i < NumVecs; ++i) {
2658 SDValue NV = CurDAG->getTargetExtractSubreg(SRIdx: QSubs[i], DL: dl, VT: WideVT, Operand: SuperReg);
2659 if (Narrow)
2660 NV = NarrowVector(V128Reg: NV, DAG&: *CurDAG);
2661 ReplaceUses(F: SDValue(N, i), T: NV);
2662 }
2663
2664 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 1));
2665 CurDAG->RemoveDeadNode(N);
2666}
2667
2668void AArch64DAGToDAGISel::SelectPostLoadLane(SDNode *N, unsigned NumVecs,
2669 unsigned Opc) {
2670 SDLoc dl(N);
2671 EVT VT = N->getValueType(ResNo: 0);
2672 bool Narrow = VT.getSizeInBits() == 64;
2673
2674 // Form a REG_SEQUENCE to force register allocation.
2675 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2676
2677 if (Narrow)
2678 transform(Range&: Regs, d_first: Regs.begin(),
2679 F: WidenVector(*CurDAG));
2680
2681 SDValue RegSeq = createQTuple(Regs);
2682
2683 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2684 RegSeq->getValueType(ResNo: 0), MVT::Other};
2685
2686 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 1);
2687
2688 SDValue Ops[] = {RegSeq,
2689 CurDAG->getTargetConstant(Val: LaneNo, DL: dl,
2690 VT: MVT::i64), // Lane Number
2691 N->getOperand(Num: NumVecs + 2), // Base register
2692 N->getOperand(Num: NumVecs + 3), // Incremental
2693 N->getOperand(Num: 0)};
2694 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2695
2696 // Update uses of the write back register
2697 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 0));
2698
2699 // Update uses of the vector list
2700 SDValue SuperReg = SDValue(Ld, 1);
2701 if (NumVecs == 1) {
2702 ReplaceUses(F: SDValue(N, 0),
2703 T: Narrow ? NarrowVector(V128Reg: SuperReg, DAG&: *CurDAG) : SuperReg);
2704 } else {
2705 EVT WideVT = RegSeq.getOperand(i: 1)->getValueType(ResNo: 0);
2706 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2707 AArch64::qsub2, AArch64::qsub3 };
2708 for (unsigned i = 0; i < NumVecs; ++i) {
2709 SDValue NV = CurDAG->getTargetExtractSubreg(SRIdx: QSubs[i], DL: dl, VT: WideVT,
2710 Operand: SuperReg);
2711 if (Narrow)
2712 NV = NarrowVector(V128Reg: NV, DAG&: *CurDAG);
2713 ReplaceUses(F: SDValue(N, i), T: NV);
2714 }
2715 }
2716
2717 // Update the Chain
2718 ReplaceUses(F: SDValue(N, NumVecs + 1), T: SDValue(Ld, 2));
2719 CurDAG->RemoveDeadNode(N);
2720}
2721
2722void AArch64DAGToDAGISel::SelectStoreLane(SDNode *N, unsigned NumVecs,
2723 unsigned Opc) {
2724 SDLoc dl(N);
2725 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2726 bool Narrow = VT.getSizeInBits() == 64;
2727
2728 // Form a REG_SEQUENCE to force register allocation.
2729 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2730
2731 if (Narrow)
2732 transform(Range&: Regs, d_first: Regs.begin(),
2733 F: WidenVector(*CurDAG));
2734
2735 SDValue RegSeq = createQTuple(Regs);
2736
2737 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 2);
2738
2739 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(Val: LaneNo, DL: dl, VT: MVT::i64),
2740 N->getOperand(Num: NumVecs + 3), N->getOperand(Num: 0)};
2741 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, VT: MVT::Other, Ops);
2742
2743 // Transfer memoperands.
2744 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2745 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2746
2747 ReplaceNode(F: N, T: St);
2748}
2749
2750void AArch64DAGToDAGISel::SelectPostStoreLane(SDNode *N, unsigned NumVecs,
2751 unsigned Opc) {
2752 SDLoc dl(N);
2753 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2754 bool Narrow = VT.getSizeInBits() == 64;
2755
2756 // Form a REG_SEQUENCE to force register allocation.
2757 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2758
2759 if (Narrow)
2760 transform(Range&: Regs, d_first: Regs.begin(),
2761 F: WidenVector(*CurDAG));
2762
2763 SDValue RegSeq = createQTuple(Regs);
2764
2765 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2766 MVT::Other};
2767
2768 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 1);
2769
2770 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(Val: LaneNo, DL: dl, VT: MVT::i64),
2771 N->getOperand(Num: NumVecs + 2), // Base Register
2772 N->getOperand(Num: NumVecs + 3), // Incremental
2773 N->getOperand(Num: 0)};
2774 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2775
2776 // Transfer memoperands.
2777 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2778 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2779
2780 ReplaceNode(F: N, T: St);
2781}
2782
2783static bool isBitfieldExtractOpFromAnd(SelectionDAG *CurDAG, SDNode *N,
2784 unsigned &Opc, SDValue &Opd0,
2785 unsigned &LSB, unsigned &MSB,
2786 unsigned NumberOfIgnoredLowBits,
2787 bool BiggerPattern) {
2788 assert(N->getOpcode() == ISD::AND &&
2789 "N must be a AND operation to call this function");
2790
2791 EVT VT = N->getValueType(ResNo: 0);
2792
2793 // Here we can test the type of VT and return false when the type does not
2794 // match, but since it is done prior to that call in the current context
2795 // we turned that into an assert to avoid redundant code.
2796 assert((VT == MVT::i32 || VT == MVT::i64) &&
2797 "Type checking must have been done before calling this function");
2798
2799 // FIXME: simplify-demanded-bits in DAGCombine will probably have
2800 // changed the AND node to a 32-bit mask operation. We'll have to
2801 // undo that as part of the transform here if we want to catch all
2802 // the opportunities.
2803 // Currently the NumberOfIgnoredLowBits argument helps to recover
2804 // from these situations when matching bigger pattern (bitfield insert).
2805
2806 // For unsigned extracts, check for a shift right and mask
2807 uint64_t AndImm = 0;
2808 if (!isOpcWithIntImmediate(N, Opc: ISD::AND, Imm&: AndImm))
2809 return false;
2810
2811 const SDNode *Op0 = N->getOperand(Num: 0).getNode();
2812
2813 // Because of simplify-demanded-bits in DAGCombine, the mask may have been
2814 // simplified. Try to undo that
2815 AndImm |= maskTrailingOnes<uint64_t>(N: NumberOfIgnoredLowBits);
2816
2817 // The immediate is a mask of the low bits iff imm & (imm+1) == 0
2818 if (AndImm & (AndImm + 1))
2819 return false;
2820
2821 bool ClampMSB = false;
2822 uint64_t SrlImm = 0;
2823 // Handle the SRL + ANY_EXTEND case.
2824 if (VT == MVT::i64 && Op0->getOpcode() == ISD::ANY_EXTEND &&
2825 isOpcWithIntImmediate(N: Op0->getOperand(Num: 0).getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
2826 // Extend the incoming operand of the SRL to 64-bit.
2827 Opd0 = Widen(CurDAG, N: Op0->getOperand(Num: 0).getOperand(i: 0));
2828 // Make sure to clamp the MSB so that we preserve the semantics of the
2829 // original operations.
2830 ClampMSB = true;
2831 } else if (VT == MVT::i32 && Op0->getOpcode() == ISD::TRUNCATE &&
2832 isOpcWithIntImmediate(N: Op0->getOperand(Num: 0).getNode(), Opc: ISD::SRL,
2833 Imm&: SrlImm)) {
2834 // If the shift result was truncated, we can still combine them.
2835 Opd0 = Op0->getOperand(Num: 0).getOperand(i: 0);
2836
2837 // Use the type of SRL node.
2838 VT = Opd0->getValueType(ResNo: 0);
2839 } else if (isOpcWithIntImmediate(N: Op0, Opc: ISD::SRL, Imm&: SrlImm)) {
2840 Opd0 = Op0->getOperand(Num: 0);
2841 ClampMSB = (VT == MVT::i32);
2842 } else if (BiggerPattern) {
2843 // Let's pretend a 0 shift right has been performed.
2844 // The resulting code will be at least as good as the original one
2845 // plus it may expose more opportunities for bitfield insert pattern.
2846 // FIXME: Currently we limit this to the bigger pattern, because
2847 // some optimizations expect AND and not UBFM.
2848 Opd0 = N->getOperand(Num: 0);
2849 } else
2850 return false;
2851
2852 // Bail out on large immediates. This happens when no proper
2853 // combining/constant folding was performed.
2854 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.getSizeInBits())) {
2855 LLVM_DEBUG(
2856 (dbgs() << N
2857 << ": Found large shift immediate, this should not happen\n"));
2858 return false;
2859 }
2860
2861 LSB = SrlImm;
2862 MSB = SrlImm +
2863 (VT == MVT::i32 ? llvm::countr_one<uint32_t>(Value: AndImm)
2864 : llvm::countr_one<uint64_t>(Value: AndImm)) -
2865 1;
2866 if (ClampMSB)
2867 // Since we're moving the extend before the right shift operation, we need
2868 // to clamp the MSB to make sure we don't shift in undefined bits instead of
2869 // the zeros which would get shifted in with the original right shift
2870 // operation.
2871 MSB = MSB > 31 ? 31 : MSB;
2872
2873 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2874 return true;
2875}
2876
2877static bool isBitfieldExtractOpFromSExtInReg(SDNode *N, unsigned &Opc,
2878 SDValue &Opd0, unsigned &Immr,
2879 unsigned &Imms) {
2880 assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG);
2881
2882 EVT VT = N->getValueType(ResNo: 0);
2883 unsigned BitWidth = VT.getSizeInBits();
2884 assert((VT == MVT::i32 || VT == MVT::i64) &&
2885 "Type checking must have been done before calling this function");
2886
2887 SDValue Op = N->getOperand(Num: 0);
2888 if (Op->getOpcode() == ISD::TRUNCATE) {
2889 Op = Op->getOperand(Num: 0);
2890 VT = Op->getValueType(ResNo: 0);
2891 BitWidth = VT.getSizeInBits();
2892 }
2893
2894 uint64_t ShiftImm;
2895 if (!isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::SRL, Imm&: ShiftImm) &&
2896 !isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::SRA, Imm&: ShiftImm))
2897 return false;
2898
2899 unsigned Width = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getSizeInBits();
2900 if (ShiftImm + Width > BitWidth)
2901 return false;
2902
2903 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri;
2904 Opd0 = Op.getOperand(i: 0);
2905 Immr = ShiftImm;
2906 Imms = ShiftImm + Width - 1;
2907 return true;
2908}
2909
2910static bool isSeveralBitsExtractOpFromShr(SDNode *N, unsigned &Opc,
2911 SDValue &Opd0, unsigned &LSB,
2912 unsigned &MSB) {
2913 // We are looking for the following pattern which basically extracts several
2914 // continuous bits from the source value and places it from the LSB of the
2915 // destination value, all other bits of the destination value or set to zero:
2916 //
2917 // Value2 = AND Value, MaskImm
2918 // SRL Value2, ShiftImm
2919 //
2920 // with MaskImm >> ShiftImm to search for the bit width.
2921 //
2922 // This gets selected into a single UBFM:
2923 //
2924 // UBFM Value, ShiftImm, Log2_64(MaskImm)
2925 //
2926
2927 if (N->getOpcode() != ISD::SRL)
2928 return false;
2929
2930 uint64_t AndMask = 0;
2931 if (!isOpcWithIntImmediate(N: N->getOperand(Num: 0).getNode(), Opc: ISD::AND, Imm&: AndMask))
2932 return false;
2933
2934 Opd0 = N->getOperand(Num: 0).getOperand(i: 0);
2935
2936 uint64_t SrlImm = 0;
2937 if (!isIntImmediate(N: N->getOperand(Num: 1), Imm&: SrlImm))
2938 return false;
2939
2940 // Check whether we really have several bits extract here.
2941 if (!isMask_64(Value: AndMask >> SrlImm))
2942 return false;
2943
2944 Opc = N->getValueType(ResNo: 0) == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2945 LSB = SrlImm;
2946 MSB = llvm::Log2_64(Value: AndMask);
2947 return true;
2948}
2949
2950static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0,
2951 unsigned &Immr, unsigned &Imms,
2952 bool BiggerPattern) {
2953 assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) &&
2954 "N must be a SHR/SRA operation to call this function");
2955
2956 EVT VT = N->getValueType(ResNo: 0);
2957
2958 // Here we can test the type of VT and return false when the type does not
2959 // match, but since it is done prior to that call in the current context
2960 // we turned that into an assert to avoid redundant code.
2961 assert((VT == MVT::i32 || VT == MVT::i64) &&
2962 "Type checking must have been done before calling this function");
2963
2964 // Check for AND + SRL doing several bits extract.
2965 if (isSeveralBitsExtractOpFromShr(N, Opc, Opd0, LSB&: Immr, MSB&: Imms))
2966 return true;
2967
2968 // We're looking for a shift of a shift.
2969 uint64_t ShlImm = 0;
2970 uint64_t TruncBits = 0;
2971 if (isOpcWithIntImmediate(N: N->getOperand(Num: 0).getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
2972 Opd0 = N->getOperand(Num: 0).getOperand(i: 0);
2973 } else if (VT == MVT::i32 && N->getOpcode() == ISD::SRL &&
2974 N->getOperand(Num: 0).getNode()->getOpcode() == ISD::TRUNCATE) {
2975 // We are looking for a shift of truncate. Truncate from i64 to i32 could
2976 // be considered as setting high 32 bits as zero. Our strategy here is to
2977 // always generate 64bit UBFM. This consistency will help the CSE pass
2978 // later find more redundancy.
2979 Opd0 = N->getOperand(Num: 0).getOperand(i: 0);
2980 TruncBits = Opd0->getValueType(ResNo: 0).getSizeInBits() - VT.getSizeInBits();
2981 VT = Opd0.getValueType();
2982 assert(VT == MVT::i64 && "the promoted type should be i64");
2983 } else if (BiggerPattern) {
2984 // Let's pretend a 0 shift left has been performed.
2985 // FIXME: Currently we limit this to the bigger pattern case,
2986 // because some optimizations expect AND and not UBFM
2987 Opd0 = N->getOperand(Num: 0);
2988 } else
2989 return false;
2990
2991 // Missing combines/constant folding may have left us with strange
2992 // constants.
2993 if (ShlImm >= VT.getSizeInBits()) {
2994 LLVM_DEBUG(
2995 (dbgs() << N
2996 << ": Found large shift immediate, this should not happen\n"));
2997 return false;
2998 }
2999
3000 uint64_t SrlImm = 0;
3001 if (!isIntImmediate(N: N->getOperand(Num: 1), Imm&: SrlImm))
3002 return false;
3003
3004 assert(SrlImm > 0 && SrlImm < VT.getSizeInBits() &&
3005 "bad amount in shift node!");
3006 int immr = SrlImm - ShlImm;
3007 Immr = immr < 0 ? immr + VT.getSizeInBits() : immr;
3008 Imms = VT.getSizeInBits() - ShlImm - TruncBits - 1;
3009 // SRA requires a signed extraction
3010 if (VT == MVT::i32)
3011 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
3012 else
3013 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri;
3014 return true;
3015}
3016
3017bool AArch64DAGToDAGISel::tryBitfieldExtractOpFromSExt(SDNode *N) {
3018 assert(N->getOpcode() == ISD::SIGN_EXTEND);
3019
3020 EVT VT = N->getValueType(ResNo: 0);
3021 EVT NarrowVT = N->getOperand(Num: 0)->getValueType(ResNo: 0);
3022 if (VT != MVT::i64 || NarrowVT != MVT::i32)
3023 return false;
3024
3025 uint64_t ShiftImm;
3026 SDValue Op = N->getOperand(Num: 0);
3027 if (!isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::SRA, Imm&: ShiftImm))
3028 return false;
3029
3030 SDLoc dl(N);
3031 // Extend the incoming operand of the shift to 64-bits.
3032 SDValue Opd0 = Widen(CurDAG, N: Op.getOperand(i: 0));
3033 unsigned Immr = ShiftImm;
3034 unsigned Imms = NarrowVT.getSizeInBits() - 1;
3035 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Val: Immr, DL: dl, VT),
3036 CurDAG->getTargetConstant(Val: Imms, DL: dl, VT)};
3037 CurDAG->SelectNodeTo(N, MachineOpc: AArch64::SBFMXri, VT, Ops);
3038 return true;
3039}
3040
3041static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc,
3042 SDValue &Opd0, unsigned &Immr, unsigned &Imms,
3043 unsigned NumberOfIgnoredLowBits = 0,
3044 bool BiggerPattern = false) {
3045 if (N->getValueType(ResNo: 0) != MVT::i32 && N->getValueType(ResNo: 0) != MVT::i64)
3046 return false;
3047
3048 switch (N->getOpcode()) {
3049 default:
3050 if (!N->isMachineOpcode())
3051 return false;
3052 break;
3053 case ISD::AND:
3054 return isBitfieldExtractOpFromAnd(CurDAG, N, Opc, Opd0, LSB&: Immr, MSB&: Imms,
3055 NumberOfIgnoredLowBits, BiggerPattern);
3056 case ISD::SRL:
3057 case ISD::SRA:
3058 return isBitfieldExtractOpFromShr(N, Opc, Opd0, Immr, Imms, BiggerPattern);
3059
3060 case ISD::SIGN_EXTEND_INREG:
3061 return isBitfieldExtractOpFromSExtInReg(N, Opc, Opd0, Immr, Imms);
3062 }
3063
3064 unsigned NOpc = N->getMachineOpcode();
3065 switch (NOpc) {
3066 default:
3067 return false;
3068 case AArch64::SBFMWri:
3069 case AArch64::UBFMWri:
3070 case AArch64::SBFMXri:
3071 case AArch64::UBFMXri:
3072 Opc = NOpc;
3073 Opd0 = N->getOperand(Num: 0);
3074 Immr = N->getConstantOperandVal(Num: 1);
3075 Imms = N->getConstantOperandVal(Num: 2);
3076 return true;
3077 }
3078 // Unreachable
3079 return false;
3080}
3081
3082bool AArch64DAGToDAGISel::tryBitfieldExtractOp(SDNode *N) {
3083 unsigned Opc, Immr, Imms;
3084 SDValue Opd0;
3085 if (!isBitfieldExtractOp(CurDAG, N, Opc, Opd0, Immr, Imms))
3086 return false;
3087
3088 EVT VT = N->getValueType(ResNo: 0);
3089 SDLoc dl(N);
3090
3091 // If the bit extract operation is 64bit but the original type is 32bit, we
3092 // need to add one EXTRACT_SUBREG.
3093 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) {
3094 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Val: Immr, DL: dl, VT: MVT::i64),
3095 CurDAG->getTargetConstant(Val: Imms, DL: dl, VT: MVT::i64)};
3096
3097 SDNode *BFM = CurDAG->getMachineNode(Opcode: Opc, dl, VT: MVT::i64, Ops: Ops64);
3098 SDValue Inner = CurDAG->getTargetExtractSubreg(SRIdx: AArch64::sub_32, DL: dl,
3099 VT: MVT::i32, Operand: SDValue(BFM, 0));
3100 ReplaceNode(F: N, T: Inner.getNode());
3101 return true;
3102 }
3103
3104 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Val: Immr, DL: dl, VT),
3105 CurDAG->getTargetConstant(Val: Imms, DL: dl, VT)};
3106 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
3107 return true;
3108}
3109
3110/// Does DstMask form a complementary pair with the mask provided by
3111/// BitsToBeInserted, suitable for use in a BFI instruction. Roughly speaking,
3112/// this asks whether DstMask zeroes precisely those bits that will be set by
3113/// the other half.
3114static bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted,
3115 unsigned NumberOfIgnoredHighBits, EVT VT) {
3116 assert((VT == MVT::i32 || VT == MVT::i64) &&
3117 "i32 or i64 mask type expected!");
3118 unsigned BitWidth = VT.getSizeInBits() - NumberOfIgnoredHighBits;
3119
3120 // Enable implicitTrunc as we're intentionally ignoring high bits.
3121 APInt SignificantDstMask =
3122 APInt(BitWidth, DstMask, /*isSigned=*/false, /*implicitTrunc=*/true);
3123 APInt SignificantBitsToBeInserted = BitsToBeInserted.zextOrTrunc(width: BitWidth);
3124
3125 return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
3126 (SignificantDstMask | SignificantBitsToBeInserted).isAllOnes();
3127}
3128
3129// Look for bits that will be useful for later uses.
3130// A bit is consider useless as soon as it is dropped and never used
3131// before it as been dropped.
3132// E.g., looking for useful bit of x
3133// 1. y = x & 0x7
3134// 2. z = y >> 2
3135// After #1, x useful bits are 0x7, then the useful bits of x, live through
3136// y.
3137// After #2, the useful bits of x are 0x4.
3138// However, if x is used on an unpredictable instruction, then all its bits
3139// are useful.
3140// E.g.
3141// 1. y = x & 0x7
3142// 2. z = y >> 2
3143// 3. str x, [@x]
3144static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth = 0);
3145
3146static void getUsefulBitsFromAndWithImmediate(SDValue Op, APInt &UsefulBits,
3147 unsigned Depth) {
3148 uint64_t Imm =
3149 cast<const ConstantSDNode>(Val: Op.getOperand(i: 1).getNode())->getZExtValue();
3150 Imm = AArch64_AM::decodeLogicalImmediate(val: Imm, regSize: UsefulBits.getBitWidth());
3151 UsefulBits &= APInt(UsefulBits.getBitWidth(), Imm);
3152 getUsefulBits(Op, UsefulBits, Depth: Depth + 1);
3153}
3154
3155static void getUsefulBitsFromBitfieldMoveOpd(SDValue Op, APInt &UsefulBits,
3156 uint64_t Imm, uint64_t MSB,
3157 unsigned Depth) {
3158 // inherit the bitwidth value
3159 APInt OpUsefulBits(UsefulBits);
3160 OpUsefulBits = 1;
3161
3162 if (MSB >= Imm) {
3163 OpUsefulBits <<= MSB - Imm + 1;
3164 --OpUsefulBits;
3165 // The interesting part will be in the lower part of the result
3166 getUsefulBits(Op, UsefulBits&: OpUsefulBits, Depth: Depth + 1);
3167 // The interesting part was starting at Imm in the argument
3168 OpUsefulBits <<= Imm;
3169 } else {
3170 OpUsefulBits <<= MSB + 1;
3171 --OpUsefulBits;
3172 // The interesting part will be shifted in the result
3173 OpUsefulBits <<= OpUsefulBits.getBitWidth() - Imm;
3174 getUsefulBits(Op, UsefulBits&: OpUsefulBits, Depth: Depth + 1);
3175 // The interesting part was at zero in the argument
3176 OpUsefulBits.lshrInPlace(ShiftAmt: OpUsefulBits.getBitWidth() - Imm);
3177 }
3178
3179 UsefulBits &= OpUsefulBits;
3180}
3181
3182static void getUsefulBitsFromUBFM(SDValue Op, APInt &UsefulBits,
3183 unsigned Depth) {
3184 uint64_t Imm =
3185 cast<const ConstantSDNode>(Val: Op.getOperand(i: 1).getNode())->getZExtValue();
3186 uint64_t MSB =
3187 cast<const ConstantSDNode>(Val: Op.getOperand(i: 2).getNode())->getZExtValue();
3188
3189 getUsefulBitsFromBitfieldMoveOpd(Op, UsefulBits, Imm, MSB, Depth);
3190}
3191
3192static void getUsefulBitsFromOrWithShiftedReg(SDValue Op, APInt &UsefulBits,
3193 unsigned Depth) {
3194 uint64_t ShiftTypeAndValue =
3195 cast<const ConstantSDNode>(Val: Op.getOperand(i: 2).getNode())->getZExtValue();
3196 APInt Mask(UsefulBits);
3197 Mask.clearAllBits();
3198 Mask.flipAllBits();
3199
3200 if (AArch64_AM::getShiftType(Imm: ShiftTypeAndValue) == AArch64_AM::LSL) {
3201 // Shift Left
3202 uint64_t ShiftAmt = AArch64_AM::getShiftValue(Imm: ShiftTypeAndValue);
3203 Mask <<= ShiftAmt;
3204 getUsefulBits(Op, UsefulBits&: Mask, Depth: Depth + 1);
3205 Mask.lshrInPlace(ShiftAmt);
3206 } else if (AArch64_AM::getShiftType(Imm: ShiftTypeAndValue) == AArch64_AM::LSR) {
3207 // Shift Right
3208 // We do not handle AArch64_AM::ASR, because the sign will change the
3209 // number of useful bits
3210 uint64_t ShiftAmt = AArch64_AM::getShiftValue(Imm: ShiftTypeAndValue);
3211 Mask.lshrInPlace(ShiftAmt);
3212 getUsefulBits(Op, UsefulBits&: Mask, Depth: Depth + 1);
3213 Mask <<= ShiftAmt;
3214 } else
3215 return;
3216
3217 UsefulBits &= Mask;
3218}
3219
3220static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits,
3221 unsigned Depth) {
3222 uint64_t Imm =
3223 cast<const ConstantSDNode>(Val: Op.getOperand(i: 2).getNode())->getZExtValue();
3224 uint64_t MSB =
3225 cast<const ConstantSDNode>(Val: Op.getOperand(i: 3).getNode())->getZExtValue();
3226
3227 APInt OpUsefulBits(UsefulBits);
3228 OpUsefulBits = 1;
3229
3230 APInt ResultUsefulBits(UsefulBits.getBitWidth(), 0);
3231 ResultUsefulBits.flipAllBits();
3232 APInt Mask(UsefulBits.getBitWidth(), 0);
3233
3234 getUsefulBits(Op, UsefulBits&: ResultUsefulBits, Depth: Depth + 1);
3235
3236 if (MSB >= Imm) {
3237 // The instruction is a BFXIL.
3238 uint64_t Width = MSB - Imm + 1;
3239 uint64_t LSB = Imm;
3240
3241 OpUsefulBits <<= Width;
3242 --OpUsefulBits;
3243
3244 if (Op.getOperand(i: 1) == Orig) {
3245 // Copy the low bits from the result to bits starting from LSB.
3246 Mask = ResultUsefulBits & OpUsefulBits;
3247 Mask <<= LSB;
3248 }
3249
3250 if (Op.getOperand(i: 0) == Orig)
3251 // Bits starting from LSB in the input contribute to the result.
3252 Mask |= (ResultUsefulBits & ~OpUsefulBits);
3253 } else {
3254 // The instruction is a BFI.
3255 uint64_t Width = MSB + 1;
3256 uint64_t LSB = UsefulBits.getBitWidth() - Imm;
3257
3258 OpUsefulBits <<= Width;
3259 --OpUsefulBits;
3260 OpUsefulBits <<= LSB;
3261
3262 if (Op.getOperand(i: 1) == Orig) {
3263 // Copy the bits from the result to the zero bits.
3264 Mask = ResultUsefulBits & OpUsefulBits;
3265 Mask.lshrInPlace(ShiftAmt: LSB);
3266 }
3267
3268 if (Op.getOperand(i: 0) == Orig)
3269 Mask |= (ResultUsefulBits & ~OpUsefulBits);
3270 }
3271
3272 UsefulBits &= Mask;
3273}
3274
3275static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits,
3276 SDValue Orig, unsigned Depth) {
3277
3278 // Users of this node should have already been instruction selected
3279 // FIXME: Can we turn that into an assert?
3280 if (!UserNode->isMachineOpcode())
3281 return;
3282
3283 switch (UserNode->getMachineOpcode()) {
3284 default:
3285 return;
3286 case AArch64::ANDSWri:
3287 case AArch64::ANDSXri:
3288 case AArch64::ANDWri:
3289 case AArch64::ANDXri:
3290 // We increment Depth only when we call the getUsefulBits
3291 return getUsefulBitsFromAndWithImmediate(Op: SDValue(UserNode, 0), UsefulBits,
3292 Depth);
3293 case AArch64::UBFMWri:
3294 case AArch64::UBFMXri:
3295 return getUsefulBitsFromUBFM(Op: SDValue(UserNode, 0), UsefulBits, Depth);
3296
3297 case AArch64::ORRWrs:
3298 case AArch64::ORRXrs:
3299 if (UserNode->getOperand(Num: 0) != Orig && UserNode->getOperand(Num: 1) == Orig)
3300 getUsefulBitsFromOrWithShiftedReg(Op: SDValue(UserNode, 0), UsefulBits,
3301 Depth);
3302 return;
3303 case AArch64::BFMWri:
3304 case AArch64::BFMXri:
3305 return getUsefulBitsFromBFM(Op: SDValue(UserNode, 0), Orig, UsefulBits, Depth);
3306
3307 case AArch64::STRBBui:
3308 case AArch64::STURBBi:
3309 if (UserNode->getOperand(Num: 0) != Orig)
3310 return;
3311 UsefulBits &= APInt(UsefulBits.getBitWidth(), 0xff);
3312 return;
3313
3314 case AArch64::STRHHui:
3315 case AArch64::STURHHi:
3316 if (UserNode->getOperand(Num: 0) != Orig)
3317 return;
3318 UsefulBits &= APInt(UsefulBits.getBitWidth(), 0xffff);
3319 return;
3320 }
3321}
3322
3323static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth) {
3324 if (Depth >= SelectionDAG::MaxRecursionDepth)
3325 return;
3326 // Initialize UsefulBits
3327 if (!Depth) {
3328 unsigned Bitwidth = Op.getScalarValueSizeInBits();
3329 // At the beginning, assume every produced bits is useful
3330 UsefulBits = APInt(Bitwidth, 0);
3331 UsefulBits.flipAllBits();
3332 }
3333 APInt UsersUsefulBits(UsefulBits.getBitWidth(), 0);
3334
3335 for (SDNode *Node : Op.getNode()->users()) {
3336 // A use cannot produce useful bits
3337 APInt UsefulBitsForUse = APInt(UsefulBits);
3338 getUsefulBitsForUse(UserNode: Node, UsefulBits&: UsefulBitsForUse, Orig: Op, Depth);
3339 UsersUsefulBits |= UsefulBitsForUse;
3340 }
3341 // UsefulBits contains the produced bits that are meaningful for the
3342 // current definition, thus a user cannot make a bit meaningful at
3343 // this point
3344 UsefulBits &= UsersUsefulBits;
3345}
3346
3347/// Create a machine node performing a notional SHL of Op by ShlAmount. If
3348/// ShlAmount is negative, do a (logical) right-shift instead. If ShlAmount is
3349/// 0, return Op unchanged.
3350static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount) {
3351 if (ShlAmount == 0)
3352 return Op;
3353
3354 EVT VT = Op.getValueType();
3355 SDLoc dl(Op);
3356 unsigned BitWidth = VT.getSizeInBits();
3357 unsigned UBFMOpc = BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
3358
3359 SDNode *ShiftNode;
3360 if (ShlAmount > 0) {
3361 // LSL wD, wN, #Amt == UBFM wD, wN, #32-Amt, #31-Amt
3362 ShiftNode = CurDAG->getMachineNode(
3363 Opcode: UBFMOpc, dl, VT, Op1: Op,
3364 Op2: CurDAG->getTargetConstant(Val: BitWidth - ShlAmount, DL: dl, VT),
3365 Op3: CurDAG->getTargetConstant(Val: BitWidth - 1 - ShlAmount, DL: dl, VT));
3366 } else {
3367 // LSR wD, wN, #Amt == UBFM wD, wN, #Amt, #32-1
3368 assert(ShlAmount < 0 && "expected right shift");
3369 int ShrAmount = -ShlAmount;
3370 ShiftNode = CurDAG->getMachineNode(
3371 Opcode: UBFMOpc, dl, VT, Op1: Op, Op2: CurDAG->getTargetConstant(Val: ShrAmount, DL: dl, VT),
3372 Op3: CurDAG->getTargetConstant(Val: BitWidth - 1, DL: dl, VT));
3373 }
3374
3375 return SDValue(ShiftNode, 0);
3376}
3377
3378// For bit-field-positioning pattern "(and (shl VAL, N), ShiftedMask)".
3379static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op,
3380 bool BiggerPattern,
3381 const uint64_t NonZeroBits,
3382 SDValue &Src, int &DstLSB,
3383 int &Width);
3384
3385// For bit-field-positioning pattern "shl VAL, N)".
3386static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op,
3387 bool BiggerPattern,
3388 const uint64_t NonZeroBits,
3389 SDValue &Src, int &DstLSB,
3390 int &Width);
3391
3392/// Does this tree qualify as an attempt to move a bitfield into position,
3393/// essentially "(and (shl VAL, N), Mask)" or (shl VAL, N).
3394static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op,
3395 bool BiggerPattern, SDValue &Src,
3396 int &DstLSB, int &Width) {
3397 EVT VT = Op.getValueType();
3398 unsigned BitWidth = VT.getSizeInBits();
3399 (void)BitWidth;
3400 assert(BitWidth == 32 || BitWidth == 64);
3401
3402 KnownBits Known = CurDAG->computeKnownBits(Op);
3403
3404 // Non-zero in the sense that they're not provably zero, which is the key
3405 // point if we want to use this value
3406 const uint64_t NonZeroBits = (~Known.Zero).getZExtValue();
3407 if (!isShiftedMask_64(Value: NonZeroBits))
3408 return false;
3409
3410 switch (Op.getOpcode()) {
3411 default:
3412 break;
3413 case ISD::AND:
3414 return isBitfieldPositioningOpFromAnd(CurDAG, Op, BiggerPattern,
3415 NonZeroBits, Src, DstLSB, Width);
3416 case ISD::SHL:
3417 return isBitfieldPositioningOpFromShl(CurDAG, Op, BiggerPattern,
3418 NonZeroBits, Src, DstLSB, Width);
3419 }
3420
3421 return false;
3422}
3423
3424static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op,
3425 bool BiggerPattern,
3426 const uint64_t NonZeroBits,
3427 SDValue &Src, int &DstLSB,
3428 int &Width) {
3429 assert(isShiftedMask_64(NonZeroBits) && "Caller guaranteed");
3430
3431 EVT VT = Op.getValueType();
3432 assert((VT == MVT::i32 || VT == MVT::i64) &&
3433 "Caller guarantees VT is one of i32 or i64");
3434 (void)VT;
3435
3436 uint64_t AndImm;
3437 if (!isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::AND, Imm&: AndImm))
3438 return false;
3439
3440 // If (~AndImm & NonZeroBits) is not zero at POS, we know that
3441 // 1) (AndImm & (1 << POS) == 0)
3442 // 2) the result of AND is not zero at POS bit (according to NonZeroBits)
3443 //
3444 // 1) and 2) don't agree so something must be wrong (e.g., in
3445 // 'SelectionDAG::computeKnownBits')
3446 assert((~AndImm & NonZeroBits) == 0 &&
3447 "Something must be wrong (e.g., in SelectionDAG::computeKnownBits)");
3448
3449 SDValue AndOp0 = Op.getOperand(i: 0);
3450
3451 uint64_t ShlImm;
3452 SDValue ShlOp0;
3453 if (isOpcWithIntImmediate(N: AndOp0.getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
3454 // For pattern "and(shl(val, N), shifted-mask)", 'ShlOp0' is set to 'val'.
3455 ShlOp0 = AndOp0.getOperand(i: 0);
3456 } else if (VT == MVT::i64 && AndOp0.getOpcode() == ISD::ANY_EXTEND &&
3457 isOpcWithIntImmediate(N: AndOp0.getOperand(i: 0).getNode(), Opc: ISD::SHL,
3458 Imm&: ShlImm)) {
3459 // For pattern "and(any_extend(shl(val, N)), shifted-mask)"
3460
3461 // ShlVal == shl(val, N), which is a left shift on a smaller type.
3462 SDValue ShlVal = AndOp0.getOperand(i: 0);
3463
3464 // Since this is after type legalization and ShlVal is extended to MVT::i64,
3465 // expect VT to be MVT::i32.
3466 assert((ShlVal.getValueType() == MVT::i32) && "Expect VT to be MVT::i32.");
3467
3468 // Widens 'val' to MVT::i64 as the source of bit field positioning.
3469 ShlOp0 = Widen(CurDAG, N: ShlVal.getOperand(i: 0));
3470 } else
3471 return false;
3472
3473 // For !BiggerPattern, bail out if the AndOp0 has more than one use, since
3474 // then we'll end up generating AndOp0+UBFIZ instead of just keeping
3475 // AndOp0+AND.
3476 if (!BiggerPattern && !AndOp0.hasOneUse())
3477 return false;
3478
3479 DstLSB = llvm::countr_zero(Val: NonZeroBits);
3480 Width = llvm::countr_one(Value: NonZeroBits >> DstLSB);
3481
3482 // Bail out on large Width. This happens when no proper combining / constant
3483 // folding was performed.
3484 if (Width >= (int)VT.getSizeInBits()) {
3485 // If VT is i64, Width > 64 is insensible since NonZeroBits is uint64_t, and
3486 // Width == 64 indicates a missed dag-combine from "(and val, AllOnes)" to
3487 // "val".
3488 // If VT is i32, what Width >= 32 means:
3489 // - For "(and (any_extend(shl val, N)), shifted-mask)", the`and` Op
3490 // demands at least 'Width' bits (after dag-combiner). This together with
3491 // `any_extend` Op (undefined higher bits) indicates missed combination
3492 // when lowering the 'and' IR instruction to an machine IR instruction.
3493 LLVM_DEBUG(
3494 dbgs()
3495 << "Found large Width in bit-field-positioning -- this indicates no "
3496 "proper combining / constant folding was performed\n");
3497 return false;
3498 }
3499
3500 // BFI encompasses sufficiently many nodes that it's worth inserting an extra
3501 // LSL/LSR if the mask in NonZeroBits doesn't quite match up with the ISD::SHL
3502 // amount. BiggerPattern is true when this pattern is being matched for BFI,
3503 // BiggerPattern is false when this pattern is being matched for UBFIZ, in
3504 // which case it is not profitable to insert an extra shift.
3505 if (ShlImm != uint64_t(DstLSB) && !BiggerPattern)
3506 return false;
3507
3508 Src = getLeftShift(CurDAG, Op: ShlOp0, ShlAmount: ShlImm - DstLSB);
3509 return true;
3510}
3511
3512// For node (shl (and val, mask), N)), returns true if the node is equivalent to
3513// UBFIZ.
3514static bool isSeveralBitsPositioningOpFromShl(const uint64_t ShlImm, SDValue Op,
3515 SDValue &Src, int &DstLSB,
3516 int &Width) {
3517 // Caller should have verified that N is a left shift with constant shift
3518 // amount; asserts that.
3519 assert(Op.getOpcode() == ISD::SHL &&
3520 "Op.getNode() should be a SHL node to call this function");
3521 assert(isIntImmediateEq(Op.getOperand(1), ShlImm) &&
3522 "Op.getNode() should shift ShlImm to call this function");
3523
3524 uint64_t AndImm = 0;
3525 SDValue Op0 = Op.getOperand(i: 0);
3526 if (!isOpcWithIntImmediate(N: Op0.getNode(), Opc: ISD::AND, Imm&: AndImm))
3527 return false;
3528
3529 const uint64_t ShiftedAndImm = ((AndImm << ShlImm) >> ShlImm);
3530 if (isMask_64(Value: ShiftedAndImm)) {
3531 // AndImm is a superset of (AllOnes >> ShlImm); in other words, AndImm
3532 // should end with Mask, and could be prefixed with random bits if those
3533 // bits are shifted out.
3534 //
3535 // For example, xyz11111 (with {x,y,z} being 0 or 1) is fine if ShlImm >= 3;
3536 // the AND result corresponding to those bits are shifted out, so it's fine
3537 // to not extract them.
3538 Width = llvm::countr_one(Value: ShiftedAndImm);
3539 DstLSB = ShlImm;
3540 Src = Op0.getOperand(i: 0);
3541 return true;
3542 }
3543 return false;
3544}
3545
3546static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op,
3547 bool BiggerPattern,
3548 const uint64_t NonZeroBits,
3549 SDValue &Src, int &DstLSB,
3550 int &Width) {
3551 assert(isShiftedMask_64(NonZeroBits) && "Caller guaranteed");
3552
3553 EVT VT = Op.getValueType();
3554 assert((VT == MVT::i32 || VT == MVT::i64) &&
3555 "Caller guarantees that type is i32 or i64");
3556 (void)VT;
3557
3558 uint64_t ShlImm;
3559 if (!isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::SHL, Imm&: ShlImm))
3560 return false;
3561
3562 if (!BiggerPattern && !Op.hasOneUse())
3563 return false;
3564
3565 if (isSeveralBitsPositioningOpFromShl(ShlImm, Op, Src, DstLSB, Width))
3566 return true;
3567
3568 DstLSB = llvm::countr_zero(Val: NonZeroBits);
3569 Width = llvm::countr_one(Value: NonZeroBits >> DstLSB);
3570
3571 if (ShlImm != uint64_t(DstLSB) && !BiggerPattern)
3572 return false;
3573
3574 Src = getLeftShift(CurDAG, Op: Op.getOperand(i: 0), ShlAmount: ShlImm - DstLSB);
3575 return true;
3576}
3577
3578static bool isShiftedMask(uint64_t Mask, EVT VT) {
3579 assert(VT == MVT::i32 || VT == MVT::i64);
3580 if (VT == MVT::i32)
3581 return isShiftedMask_32(Value: Mask);
3582 return isShiftedMask_64(Value: Mask);
3583}
3584
3585// Generate a BFI/BFXIL from 'or (and X, MaskImm), OrImm' iff the value being
3586// inserted only sets known zero bits.
3587static bool tryBitfieldInsertOpFromOrAndImm(SDNode *N, SelectionDAG *CurDAG) {
3588 assert(N->getOpcode() == ISD::OR && "Expect a OR operation");
3589
3590 EVT VT = N->getValueType(ResNo: 0);
3591 if (VT != MVT::i32 && VT != MVT::i64)
3592 return false;
3593
3594 unsigned BitWidth = VT.getSizeInBits();
3595
3596 uint64_t OrImm;
3597 if (!isOpcWithIntImmediate(N, Opc: ISD::OR, Imm&: OrImm))
3598 return false;
3599
3600 // Skip this transformation if the ORR immediate can be encoded in the ORR.
3601 // Otherwise, we'll trade an AND+ORR for ORR+BFI/BFXIL, which is most likely
3602 // performance neutral.
3603 if (AArch64_AM::isLogicalImmediate(imm: OrImm, regSize: BitWidth))
3604 return false;
3605
3606 uint64_t MaskImm;
3607 SDValue And = N->getOperand(Num: 0);
3608 // Must be a single use AND with an immediate operand.
3609 if (!And.hasOneUse() ||
3610 !isOpcWithIntImmediate(N: And.getNode(), Opc: ISD::AND, Imm&: MaskImm))
3611 return false;
3612
3613 // Compute the Known Zero for the AND as this allows us to catch more general
3614 // cases than just looking for AND with imm.
3615 KnownBits Known = CurDAG->computeKnownBits(Op: And);
3616
3617 // Non-zero in the sense that they're not provably zero, which is the key
3618 // point if we want to use this value.
3619 uint64_t NotKnownZero = (~Known.Zero).getZExtValue();
3620
3621 // The KnownZero mask must be a shifted mask (e.g., 1110..011, 11100..00).
3622 if (!isShiftedMask(Mask: Known.Zero.getZExtValue(), VT))
3623 return false;
3624
3625 // The bits being inserted must only set those bits that are known to be zero.
3626 if ((OrImm & NotKnownZero) != 0) {
3627 // FIXME: It's okay if the OrImm sets NotKnownZero bits to 1, but we don't
3628 // currently handle this case.
3629 return false;
3630 }
3631
3632 // BFI/BFXIL dst, src, #lsb, #width.
3633 int LSB = llvm::countr_one(Value: NotKnownZero);
3634 int Width = BitWidth - APInt(BitWidth, NotKnownZero).popcount();
3635
3636 // BFI/BFXIL is an alias of BFM, so translate to BFM operands.
3637 unsigned ImmR = (BitWidth - LSB) % BitWidth;
3638 unsigned ImmS = Width - 1;
3639
3640 // If we're creating a BFI instruction avoid cases where we need more
3641 // instructions to materialize the BFI constant as compared to the original
3642 // ORR. A BFXIL will use the same constant as the original ORR, so the code
3643 // should be no worse in this case.
3644 bool IsBFI = LSB != 0;
3645 uint64_t BFIImm = OrImm >> LSB;
3646 if (IsBFI && !AArch64_AM::isLogicalImmediate(imm: BFIImm, regSize: BitWidth)) {
3647 // We have a BFI instruction and we know the constant can't be materialized
3648 // with a ORR-immediate with the zero register.
3649 unsigned OrChunks = 0, BFIChunks = 0;
3650 for (unsigned Shift = 0; Shift < BitWidth; Shift += 16) {
3651 if (((OrImm >> Shift) & 0xFFFF) != 0)
3652 ++OrChunks;
3653 if (((BFIImm >> Shift) & 0xFFFF) != 0)
3654 ++BFIChunks;
3655 }
3656 if (BFIChunks > OrChunks)
3657 return false;
3658 }
3659
3660 // Materialize the constant to be inserted.
3661 SDLoc DL(N);
3662 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
3663 SDNode *MOVI = CurDAG->getMachineNode(
3664 Opcode: MOVIOpc, dl: DL, VT, Op1: CurDAG->getTargetConstant(Val: BFIImm, DL, VT));
3665
3666 // Create the BFI/BFXIL instruction.
3667 SDValue Ops[] = {And.getOperand(i: 0), SDValue(MOVI, 0),
3668 CurDAG->getTargetConstant(Val: ImmR, DL, VT),
3669 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
3670 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3671 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
3672 return true;
3673}
3674
3675static bool isWorthFoldingIntoOrrWithShift(SDValue Dst, SelectionDAG *CurDAG,
3676 SDValue &ShiftedOperand,
3677 uint64_t &EncodedShiftImm) {
3678 // Avoid folding Dst into ORR-with-shift if Dst has other uses than ORR.
3679 if (!Dst.hasOneUse())
3680 return false;
3681
3682 EVT VT = Dst.getValueType();
3683 assert((VT == MVT::i32 || VT == MVT::i64) &&
3684 "Caller should guarantee that VT is one of i32 or i64");
3685 const unsigned SizeInBits = VT.getSizeInBits();
3686
3687 SDLoc DL(Dst.getNode());
3688 uint64_t AndImm, ShlImm;
3689 if (isOpcWithIntImmediate(N: Dst.getNode(), Opc: ISD::AND, Imm&: AndImm) &&
3690 isShiftedMask_64(Value: AndImm)) {
3691 // Avoid transforming 'DstOp0' if it has other uses than the AND node.
3692 SDValue DstOp0 = Dst.getOperand(i: 0);
3693 if (!DstOp0.hasOneUse())
3694 return false;
3695
3696 // An example to illustrate the transformation
3697 // From:
3698 // lsr x8, x1, #1
3699 // and x8, x8, #0x3f80
3700 // bfxil x8, x1, #0, #7
3701 // To:
3702 // and x8, x23, #0x7f
3703 // ubfx x9, x23, #8, #7
3704 // orr x23, x8, x9, lsl #7
3705 //
3706 // The number of instructions remains the same, but ORR is faster than BFXIL
3707 // on many AArch64 processors (or as good as BFXIL if not faster). Besides,
3708 // the dependency chain is improved after the transformation.
3709 uint64_t SrlImm;
3710 if (isOpcWithIntImmediate(N: DstOp0.getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
3711 uint64_t NumTrailingZeroInShiftedMask = llvm::countr_zero(Val: AndImm);
3712 if ((SrlImm + NumTrailingZeroInShiftedMask) < SizeInBits) {
3713 unsigned MaskWidth =
3714 llvm::countr_one(Value: AndImm >> NumTrailingZeroInShiftedMask);
3715 unsigned UBFMOpc =
3716 (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3717 SDNode *UBFMNode = CurDAG->getMachineNode(
3718 Opcode: UBFMOpc, dl: DL, VT, Op1: DstOp0.getOperand(i: 0),
3719 Op2: CurDAG->getTargetConstant(Val: SrlImm + NumTrailingZeroInShiftedMask, DL,
3720 VT),
3721 Op3: CurDAG->getTargetConstant(
3722 Val: SrlImm + NumTrailingZeroInShiftedMask + MaskWidth - 1, DL, VT));
3723 ShiftedOperand = SDValue(UBFMNode, 0);
3724 EncodedShiftImm = AArch64_AM::getShifterImm(
3725 ST: AArch64_AM::LSL, Imm: NumTrailingZeroInShiftedMask);
3726 return true;
3727 }
3728 }
3729 return false;
3730 }
3731
3732 if (isOpcWithIntImmediate(N: Dst.getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
3733 ShiftedOperand = Dst.getOperand(i: 0);
3734 EncodedShiftImm = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShlImm);
3735 return true;
3736 }
3737
3738 uint64_t SrlImm;
3739 if (isOpcWithIntImmediate(N: Dst.getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
3740 ShiftedOperand = Dst.getOperand(i: 0);
3741 EncodedShiftImm = AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: SrlImm);
3742 return true;
3743 }
3744 return false;
3745}
3746
3747// Given an 'ISD::OR' node that is going to be selected as BFM, analyze
3748// the operands and select it to AArch64::ORR with shifted registers if
3749// that's more efficient. Returns true iff selection to AArch64::ORR happens.
3750static bool tryOrrWithShift(SDNode *N, SDValue OrOpd0, SDValue OrOpd1,
3751 SDValue Src, SDValue Dst, SelectionDAG *CurDAG,
3752 const bool BiggerPattern) {
3753 EVT VT = N->getValueType(ResNo: 0);
3754 assert(N->getOpcode() == ISD::OR && "Expect N to be an OR node");
3755 assert(((N->getOperand(0) == OrOpd0 && N->getOperand(1) == OrOpd1) ||
3756 (N->getOperand(1) == OrOpd0 && N->getOperand(0) == OrOpd1)) &&
3757 "Expect OrOpd0 and OrOpd1 to be operands of ISD::OR");
3758 assert((VT == MVT::i32 || VT == MVT::i64) &&
3759 "Expect result type to be i32 or i64 since N is combinable to BFM");
3760 SDLoc DL(N);
3761
3762 // Bail out if BFM simplifies away one node in BFM Dst.
3763 if (OrOpd1 != Dst)
3764 return false;
3765
3766 const unsigned OrrOpc = (VT == MVT::i32) ? AArch64::ORRWrs : AArch64::ORRXrs;
3767 // For "BFM Rd, Rn, #immr, #imms", it's known that BFM simplifies away fewer
3768 // nodes from Rn (or inserts additional shift node) if BiggerPattern is true.
3769 if (BiggerPattern) {
3770 uint64_t SrcAndImm;
3771 if (isOpcWithIntImmediate(N: OrOpd0.getNode(), Opc: ISD::AND, Imm&: SrcAndImm) &&
3772 isMask_64(Value: SrcAndImm) && OrOpd0.getOperand(i: 0) == Src) {
3773 // OrOpd0 = AND Src, #Mask
3774 // So BFM simplifies away one AND node from Src and doesn't simplify away
3775 // nodes from Dst. If ORR with left-shifted operand also simplifies away
3776 // one node (from Rd), ORR is better since it has higher throughput and
3777 // smaller latency than BFM on many AArch64 processors (and for the rest
3778 // ORR is at least as good as BFM).
3779 SDValue ShiftedOperand;
3780 uint64_t EncodedShiftImm;
3781 if (isWorthFoldingIntoOrrWithShift(Dst, CurDAG, ShiftedOperand,
3782 EncodedShiftImm)) {
3783 SDValue Ops[] = {OrOpd0, ShiftedOperand,
3784 CurDAG->getTargetConstant(Val: EncodedShiftImm, DL, VT)};
3785 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
3786 return true;
3787 }
3788 }
3789 return false;
3790 }
3791
3792 assert((!BiggerPattern) && "BiggerPattern should be handled above");
3793
3794 uint64_t ShlImm;
3795 if (isOpcWithIntImmediate(N: OrOpd0.getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
3796 if (OrOpd0.getOperand(i: 0) == Src && OrOpd0.hasOneUse()) {
3797 SDValue Ops[] = {
3798 Dst, Src,
3799 CurDAG->getTargetConstant(
3800 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShlImm), DL, VT)};
3801 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
3802 return true;
3803 }
3804
3805 // Select the following pattern to left-shifted operand rather than BFI.
3806 // %val1 = op ..
3807 // %val2 = shl %val1, #imm
3808 // %res = or %val1, %val2
3809 //
3810 // If N is selected to be BFI, we know that
3811 // 1) OrOpd0 would be the operand from which extract bits (i.e., folded into
3812 // BFI) 2) OrOpd1 would be the destination operand (i.e., preserved)
3813 //
3814 // Instead of selecting N to BFI, fold OrOpd0 as a left shift directly.
3815 if (OrOpd0.getOperand(i: 0) == OrOpd1) {
3816 SDValue Ops[] = {
3817 OrOpd1, OrOpd1,
3818 CurDAG->getTargetConstant(
3819 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShlImm), DL, VT)};
3820 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
3821 return true;
3822 }
3823 }
3824
3825 uint64_t SrlImm;
3826 if (isOpcWithIntImmediate(N: OrOpd0.getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
3827 // Select the following pattern to right-shifted operand rather than BFXIL.
3828 // %val1 = op ..
3829 // %val2 = lshr %val1, #imm
3830 // %res = or %val1, %val2
3831 //
3832 // If N is selected to be BFXIL, we know that
3833 // 1) OrOpd0 would be the operand from which extract bits (i.e., folded into
3834 // BFXIL) 2) OrOpd1 would be the destination operand (i.e., preserved)
3835 //
3836 // Instead of selecting N to BFXIL, fold OrOpd0 as a right shift directly.
3837 if (OrOpd0.getOperand(i: 0) == OrOpd1) {
3838 SDValue Ops[] = {
3839 OrOpd1, OrOpd1,
3840 CurDAG->getTargetConstant(
3841 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: SrlImm), DL, VT)};
3842 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
3843 return true;
3844 }
3845 }
3846
3847 return false;
3848}
3849
3850static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits,
3851 SelectionDAG *CurDAG) {
3852 assert(N->getOpcode() == ISD::OR && "Expect a OR operation");
3853
3854 EVT VT = N->getValueType(ResNo: 0);
3855 if (VT != MVT::i32 && VT != MVT::i64)
3856 return false;
3857
3858 unsigned BitWidth = VT.getSizeInBits();
3859
3860 // Because of simplify-demanded-bits in DAGCombine, involved masks may not
3861 // have the expected shape. Try to undo that.
3862
3863 unsigned NumberOfIgnoredLowBits = UsefulBits.countr_zero();
3864 unsigned NumberOfIgnoredHighBits = UsefulBits.countl_zero();
3865
3866 // Given a OR operation, check if we have the following pattern
3867 // ubfm c, b, imm, imm2 (or something that does the same jobs, see
3868 // isBitfieldExtractOp)
3869 // d = e & mask2 ; where mask is a binary sequence of 1..10..0 and
3870 // countTrailingZeros(mask2) == imm2 - imm + 1
3871 // f = d | c
3872 // if yes, replace the OR instruction with:
3873 // f = BFM Opd0, Opd1, LSB, MSB ; where LSB = imm, and MSB = imm2
3874
3875 // OR is commutative, check all combinations of operand order and values of
3876 // BiggerPattern, i.e.
3877 // Opd0, Opd1, BiggerPattern=false
3878 // Opd1, Opd0, BiggerPattern=false
3879 // Opd0, Opd1, BiggerPattern=true
3880 // Opd1, Opd0, BiggerPattern=true
3881 // Several of these combinations may match, so check with BiggerPattern=false
3882 // first since that will produce better results by matching more instructions
3883 // and/or inserting fewer extra instructions.
3884 for (int I = 0; I < 4; ++I) {
3885
3886 SDValue Dst, Src;
3887 unsigned ImmR, ImmS;
3888 bool BiggerPattern = I / 2;
3889 SDValue OrOpd0Val = N->getOperand(Num: I % 2);
3890 SDNode *OrOpd0 = OrOpd0Val.getNode();
3891 SDValue OrOpd1Val = N->getOperand(Num: (I + 1) % 2);
3892 SDNode *OrOpd1 = OrOpd1Val.getNode();
3893
3894 unsigned BFXOpc;
3895 int DstLSB, Width;
3896 if (isBitfieldExtractOp(CurDAG, N: OrOpd0, Opc&: BFXOpc, Opd0&: Src, Immr&: ImmR, Imms&: ImmS,
3897 NumberOfIgnoredLowBits, BiggerPattern)) {
3898 // Check that the returned opcode is compatible with the pattern,
3899 // i.e., same type and zero extended (U and not S)
3900 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) ||
3901 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32))
3902 continue;
3903
3904 // Compute the width of the bitfield insertion
3905 DstLSB = 0;
3906 Width = ImmS - ImmR + 1;
3907 // FIXME: This constraint is to catch bitfield insertion we may
3908 // want to widen the pattern if we want to grab general bitfield
3909 // move case
3910 if (Width <= 0)
3911 continue;
3912
3913 // If the mask on the insertee is correct, we have a BFXIL operation. We
3914 // can share the ImmR and ImmS values from the already-computed UBFM.
3915 } else if (isBitfieldPositioningOp(CurDAG, Op: OrOpd0Val,
3916 BiggerPattern,
3917 Src, DstLSB, Width)) {
3918 ImmR = (BitWidth - DstLSB) % BitWidth;
3919 ImmS = Width - 1;
3920 } else
3921 continue;
3922
3923 // Check the second part of the pattern
3924 EVT VT = OrOpd1Val.getValueType();
3925 assert((VT == MVT::i32 || VT == MVT::i64) && "unexpected OR operand");
3926
3927 // Compute the Known Zero for the candidate of the first operand.
3928 // This allows to catch more general case than just looking for
3929 // AND with imm. Indeed, simplify-demanded-bits may have removed
3930 // the AND instruction because it proves it was useless.
3931 KnownBits Known = CurDAG->computeKnownBits(Op: OrOpd1Val);
3932
3933 // Check if there is enough room for the second operand to appear
3934 // in the first one
3935 APInt BitsToBeInserted =
3936 APInt::getBitsSet(numBits: Known.getBitWidth(), loBit: DstLSB, hiBit: DstLSB + Width);
3937
3938 if ((BitsToBeInserted & ~Known.Zero) != 0)
3939 continue;
3940
3941 // Set the first operand
3942 uint64_t Imm;
3943 if (isOpcWithIntImmediate(N: OrOpd1, Opc: ISD::AND, Imm) &&
3944 isBitfieldDstMask(DstMask: Imm, BitsToBeInserted, NumberOfIgnoredHighBits, VT))
3945 // In that case, we can eliminate the AND
3946 Dst = OrOpd1->getOperand(Num: 0);
3947 else
3948 // Maybe the AND has been removed by simplify-demanded-bits
3949 // or is useful because it discards more bits
3950 Dst = OrOpd1Val;
3951
3952 // Before selecting ISD::OR node to AArch64::BFM, see if an AArch64::ORR
3953 // with shifted operand is more efficient.
3954 if (tryOrrWithShift(N, OrOpd0: OrOpd0Val, OrOpd1: OrOpd1Val, Src, Dst, CurDAG,
3955 BiggerPattern))
3956 return true;
3957
3958 // both parts match
3959 SDLoc DL(N);
3960 SDValue Ops[] = {Dst, Src, CurDAG->getTargetConstant(Val: ImmR, DL, VT),
3961 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
3962 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3963 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
3964 return true;
3965 }
3966
3967 // Generate a BFXIL from 'or (and X, Mask0Imm), (and Y, Mask1Imm)' iff
3968 // Mask0Imm and ~Mask1Imm are equivalent and one of the MaskImms is a shifted
3969 // mask (e.g., 0x000ffff0).
3970 uint64_t Mask0Imm, Mask1Imm;
3971 SDValue And0 = N->getOperand(Num: 0);
3972 SDValue And1 = N->getOperand(Num: 1);
3973 if (And0.hasOneUse() && And1.hasOneUse() &&
3974 isOpcWithIntImmediate(N: And0.getNode(), Opc: ISD::AND, Imm&: Mask0Imm) &&
3975 isOpcWithIntImmediate(N: And1.getNode(), Opc: ISD::AND, Imm&: Mask1Imm) &&
3976 APInt(BitWidth, Mask0Imm) == ~APInt(BitWidth, Mask1Imm) &&
3977 (isShiftedMask(Mask: Mask0Imm, VT) || isShiftedMask(Mask: Mask1Imm, VT))) {
3978
3979 // ORR is commutative, so canonicalize to the form 'or (and X, Mask0Imm),
3980 // (and Y, Mask1Imm)' where Mask1Imm is the shifted mask masking off the
3981 // bits to be inserted.
3982 if (isShiftedMask(Mask: Mask0Imm, VT)) {
3983 std::swap(a&: And0, b&: And1);
3984 std::swap(a&: Mask0Imm, b&: Mask1Imm);
3985 }
3986
3987 SDValue Src = And1->getOperand(Num: 0);
3988 SDValue Dst = And0->getOperand(Num: 0);
3989 unsigned LSB = llvm::countr_zero(Val: Mask1Imm);
3990 int Width = BitWidth - APInt(BitWidth, Mask0Imm).popcount();
3991
3992 // The BFXIL inserts the low-order bits from a source register, so right
3993 // shift the needed bits into place.
3994 SDLoc DL(N);
3995 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3996 uint64_t LsrImm = LSB;
3997 if (Src->hasOneUse() &&
3998 isOpcWithIntImmediate(N: Src.getNode(), Opc: ISD::SRL, Imm&: LsrImm) &&
3999 (LsrImm + LSB) < BitWidth) {
4000 Src = Src->getOperand(Num: 0);
4001 LsrImm += LSB;
4002 }
4003
4004 SDNode *LSR = CurDAG->getMachineNode(
4005 Opcode: ShiftOpc, dl: DL, VT, Op1: Src, Op2: CurDAG->getTargetConstant(Val: LsrImm, DL, VT),
4006 Op3: CurDAG->getTargetConstant(Val: BitWidth - 1, DL, VT));
4007
4008 // BFXIL is an alias of BFM, so translate to BFM operands.
4009 unsigned ImmR = (BitWidth - LSB) % BitWidth;
4010 unsigned ImmS = Width - 1;
4011
4012 // Create the BFXIL instruction.
4013 SDValue Ops[] = {Dst, SDValue(LSR, 0),
4014 CurDAG->getTargetConstant(Val: ImmR, DL, VT),
4015 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
4016 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
4017 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
4018 return true;
4019 }
4020
4021 return false;
4022}
4023
4024bool AArch64DAGToDAGISel::tryBitfieldInsertOp(SDNode *N) {
4025 if (N->getOpcode() != ISD::OR)
4026 return false;
4027
4028 APInt NUsefulBits;
4029 getUsefulBits(Op: SDValue(N, 0), UsefulBits&: NUsefulBits);
4030
4031 // If all bits are not useful, just return UNDEF.
4032 if (!NUsefulBits) {
4033 CurDAG->SelectNodeTo(N, MachineOpc: TargetOpcode::IMPLICIT_DEF, VT: N->getValueType(ResNo: 0));
4034 return true;
4035 }
4036
4037 if (tryBitfieldInsertOpFromOr(N, UsefulBits: NUsefulBits, CurDAG))
4038 return true;
4039
4040 return tryBitfieldInsertOpFromOrAndImm(N, CurDAG);
4041}
4042
4043/// SelectBitfieldInsertInZeroOp - Match a UBFIZ instruction that is the
4044/// equivalent of a left shift by a constant amount followed by an and masking
4045/// out a contiguous set of bits.
4046bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(SDNode *N) {
4047 if (N->getOpcode() != ISD::AND)
4048 return false;
4049
4050 EVT VT = N->getValueType(ResNo: 0);
4051 if (VT != MVT::i32 && VT != MVT::i64)
4052 return false;
4053
4054 SDValue Op0;
4055 int DstLSB, Width;
4056 if (!isBitfieldPositioningOp(CurDAG, Op: SDValue(N, 0), /*BiggerPattern=*/false,
4057 Src&: Op0, DstLSB, Width))
4058 return false;
4059
4060 // ImmR is the rotate right amount.
4061 unsigned ImmR = (VT.getSizeInBits() - DstLSB) % VT.getSizeInBits();
4062 // ImmS is the most significant bit of the source to be moved.
4063 unsigned ImmS = Width - 1;
4064
4065 SDLoc DL(N);
4066 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(Val: ImmR, DL, VT),
4067 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
4068 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
4069 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
4070 return true;
4071}
4072
4073/// tryShiftAmountMod - Take advantage of built-in mod of shift amount in
4074/// variable shift/rotate instructions.
4075bool AArch64DAGToDAGISel::tryShiftAmountMod(SDNode *N) {
4076 EVT VT = N->getValueType(ResNo: 0);
4077
4078 unsigned Opc;
4079 switch (N->getOpcode()) {
4080 case ISD::ROTR:
4081 Opc = (VT == MVT::i32) ? AArch64::RORVWr : AArch64::RORVXr;
4082 break;
4083 case ISD::SHL:
4084 Opc = (VT == MVT::i32) ? AArch64::LSLVWr : AArch64::LSLVXr;
4085 break;
4086 case ISD::SRL:
4087 Opc = (VT == MVT::i32) ? AArch64::LSRVWr : AArch64::LSRVXr;
4088 break;
4089 case ISD::SRA:
4090 Opc = (VT == MVT::i32) ? AArch64::ASRVWr : AArch64::ASRVXr;
4091 break;
4092 default:
4093 return false;
4094 }
4095
4096 uint64_t Size;
4097 uint64_t Bits;
4098 if (VT == MVT::i32) {
4099 Bits = 5;
4100 Size = 32;
4101 } else if (VT == MVT::i64) {
4102 Bits = 6;
4103 Size = 64;
4104 } else
4105 return false;
4106
4107 SDValue ShiftAmt = N->getOperand(Num: 1);
4108 SDLoc DL(N);
4109 SDValue NewShiftAmt;
4110
4111 // Skip over an extend of the shift amount.
4112 if (ShiftAmt->getOpcode() == ISD::ZERO_EXTEND ||
4113 ShiftAmt->getOpcode() == ISD::ANY_EXTEND)
4114 ShiftAmt = ShiftAmt->getOperand(Num: 0);
4115
4116 if (ShiftAmt->getOpcode() == ISD::ADD || ShiftAmt->getOpcode() == ISD::SUB) {
4117 SDValue Add0 = ShiftAmt->getOperand(Num: 0);
4118 SDValue Add1 = ShiftAmt->getOperand(Num: 1);
4119 uint64_t Add0Imm;
4120 uint64_t Add1Imm;
4121 if (isIntImmediate(N: Add1, Imm&: Add1Imm) && (Add1Imm % Size == 0)) {
4122 // If we are shifting by X+/-N where N == 0 mod Size, then just shift by X
4123 // to avoid the ADD/SUB.
4124 NewShiftAmt = Add0;
4125 } else if (ShiftAmt->getOpcode() == ISD::SUB &&
4126 isIntImmediate(N: Add0, Imm&: Add0Imm) && Add0Imm != 0 &&
4127 (Add0Imm % Size == 0)) {
4128 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
4129 // to generate a NEG instead of a SUB from a constant.
4130 unsigned NegOpc;
4131 unsigned ZeroReg;
4132 EVT SubVT = ShiftAmt->getValueType(ResNo: 0);
4133 if (SubVT == MVT::i32) {
4134 NegOpc = AArch64::SUBWrr;
4135 ZeroReg = AArch64::WZR;
4136 } else {
4137 assert(SubVT == MVT::i64);
4138 NegOpc = AArch64::SUBXrr;
4139 ZeroReg = AArch64::XZR;
4140 }
4141 SDValue Zero =
4142 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: ZeroReg, VT: SubVT);
4143 MachineSDNode *Neg =
4144 CurDAG->getMachineNode(Opcode: NegOpc, dl: DL, VT: SubVT, Op1: Zero, Op2: Add1);
4145 NewShiftAmt = SDValue(Neg, 0);
4146 } else if (ShiftAmt->getOpcode() == ISD::SUB &&
4147 isIntImmediate(N: Add0, Imm&: Add0Imm) && (Add0Imm % Size == Size - 1)) {
4148 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
4149 // to generate a NOT instead of a SUB from a constant.
4150 unsigned NotOpc;
4151 unsigned ZeroReg;
4152 EVT SubVT = ShiftAmt->getValueType(ResNo: 0);
4153 if (SubVT == MVT::i32) {
4154 NotOpc = AArch64::ORNWrr;
4155 ZeroReg = AArch64::WZR;
4156 } else {
4157 assert(SubVT == MVT::i64);
4158 NotOpc = AArch64::ORNXrr;
4159 ZeroReg = AArch64::XZR;
4160 }
4161 SDValue Zero =
4162 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: ZeroReg, VT: SubVT);
4163 MachineSDNode *Not =
4164 CurDAG->getMachineNode(Opcode: NotOpc, dl: DL, VT: SubVT, Op1: Zero, Op2: Add1);
4165 NewShiftAmt = SDValue(Not, 0);
4166 } else
4167 return false;
4168 } else {
4169 // If the shift amount is masked with an AND, check that the mask covers the
4170 // bits that are implicitly ANDed off by the above opcodes and if so, skip
4171 // the AND.
4172 uint64_t MaskImm;
4173 if (!isOpcWithIntImmediate(N: ShiftAmt.getNode(), Opc: ISD::AND, Imm&: MaskImm) &&
4174 !isOpcWithIntImmediate(N: ShiftAmt.getNode(), Opc: AArch64ISD::ANDS, Imm&: MaskImm))
4175 return false;
4176
4177 if ((unsigned)llvm::countr_one(Value: MaskImm) < Bits)
4178 return false;
4179
4180 NewShiftAmt = ShiftAmt->getOperand(Num: 0);
4181 }
4182
4183 // Narrow/widen the shift amount to match the size of the shift operation.
4184 if (VT == MVT::i32)
4185 NewShiftAmt = narrowIfNeeded(CurDAG, N: NewShiftAmt);
4186 else if (VT == MVT::i64 && NewShiftAmt->getValueType(ResNo: 0) == MVT::i32) {
4187 SDValue SubReg = CurDAG->getTargetConstant(Val: AArch64::sub_32, DL, VT: MVT::i32);
4188 MachineSDNode *Ext = CurDAG->getMachineNode(Opcode: AArch64::SUBREG_TO_REG, dl: DL, VT,
4189 Op1: NewShiftAmt, Op2: SubReg);
4190 NewShiftAmt = SDValue(Ext, 0);
4191 }
4192
4193 SDValue Ops[] = {N->getOperand(Num: 0), NewShiftAmt};
4194 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
4195 return true;
4196}
4197
4198static bool checkCVTFixedPointOperandWithFBits(SelectionDAG *CurDAG, SDValue N,
4199 SDValue &FixedPos,
4200 unsigned RegWidth,
4201 bool isReciprocal) {
4202 APFloat FVal(0.0);
4203 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: N))
4204 FVal = CN->getValueAPF();
4205 else if (LoadSDNode *LN = dyn_cast<LoadSDNode>(Val&: N)) {
4206 // Some otherwise illegal constants are allowed in this case.
4207 if (LN->getOperand(Num: 1).getOpcode() != AArch64ISD::ADDlow ||
4208 !isa<ConstantPoolSDNode>(Val: LN->getOperand(Num: 1)->getOperand(Num: 1)))
4209 return false;
4210
4211 ConstantPoolSDNode *CN =
4212 dyn_cast<ConstantPoolSDNode>(Val: LN->getOperand(Num: 1)->getOperand(Num: 1));
4213 FVal = cast<ConstantFP>(Val: CN->getConstVal())->getValueAPF();
4214 } else
4215 return false;
4216
4217 if (unsigned FBits =
4218 CheckFixedPointOperandConstant(FVal, RegWidth, isReciprocal)) {
4219 FixedPos = CurDAG->getTargetConstant(Val: FBits, DL: SDLoc(N), VT: MVT::i32);
4220 return true;
4221 }
4222
4223 return false;
4224}
4225
4226static bool checkCVTFixedPointOperandWithFBitsForVectors(SelectionDAG *CurDAG,
4227 SDValue N,
4228 SDValue &FixedPos,
4229 unsigned RegWidth,
4230 bool isReciprocal) {
4231 if ((N.getOpcode() == AArch64ISD::NVCAST || N.getOpcode() == ISD::BITCAST) &&
4232 N.getValueType().getScalarSizeInBits() ==
4233 N.getOperand(i: 0).getValueType().getScalarSizeInBits())
4234 N = N.getOperand(i: 0);
4235
4236 auto ImmToFloat = [RegWidth](APInt Imm) {
4237 switch (RegWidth) {
4238 case 16:
4239 return APFloat(APFloat::IEEEhalf(), Imm);
4240 case 32:
4241 return APFloat(APFloat::IEEEsingle(), Imm);
4242 case 64:
4243 return APFloat(APFloat::IEEEdouble(), Imm);
4244 default:
4245 llvm_unreachable("Unexpected RegWidth!");
4246 };
4247 };
4248
4249 APFloat FVal(0.0);
4250 switch (N->getOpcode()) {
4251 case AArch64ISD::MOVIshift:
4252 FVal = ImmToFloat(APInt(RegWidth, N.getConstantOperandVal(i: 0)
4253 << N.getConstantOperandVal(i: 1)));
4254 break;
4255 case AArch64ISD::FMOV:
4256 FVal = ImmToFloat(DecodeFMOVImm(Imm: N.getConstantOperandVal(i: 0), RegWidth));
4257 break;
4258 case AArch64ISD::DUP:
4259 if (isa<ConstantSDNode>(Val: N.getOperand(i: 0)))
4260 FVal = ImmToFloat(N.getConstantOperandAPInt(i: 0).trunc(width: RegWidth));
4261 else
4262 return false;
4263 break;
4264 default:
4265 return false;
4266 }
4267
4268 if (unsigned FBits =
4269 CheckFixedPointOperandConstant(FVal, RegWidth, isReciprocal)) {
4270 FixedPos = CurDAG->getTargetConstant(Val: FBits, DL: SDLoc(N), VT: MVT::i32);
4271 return true;
4272 }
4273
4274 return false;
4275}
4276
4277bool AArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos,
4278 unsigned RegWidth) {
4279 return checkCVTFixedPointOperandWithFBits(CurDAG, N, FixedPos, RegWidth,
4280 /*isReciprocal*/ false);
4281}
4282
4283bool AArch64DAGToDAGISel::SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos,
4284 unsigned RegWidth) {
4285 return checkCVTFixedPointOperandWithFBitsForVectors(
4286 CurDAG, N, FixedPos, RegWidth, /*isReciprocal*/ false);
4287}
4288
4289bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperandVec(SDValue N,
4290 SDValue &FixedPos,
4291 unsigned RegWidth) {
4292 return checkCVTFixedPointOperandWithFBitsForVectors(
4293 CurDAG, N, FixedPos, RegWidth, /*isReciprocal*/ true);
4294}
4295
4296bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperand(SDValue N,
4297 SDValue &FixedPos,
4298 unsigned RegWidth) {
4299 return checkCVTFixedPointOperandWithFBits(CurDAG, N, FixedPos, RegWidth,
4300 /*isReciprocal*/ true);
4301}
4302
4303// Inspects a register string of the form o0:op1:CRn:CRm:op2 gets the fields
4304// of the string and obtains the integer values from them and combines these
4305// into a single value to be used in the MRS/MSR instruction.
4306static int getIntOperandFromRegisterString(StringRef RegString) {
4307 SmallVector<StringRef, 5> Fields;
4308 RegString.split(A&: Fields, Separator: ':');
4309
4310 if (Fields.size() == 1)
4311 return -1;
4312
4313 assert(Fields.size() == 5
4314 && "Invalid number of fields in read register string");
4315
4316 SmallVector<int, 5> Ops;
4317 bool AllIntFields = true;
4318
4319 for (StringRef Field : Fields) {
4320 unsigned IntField;
4321 AllIntFields &= !Field.getAsInteger(Radix: 10, Result&: IntField);
4322 Ops.push_back(Elt: IntField);
4323 }
4324
4325 assert(AllIntFields &&
4326 "Unexpected non-integer value in special register string.");
4327 (void)AllIntFields;
4328
4329 // Need to combine the integer fields of the string into a single value
4330 // based on the bit encoding of MRS/MSR instruction.
4331 return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) | (Ops[3] << 3) |
4332 (Ops[4]);
4333}
4334
4335// Lower the read_register intrinsic to an MRS instruction node if the special
4336// register string argument is either of the form detailed in the ALCE (the
4337// form described in getIntOperandsFromRegisterString) or is a named register
4338// known by the MRS SysReg mapper.
4339bool AArch64DAGToDAGISel::tryReadRegister(SDNode *N) {
4340 const auto *MD = cast<MDNodeSDNode>(Val: N->getOperand(Num: 1));
4341 const auto *RegString = cast<MDString>(Val: MD->getMD()->getOperand(I: 0));
4342 SDLoc DL(N);
4343
4344 bool ReadIs128Bit = N->getOpcode() == AArch64ISD::MRRS;
4345
4346 unsigned Opcode64Bit = AArch64::MRS;
4347 int Imm = getIntOperandFromRegisterString(RegString: RegString->getString());
4348 if (Imm == -1) {
4349 // No match, Use the sysreg mapper to map the remaining possible strings to
4350 // the value for the register to be used for the instruction operand.
4351 const auto *TheReg =
4352 AArch64SysReg::lookupSysRegByName(Name: RegString->getString());
4353 if (TheReg && TheReg->Readable &&
4354 TheReg->haveFeatures(ActiveFeatures: Subtarget->getFeatureBits()))
4355 Imm = TheReg->Encoding;
4356 else
4357 Imm = AArch64SysReg::parseGenericRegister(Name: RegString->getString());
4358
4359 if (Imm == -1) {
4360 // Still no match, see if this is "pc" or give up.
4361 if (!ReadIs128Bit && RegString->getString() == "pc") {
4362 Opcode64Bit = AArch64::ADR;
4363 Imm = 0;
4364 } else {
4365 // Not a system register. It may name an allocatable 64-bit GPR/FPR read
4366 // by the MSVC __getReg/__getRegFp intrinsics. Emit a pseudo that
4367 // carries the source register as an immediate so the read does not
4368 // reference an undefined physical register (which the machine verifier
4369 // rejects); the AsmPrinter materializes the real mov/fmov.
4370 Register PReg = Subtarget->getTargetLowering()->matchRegisterName(
4371 RegName: RegString->getString());
4372 unsigned PseudoOp = 0;
4373 if (AArch64::GPR64RegClass.contains(Reg: PReg))
4374 PseudoOp = AArch64::READ_REGISTER_GPR64;
4375 else if (AArch64::FPR64RegClass.contains(Reg: PReg))
4376 PseudoOp = AArch64::READ_REGISTER_FPR64;
4377 if (!ReadIs128Bit && PseudoOp && N->getValueType(ResNo: 0) == MVT::i64) {
4378 CurDAG->SelectNodeTo(N, MachineOpc: PseudoOp, VT1: MVT::i64, VT2: MVT::Other,
4379 Ops: {CurDAG->getTargetConstant(Val: PReg, DL, VT: MVT::i32),
4380 N->getOperand(Num: 0)});
4381 return true;
4382 }
4383 return false;
4384 }
4385 }
4386 }
4387
4388 SDValue InChain = N->getOperand(Num: 0);
4389 SDValue SysRegImm = CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i32);
4390 if (!ReadIs128Bit) {
4391 CurDAG->SelectNodeTo(N, MachineOpc: Opcode64Bit, VT1: MVT::i64, VT2: MVT::Other /* Chain */,
4392 Ops: {SysRegImm, InChain});
4393 } else {
4394 SDNode *MRRS = CurDAG->getMachineNode(
4395 Opcode: AArch64::MRRS, dl: DL,
4396 ResultTys: {MVT::Untyped /* XSeqPair */, MVT::Other /* Chain */},
4397 Ops: {SysRegImm, InChain});
4398
4399 // Sysregs are not endian. The even register always contains the low half
4400 // of the register.
4401 SDValue Lo = CurDAG->getTargetExtractSubreg(SRIdx: AArch64::sube64, DL, VT: MVT::i64,
4402 Operand: SDValue(MRRS, 0));
4403 SDValue Hi = CurDAG->getTargetExtractSubreg(SRIdx: AArch64::subo64, DL, VT: MVT::i64,
4404 Operand: SDValue(MRRS, 0));
4405 SDValue OutChain = SDValue(MRRS, 1);
4406
4407 ReplaceUses(F: SDValue(N, 0), T: Lo);
4408 ReplaceUses(F: SDValue(N, 1), T: Hi);
4409 ReplaceUses(F: SDValue(N, 2), T: OutChain);
4410 };
4411 return true;
4412}
4413
4414// Lower the write_register intrinsic to an MSR instruction node if the special
4415// register string argument is either of the form detailed in the ALCE (the
4416// form described in getIntOperandsFromRegisterString) or is a named register
4417// known by the MSR SysReg mapper.
4418bool AArch64DAGToDAGISel::tryWriteRegister(SDNode *N) {
4419 const auto *MD = cast<MDNodeSDNode>(Val: N->getOperand(Num: 1));
4420 const auto *RegString = cast<MDString>(Val: MD->getMD()->getOperand(I: 0));
4421 SDLoc DL(N);
4422
4423 bool WriteIs128Bit = N->getOpcode() == AArch64ISD::MSRR;
4424
4425 if (!WriteIs128Bit) {
4426 // Check if the register was one of those allowed as the pstatefield value
4427 // in the MSR (immediate) instruction. To accept the values allowed in the
4428 // pstatefield for the MSR (immediate) instruction, we also require that an
4429 // immediate value has been provided as an argument, we know that this is
4430 // the case as it has been ensured by semantic checking.
4431 auto trySelectPState = [&](auto PMapper, unsigned State) {
4432 if (PMapper) {
4433 assert(isa<ConstantSDNode>(N->getOperand(2)) &&
4434 "Expected a constant integer expression.");
4435 unsigned Reg = PMapper->Encoding;
4436 uint64_t Immed = N->getConstantOperandVal(Num: 2);
4437 CurDAG->SelectNodeTo(
4438 N, MachineOpc: State, VT: MVT::Other, Op1: CurDAG->getTargetConstant(Val: Reg, DL, VT: MVT::i32),
4439 Op2: CurDAG->getTargetConstant(Val: Immed, DL, VT: MVT::i16), Op3: N->getOperand(Num: 0));
4440 return true;
4441 }
4442 return false;
4443 };
4444
4445 if (trySelectPState(
4446 AArch64PState::lookupPStateImm0_15ByName(Name: RegString->getString()),
4447 AArch64::MSRpstateImm4))
4448 return true;
4449 if (trySelectPState(
4450 AArch64PState::lookupPStateImm0_1ByName(Name: RegString->getString()),
4451 AArch64::MSRpstateImm1))
4452 return true;
4453 }
4454
4455 int Imm = getIntOperandFromRegisterString(RegString: RegString->getString());
4456 if (Imm == -1) {
4457 // Use the sysreg mapper to attempt to map the remaining possible strings
4458 // to the value for the register to be used for the MSR (register)
4459 // instruction operand.
4460 auto TheReg = AArch64SysReg::lookupSysRegByName(Name: RegString->getString());
4461 if (TheReg && TheReg->Writeable &&
4462 TheReg->haveFeatures(ActiveFeatures: Subtarget->getFeatureBits()))
4463 Imm = TheReg->Encoding;
4464 else
4465 Imm = AArch64SysReg::parseGenericRegister(Name: RegString->getString());
4466
4467 if (Imm == -1) {
4468 // Used by the MSVC __setReg/__setRegFp intrinsics. Copy the value into
4469 // the physical register and keep it live with a FAKE_USE so the write is
4470 // not dead-eliminated. (getRegisterByName rejects allocatable registers,
4471 // so the generic write path cannot handle these.)
4472 Register PReg = Subtarget->getTargetLowering()->matchRegisterName(
4473 RegName: RegString->getString());
4474 bool IsGPR = AArch64::GPR64RegClass.contains(Reg: PReg);
4475 bool IsFPR = AArch64::FPR64RegClass.contains(Reg: PReg);
4476 if (!WriteIs128Bit && (IsGPR || IsFPR) &&
4477 N->getOperand(Num: 2).getValueType() == MVT::i64) {
4478 SDValue Copy =
4479 CurDAG->getCopyToReg(Chain: N->getOperand(Num: 0), dl: DL, Reg: PReg, N: N->getOperand(Num: 2));
4480 SDValue RegOp = CurDAG->getRegister(Reg: PReg, VT: MVT::i64);
4481 SDNode *FakeUse = CurDAG->getMachineNode(Opcode: TargetOpcode::FAKE_USE, dl: DL,
4482 VT: MVT::Other, Ops: {RegOp, Copy});
4483 ReplaceUses(F: SDValue(N, 0), T: SDValue(FakeUse, 0));
4484 CurDAG->RemoveDeadNode(N);
4485 return true;
4486 }
4487 return false;
4488 }
4489 }
4490
4491 SDValue InChain = N->getOperand(Num: 0);
4492 if (!WriteIs128Bit) {
4493 CurDAG->SelectNodeTo(N, MachineOpc: AArch64::MSR, VT: MVT::Other,
4494 Op1: CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i32),
4495 Op2: N->getOperand(Num: 2), Op3: InChain);
4496 } else {
4497 // No endian swap. The lower half always goes into the even subreg, and the
4498 // higher half always into the odd supreg.
4499 SDNode *Pair = CurDAG->getMachineNode(
4500 Opcode: TargetOpcode::REG_SEQUENCE, dl: DL, VT: MVT::Untyped /* XSeqPair */,
4501 Ops: {CurDAG->getTargetConstant(Val: AArch64::XSeqPairsClassRegClass.getID(), DL,
4502 VT: MVT::i32),
4503 N->getOperand(Num: 2),
4504 CurDAG->getTargetConstant(Val: AArch64::sube64, DL, VT: MVT::i32),
4505 N->getOperand(Num: 3),
4506 CurDAG->getTargetConstant(Val: AArch64::subo64, DL, VT: MVT::i32)});
4507
4508 CurDAG->SelectNodeTo(N, MachineOpc: AArch64::MSRR, VT: MVT::Other,
4509 Op1: CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i32),
4510 Op2: SDValue(Pair, 0), Op3: InChain);
4511 }
4512
4513 return true;
4514}
4515
4516/// We've got special pseudo-instructions for these
4517bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
4518 unsigned Opcode;
4519 EVT MemTy = cast<MemSDNode>(Val: N)->getMemoryVT();
4520
4521 // Leave IR for LSE if subtarget supports it.
4522 if (Subtarget->hasLSE()) return false;
4523
4524 if (MemTy == MVT::i8)
4525 Opcode = AArch64::CMP_SWAP_8;
4526 else if (MemTy == MVT::i16)
4527 Opcode = AArch64::CMP_SWAP_16;
4528 else if (MemTy == MVT::i32)
4529 Opcode = AArch64::CMP_SWAP_32;
4530 else if (MemTy == MVT::i64)
4531 Opcode = AArch64::CMP_SWAP_64;
4532 else
4533 llvm_unreachable("Unknown AtomicCmpSwap type");
4534
4535 MVT RegTy = MemTy == MVT::i64 ? MVT::i64 : MVT::i32;
4536 SDValue Ops[] = {N->getOperand(Num: 1), N->getOperand(Num: 2), N->getOperand(Num: 3),
4537 N->getOperand(Num: 0)};
4538 SDNode *CmpSwap = CurDAG->getMachineNode(
4539 Opcode, dl: SDLoc(N),
4540 VTs: CurDAG->getVTList(VT1: RegTy, VT2: MVT::i32, VT3: MVT::Other), Ops);
4541
4542 MachineMemOperand *MemOp = cast<MemSDNode>(Val: N)->getMemOperand();
4543 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: CmpSwap), NewMemRefs: {MemOp});
4544
4545 ReplaceUses(F: SDValue(N, 0), T: SDValue(CmpSwap, 0));
4546 ReplaceUses(F: SDValue(N, 1), T: SDValue(CmpSwap, 2));
4547 CurDAG->RemoveDeadNode(N);
4548
4549 return true;
4550}
4551
4552bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm,
4553 SDValue &Shift, bool Negate) {
4554 if (!isa<ConstantSDNode>(Val: N))
4555 return false;
4556
4557 APInt Val =
4558 cast<ConstantSDNode>(Val&: N)->getAPIntValue().trunc(width: VT.getFixedSizeInBits());
4559
4560 return SelectSVEAddSubImm(DL: SDLoc(N), Value: Val, VT, Imm, Shift, Negate);
4561}
4562
4563bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDLoc DL, APInt Val, MVT VT,
4564 SDValue &Imm, SDValue &Shift,
4565 bool Negate) {
4566 if (Negate)
4567 Val = -Val;
4568
4569 switch (VT.SimpleTy) {
4570 case MVT::i8:
4571 // All immediates are supported.
4572 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4573 Imm = CurDAG->getTargetConstant(Val: Val.getZExtValue(), DL, VT: MVT::i32);
4574 return true;
4575 case MVT::i16:
4576 case MVT::i32:
4577 case MVT::i64:
4578 // Support 8bit unsigned immediates.
4579 if ((Val & ~0xff) == 0) {
4580 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4581 Imm = CurDAG->getTargetConstant(Val: Val.getZExtValue(), DL, VT: MVT::i32);
4582 return true;
4583 }
4584 // Support 16bit unsigned immediates that are a multiple of 256.
4585 if ((Val & ~0xff00) == 0) {
4586 Shift = CurDAG->getTargetConstant(Val: 8, DL, VT: MVT::i32);
4587 Imm = CurDAG->getTargetConstant(Val: Val.lshr(shiftAmt: 8).getZExtValue(), DL, VT: MVT::i32);
4588 return true;
4589 }
4590 break;
4591 default:
4592 break;
4593 }
4594
4595 return false;
4596}
4597
4598bool AArch64DAGToDAGISel::SelectSVEAddSubSSatImm(SDValue N, MVT VT,
4599 SDValue &Imm, SDValue &Shift,
4600 bool Negate) {
4601 if (!isa<ConstantSDNode>(Val: N))
4602 return false;
4603
4604 SDLoc DL(N);
4605 int64_t Val = cast<ConstantSDNode>(Val&: N)
4606 ->getAPIntValue()
4607 .trunc(width: VT.getFixedSizeInBits())
4608 .getSExtValue();
4609
4610 if (Negate)
4611 Val = -Val;
4612
4613 // Signed saturating instructions treat their immediate operand as unsigned,
4614 // whereas the related intrinsics define their operands to be signed. This
4615 // means we can only use the immediate form when the operand is non-negative.
4616 if (Val < 0)
4617 return false;
4618
4619 switch (VT.SimpleTy) {
4620 case MVT::i8:
4621 // All positive immediates are supported.
4622 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4623 Imm = CurDAG->getTargetConstant(Val, DL, VT: MVT::i32);
4624 return true;
4625 case MVT::i16:
4626 case MVT::i32:
4627 case MVT::i64:
4628 // Support 8bit positive immediates.
4629 if (Val <= 255) {
4630 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4631 Imm = CurDAG->getTargetConstant(Val, DL, VT: MVT::i32);
4632 return true;
4633 }
4634 // Support 16bit positive immediates that are a multiple of 256.
4635 if (Val <= 65280 && Val % 256 == 0) {
4636 Shift = CurDAG->getTargetConstant(Val: 8, DL, VT: MVT::i32);
4637 Imm = CurDAG->getTargetConstant(Val: Val >> 8, DL, VT: MVT::i32);
4638 return true;
4639 }
4640 break;
4641 default:
4642 break;
4643 }
4644
4645 return false;
4646}
4647
4648bool AArch64DAGToDAGISel::SelectSVECpyDupImm(SDValue N, MVT VT, SDValue &Imm,
4649 SDValue &Shift) {
4650 if (!isa<ConstantSDNode>(Val: N))
4651 return false;
4652
4653 SDLoc DL(N);
4654 int64_t Val = cast<ConstantSDNode>(Val&: N)
4655 ->getAPIntValue()
4656 .trunc(width: VT.getFixedSizeInBits())
4657 .getSExtValue();
4658 int32_t ImmVal, ShiftVal;
4659 if (!AArch64_AM::isSVECpyDupImm(SizeInBits: VT.getScalarSizeInBits(), Val, Imm&: ImmVal,
4660 Shift&: ShiftVal))
4661 return false;
4662
4663 Shift = CurDAG->getTargetConstant(Val: ShiftVal, DL, VT: MVT::i32);
4664 Imm = CurDAG->getTargetConstant(Val: ImmVal, DL, VT: MVT::i32);
4665 return true;
4666}
4667
4668bool AArch64DAGToDAGISel::SelectSVESignedArithImm(SDValue N, SDValue &Imm) {
4669 if (auto CNode = dyn_cast<ConstantSDNode>(Val&: N))
4670 return SelectSVESignedArithImm(DL: SDLoc(N), Value: CNode->getAPIntValue(), Imm);
4671 return false;
4672}
4673
4674bool AArch64DAGToDAGISel::SelectSVESignedArithImm(SDLoc DL, APInt Val,
4675 SDValue &Imm) {
4676 int64_t ImmVal = Val.getSExtValue();
4677 if (ImmVal >= -128 && ImmVal < 128) {
4678 Imm = CurDAG->getSignedTargetConstant(Val: ImmVal, DL, VT: MVT::i32);
4679 return true;
4680 }
4681 return false;
4682}
4683
4684bool AArch64DAGToDAGISel::SelectSVEArithImm(SDValue N, MVT VT, SDValue &Imm) {
4685 if (auto CNode = dyn_cast<ConstantSDNode>(Val&: N)) {
4686 uint64_t ImmVal = CNode->getZExtValue();
4687
4688 switch (VT.SimpleTy) {
4689 case MVT::i8:
4690 ImmVal &= 0xFF;
4691 break;
4692 case MVT::i16:
4693 ImmVal &= 0xFFFF;
4694 break;
4695 case MVT::i32:
4696 ImmVal &= 0xFFFFFFFF;
4697 break;
4698 case MVT::i64:
4699 break;
4700 default:
4701 llvm_unreachable("Unexpected type");
4702 }
4703
4704 if (ImmVal < 256) {
4705 Imm = CurDAG->getTargetConstant(Val: ImmVal, DL: SDLoc(N), VT: MVT::i32);
4706 return true;
4707 }
4708 }
4709 return false;
4710}
4711
4712bool AArch64DAGToDAGISel::SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm,
4713 bool Invert) {
4714 uint64_t ImmVal;
4715 if (auto CI = dyn_cast<ConstantSDNode>(Val&: N))
4716 ImmVal = CI->getZExtValue();
4717 else if (auto CFP = dyn_cast<ConstantFPSDNode>(Val&: N))
4718 ImmVal = CFP->getValueAPF().bitcastToAPInt().getZExtValue();
4719 else
4720 return false;
4721
4722 if (Invert)
4723 ImmVal = ~ImmVal;
4724
4725 uint64_t encoding;
4726 if (!AArch64_AM::isSVELogicalImm(SizeInBits: VT.getScalarSizeInBits(), ImmVal, Encoding&: encoding))
4727 return false;
4728
4729 Imm = CurDAG->getTargetConstant(Val: encoding, DL: SDLoc(N), VT: MVT::i64);
4730 return true;
4731}
4732
4733// SVE shift intrinsics allow shift amounts larger than the element's bitwidth.
4734// Rather than attempt to normalise everything we can sometimes saturate the
4735// shift amount during selection. This function also allows for consistent
4736// isel patterns by ensuring the resulting "Imm" node is of the i32 type
4737// required by the instructions.
4738bool AArch64DAGToDAGISel::SelectSVEShiftImm(SDValue N, uint64_t Low,
4739 uint64_t High, bool AllowSaturation,
4740 SDValue &Imm) {
4741 if (auto *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
4742 uint64_t ImmVal = CN->getZExtValue();
4743
4744 // Reject shift amounts that are too small.
4745 if (ImmVal < Low)
4746 return false;
4747
4748 // Reject or saturate shift amounts that are too big.
4749 if (ImmVal > High) {
4750 if (!AllowSaturation)
4751 return false;
4752 ImmVal = High;
4753 }
4754
4755 Imm = CurDAG->getTargetConstant(Val: ImmVal, DL: SDLoc(N), VT: MVT::i32);
4756 return true;
4757 }
4758
4759 return false;
4760}
4761
4762bool AArch64DAGToDAGISel::trySelectStackSlotTagP(SDNode *N) {
4763 // tagp(FrameIndex, IRGstack, tag_offset):
4764 // since the offset between FrameIndex and IRGstack is a compile-time
4765 // constant, this can be lowered to a single ADDG instruction.
4766 if (!(isa<FrameIndexSDNode>(Val: N->getOperand(Num: 1)))) {
4767 return false;
4768 }
4769
4770 SDValue IRG_SP = N->getOperand(Num: 2);
4771 if (IRG_SP->getOpcode() != ISD::INTRINSIC_W_CHAIN ||
4772 IRG_SP->getConstantOperandVal(Num: 1) != Intrinsic::aarch64_irg_sp) {
4773 return false;
4774 }
4775
4776 const TargetLowering *TLI = getTargetLowering();
4777 SDLoc DL(N);
4778 int FI = cast<FrameIndexSDNode>(Val: N->getOperand(Num: 1))->getIndex();
4779 SDValue FiOp = CurDAG->getTargetFrameIndex(
4780 FI, VT: TLI->getPointerTy(DL: CurDAG->getDataLayout()));
4781 int TagOffset = N->getConstantOperandVal(Num: 3);
4782
4783 SDNode *Out = CurDAG->getMachineNode(
4784 Opcode: AArch64::TAGPstack, dl: DL, VT: MVT::i64,
4785 Ops: {FiOp, CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64), N->getOperand(Num: 2),
4786 CurDAG->getTargetConstant(Val: TagOffset, DL, VT: MVT::i64)});
4787 ReplaceNode(F: N, T: Out);
4788 return true;
4789}
4790
4791void AArch64DAGToDAGISel::SelectTagP(SDNode *N) {
4792 assert(isa<ConstantSDNode>(N->getOperand(3)) &&
4793 "llvm.aarch64.tagp third argument must be an immediate");
4794 if (trySelectStackSlotTagP(N))
4795 return;
4796 // FIXME: above applies in any case when offset between Op1 and Op2 is a
4797 // compile-time constant, not just for stack allocations.
4798
4799 // General case for unrelated pointers in Op1 and Op2.
4800 SDLoc DL(N);
4801 int TagOffset = N->getConstantOperandVal(Num: 3);
4802 SDNode *N1 = CurDAG->getMachineNode(Opcode: AArch64::SUBP, dl: DL, VT: MVT::i64,
4803 Ops: {N->getOperand(Num: 1), N->getOperand(Num: 2)});
4804 SDNode *N2 = CurDAG->getMachineNode(Opcode: AArch64::ADDXrr, dl: DL, VT: MVT::i64,
4805 Ops: {SDValue(N1, 0), N->getOperand(Num: 2)});
4806 SDNode *N3 = CurDAG->getMachineNode(
4807 Opcode: AArch64::ADDG, dl: DL, VT: MVT::i64,
4808 Ops: {SDValue(N2, 0), CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64),
4809 CurDAG->getTargetConstant(Val: TagOffset, DL, VT: MVT::i64)});
4810 ReplaceNode(F: N, T: N3);
4811}
4812
4813bool AArch64DAGToDAGISel::trySelectCastFixedLengthToScalableVector(SDNode *N) {
4814 assert(N->getOpcode() == ISD::INSERT_SUBVECTOR && "Invalid Node!");
4815
4816 // Bail when not a "cast" like insert_subvector.
4817 if (N->getConstantOperandVal(Num: 2) != 0)
4818 return false;
4819 if (!N->getOperand(Num: 0).isUndef())
4820 return false;
4821
4822 // Bail when normal isel should do the job.
4823 EVT VT = N->getValueType(ResNo: 0);
4824 EVT InVT = N->getOperand(Num: 1).getValueType();
4825 if (VT.isFixedLengthVector() || InVT.isScalableVector())
4826 return false;
4827 if (InVT.getSizeInBits() <= 128)
4828 return false;
4829
4830 // NOTE: We can only get here when doing fixed length SVE code generation.
4831 // We do manual selection because the types involved are not linked to real
4832 // registers (despite being legal) and must be coerced into SVE registers.
4833
4834 assert(VT.getSizeInBits().getKnownMinValue() == AArch64::SVEBitsPerBlock &&
4835 "Expected to insert into a packed scalable vector!");
4836
4837 SDLoc DL(N);
4838 auto RC = CurDAG->getTargetConstant(Val: AArch64::ZPRRegClassID, DL, VT: MVT::i64);
4839 ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS, dl: DL, VT,
4840 Op1: N->getOperand(Num: 1), Op2: RC));
4841 return true;
4842}
4843
4844bool AArch64DAGToDAGISel::trySelectCastScalableToFixedLengthVector(SDNode *N) {
4845 assert(N->getOpcode() == ISD::EXTRACT_SUBVECTOR && "Invalid Node!");
4846
4847 // Bail when not a "cast" like extract_subvector.
4848 if (N->getConstantOperandVal(Num: 1) != 0)
4849 return false;
4850
4851 // Bail when normal isel can do the job.
4852 EVT VT = N->getValueType(ResNo: 0);
4853 EVT InVT = N->getOperand(Num: 0).getValueType();
4854 if (VT.isScalableVector() || InVT.isFixedLengthVector())
4855 return false;
4856 if (VT.getSizeInBits() <= 128)
4857 return false;
4858
4859 // NOTE: We can only get here when doing fixed length SVE code generation.
4860 // We do manual selection because the types involved are not linked to real
4861 // registers (despite being legal) and must be coerced into SVE registers.
4862
4863 assert(InVT.getSizeInBits().getKnownMinValue() == AArch64::SVEBitsPerBlock &&
4864 "Expected to extract from a packed scalable vector!");
4865
4866 SDLoc DL(N);
4867 auto RC = CurDAG->getTargetConstant(Val: AArch64::ZPRRegClassID, DL, VT: MVT::i64);
4868 ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS, dl: DL, VT,
4869 Op1: N->getOperand(Num: 0), Op2: RC));
4870 return true;
4871}
4872
4873bool AArch64DAGToDAGISel::trySelectXAR(SDNode *N) {
4874 assert(N->getOpcode() == ISD::OR && "Expected OR instruction");
4875
4876 SDValue N0 = N->getOperand(Num: 0);
4877 SDValue N1 = N->getOperand(Num: 1);
4878
4879 EVT VT = N->getValueType(ResNo: 0);
4880 SDLoc DL(N);
4881
4882 // Essentially: rotr (xor(x, y), imm) -> xar (x, y, imm)
4883 // Rotate by a constant is a funnel shift in IR which is expanded to
4884 // an OR with shifted operands.
4885 // We do the following transform:
4886 // OR N0, N1 -> xar (x, y, imm)
4887 // Where:
4888 // N1 = SRL_PRED true, V, splat(imm) --> rotr amount
4889 // N0 = SHL_PRED true, V, splat(bits-imm)
4890 // V = (xor x, y)
4891 if (VT.isScalableVector() &&
4892 (Subtarget->hasSVE2() ||
4893 (Subtarget->hasSME() && Subtarget->isStreaming()))) {
4894 if (N0.getOpcode() != AArch64ISD::SHL_PRED ||
4895 N1.getOpcode() != AArch64ISD::SRL_PRED)
4896 std::swap(a&: N0, b&: N1);
4897 if (N0.getOpcode() != AArch64ISD::SHL_PRED ||
4898 N1.getOpcode() != AArch64ISD::SRL_PRED)
4899 return false;
4900
4901 auto *TLI = static_cast<const AArch64TargetLowering *>(getTargetLowering());
4902 if (!TLI->isAllActivePredicate(DAG: *CurDAG, N: N0.getOperand(i: 0)) ||
4903 !TLI->isAllActivePredicate(DAG: *CurDAG, N: N1.getOperand(i: 0)))
4904 return false;
4905
4906 if (N0.getOperand(i: 1) != N1.getOperand(i: 1))
4907 return false;
4908
4909 SDValue R1, R2;
4910 bool IsXOROperand = true;
4911 if (N0.getOperand(i: 1).getOpcode() != ISD::XOR) {
4912 IsXOROperand = false;
4913 } else {
4914 R1 = N0.getOperand(i: 1).getOperand(i: 0);
4915 R2 = N1.getOperand(i: 1).getOperand(i: 1);
4916 }
4917
4918 APInt ShlAmt, ShrAmt;
4919 if (!ISD::isConstantSplatVector(N: N0.getOperand(i: 2).getNode(), SplatValue&: ShlAmt) ||
4920 !ISD::isConstantSplatVector(N: N1.getOperand(i: 2).getNode(), SplatValue&: ShrAmt))
4921 return false;
4922
4923 if (ShlAmt + ShrAmt != VT.getScalarSizeInBits())
4924 return false;
4925
4926 if (!IsXOROperand) {
4927 SDValue Zero = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64);
4928 SDNode *MOV = CurDAG->getMachineNode(Opcode: AArch64::MOVIv2d_ns, dl: DL, VT, Op1: Zero);
4929 SDValue MOVIV = SDValue(MOV, 0);
4930
4931 SDValue ZSub = CurDAG->getTargetConstant(Val: AArch64::zsub, DL, VT: MVT::i32);
4932 SDNode *SubRegToReg =
4933 CurDAG->getMachineNode(Opcode: AArch64::SUBREG_TO_REG, dl: DL, VT, Op1: MOVIV, Op2: ZSub);
4934
4935 R1 = N1->getOperand(Num: 1);
4936 R2 = SDValue(SubRegToReg, 0);
4937 }
4938
4939 SDValue Imm =
4940 CurDAG->getTargetConstant(Val: ShrAmt.getZExtValue(), DL, VT: MVT::i32);
4941
4942 SDValue Ops[] = {R1, R2, Imm};
4943 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::Int>(
4944 VT, Opcodes: {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
4945 AArch64::XAR_ZZZI_D})) {
4946 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
4947 return true;
4948 }
4949 return false;
4950 }
4951
4952 // We have Neon SHA3 XAR operation for v2i64 but for types
4953 // v4i32, v8i16, v16i8 we can use SVE operations when SVE2-SHA3
4954 // is available.
4955 EVT SVT;
4956 switch (VT.getSimpleVT().SimpleTy) {
4957 case MVT::v4i32:
4958 case MVT::v2i32:
4959 SVT = MVT::nxv4i32;
4960 break;
4961 case MVT::v8i16:
4962 case MVT::v4i16:
4963 SVT = MVT::nxv8i16;
4964 break;
4965 case MVT::v16i8:
4966 case MVT::v8i8:
4967 SVT = MVT::nxv16i8;
4968 break;
4969 case MVT::v2i64:
4970 case MVT::v1i64:
4971 SVT = Subtarget->hasSHA3() ? MVT::v2i64 : MVT::nxv2i64;
4972 break;
4973 default:
4974 return false;
4975 }
4976
4977 if ((!SVT.isScalableVector() && !Subtarget->hasSHA3()) ||
4978 (SVT.isScalableVector() && !Subtarget->hasSVE2()))
4979 return false;
4980
4981 if (N0->getOpcode() != AArch64ISD::VSHL ||
4982 N1->getOpcode() != AArch64ISD::VLSHR)
4983 return false;
4984
4985 if (N0->getOperand(Num: 0) != N1->getOperand(Num: 0))
4986 return false;
4987
4988 SDValue R1, R2;
4989 bool IsXOROperand = true;
4990 if (N1->getOperand(Num: 0)->getOpcode() != ISD::XOR) {
4991 IsXOROperand = false;
4992 } else {
4993 SDValue XOR = N0.getOperand(i: 0);
4994 R1 = XOR.getOperand(i: 0);
4995 R2 = XOR.getOperand(i: 1);
4996 }
4997
4998 unsigned HsAmt = N0.getConstantOperandVal(i: 1);
4999 unsigned ShAmt = N1.getConstantOperandVal(i: 1);
5000
5001 SDValue Imm = CurDAG->getTargetConstant(
5002 Val: ShAmt, DL, VT: N0.getOperand(i: 1).getValueType(), isOpaque: false);
5003
5004 unsigned VTSizeInBits = VT.getScalarSizeInBits();
5005 if (ShAmt + HsAmt != VTSizeInBits)
5006 return false;
5007
5008 if (!IsXOROperand) {
5009 SDValue Zero = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64);
5010 SDNode *MOV =
5011 CurDAG->getMachineNode(Opcode: AArch64::MOVIv2d_ns, dl: DL, VT: MVT::v2i64, Op1: Zero);
5012 SDValue MOVIV = SDValue(MOV, 0);
5013
5014 R1 = N1->getOperand(Num: 0);
5015 R2 = MOVIV;
5016 }
5017
5018 if (SVT != VT) {
5019 SDValue Undef =
5020 SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: SVT), 0);
5021
5022 if (SVT.isScalableVector() && VT.is64BitVector()) {
5023 EVT QVT = VT.getDoubleNumVectorElementsVT(Context&: *CurDAG->getContext());
5024
5025 SDValue UndefQ = SDValue(
5026 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: QVT), 0);
5027 SDValue DSub = CurDAG->getTargetConstant(Val: AArch64::dsub, DL, VT: MVT::i32);
5028
5029 R1 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: QVT,
5030 Op1: UndefQ, Op2: R1, Op3: DSub),
5031 0);
5032 if (R2.getValueType() == VT)
5033 R2 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: QVT,
5034 Op1: UndefQ, Op2: R2, Op3: DSub),
5035 0);
5036 }
5037
5038 SDValue SubReg = CurDAG->getTargetConstant(
5039 Val: (SVT.isScalableVector() ? AArch64::zsub : AArch64::dsub), DL, VT: MVT::i32);
5040
5041 R1 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: SVT, Op1: Undef,
5042 Op2: R1, Op3: SubReg),
5043 0);
5044
5045 if (SVT.isScalableVector() || R2.getValueType() != SVT)
5046 R2 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: SVT,
5047 Op1: Undef, Op2: R2, Op3: SubReg),
5048 0);
5049 }
5050
5051 SDValue Ops[] = {R1, R2, Imm};
5052 SDNode *XAR = nullptr;
5053
5054 if (SVT.isScalableVector()) {
5055 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::Int>(
5056 VT: SVT, Opcodes: {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
5057 AArch64::XAR_ZZZI_D}))
5058 XAR = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: SVT, Ops);
5059 } else {
5060 XAR = CurDAG->getMachineNode(Opcode: AArch64::XAR, dl: DL, VT: SVT, Ops);
5061 }
5062
5063 assert(XAR && "Unexpected NULL value for XAR instruction in DAG");
5064
5065 if (SVT != VT) {
5066 if (VT.is64BitVector() && SVT.isScalableVector()) {
5067 EVT QVT = VT.getDoubleNumVectorElementsVT(Context&: *CurDAG->getContext());
5068
5069 SDValue ZSub = CurDAG->getTargetConstant(Val: AArch64::zsub, DL, VT: MVT::i32);
5070 SDNode *Q = CurDAG->getMachineNode(Opcode: AArch64::EXTRACT_SUBREG, dl: DL, VT: QVT,
5071 Op1: SDValue(XAR, 0), Op2: ZSub);
5072
5073 SDValue DSub = CurDAG->getTargetConstant(Val: AArch64::dsub, DL, VT: MVT::i32);
5074 XAR = CurDAG->getMachineNode(Opcode: AArch64::EXTRACT_SUBREG, dl: DL, VT,
5075 Op1: SDValue(Q, 0), Op2: DSub);
5076 } else {
5077 SDValue SubReg = CurDAG->getTargetConstant(
5078 Val: (SVT.isScalableVector() ? AArch64::zsub : AArch64::dsub), DL,
5079 VT: MVT::i32);
5080 XAR = CurDAG->getMachineNode(Opcode: AArch64::EXTRACT_SUBREG, dl: DL, VT,
5081 Op1: SDValue(XAR, 0), Op2: SubReg);
5082 }
5083 }
5084 ReplaceNode(F: N, T: XAR);
5085 return true;
5086}
5087
5088/// Returns a copy from WZR or XZR. This can be used during instruction
5089/// selection (it does not require any further selection/legalization).
5090static SDValue getZeroRegister(SelectionDAG &DAG, SDLoc DL, EVT VT) {
5091 assert(VT == MVT::i32 || VT == MVT::i64);
5092 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL,
5093 Reg: VT == MVT::i32 ? AArch64::WZR : AArch64::XZR, VT);
5094}
5095
5096void AArch64DAGToDAGISel::Select(SDNode *Node) {
5097 // If we have a custom node, we already have selected!
5098 if (Node->isMachineOpcode()) {
5099 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
5100 Node->setNodeId(-1);
5101 return;
5102 }
5103
5104 // Few custom selection stuff.
5105 EVT VT = Node->getValueType(ResNo: 0);
5106
5107 switch (Node->getOpcode()) {
5108 default:
5109 break;
5110
5111 case ISD::ATOMIC_CMP_SWAP:
5112 if (SelectCMP_SWAP(N: Node))
5113 return;
5114 break;
5115
5116 case ISD::READ_REGISTER:
5117 case AArch64ISD::MRRS:
5118 if (tryReadRegister(N: Node))
5119 return;
5120 break;
5121
5122 case ISD::WRITE_REGISTER:
5123 case AArch64ISD::MSRR:
5124 if (tryWriteRegister(N: Node))
5125 return;
5126 break;
5127
5128 case ISD::LOAD: {
5129 // Try to select as an indexed load. Fall through to normal processing
5130 // if we can't.
5131 if (tryIndexedLoad(N: Node))
5132 return;
5133 break;
5134 }
5135
5136 case ISD::SRL:
5137 case ISD::AND:
5138 case ISD::SRA:
5139 case ISD::SIGN_EXTEND_INREG:
5140 if (tryBitfieldExtractOp(N: Node))
5141 return;
5142 if (tryBitfieldInsertInZeroOp(N: Node))
5143 return;
5144 [[fallthrough]];
5145 case ISD::ROTR:
5146 case ISD::SHL:
5147 if (tryShiftAmountMod(N: Node))
5148 return;
5149 break;
5150
5151 case ISD::SIGN_EXTEND:
5152 if (tryBitfieldExtractOpFromSExt(N: Node))
5153 return;
5154 break;
5155
5156 case ISD::OR:
5157 if (tryBitfieldInsertOp(N: Node))
5158 return;
5159 if (trySelectXAR(N: Node))
5160 return;
5161 break;
5162
5163 case ISD::EXTRACT_SUBVECTOR: {
5164 if (trySelectCastScalableToFixedLengthVector(N: Node))
5165 return;
5166 break;
5167 }
5168
5169 case ISD::INSERT_SUBVECTOR: {
5170 if (trySelectCastFixedLengthToScalableVector(N: Node))
5171 return;
5172 break;
5173 }
5174
5175 case ISD::Constant: {
5176 // Materialize zero constants as copies from WZR/XZR. This allows
5177 // the coalescer to propagate these into other instructions.
5178 ConstantSDNode *ConstNode = cast<ConstantSDNode>(Val: Node);
5179 if (ConstNode->isZero() && (VT == MVT::i32 || VT == MVT::i64)) {
5180 ReplaceNode(F: Node, T: getZeroRegister(DAG&: *CurDAG, DL: SDLoc(Node), VT).getNode());
5181 return;
5182 }
5183 break;
5184 }
5185
5186 case ISD::FrameIndex: {
5187 // Selects to ADDXri FI, 0 which in turn will become ADDXri SP, imm.
5188 int FI = cast<FrameIndexSDNode>(Val: Node)->getIndex();
5189 unsigned Shifter = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0);
5190 const TargetLowering *TLI = getTargetLowering();
5191 SDValue TFI = CurDAG->getTargetFrameIndex(
5192 FI, VT: TLI->getPointerTy(DL: CurDAG->getDataLayout()));
5193 SDLoc DL(Node);
5194 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32),
5195 CurDAG->getTargetConstant(Val: Shifter, DL, VT: MVT::i32) };
5196 CurDAG->SelectNodeTo(N: Node, MachineOpc: AArch64::ADDXri, VT: MVT::i64, Ops);
5197 return;
5198 }
5199 case ISD::INTRINSIC_W_CHAIN: {
5200 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
5201 switch (IntNo) {
5202 default:
5203 break;
5204 case Intrinsic::aarch64_gcsss: {
5205 SDLoc DL(Node);
5206 SDValue Chain = Node->getOperand(Num: 0);
5207 SDValue Val = Node->getOperand(Num: 2);
5208 SDValue Zero = CurDAG->getCopyFromReg(Chain, dl: DL, Reg: AArch64::XZR, VT: MVT::i64);
5209 SDNode *SS1 =
5210 CurDAG->getMachineNode(Opcode: AArch64::GCSSS1, dl: DL, VT: MVT::Other, Op1: Val, Op2: Chain);
5211 SDNode *SS2 = CurDAG->getMachineNode(Opcode: AArch64::GCSSS2, dl: DL, VT1: MVT::i64,
5212 VT2: MVT::Other, Op1: Zero, Op2: SDValue(SS1, 0));
5213 ReplaceNode(F: Node, T: SS2);
5214 return;
5215 }
5216 case Intrinsic::aarch64_ldaxp:
5217 case Intrinsic::aarch64_ldxp: {
5218 unsigned Op =
5219 IntNo == Intrinsic::aarch64_ldaxp ? AArch64::LDAXPX : AArch64::LDXPX;
5220 SDValue MemAddr = Node->getOperand(Num: 2);
5221 SDLoc DL(Node);
5222 SDValue Chain = Node->getOperand(Num: 0);
5223
5224 SDNode *Ld = CurDAG->getMachineNode(Opcode: Op, dl: DL, VT1: MVT::i64, VT2: MVT::i64,
5225 VT3: MVT::Other, Op1: MemAddr, Op2: Chain);
5226
5227 // Transfer memoperands.
5228 MachineMemOperand *MemOp =
5229 cast<MemIntrinsicSDNode>(Val: Node)->getMemOperand();
5230 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Ld), NewMemRefs: {MemOp});
5231 ReplaceNode(F: Node, T: Ld);
5232 return;
5233 }
5234 case Intrinsic::aarch64_stlxp:
5235 case Intrinsic::aarch64_stxp: {
5236 unsigned Op =
5237 IntNo == Intrinsic::aarch64_stlxp ? AArch64::STLXPX : AArch64::STXPX;
5238 SDLoc DL(Node);
5239 SDValue Chain = Node->getOperand(Num: 0);
5240 SDValue ValLo = Node->getOperand(Num: 2);
5241 SDValue ValHi = Node->getOperand(Num: 3);
5242 SDValue MemAddr = Node->getOperand(Num: 4);
5243
5244 // Place arguments in the right order.
5245 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
5246
5247 SDNode *St = CurDAG->getMachineNode(Opcode: Op, dl: DL, VT1: MVT::i32, VT2: MVT::Other, Ops);
5248 // Transfer memoperands.
5249 MachineMemOperand *MemOp =
5250 cast<MemIntrinsicSDNode>(Val: Node)->getMemOperand();
5251 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
5252
5253 ReplaceNode(F: Node, T: St);
5254 return;
5255 }
5256 case Intrinsic::aarch64_neon_ld1x2:
5257 if (VT == MVT::v8i8) {
5258 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8b, SubRegIdx: AArch64::dsub0);
5259 return;
5260 } else if (VT == MVT::v16i8) {
5261 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov16b, SubRegIdx: AArch64::qsub0);
5262 return;
5263 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5264 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4h, SubRegIdx: AArch64::dsub0);
5265 return;
5266 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5267 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8h, SubRegIdx: AArch64::qsub0);
5268 return;
5269 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5270 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2s, SubRegIdx: AArch64::dsub0);
5271 return;
5272 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5273 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4s, SubRegIdx: AArch64::qsub0);
5274 return;
5275 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5276 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d, SubRegIdx: AArch64::dsub0);
5277 return;
5278 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5279 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2d, SubRegIdx: AArch64::qsub0);
5280 return;
5281 }
5282 break;
5283 case Intrinsic::aarch64_neon_ld1x3:
5284 if (VT == MVT::v8i8) {
5285 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8b, SubRegIdx: AArch64::dsub0);
5286 return;
5287 } else if (VT == MVT::v16i8) {
5288 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev16b, SubRegIdx: AArch64::qsub0);
5289 return;
5290 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5291 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4h, SubRegIdx: AArch64::dsub0);
5292 return;
5293 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5294 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8h, SubRegIdx: AArch64::qsub0);
5295 return;
5296 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5297 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2s, SubRegIdx: AArch64::dsub0);
5298 return;
5299 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5300 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4s, SubRegIdx: AArch64::qsub0);
5301 return;
5302 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5303 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d, SubRegIdx: AArch64::dsub0);
5304 return;
5305 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5306 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2d, SubRegIdx: AArch64::qsub0);
5307 return;
5308 }
5309 break;
5310 case Intrinsic::aarch64_neon_ld1x4:
5311 if (VT == MVT::v8i8) {
5312 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8b, SubRegIdx: AArch64::dsub0);
5313 return;
5314 } else if (VT == MVT::v16i8) {
5315 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv16b, SubRegIdx: AArch64::qsub0);
5316 return;
5317 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5318 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4h, SubRegIdx: AArch64::dsub0);
5319 return;
5320 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5321 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8h, SubRegIdx: AArch64::qsub0);
5322 return;
5323 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5324 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2s, SubRegIdx: AArch64::dsub0);
5325 return;
5326 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5327 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4s, SubRegIdx: AArch64::qsub0);
5328 return;
5329 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5330 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d, SubRegIdx: AArch64::dsub0);
5331 return;
5332 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5333 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2d, SubRegIdx: AArch64::qsub0);
5334 return;
5335 }
5336 break;
5337 case Intrinsic::aarch64_neon_ld2:
5338 if (VT == MVT::v8i8) {
5339 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8b, SubRegIdx: AArch64::dsub0);
5340 return;
5341 } else if (VT == MVT::v16i8) {
5342 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov16b, SubRegIdx: AArch64::qsub0);
5343 return;
5344 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5345 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4h, SubRegIdx: AArch64::dsub0);
5346 return;
5347 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5348 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8h, SubRegIdx: AArch64::qsub0);
5349 return;
5350 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5351 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2s, SubRegIdx: AArch64::dsub0);
5352 return;
5353 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5354 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4s, SubRegIdx: AArch64::qsub0);
5355 return;
5356 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5357 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d, SubRegIdx: AArch64::dsub0);
5358 return;
5359 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5360 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2d, SubRegIdx: AArch64::qsub0);
5361 return;
5362 }
5363 break;
5364 case Intrinsic::aarch64_neon_ld3:
5365 if (VT == MVT::v8i8) {
5366 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8b, SubRegIdx: AArch64::dsub0);
5367 return;
5368 } else if (VT == MVT::v16i8) {
5369 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev16b, SubRegIdx: AArch64::qsub0);
5370 return;
5371 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5372 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4h, SubRegIdx: AArch64::dsub0);
5373 return;
5374 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5375 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8h, SubRegIdx: AArch64::qsub0);
5376 return;
5377 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5378 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2s, SubRegIdx: AArch64::dsub0);
5379 return;
5380 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5381 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4s, SubRegIdx: AArch64::qsub0);
5382 return;
5383 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5384 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d, SubRegIdx: AArch64::dsub0);
5385 return;
5386 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5387 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2d, SubRegIdx: AArch64::qsub0);
5388 return;
5389 }
5390 break;
5391 case Intrinsic::aarch64_neon_ld4:
5392 if (VT == MVT::v8i8) {
5393 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8b, SubRegIdx: AArch64::dsub0);
5394 return;
5395 } else if (VT == MVT::v16i8) {
5396 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv16b, SubRegIdx: AArch64::qsub0);
5397 return;
5398 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5399 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4h, SubRegIdx: AArch64::dsub0);
5400 return;
5401 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5402 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8h, SubRegIdx: AArch64::qsub0);
5403 return;
5404 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5405 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2s, SubRegIdx: AArch64::dsub0);
5406 return;
5407 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5408 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4s, SubRegIdx: AArch64::qsub0);
5409 return;
5410 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5411 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d, SubRegIdx: AArch64::dsub0);
5412 return;
5413 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5414 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2d, SubRegIdx: AArch64::qsub0);
5415 return;
5416 }
5417 break;
5418 case Intrinsic::aarch64_neon_ld2r:
5419 if (VT == MVT::v8i8) {
5420 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8b, SubRegIdx: AArch64::dsub0);
5421 return;
5422 } else if (VT == MVT::v16i8) {
5423 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv16b, SubRegIdx: AArch64::qsub0);
5424 return;
5425 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5426 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4h, SubRegIdx: AArch64::dsub0);
5427 return;
5428 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5429 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8h, SubRegIdx: AArch64::qsub0);
5430 return;
5431 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5432 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2s, SubRegIdx: AArch64::dsub0);
5433 return;
5434 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5435 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4s, SubRegIdx: AArch64::qsub0);
5436 return;
5437 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5438 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv1d, SubRegIdx: AArch64::dsub0);
5439 return;
5440 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5441 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2d, SubRegIdx: AArch64::qsub0);
5442 return;
5443 }
5444 break;
5445 case Intrinsic::aarch64_neon_ld3r:
5446 if (VT == MVT::v8i8) {
5447 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8b, SubRegIdx: AArch64::dsub0);
5448 return;
5449 } else if (VT == MVT::v16i8) {
5450 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv16b, SubRegIdx: AArch64::qsub0);
5451 return;
5452 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5453 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4h, SubRegIdx: AArch64::dsub0);
5454 return;
5455 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5456 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8h, SubRegIdx: AArch64::qsub0);
5457 return;
5458 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5459 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2s, SubRegIdx: AArch64::dsub0);
5460 return;
5461 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5462 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4s, SubRegIdx: AArch64::qsub0);
5463 return;
5464 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5465 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv1d, SubRegIdx: AArch64::dsub0);
5466 return;
5467 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5468 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2d, SubRegIdx: AArch64::qsub0);
5469 return;
5470 }
5471 break;
5472 case Intrinsic::aarch64_neon_ld4r:
5473 if (VT == MVT::v8i8) {
5474 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8b, SubRegIdx: AArch64::dsub0);
5475 return;
5476 } else if (VT == MVT::v16i8) {
5477 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv16b, SubRegIdx: AArch64::qsub0);
5478 return;
5479 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5480 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4h, SubRegIdx: AArch64::dsub0);
5481 return;
5482 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5483 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8h, SubRegIdx: AArch64::qsub0);
5484 return;
5485 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5486 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2s, SubRegIdx: AArch64::dsub0);
5487 return;
5488 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5489 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4s, SubRegIdx: AArch64::qsub0);
5490 return;
5491 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5492 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv1d, SubRegIdx: AArch64::dsub0);
5493 return;
5494 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5495 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2d, SubRegIdx: AArch64::qsub0);
5496 return;
5497 }
5498 break;
5499 case Intrinsic::aarch64_neon_ld2lane:
5500 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5501 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i8);
5502 return;
5503 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5504 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5505 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i16);
5506 return;
5507 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5508 VT == MVT::v2f32) {
5509 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i32);
5510 return;
5511 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5512 VT == MVT::v1f64) {
5513 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i64);
5514 return;
5515 }
5516 break;
5517 case Intrinsic::aarch64_neon_ld3lane:
5518 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5519 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i8);
5520 return;
5521 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5522 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5523 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i16);
5524 return;
5525 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5526 VT == MVT::v2f32) {
5527 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i32);
5528 return;
5529 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5530 VT == MVT::v1f64) {
5531 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i64);
5532 return;
5533 }
5534 break;
5535 case Intrinsic::aarch64_neon_ld4lane:
5536 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5537 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i8);
5538 return;
5539 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5540 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5541 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i16);
5542 return;
5543 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5544 VT == MVT::v2f32) {
5545 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i32);
5546 return;
5547 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5548 VT == MVT::v1f64) {
5549 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i64);
5550 return;
5551 }
5552 break;
5553 case Intrinsic::aarch64_ld64b:
5554 SelectLoad(N: Node, NumVecs: 8, Opc: AArch64::LD64B, SubRegIdx: AArch64::x8sub_0);
5555 return;
5556 case Intrinsic::aarch64_sve_ld2q_sret: {
5557 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 4, Opc_ri: AArch64::LD2Q_IMM, Opc_rr: AArch64::LD2Q, IsIntr: true);
5558 return;
5559 }
5560 case Intrinsic::aarch64_sve_ld3q_sret: {
5561 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 4, Opc_ri: AArch64::LD3Q_IMM, Opc_rr: AArch64::LD3Q, IsIntr: true);
5562 return;
5563 }
5564 case Intrinsic::aarch64_sve_ld4q_sret: {
5565 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 4, Opc_ri: AArch64::LD4Q_IMM, Opc_rr: AArch64::LD4Q, IsIntr: true);
5566 return;
5567 }
5568 case Intrinsic::aarch64_sve_ld2_sret: {
5569 if (VT == MVT::nxv16i8) {
5570 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LD2B_IMM, Opc_rr: AArch64::LD2B,
5571 IsIntr: true);
5572 return;
5573 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5574 VT == MVT::nxv8bf16) {
5575 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LD2H_IMM, Opc_rr: AArch64::LD2H,
5576 IsIntr: true);
5577 return;
5578 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5579 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LD2W_IMM, Opc_rr: AArch64::LD2W,
5580 IsIntr: true);
5581 return;
5582 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5583 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LD2D_IMM, Opc_rr: AArch64::LD2D,
5584 IsIntr: true);
5585 return;
5586 }
5587 break;
5588 }
5589 case Intrinsic::aarch64_sve_ld1_pn_x2: {
5590 if (VT == MVT::nxv16i8) {
5591 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5592 SelectContiguousMultiVectorLoad(
5593 N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LD1B_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1B_2Z_PSEUDO);
5594 else if (Subtarget->hasSVE2p1())
5595 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LD1B_2Z_IMM,
5596 Opc_rr: AArch64::LD1B_2Z);
5597 else
5598 break;
5599 return;
5600 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5601 VT == MVT::nxv8bf16) {
5602 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5603 SelectContiguousMultiVectorLoad(
5604 N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LD1H_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1H_2Z_PSEUDO);
5605 else if (Subtarget->hasSVE2p1())
5606 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LD1H_2Z_IMM,
5607 Opc_rr: AArch64::LD1H_2Z);
5608 else
5609 break;
5610 return;
5611 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5612 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5613 SelectContiguousMultiVectorLoad(
5614 N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LD1W_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1W_2Z_PSEUDO);
5615 else if (Subtarget->hasSVE2p1())
5616 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LD1W_2Z_IMM,
5617 Opc_rr: AArch64::LD1W_2Z);
5618 else
5619 break;
5620 return;
5621 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5622 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5623 SelectContiguousMultiVectorLoad(
5624 N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LD1D_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1D_2Z_PSEUDO);
5625 else if (Subtarget->hasSVE2p1())
5626 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LD1D_2Z_IMM,
5627 Opc_rr: AArch64::LD1D_2Z);
5628 else
5629 break;
5630 return;
5631 }
5632 break;
5633 }
5634 case Intrinsic::aarch64_sve_ld1_pn_x4: {
5635 if (VT == MVT::nxv16i8) {
5636 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5637 SelectContiguousMultiVectorLoad(
5638 N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LD1B_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1B_4Z_PSEUDO);
5639 else if (Subtarget->hasSVE2p1())
5640 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LD1B_4Z_IMM,
5641 Opc_rr: AArch64::LD1B_4Z);
5642 else
5643 break;
5644 return;
5645 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5646 VT == MVT::nxv8bf16) {
5647 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5648 SelectContiguousMultiVectorLoad(
5649 N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LD1H_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1H_4Z_PSEUDO);
5650 else if (Subtarget->hasSVE2p1())
5651 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LD1H_4Z_IMM,
5652 Opc_rr: AArch64::LD1H_4Z);
5653 else
5654 break;
5655 return;
5656 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5657 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5658 SelectContiguousMultiVectorLoad(
5659 N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LD1W_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1W_4Z_PSEUDO);
5660 else if (Subtarget->hasSVE2p1())
5661 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LD1W_4Z_IMM,
5662 Opc_rr: AArch64::LD1W_4Z);
5663 else
5664 break;
5665 return;
5666 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5667 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5668 SelectContiguousMultiVectorLoad(
5669 N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LD1D_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1D_4Z_PSEUDO);
5670 else if (Subtarget->hasSVE2p1())
5671 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LD1D_4Z_IMM,
5672 Opc_rr: AArch64::LD1D_4Z);
5673 else
5674 break;
5675 return;
5676 }
5677 break;
5678 }
5679 case Intrinsic::aarch64_sve_ldnt1_pn_x2: {
5680 if (VT == MVT::nxv16i8) {
5681 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5682 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 0,
5683 Opc_ri: AArch64::LDNT1B_2Z_IMM_PSEUDO,
5684 Opc_rr: AArch64::LDNT1B_2Z_PSEUDO);
5685 else if (Subtarget->hasSVE2p1())
5686 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LDNT1B_2Z_IMM,
5687 Opc_rr: AArch64::LDNT1B_2Z);
5688 else
5689 break;
5690 return;
5691 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5692 VT == MVT::nxv8bf16) {
5693 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5694 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 1,
5695 Opc_ri: AArch64::LDNT1H_2Z_IMM_PSEUDO,
5696 Opc_rr: AArch64::LDNT1H_2Z_PSEUDO);
5697 else if (Subtarget->hasSVE2p1())
5698 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LDNT1H_2Z_IMM,
5699 Opc_rr: AArch64::LDNT1H_2Z);
5700 else
5701 break;
5702 return;
5703 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5704 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5705 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 2,
5706 Opc_ri: AArch64::LDNT1W_2Z_IMM_PSEUDO,
5707 Opc_rr: AArch64::LDNT1W_2Z_PSEUDO);
5708 else if (Subtarget->hasSVE2p1())
5709 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LDNT1W_2Z_IMM,
5710 Opc_rr: AArch64::LDNT1W_2Z);
5711 else
5712 break;
5713 return;
5714 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5715 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5716 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 3,
5717 Opc_ri: AArch64::LDNT1D_2Z_IMM_PSEUDO,
5718 Opc_rr: AArch64::LDNT1D_2Z_PSEUDO);
5719 else if (Subtarget->hasSVE2p1())
5720 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LDNT1D_2Z_IMM,
5721 Opc_rr: AArch64::LDNT1D_2Z);
5722 else
5723 break;
5724 return;
5725 }
5726 break;
5727 }
5728 case Intrinsic::aarch64_sve_ldnt1_pn_x4: {
5729 if (VT == MVT::nxv16i8) {
5730 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5731 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 0,
5732 Opc_ri: AArch64::LDNT1B_4Z_IMM_PSEUDO,
5733 Opc_rr: AArch64::LDNT1B_4Z_PSEUDO);
5734 else if (Subtarget->hasSVE2p1())
5735 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LDNT1B_4Z_IMM,
5736 Opc_rr: AArch64::LDNT1B_4Z);
5737 else
5738 break;
5739 return;
5740 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5741 VT == MVT::nxv8bf16) {
5742 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5743 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 1,
5744 Opc_ri: AArch64::LDNT1H_4Z_IMM_PSEUDO,
5745 Opc_rr: AArch64::LDNT1H_4Z_PSEUDO);
5746 else if (Subtarget->hasSVE2p1())
5747 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LDNT1H_4Z_IMM,
5748 Opc_rr: AArch64::LDNT1H_4Z);
5749 else
5750 break;
5751 return;
5752 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5753 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5754 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 2,
5755 Opc_ri: AArch64::LDNT1W_4Z_IMM_PSEUDO,
5756 Opc_rr: AArch64::LDNT1W_4Z_PSEUDO);
5757 else if (Subtarget->hasSVE2p1())
5758 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LDNT1W_4Z_IMM,
5759 Opc_rr: AArch64::LDNT1W_4Z);
5760 else
5761 break;
5762 return;
5763 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5764 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5765 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 3,
5766 Opc_ri: AArch64::LDNT1D_4Z_IMM_PSEUDO,
5767 Opc_rr: AArch64::LDNT1D_4Z_PSEUDO);
5768 else if (Subtarget->hasSVE2p1())
5769 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LDNT1D_4Z_IMM,
5770 Opc_rr: AArch64::LDNT1D_4Z);
5771 else
5772 break;
5773 return;
5774 }
5775 break;
5776 }
5777 case Intrinsic::aarch64_sve_ld3_sret: {
5778 if (VT == MVT::nxv16i8) {
5779 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 0, Opc_ri: AArch64::LD3B_IMM, Opc_rr: AArch64::LD3B,
5780 IsIntr: true);
5781 return;
5782 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5783 VT == MVT::nxv8bf16) {
5784 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 1, Opc_ri: AArch64::LD3H_IMM, Opc_rr: AArch64::LD3H,
5785 IsIntr: true);
5786 return;
5787 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5788 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 2, Opc_ri: AArch64::LD3W_IMM, Opc_rr: AArch64::LD3W,
5789 IsIntr: true);
5790 return;
5791 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5792 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 3, Opc_ri: AArch64::LD3D_IMM, Opc_rr: AArch64::LD3D,
5793 IsIntr: true);
5794 return;
5795 }
5796 break;
5797 }
5798 case Intrinsic::aarch64_sve_ld4_sret: {
5799 if (VT == MVT::nxv16i8) {
5800 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LD4B_IMM, Opc_rr: AArch64::LD4B,
5801 IsIntr: true);
5802 return;
5803 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5804 VT == MVT::nxv8bf16) {
5805 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LD4H_IMM, Opc_rr: AArch64::LD4H,
5806 IsIntr: true);
5807 return;
5808 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5809 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LD4W_IMM, Opc_rr: AArch64::LD4W,
5810 IsIntr: true);
5811 return;
5812 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5813 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LD4D_IMM, Opc_rr: AArch64::LD4D,
5814 IsIntr: true);
5815 return;
5816 }
5817 break;
5818 }
5819 case Intrinsic::aarch64_sme_read_hor_vg2: {
5820 if (VT == MVT::nxv16i8) {
5821 SelectMultiVectorMove<14, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAB0,
5822 Op: AArch64::MOVA_2ZMXI_H_B);
5823 return;
5824 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5825 VT == MVT::nxv8bf16) {
5826 SelectMultiVectorMove<6, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAH0,
5827 Op: AArch64::MOVA_2ZMXI_H_H);
5828 return;
5829 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5830 SelectMultiVectorMove<2, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAS0,
5831 Op: AArch64::MOVA_2ZMXI_H_S);
5832 return;
5833 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5834 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAD0,
5835 Op: AArch64::MOVA_2ZMXI_H_D);
5836 return;
5837 }
5838 break;
5839 }
5840 case Intrinsic::aarch64_sme_read_ver_vg2: {
5841 if (VT == MVT::nxv16i8) {
5842 SelectMultiVectorMove<14, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAB0,
5843 Op: AArch64::MOVA_2ZMXI_V_B);
5844 return;
5845 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5846 VT == MVT::nxv8bf16) {
5847 SelectMultiVectorMove<6, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAH0,
5848 Op: AArch64::MOVA_2ZMXI_V_H);
5849 return;
5850 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5851 SelectMultiVectorMove<2, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAS0,
5852 Op: AArch64::MOVA_2ZMXI_V_S);
5853 return;
5854 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5855 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAD0,
5856 Op: AArch64::MOVA_2ZMXI_V_D);
5857 return;
5858 }
5859 break;
5860 }
5861 case Intrinsic::aarch64_sme_read_hor_vg4: {
5862 if (VT == MVT::nxv16i8) {
5863 SelectMultiVectorMove<12, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAB0,
5864 Op: AArch64::MOVA_4ZMXI_H_B);
5865 return;
5866 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5867 VT == MVT::nxv8bf16) {
5868 SelectMultiVectorMove<4, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAH0,
5869 Op: AArch64::MOVA_4ZMXI_H_H);
5870 return;
5871 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5872 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAS0,
5873 Op: AArch64::MOVA_4ZMXI_H_S);
5874 return;
5875 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5876 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAD0,
5877 Op: AArch64::MOVA_4ZMXI_H_D);
5878 return;
5879 }
5880 break;
5881 }
5882 case Intrinsic::aarch64_sme_read_ver_vg4: {
5883 if (VT == MVT::nxv16i8) {
5884 SelectMultiVectorMove<12, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAB0,
5885 Op: AArch64::MOVA_4ZMXI_V_B);
5886 return;
5887 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5888 VT == MVT::nxv8bf16) {
5889 SelectMultiVectorMove<4, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAH0,
5890 Op: AArch64::MOVA_4ZMXI_V_H);
5891 return;
5892 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5893 SelectMultiVectorMove<0, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAS0,
5894 Op: AArch64::MOVA_4ZMXI_V_S);
5895 return;
5896 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5897 SelectMultiVectorMove<0, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAD0,
5898 Op: AArch64::MOVA_4ZMXI_V_D);
5899 return;
5900 }
5901 break;
5902 }
5903 case Intrinsic::aarch64_sme_read_vg1x2: {
5904 SelectMultiVectorMove<7, 1>(N: Node, NumVecs: 2, BaseReg: AArch64::ZA,
5905 Op: AArch64::MOVA_VG2_2ZMXI);
5906 return;
5907 }
5908 case Intrinsic::aarch64_sme_read_vg1x4: {
5909 SelectMultiVectorMove<7, 1>(N: Node, NumVecs: 4, BaseReg: AArch64::ZA,
5910 Op: AArch64::MOVA_VG4_4ZMXI);
5911 return;
5912 }
5913 case Intrinsic::aarch64_sme_readz_horiz_x2: {
5914 if (VT == MVT::nxv16i8) {
5915 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_B_PSEUDO, MaxIdx: 14, Scale: 2);
5916 return;
5917 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5918 VT == MVT::nxv8bf16) {
5919 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_H_PSEUDO, MaxIdx: 6, Scale: 2);
5920 return;
5921 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5922 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_S_PSEUDO, MaxIdx: 2, Scale: 2);
5923 return;
5924 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5925 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_D_PSEUDO, MaxIdx: 0, Scale: 2);
5926 return;
5927 }
5928 break;
5929 }
5930 case Intrinsic::aarch64_sme_readz_vert_x2: {
5931 if (VT == MVT::nxv16i8) {
5932 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_B_PSEUDO, MaxIdx: 14, Scale: 2);
5933 return;
5934 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5935 VT == MVT::nxv8bf16) {
5936 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_H_PSEUDO, MaxIdx: 6, Scale: 2);
5937 return;
5938 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5939 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_S_PSEUDO, MaxIdx: 2, Scale: 2);
5940 return;
5941 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5942 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_D_PSEUDO, MaxIdx: 0, Scale: 2);
5943 return;
5944 }
5945 break;
5946 }
5947 case Intrinsic::aarch64_sme_readz_horiz_x4: {
5948 if (VT == MVT::nxv16i8) {
5949 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_B_PSEUDO, MaxIdx: 12, Scale: 4);
5950 return;
5951 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5952 VT == MVT::nxv8bf16) {
5953 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_H_PSEUDO, MaxIdx: 4, Scale: 4);
5954 return;
5955 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5956 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_S_PSEUDO, MaxIdx: 0, Scale: 4);
5957 return;
5958 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5959 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_D_PSEUDO, MaxIdx: 0, Scale: 4);
5960 return;
5961 }
5962 break;
5963 }
5964 case Intrinsic::aarch64_sme_readz_vert_x4: {
5965 if (VT == MVT::nxv16i8) {
5966 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_B_PSEUDO, MaxIdx: 12, Scale: 4);
5967 return;
5968 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5969 VT == MVT::nxv8bf16) {
5970 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_H_PSEUDO, MaxIdx: 4, Scale: 4);
5971 return;
5972 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5973 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_S_PSEUDO, MaxIdx: 0, Scale: 4);
5974 return;
5975 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5976 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_D_PSEUDO, MaxIdx: 0, Scale: 4);
5977 return;
5978 }
5979 break;
5980 }
5981 case Intrinsic::aarch64_sme_readz_x2: {
5982 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_VG2_2ZMXI_PSEUDO, MaxIdx: 7, Scale: 1,
5983 BaseReg: AArch64::ZA);
5984 return;
5985 }
5986 case Intrinsic::aarch64_sme_readz_x4: {
5987 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_VG4_4ZMXI_PSEUDO, MaxIdx: 7, Scale: 1,
5988 BaseReg: AArch64::ZA);
5989 return;
5990 }
5991 case Intrinsic::swift_async_context_addr: {
5992 SDLoc DL(Node);
5993 SDValue Chain = Node->getOperand(Num: 0);
5994 SDValue CopyFP = CurDAG->getCopyFromReg(Chain, dl: DL, Reg: AArch64::FP, VT: MVT::i64);
5995 SDValue Res = SDValue(
5996 CurDAG->getMachineNode(Opcode: AArch64::SUBXri, dl: DL, VT: MVT::i64, Op1: CopyFP,
5997 Op2: CurDAG->getTargetConstant(Val: 8, DL, VT: MVT::i32),
5998 Op3: CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32)),
5999 0);
6000 ReplaceUses(F: SDValue(Node, 0), T: Res);
6001 ReplaceUses(F: SDValue(Node, 1), T: CopyFP.getValue(R: 1));
6002 CurDAG->RemoveDeadNode(N: Node);
6003
6004 auto &MF = CurDAG->getMachineFunction();
6005 MF.getFrameInfo().setFrameAddressIsTaken(true);
6006 MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
6007 return;
6008 }
6009 case Intrinsic::aarch64_sme_luti2_lane_zt_x4: {
6010 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6011 VT: Node->getValueType(ResNo: 0),
6012 Opcodes: {AArch64::LUTI2_4ZTZI_B, AArch64::LUTI2_4ZTZI_H,
6013 AArch64::LUTI2_4ZTZI_S}))
6014 // Second Immediate must be <= 3:
6015 SelectMultiVectorLutiLane(Node, NumOutVecs: 4, Opc, MaxImm: 3);
6016 return;
6017 }
6018 case Intrinsic::aarch64_sme_luti4_lane_zt_x4: {
6019 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6020 VT: Node->getValueType(ResNo: 0),
6021 Opcodes: {0, AArch64::LUTI4_4ZTZI_H, AArch64::LUTI4_4ZTZI_S}))
6022 // Second Immediate must be <= 1:
6023 SelectMultiVectorLutiLane(Node, NumOutVecs: 4, Opc, MaxImm: 1);
6024 return;
6025 }
6026 case Intrinsic::aarch64_sme_luti2_lane_zt_x2: {
6027 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6028 VT: Node->getValueType(ResNo: 0),
6029 Opcodes: {AArch64::LUTI2_2ZTZI_B, AArch64::LUTI2_2ZTZI_H,
6030 AArch64::LUTI2_2ZTZI_S}))
6031 // Second Immediate must be <= 7:
6032 SelectMultiVectorLutiLane(Node, NumOutVecs: 2, Opc, MaxImm: 7);
6033 return;
6034 }
6035 case Intrinsic::aarch64_sme_luti4_lane_zt_x2: {
6036 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6037 VT: Node->getValueType(ResNo: 0),
6038 Opcodes: {AArch64::LUTI4_2ZTZI_B, AArch64::LUTI4_2ZTZI_H,
6039 AArch64::LUTI4_2ZTZI_S}))
6040 // Second Immediate must be <= 3:
6041 SelectMultiVectorLutiLane(Node, NumOutVecs: 2, Opc, MaxImm: 3);
6042 return;
6043 }
6044 case Intrinsic::aarch64_sme_luti4_zt_x4: {
6045 SelectMultiVectorLuti(Node, NumOutVecs: 4, Opc: AArch64::LUTI4_4ZZT2Z, NumInVecs: 2);
6046 return;
6047 }
6048 case Intrinsic::aarch64_sme_luti6_zt_x4: {
6049 SelectMultiVectorLuti(Node, NumOutVecs: 4, Opc: AArch64::LUTI6_4ZT3Z, NumInVecs: 3);
6050 return;
6051 }
6052 case Intrinsic::aarch64_sve_fp8_cvtl1_x2:
6053 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6054 VT: Node->getValueType(ResNo: 0),
6055 Opcodes: {AArch64::BF1CVTL_2ZZ_BtoH, AArch64::F1CVTL_2ZZ_BtoH}))
6056 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6057 return;
6058 case Intrinsic::aarch64_sve_fp8_cvtl2_x2:
6059 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6060 VT: Node->getValueType(ResNo: 0),
6061 Opcodes: {AArch64::BF2CVTL_2ZZ_BtoH, AArch64::F2CVTL_2ZZ_BtoH}))
6062 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6063 return;
6064 case Intrinsic::aarch64_sve_fp8_cvt1_x2:
6065 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6066 VT: Node->getValueType(ResNo: 0),
6067 Opcodes: {AArch64::BF1CVT_2ZZ_BtoH, AArch64::F1CVT_2ZZ_BtoH}))
6068 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6069 return;
6070 case Intrinsic::aarch64_sve_fp8_cvt2_x2:
6071 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6072 VT: Node->getValueType(ResNo: 0),
6073 Opcodes: {AArch64::BF2CVT_2ZZ_BtoH, AArch64::F2CVT_2ZZ_BtoH}))
6074 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6075 return;
6076 case Intrinsic::ptrauth_resign_load_relative:
6077 SelectPtrauthResign(N: Node);
6078 return;
6079 }
6080 } break;
6081 case ISD::INTRINSIC_WO_CHAIN: {
6082 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
6083 switch (IntNo) {
6084 default:
6085 break;
6086 case Intrinsic::aarch64_tagp:
6087 SelectTagP(N: Node);
6088 return;
6089
6090 case Intrinsic::ptrauth_auth:
6091 SelectPtrauthAuth(N: Node);
6092 return;
6093
6094 case Intrinsic::ptrauth_resign:
6095 SelectPtrauthResign(N: Node);
6096 return;
6097
6098 case Intrinsic::ptrauth_auth_with_pc_and_resign:
6099 SelectPtrauthResignWithPC(N: Node);
6100 return;
6101
6102 case Intrinsic::aarch64_neon_tbl2:
6103 SelectTable(N: Node, NumVecs: 2,
6104 Opc: VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two,
6105 isExt: false);
6106 return;
6107 case Intrinsic::aarch64_neon_tbl3:
6108 SelectTable(N: Node, NumVecs: 3, Opc: VT == MVT::v8i8 ? AArch64::TBLv8i8Three
6109 : AArch64::TBLv16i8Three,
6110 isExt: false);
6111 return;
6112 case Intrinsic::aarch64_neon_tbl4:
6113 SelectTable(N: Node, NumVecs: 4, Opc: VT == MVT::v8i8 ? AArch64::TBLv8i8Four
6114 : AArch64::TBLv16i8Four,
6115 isExt: false);
6116 return;
6117 case Intrinsic::aarch64_neon_tbx2:
6118 SelectTable(N: Node, NumVecs: 2,
6119 Opc: VT == MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two,
6120 isExt: true);
6121 return;
6122 case Intrinsic::aarch64_neon_tbx3:
6123 SelectTable(N: Node, NumVecs: 3, Opc: VT == MVT::v8i8 ? AArch64::TBXv8i8Three
6124 : AArch64::TBXv16i8Three,
6125 isExt: true);
6126 return;
6127 case Intrinsic::aarch64_neon_tbx4:
6128 SelectTable(N: Node, NumVecs: 4, Opc: VT == MVT::v8i8 ? AArch64::TBXv8i8Four
6129 : AArch64::TBXv16i8Four,
6130 isExt: true);
6131 return;
6132 case Intrinsic::aarch64_sve_srshl_single_x2:
6133 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6134 VT: Node->getValueType(ResNo: 0),
6135 Opcodes: {AArch64::SRSHL_VG2_2ZZ_B, AArch64::SRSHL_VG2_2ZZ_H,
6136 AArch64::SRSHL_VG2_2ZZ_S, AArch64::SRSHL_VG2_2ZZ_D}))
6137 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6138 return;
6139 case Intrinsic::aarch64_sve_srshl_single_x4:
6140 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6141 VT: Node->getValueType(ResNo: 0),
6142 Opcodes: {AArch64::SRSHL_VG4_4ZZ_B, AArch64::SRSHL_VG4_4ZZ_H,
6143 AArch64::SRSHL_VG4_4ZZ_S, AArch64::SRSHL_VG4_4ZZ_D}))
6144 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6145 return;
6146 case Intrinsic::aarch64_sme_luti6_lane_x4_x2:
6147 SelectMultiVectorLuti6LaneX4(Node, NumIndexVecs: 2);
6148 return;
6149 case Intrinsic::aarch64_sme_luti6_lane_x4_x3:
6150 SelectMultiVectorLuti6LaneX4(Node, NumIndexVecs: 3);
6151 return;
6152 case Intrinsic::aarch64_sve_urshl_single_x2:
6153 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6154 VT: Node->getValueType(ResNo: 0),
6155 Opcodes: {AArch64::URSHL_VG2_2ZZ_B, AArch64::URSHL_VG2_2ZZ_H,
6156 AArch64::URSHL_VG2_2ZZ_S, AArch64::URSHL_VG2_2ZZ_D}))
6157 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6158 return;
6159 case Intrinsic::aarch64_sve_urshl_single_x4:
6160 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6161 VT: Node->getValueType(ResNo: 0),
6162 Opcodes: {AArch64::URSHL_VG4_4ZZ_B, AArch64::URSHL_VG4_4ZZ_H,
6163 AArch64::URSHL_VG4_4ZZ_S, AArch64::URSHL_VG4_4ZZ_D}))
6164 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6165 return;
6166 case Intrinsic::aarch64_sve_srshl_x2:
6167 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6168 VT: Node->getValueType(ResNo: 0),
6169 Opcodes: {AArch64::SRSHL_VG2_2Z2Z_B, AArch64::SRSHL_VG2_2Z2Z_H,
6170 AArch64::SRSHL_VG2_2Z2Z_S, AArch64::SRSHL_VG2_2Z2Z_D}))
6171 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6172 return;
6173 case Intrinsic::aarch64_sve_srshl_x4:
6174 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6175 VT: Node->getValueType(ResNo: 0),
6176 Opcodes: {AArch64::SRSHL_VG4_4Z4Z_B, AArch64::SRSHL_VG4_4Z4Z_H,
6177 AArch64::SRSHL_VG4_4Z4Z_S, AArch64::SRSHL_VG4_4Z4Z_D}))
6178 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6179 return;
6180 case Intrinsic::aarch64_sve_urshl_x2:
6181 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6182 VT: Node->getValueType(ResNo: 0),
6183 Opcodes: {AArch64::URSHL_VG2_2Z2Z_B, AArch64::URSHL_VG2_2Z2Z_H,
6184 AArch64::URSHL_VG2_2Z2Z_S, AArch64::URSHL_VG2_2Z2Z_D}))
6185 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6186 return;
6187 case Intrinsic::aarch64_sve_urshl_x4:
6188 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6189 VT: Node->getValueType(ResNo: 0),
6190 Opcodes: {AArch64::URSHL_VG4_4Z4Z_B, AArch64::URSHL_VG4_4Z4Z_H,
6191 AArch64::URSHL_VG4_4Z4Z_S, AArch64::URSHL_VG4_4Z4Z_D}))
6192 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6193 return;
6194 case Intrinsic::aarch64_sve_sqdmulh_single_vgx2:
6195 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6196 VT: Node->getValueType(ResNo: 0),
6197 Opcodes: {AArch64::SQDMULH_VG2_2ZZ_B, AArch64::SQDMULH_VG2_2ZZ_H,
6198 AArch64::SQDMULH_VG2_2ZZ_S, AArch64::SQDMULH_VG2_2ZZ_D}))
6199 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6200 return;
6201 case Intrinsic::aarch64_sve_sqdmulh_single_vgx4:
6202 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6203 VT: Node->getValueType(ResNo: 0),
6204 Opcodes: {AArch64::SQDMULH_VG4_4ZZ_B, AArch64::SQDMULH_VG4_4ZZ_H,
6205 AArch64::SQDMULH_VG4_4ZZ_S, AArch64::SQDMULH_VG4_4ZZ_D}))
6206 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6207 return;
6208 case Intrinsic::aarch64_sve_sqdmulh_vgx2:
6209 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6210 VT: Node->getValueType(ResNo: 0),
6211 Opcodes: {AArch64::SQDMULH_VG2_2Z2Z_B, AArch64::SQDMULH_VG2_2Z2Z_H,
6212 AArch64::SQDMULH_VG2_2Z2Z_S, AArch64::SQDMULH_VG2_2Z2Z_D}))
6213 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6214 return;
6215 case Intrinsic::aarch64_sve_sqdmulh_vgx4:
6216 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6217 VT: Node->getValueType(ResNo: 0),
6218 Opcodes: {AArch64::SQDMULH_VG4_4Z4Z_B, AArch64::SQDMULH_VG4_4Z4Z_H,
6219 AArch64::SQDMULH_VG4_4Z4Z_S, AArch64::SQDMULH_VG4_4Z4Z_D}))
6220 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6221 return;
6222 case Intrinsic::aarch64_sme_fp8_scale_single_x2:
6223 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6224 VT: Node->getValueType(ResNo: 0),
6225 Opcodes: {0, AArch64::FSCALE_2ZZ_H, AArch64::FSCALE_2ZZ_S,
6226 AArch64::FSCALE_2ZZ_D}))
6227 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6228 return;
6229 case Intrinsic::aarch64_sme_fp8_scale_single_x4:
6230 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6231 VT: Node->getValueType(ResNo: 0),
6232 Opcodes: {0, AArch64::FSCALE_4ZZ_H, AArch64::FSCALE_4ZZ_S,
6233 AArch64::FSCALE_4ZZ_D}))
6234 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6235 return;
6236 case Intrinsic::aarch64_sme_fp8_scale_x2:
6237 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6238 VT: Node->getValueType(ResNo: 0),
6239 Opcodes: {0, AArch64::FSCALE_2Z2Z_H, AArch64::FSCALE_2Z2Z_S,
6240 AArch64::FSCALE_2Z2Z_D}))
6241 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6242 return;
6243 case Intrinsic::aarch64_sme_fp8_scale_x4:
6244 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6245 VT: Node->getValueType(ResNo: 0),
6246 Opcodes: {0, AArch64::FSCALE_4Z4Z_H, AArch64::FSCALE_4Z4Z_S,
6247 AArch64::FSCALE_4Z4Z_D}))
6248 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6249 return;
6250 case Intrinsic::aarch64_sve_whilege_x2:
6251 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6252 VT: Node->getValueType(ResNo: 0),
6253 Opcodes: {AArch64::WHILEGE_2PXX_B, AArch64::WHILEGE_2PXX_H,
6254 AArch64::WHILEGE_2PXX_S, AArch64::WHILEGE_2PXX_D}))
6255 SelectWhilePair(N: Node, Opc: Op);
6256 return;
6257 case Intrinsic::aarch64_sve_whilegt_x2:
6258 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6259 VT: Node->getValueType(ResNo: 0),
6260 Opcodes: {AArch64::WHILEGT_2PXX_B, AArch64::WHILEGT_2PXX_H,
6261 AArch64::WHILEGT_2PXX_S, AArch64::WHILEGT_2PXX_D}))
6262 SelectWhilePair(N: Node, Opc: Op);
6263 return;
6264 case Intrinsic::aarch64_sve_whilehi_x2:
6265 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6266 VT: Node->getValueType(ResNo: 0),
6267 Opcodes: {AArch64::WHILEHI_2PXX_B, AArch64::WHILEHI_2PXX_H,
6268 AArch64::WHILEHI_2PXX_S, AArch64::WHILEHI_2PXX_D}))
6269 SelectWhilePair(N: Node, Opc: Op);
6270 return;
6271 case Intrinsic::aarch64_sve_whilehs_x2:
6272 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6273 VT: Node->getValueType(ResNo: 0),
6274 Opcodes: {AArch64::WHILEHS_2PXX_B, AArch64::WHILEHS_2PXX_H,
6275 AArch64::WHILEHS_2PXX_S, AArch64::WHILEHS_2PXX_D}))
6276 SelectWhilePair(N: Node, Opc: Op);
6277 return;
6278 case Intrinsic::aarch64_sve_whilele_x2:
6279 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6280 VT: Node->getValueType(ResNo: 0),
6281 Opcodes: {AArch64::WHILELE_2PXX_B, AArch64::WHILELE_2PXX_H,
6282 AArch64::WHILELE_2PXX_S, AArch64::WHILELE_2PXX_D}))
6283 SelectWhilePair(N: Node, Opc: Op);
6284 return;
6285 case Intrinsic::aarch64_sve_whilelo_x2:
6286 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6287 VT: Node->getValueType(ResNo: 0),
6288 Opcodes: {AArch64::WHILELO_2PXX_B, AArch64::WHILELO_2PXX_H,
6289 AArch64::WHILELO_2PXX_S, AArch64::WHILELO_2PXX_D}))
6290 SelectWhilePair(N: Node, Opc: Op);
6291 return;
6292 case Intrinsic::aarch64_sve_whilels_x2:
6293 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6294 VT: Node->getValueType(ResNo: 0),
6295 Opcodes: {AArch64::WHILELS_2PXX_B, AArch64::WHILELS_2PXX_H,
6296 AArch64::WHILELS_2PXX_S, AArch64::WHILELS_2PXX_D}))
6297 SelectWhilePair(N: Node, Opc: Op);
6298 return;
6299 case Intrinsic::aarch64_sve_whilelt_x2:
6300 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6301 VT: Node->getValueType(ResNo: 0),
6302 Opcodes: {AArch64::WHILELT_2PXX_B, AArch64::WHILELT_2PXX_H,
6303 AArch64::WHILELT_2PXX_S, AArch64::WHILELT_2PXX_D}))
6304 SelectWhilePair(N: Node, Opc: Op);
6305 return;
6306 case Intrinsic::aarch64_sve_smax_single_x2:
6307 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6308 VT: Node->getValueType(ResNo: 0),
6309 Opcodes: {AArch64::SMAX_VG2_2ZZ_B, AArch64::SMAX_VG2_2ZZ_H,
6310 AArch64::SMAX_VG2_2ZZ_S, AArch64::SMAX_VG2_2ZZ_D}))
6311 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6312 return;
6313 case Intrinsic::aarch64_sve_umax_single_x2:
6314 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6315 VT: Node->getValueType(ResNo: 0),
6316 Opcodes: {AArch64::UMAX_VG2_2ZZ_B, AArch64::UMAX_VG2_2ZZ_H,
6317 AArch64::UMAX_VG2_2ZZ_S, AArch64::UMAX_VG2_2ZZ_D}))
6318 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6319 return;
6320 case Intrinsic::aarch64_sve_fmax_single_x2:
6321 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6322 VT: Node->getValueType(ResNo: 0),
6323 Opcodes: {AArch64::BFMAX_VG2_2ZZ_H, AArch64::FMAX_VG2_2ZZ_H,
6324 AArch64::FMAX_VG2_2ZZ_S, AArch64::FMAX_VG2_2ZZ_D}))
6325 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6326 return;
6327 case Intrinsic::aarch64_sve_smax_single_x4:
6328 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6329 VT: Node->getValueType(ResNo: 0),
6330 Opcodes: {AArch64::SMAX_VG4_4ZZ_B, AArch64::SMAX_VG4_4ZZ_H,
6331 AArch64::SMAX_VG4_4ZZ_S, AArch64::SMAX_VG4_4ZZ_D}))
6332 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6333 return;
6334 case Intrinsic::aarch64_sve_umax_single_x4:
6335 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6336 VT: Node->getValueType(ResNo: 0),
6337 Opcodes: {AArch64::UMAX_VG4_4ZZ_B, AArch64::UMAX_VG4_4ZZ_H,
6338 AArch64::UMAX_VG4_4ZZ_S, AArch64::UMAX_VG4_4ZZ_D}))
6339 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6340 return;
6341 case Intrinsic::aarch64_sve_fmax_single_x4:
6342 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6343 VT: Node->getValueType(ResNo: 0),
6344 Opcodes: {AArch64::BFMAX_VG4_4ZZ_H, AArch64::FMAX_VG4_4ZZ_H,
6345 AArch64::FMAX_VG4_4ZZ_S, AArch64::FMAX_VG4_4ZZ_D}))
6346 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6347 return;
6348 case Intrinsic::aarch64_sve_smin_single_x2:
6349 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6350 VT: Node->getValueType(ResNo: 0),
6351 Opcodes: {AArch64::SMIN_VG2_2ZZ_B, AArch64::SMIN_VG2_2ZZ_H,
6352 AArch64::SMIN_VG2_2ZZ_S, AArch64::SMIN_VG2_2ZZ_D}))
6353 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6354 return;
6355 case Intrinsic::aarch64_sve_umin_single_x2:
6356 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6357 VT: Node->getValueType(ResNo: 0),
6358 Opcodes: {AArch64::UMIN_VG2_2ZZ_B, AArch64::UMIN_VG2_2ZZ_H,
6359 AArch64::UMIN_VG2_2ZZ_S, AArch64::UMIN_VG2_2ZZ_D}))
6360 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6361 return;
6362 case Intrinsic::aarch64_sve_fmin_single_x2:
6363 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6364 VT: Node->getValueType(ResNo: 0),
6365 Opcodes: {AArch64::BFMIN_VG2_2ZZ_H, AArch64::FMIN_VG2_2ZZ_H,
6366 AArch64::FMIN_VG2_2ZZ_S, AArch64::FMIN_VG2_2ZZ_D}))
6367 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6368 return;
6369 case Intrinsic::aarch64_sve_smin_single_x4:
6370 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6371 VT: Node->getValueType(ResNo: 0),
6372 Opcodes: {AArch64::SMIN_VG4_4ZZ_B, AArch64::SMIN_VG4_4ZZ_H,
6373 AArch64::SMIN_VG4_4ZZ_S, AArch64::SMIN_VG4_4ZZ_D}))
6374 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6375 return;
6376 case Intrinsic::aarch64_sve_umin_single_x4:
6377 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6378 VT: Node->getValueType(ResNo: 0),
6379 Opcodes: {AArch64::UMIN_VG4_4ZZ_B, AArch64::UMIN_VG4_4ZZ_H,
6380 AArch64::UMIN_VG4_4ZZ_S, AArch64::UMIN_VG4_4ZZ_D}))
6381 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6382 return;
6383 case Intrinsic::aarch64_sve_fmin_single_x4:
6384 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6385 VT: Node->getValueType(ResNo: 0),
6386 Opcodes: {AArch64::BFMIN_VG4_4ZZ_H, AArch64::FMIN_VG4_4ZZ_H,
6387 AArch64::FMIN_VG4_4ZZ_S, AArch64::FMIN_VG4_4ZZ_D}))
6388 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6389 return;
6390 case Intrinsic::aarch64_sve_smax_x2:
6391 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6392 VT: Node->getValueType(ResNo: 0),
6393 Opcodes: {AArch64::SMAX_VG2_2Z2Z_B, AArch64::SMAX_VG2_2Z2Z_H,
6394 AArch64::SMAX_VG2_2Z2Z_S, AArch64::SMAX_VG2_2Z2Z_D}))
6395 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6396 return;
6397 case Intrinsic::aarch64_sve_umax_x2:
6398 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6399 VT: Node->getValueType(ResNo: 0),
6400 Opcodes: {AArch64::UMAX_VG2_2Z2Z_B, AArch64::UMAX_VG2_2Z2Z_H,
6401 AArch64::UMAX_VG2_2Z2Z_S, AArch64::UMAX_VG2_2Z2Z_D}))
6402 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6403 return;
6404 case Intrinsic::aarch64_sve_fmax_x2:
6405 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6406 VT: Node->getValueType(ResNo: 0),
6407 Opcodes: {AArch64::BFMAX_VG2_2Z2Z_H, AArch64::FMAX_VG2_2Z2Z_H,
6408 AArch64::FMAX_VG2_2Z2Z_S, AArch64::FMAX_VG2_2Z2Z_D}))
6409 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6410 return;
6411 case Intrinsic::aarch64_sve_smax_x4:
6412 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6413 VT: Node->getValueType(ResNo: 0),
6414 Opcodes: {AArch64::SMAX_VG4_4Z4Z_B, AArch64::SMAX_VG4_4Z4Z_H,
6415 AArch64::SMAX_VG4_4Z4Z_S, AArch64::SMAX_VG4_4Z4Z_D}))
6416 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6417 return;
6418 case Intrinsic::aarch64_sve_umax_x4:
6419 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6420 VT: Node->getValueType(ResNo: 0),
6421 Opcodes: {AArch64::UMAX_VG4_4Z4Z_B, AArch64::UMAX_VG4_4Z4Z_H,
6422 AArch64::UMAX_VG4_4Z4Z_S, AArch64::UMAX_VG4_4Z4Z_D}))
6423 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6424 return;
6425 case Intrinsic::aarch64_sve_fmax_x4:
6426 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6427 VT: Node->getValueType(ResNo: 0),
6428 Opcodes: {AArch64::BFMAX_VG4_4Z2Z_H, AArch64::FMAX_VG4_4Z4Z_H,
6429 AArch64::FMAX_VG4_4Z4Z_S, AArch64::FMAX_VG4_4Z4Z_D}))
6430 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6431 return;
6432 case Intrinsic::aarch64_sme_famax_x2:
6433 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6434 VT: Node->getValueType(ResNo: 0),
6435 Opcodes: {0, AArch64::FAMAX_2Z2Z_H, AArch64::FAMAX_2Z2Z_S,
6436 AArch64::FAMAX_2Z2Z_D}))
6437 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6438 return;
6439 case Intrinsic::aarch64_sme_famax_x4:
6440 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6441 VT: Node->getValueType(ResNo: 0),
6442 Opcodes: {0, AArch64::FAMAX_4Z4Z_H, AArch64::FAMAX_4Z4Z_S,
6443 AArch64::FAMAX_4Z4Z_D}))
6444 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6445 return;
6446 case Intrinsic::aarch64_sme_famin_x2:
6447 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6448 VT: Node->getValueType(ResNo: 0),
6449 Opcodes: {0, AArch64::FAMIN_2Z2Z_H, AArch64::FAMIN_2Z2Z_S,
6450 AArch64::FAMIN_2Z2Z_D}))
6451 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6452 return;
6453 case Intrinsic::aarch64_sme_famin_x4:
6454 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6455 VT: Node->getValueType(ResNo: 0),
6456 Opcodes: {0, AArch64::FAMIN_4Z4Z_H, AArch64::FAMIN_4Z4Z_S,
6457 AArch64::FAMIN_4Z4Z_D}))
6458 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6459 return;
6460 case Intrinsic::aarch64_sve_smin_x2:
6461 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6462 VT: Node->getValueType(ResNo: 0),
6463 Opcodes: {AArch64::SMIN_VG2_2Z2Z_B, AArch64::SMIN_VG2_2Z2Z_H,
6464 AArch64::SMIN_VG2_2Z2Z_S, AArch64::SMIN_VG2_2Z2Z_D}))
6465 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6466 return;
6467 case Intrinsic::aarch64_sve_umin_x2:
6468 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6469 VT: Node->getValueType(ResNo: 0),
6470 Opcodes: {AArch64::UMIN_VG2_2Z2Z_B, AArch64::UMIN_VG2_2Z2Z_H,
6471 AArch64::UMIN_VG2_2Z2Z_S, AArch64::UMIN_VG2_2Z2Z_D}))
6472 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6473 return;
6474 case Intrinsic::aarch64_sve_fmin_x2:
6475 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6476 VT: Node->getValueType(ResNo: 0),
6477 Opcodes: {AArch64::BFMIN_VG2_2Z2Z_H, AArch64::FMIN_VG2_2Z2Z_H,
6478 AArch64::FMIN_VG2_2Z2Z_S, AArch64::FMIN_VG2_2Z2Z_D}))
6479 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6480 return;
6481 case Intrinsic::aarch64_sve_smin_x4:
6482 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6483 VT: Node->getValueType(ResNo: 0),
6484 Opcodes: {AArch64::SMIN_VG4_4Z4Z_B, AArch64::SMIN_VG4_4Z4Z_H,
6485 AArch64::SMIN_VG4_4Z4Z_S, AArch64::SMIN_VG4_4Z4Z_D}))
6486 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6487 return;
6488 case Intrinsic::aarch64_sve_umin_x4:
6489 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6490 VT: Node->getValueType(ResNo: 0),
6491 Opcodes: {AArch64::UMIN_VG4_4Z4Z_B, AArch64::UMIN_VG4_4Z4Z_H,
6492 AArch64::UMIN_VG4_4Z4Z_S, AArch64::UMIN_VG4_4Z4Z_D}))
6493 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6494 return;
6495 case Intrinsic::aarch64_sve_fmin_x4:
6496 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6497 VT: Node->getValueType(ResNo: 0),
6498 Opcodes: {AArch64::BFMIN_VG4_4Z2Z_H, AArch64::FMIN_VG4_4Z4Z_H,
6499 AArch64::FMIN_VG4_4Z4Z_S, AArch64::FMIN_VG4_4Z4Z_D}))
6500 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6501 return;
6502 case Intrinsic::aarch64_sve_fmaxnm_single_x2 :
6503 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6504 VT: Node->getValueType(ResNo: 0),
6505 Opcodes: {AArch64::BFMAXNM_VG2_2ZZ_H, AArch64::FMAXNM_VG2_2ZZ_H,
6506 AArch64::FMAXNM_VG2_2ZZ_S, AArch64::FMAXNM_VG2_2ZZ_D}))
6507 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6508 return;
6509 case Intrinsic::aarch64_sve_fmaxnm_single_x4 :
6510 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6511 VT: Node->getValueType(ResNo: 0),
6512 Opcodes: {AArch64::BFMAXNM_VG4_4ZZ_H, AArch64::FMAXNM_VG4_4ZZ_H,
6513 AArch64::FMAXNM_VG4_4ZZ_S, AArch64::FMAXNM_VG4_4ZZ_D}))
6514 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6515 return;
6516 case Intrinsic::aarch64_sve_fminnm_single_x2:
6517 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6518 VT: Node->getValueType(ResNo: 0),
6519 Opcodes: {AArch64::BFMINNM_VG2_2ZZ_H, AArch64::FMINNM_VG2_2ZZ_H,
6520 AArch64::FMINNM_VG2_2ZZ_S, AArch64::FMINNM_VG2_2ZZ_D}))
6521 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6522 return;
6523 case Intrinsic::aarch64_sve_fminnm_single_x4:
6524 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6525 VT: Node->getValueType(ResNo: 0),
6526 Opcodes: {AArch64::BFMINNM_VG4_4ZZ_H, AArch64::FMINNM_VG4_4ZZ_H,
6527 AArch64::FMINNM_VG4_4ZZ_S, AArch64::FMINNM_VG4_4ZZ_D}))
6528 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6529 return;
6530 case Intrinsic::aarch64_sve_fscale_single_x4:
6531 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::BFSCALE_4ZZ);
6532 return;
6533 case Intrinsic::aarch64_sve_fscale_single_x2:
6534 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::BFSCALE_2ZZ);
6535 return;
6536 case Intrinsic::aarch64_sve_fmul_single_x4:
6537 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6538 VT: Node->getValueType(ResNo: 0),
6539 Opcodes: {AArch64::BFMUL_4ZZ, AArch64::FMUL_4ZZ_H, AArch64::FMUL_4ZZ_S,
6540 AArch64::FMUL_4ZZ_D}))
6541 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6542 return;
6543 case Intrinsic::aarch64_sve_fmul_single_x2:
6544 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6545 VT: Node->getValueType(ResNo: 0),
6546 Opcodes: {AArch64::BFMUL_2ZZ, AArch64::FMUL_2ZZ_H, AArch64::FMUL_2ZZ_S,
6547 AArch64::FMUL_2ZZ_D}))
6548 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6549 return;
6550 case Intrinsic::aarch64_sve_fmaxnm_x2:
6551 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6552 VT: Node->getValueType(ResNo: 0),
6553 Opcodes: {AArch64::BFMAXNM_VG2_2Z2Z_H, AArch64::FMAXNM_VG2_2Z2Z_H,
6554 AArch64::FMAXNM_VG2_2Z2Z_S, AArch64::FMAXNM_VG2_2Z2Z_D}))
6555 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6556 return;
6557 case Intrinsic::aarch64_sve_fmaxnm_x4:
6558 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6559 VT: Node->getValueType(ResNo: 0),
6560 Opcodes: {AArch64::BFMAXNM_VG4_4Z2Z_H, AArch64::FMAXNM_VG4_4Z4Z_H,
6561 AArch64::FMAXNM_VG4_4Z4Z_S, AArch64::FMAXNM_VG4_4Z4Z_D}))
6562 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6563 return;
6564 case Intrinsic::aarch64_sve_fminnm_x2:
6565 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6566 VT: Node->getValueType(ResNo: 0),
6567 Opcodes: {AArch64::BFMINNM_VG2_2Z2Z_H, AArch64::FMINNM_VG2_2Z2Z_H,
6568 AArch64::FMINNM_VG2_2Z2Z_S, AArch64::FMINNM_VG2_2Z2Z_D}))
6569 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6570 return;
6571 case Intrinsic::aarch64_sve_fminnm_x4:
6572 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6573 VT: Node->getValueType(ResNo: 0),
6574 Opcodes: {AArch64::BFMINNM_VG4_4Z2Z_H, AArch64::FMINNM_VG4_4Z4Z_H,
6575 AArch64::FMINNM_VG4_4Z4Z_S, AArch64::FMINNM_VG4_4Z4Z_D}))
6576 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6577 return;
6578 case Intrinsic::aarch64_sve_aese_lane_x2:
6579 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESE_2ZZI_B);
6580 return;
6581 case Intrinsic::aarch64_sve_aesd_lane_x2:
6582 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESD_2ZZI_B);
6583 return;
6584 case Intrinsic::aarch64_sve_aesemc_lane_x2:
6585 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESEMC_2ZZI_B);
6586 return;
6587 case Intrinsic::aarch64_sve_aesdimc_lane_x2:
6588 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESDIMC_2ZZI_B);
6589 return;
6590 case Intrinsic::aarch64_sve_aese_lane_x4:
6591 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESE_4ZZI_B);
6592 return;
6593 case Intrinsic::aarch64_sve_aesd_lane_x4:
6594 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESD_4ZZI_B);
6595 return;
6596 case Intrinsic::aarch64_sve_aesemc_lane_x4:
6597 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESEMC_4ZZI_B);
6598 return;
6599 case Intrinsic::aarch64_sve_aesdimc_lane_x4:
6600 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESDIMC_4ZZI_B);
6601 return;
6602 case Intrinsic::aarch64_sve_pmlal_pair_x2:
6603 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::PMLAL_2ZZZ_Q);
6604 return;
6605 case Intrinsic::aarch64_sve_pmull_pair_x2: {
6606 SDLoc DL(Node);
6607 SmallVector<SDValue, 4> Regs(Node->ops().slice(N: 1, M: 2));
6608 SDNode *Res =
6609 CurDAG->getMachineNode(Opcode: AArch64::PMULL_2ZZZ_Q, dl: DL, VT: MVT::Untyped, Ops: Regs);
6610 SDValue SuperReg = SDValue(Res, 0);
6611 for (unsigned I = 0; I < 2; I++)
6612 ReplaceUses(F: SDValue(Node, I),
6613 T: CurDAG->getTargetExtractSubreg(SRIdx: AArch64::zsub0 + I, DL, VT,
6614 Operand: SuperReg));
6615 CurDAG->RemoveDeadNode(N: Node);
6616 return;
6617 }
6618 case Intrinsic::aarch64_sve_fscale_x4:
6619 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: AArch64::BFSCALE_4Z4Z);
6620 return;
6621 case Intrinsic::aarch64_sve_fscale_x2:
6622 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: AArch64::BFSCALE_2Z2Z);
6623 return;
6624 case Intrinsic::aarch64_sve_fmul_x4:
6625 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6626 VT: Node->getValueType(ResNo: 0),
6627 Opcodes: {AArch64::BFMUL_4Z4Z, AArch64::FMUL_4Z4Z_H, AArch64::FMUL_4Z4Z_S,
6628 AArch64::FMUL_4Z4Z_D}))
6629 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6630 return;
6631 case Intrinsic::aarch64_sve_fmul_x2:
6632 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6633 VT: Node->getValueType(ResNo: 0),
6634 Opcodes: {AArch64::BFMUL_2Z2Z, AArch64::FMUL_2Z2Z_H, AArch64::FMUL_2Z2Z_S,
6635 AArch64::FMUL_2Z2Z_D}))
6636 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6637 return;
6638 case Intrinsic::aarch64_sve_fcvtzs_x2:
6639 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::FCVTZS_2Z2Z_StoS);
6640 return;
6641 case Intrinsic::aarch64_sve_scvtf_x2:
6642 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::SCVTF_2Z2Z_StoS);
6643 return;
6644 case Intrinsic::aarch64_sve_fcvtzu_x2:
6645 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::FCVTZU_2Z2Z_StoS);
6646 return;
6647 case Intrinsic::aarch64_sve_ucvtf_x2:
6648 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::UCVTF_2Z2Z_StoS);
6649 return;
6650 case Intrinsic::aarch64_sve_fcvtzs_x4:
6651 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::FCVTZS_4Z4Z_StoS);
6652 return;
6653 case Intrinsic::aarch64_sve_scvtf_x4:
6654 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::SCVTF_4Z4Z_StoS);
6655 return;
6656 case Intrinsic::aarch64_sve_fcvtzu_x4:
6657 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::FCVTZU_4Z4Z_StoS);
6658 return;
6659 case Intrinsic::aarch64_sve_ucvtf_x4:
6660 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::UCVTF_4Z4Z_StoS);
6661 return;
6662 case Intrinsic::aarch64_sve_fcvt_widen_x2:
6663 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, IsTupleInput: false, Opc: AArch64::FCVT_2ZZ_H_S);
6664 return;
6665 case Intrinsic::aarch64_sve_fcvtl_widen_x2:
6666 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, IsTupleInput: false, Opc: AArch64::FCVTL_2ZZ_H_S);
6667 return;
6668 case Intrinsic::aarch64_sve_sclamp_single_x2:
6669 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6670 VT: Node->getValueType(ResNo: 0),
6671 Opcodes: {AArch64::SCLAMP_VG2_2Z2Z_B, AArch64::SCLAMP_VG2_2Z2Z_H,
6672 AArch64::SCLAMP_VG2_2Z2Z_S, AArch64::SCLAMP_VG2_2Z2Z_D}))
6673 SelectClamp(N: Node, NumVecs: 2, Op);
6674 return;
6675 case Intrinsic::aarch64_sve_uclamp_single_x2:
6676 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6677 VT: Node->getValueType(ResNo: 0),
6678 Opcodes: {AArch64::UCLAMP_VG2_2Z2Z_B, AArch64::UCLAMP_VG2_2Z2Z_H,
6679 AArch64::UCLAMP_VG2_2Z2Z_S, AArch64::UCLAMP_VG2_2Z2Z_D}))
6680 SelectClamp(N: Node, NumVecs: 2, Op);
6681 return;
6682 case Intrinsic::aarch64_sve_fclamp_single_x2:
6683 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6684 VT: Node->getValueType(ResNo: 0),
6685 Opcodes: {0, AArch64::FCLAMP_VG2_2Z2Z_H, AArch64::FCLAMP_VG2_2Z2Z_S,
6686 AArch64::FCLAMP_VG2_2Z2Z_D}))
6687 SelectClamp(N: Node, NumVecs: 2, Op);
6688 return;
6689 case Intrinsic::aarch64_sve_bfclamp_single_x2:
6690 SelectClamp(N: Node, NumVecs: 2, Op: AArch64::BFCLAMP_VG2_2ZZZ_H);
6691 return;
6692 case Intrinsic::aarch64_sve_sclamp_single_x4:
6693 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6694 VT: Node->getValueType(ResNo: 0),
6695 Opcodes: {AArch64::SCLAMP_VG4_4Z4Z_B, AArch64::SCLAMP_VG4_4Z4Z_H,
6696 AArch64::SCLAMP_VG4_4Z4Z_S, AArch64::SCLAMP_VG4_4Z4Z_D}))
6697 SelectClamp(N: Node, NumVecs: 4, Op);
6698 return;
6699 case Intrinsic::aarch64_sve_uclamp_single_x4:
6700 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6701 VT: Node->getValueType(ResNo: 0),
6702 Opcodes: {AArch64::UCLAMP_VG4_4Z4Z_B, AArch64::UCLAMP_VG4_4Z4Z_H,
6703 AArch64::UCLAMP_VG4_4Z4Z_S, AArch64::UCLAMP_VG4_4Z4Z_D}))
6704 SelectClamp(N: Node, NumVecs: 4, Op);
6705 return;
6706 case Intrinsic::aarch64_sve_fclamp_single_x4:
6707 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6708 VT: Node->getValueType(ResNo: 0),
6709 Opcodes: {0, AArch64::FCLAMP_VG4_4Z4Z_H, AArch64::FCLAMP_VG4_4Z4Z_S,
6710 AArch64::FCLAMP_VG4_4Z4Z_D}))
6711 SelectClamp(N: Node, NumVecs: 4, Op);
6712 return;
6713 case Intrinsic::aarch64_sve_bfclamp_single_x4:
6714 SelectClamp(N: Node, NumVecs: 4, Op: AArch64::BFCLAMP_VG4_4ZZZ_H);
6715 return;
6716 case Intrinsic::aarch64_sve_add_single_x2:
6717 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6718 VT: Node->getValueType(ResNo: 0),
6719 Opcodes: {AArch64::ADD_VG2_2ZZ_B, AArch64::ADD_VG2_2ZZ_H,
6720 AArch64::ADD_VG2_2ZZ_S, AArch64::ADD_VG2_2ZZ_D}))
6721 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6722 return;
6723 case Intrinsic::aarch64_sve_add_single_x4:
6724 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6725 VT: Node->getValueType(ResNo: 0),
6726 Opcodes: {AArch64::ADD_VG4_4ZZ_B, AArch64::ADD_VG4_4ZZ_H,
6727 AArch64::ADD_VG4_4ZZ_S, AArch64::ADD_VG4_4ZZ_D}))
6728 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6729 return;
6730 case Intrinsic::aarch64_sve_zip_x2:
6731 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6732 VT: Node->getValueType(ResNo: 0),
6733 Opcodes: {AArch64::ZIP_VG2_2ZZZ_B, AArch64::ZIP_VG2_2ZZZ_H,
6734 AArch64::ZIP_VG2_2ZZZ_S, AArch64::ZIP_VG2_2ZZZ_D}))
6735 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6736 return;
6737 case Intrinsic::aarch64_sve_zipq_x2:
6738 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false,
6739 Opc: AArch64::ZIP_VG2_2ZZZ_Q);
6740 return;
6741 case Intrinsic::aarch64_sve_zip_x4:
6742 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6743 VT: Node->getValueType(ResNo: 0),
6744 Opcodes: {AArch64::ZIP_VG4_4Z4Z_B, AArch64::ZIP_VG4_4Z4Z_H,
6745 AArch64::ZIP_VG4_4Z4Z_S, AArch64::ZIP_VG4_4Z4Z_D}))
6746 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6747 return;
6748 case Intrinsic::aarch64_sve_zipq_x4:
6749 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true,
6750 Opc: AArch64::ZIP_VG4_4Z4Z_Q);
6751 return;
6752 case Intrinsic::aarch64_sve_uzp_x2:
6753 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6754 VT: Node->getValueType(ResNo: 0),
6755 Opcodes: {AArch64::UZP_VG2_2ZZZ_B, AArch64::UZP_VG2_2ZZZ_H,
6756 AArch64::UZP_VG2_2ZZZ_S, AArch64::UZP_VG2_2ZZZ_D}))
6757 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6758 return;
6759 case Intrinsic::aarch64_sve_uzpq_x2:
6760 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false,
6761 Opc: AArch64::UZP_VG2_2ZZZ_Q);
6762 return;
6763 case Intrinsic::aarch64_sve_uzp_x4:
6764 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6765 VT: Node->getValueType(ResNo: 0),
6766 Opcodes: {AArch64::UZP_VG4_4Z4Z_B, AArch64::UZP_VG4_4Z4Z_H,
6767 AArch64::UZP_VG4_4Z4Z_S, AArch64::UZP_VG4_4Z4Z_D}))
6768 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6769 return;
6770 case Intrinsic::aarch64_sve_uzpq_x4:
6771 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true,
6772 Opc: AArch64::UZP_VG4_4Z4Z_Q);
6773 return;
6774 case Intrinsic::aarch64_sve_sel_x2:
6775 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6776 VT: Node->getValueType(ResNo: 0),
6777 Opcodes: {AArch64::SEL_VG2_2ZC2Z2Z_B, AArch64::SEL_VG2_2ZC2Z2Z_H,
6778 AArch64::SEL_VG2_2ZC2Z2Z_S, AArch64::SEL_VG2_2ZC2Z2Z_D}))
6779 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op, /*HasPred=*/true);
6780 return;
6781 case Intrinsic::aarch64_sve_sel_x4:
6782 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6783 VT: Node->getValueType(ResNo: 0),
6784 Opcodes: {AArch64::SEL_VG4_4ZC4Z4Z_B, AArch64::SEL_VG4_4ZC4Z4Z_H,
6785 AArch64::SEL_VG4_4ZC4Z4Z_S, AArch64::SEL_VG4_4ZC4Z4Z_D}))
6786 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op, /*HasPred=*/true);
6787 return;
6788 case Intrinsic::aarch64_sve_frinta_x2:
6789 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTA_2Z2Z_S);
6790 return;
6791 case Intrinsic::aarch64_sve_frinta_x4:
6792 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTA_4Z4Z_S);
6793 return;
6794 case Intrinsic::aarch64_sve_frintm_x2:
6795 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTM_2Z2Z_S);
6796 return;
6797 case Intrinsic::aarch64_sve_frintm_x4:
6798 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTM_4Z4Z_S);
6799 return;
6800 case Intrinsic::aarch64_sve_frintn_x2:
6801 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTN_2Z2Z_S);
6802 return;
6803 case Intrinsic::aarch64_sve_frintn_x4:
6804 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTN_4Z4Z_S);
6805 return;
6806 case Intrinsic::aarch64_sve_frintp_x2:
6807 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTP_2Z2Z_S);
6808 return;
6809 case Intrinsic::aarch64_sve_frintp_x4:
6810 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTP_4Z4Z_S);
6811 return;
6812 case Intrinsic::aarch64_sve_sunpk_x2:
6813 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6814 VT: Node->getValueType(ResNo: 0),
6815 Opcodes: {0, AArch64::SUNPK_VG2_2ZZ_H, AArch64::SUNPK_VG2_2ZZ_S,
6816 AArch64::SUNPK_VG2_2ZZ_D}))
6817 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6818 return;
6819 case Intrinsic::aarch64_sve_uunpk_x2:
6820 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6821 VT: Node->getValueType(ResNo: 0),
6822 Opcodes: {0, AArch64::UUNPK_VG2_2ZZ_H, AArch64::UUNPK_VG2_2ZZ_S,
6823 AArch64::UUNPK_VG2_2ZZ_D}))
6824 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6825 return;
6826 case Intrinsic::aarch64_sve_sunpk_x4:
6827 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6828 VT: Node->getValueType(ResNo: 0),
6829 Opcodes: {0, AArch64::SUNPK_VG4_4Z2Z_H, AArch64::SUNPK_VG4_4Z2Z_S,
6830 AArch64::SUNPK_VG4_4Z2Z_D}))
6831 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6832 return;
6833 case Intrinsic::aarch64_sve_uunpk_x4:
6834 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6835 VT: Node->getValueType(ResNo: 0),
6836 Opcodes: {0, AArch64::UUNPK_VG4_4Z2Z_H, AArch64::UUNPK_VG4_4Z2Z_S,
6837 AArch64::UUNPK_VG4_4Z2Z_D}))
6838 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6839 return;
6840 case Intrinsic::aarch64_sve_pext_x2: {
6841 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6842 VT: Node->getValueType(ResNo: 0),
6843 Opcodes: {AArch64::PEXT_2PCI_B, AArch64::PEXT_2PCI_H, AArch64::PEXT_2PCI_S,
6844 AArch64::PEXT_2PCI_D}))
6845 SelectPExtPair(N: Node, Opc: Op);
6846 return;
6847 }
6848 }
6849 break;
6850 }
6851 case ISD::INTRINSIC_VOID: {
6852 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
6853 if (Node->getNumOperands() >= 3)
6854 VT = Node->getOperand(Num: 2)->getValueType(ResNo: 0);
6855 switch (IntNo) {
6856 default:
6857 break;
6858 case Intrinsic::aarch64_neon_st1x2: {
6859 if (VT == MVT::v8i8) {
6860 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8b);
6861 return;
6862 } else if (VT == MVT::v16i8) {
6863 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov16b);
6864 return;
6865 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6866 VT == MVT::v4bf16) {
6867 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4h);
6868 return;
6869 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6870 VT == MVT::v8bf16) {
6871 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8h);
6872 return;
6873 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6874 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2s);
6875 return;
6876 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6877 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4s);
6878 return;
6879 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6880 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2d);
6881 return;
6882 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6883 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d);
6884 return;
6885 }
6886 break;
6887 }
6888 case Intrinsic::aarch64_neon_st1x3: {
6889 if (VT == MVT::v8i8) {
6890 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8b);
6891 return;
6892 } else if (VT == MVT::v16i8) {
6893 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev16b);
6894 return;
6895 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6896 VT == MVT::v4bf16) {
6897 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4h);
6898 return;
6899 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6900 VT == MVT::v8bf16) {
6901 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8h);
6902 return;
6903 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6904 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2s);
6905 return;
6906 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6907 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4s);
6908 return;
6909 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6910 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2d);
6911 return;
6912 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6913 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d);
6914 return;
6915 }
6916 break;
6917 }
6918 case Intrinsic::aarch64_neon_st1x4: {
6919 if (VT == MVT::v8i8) {
6920 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8b);
6921 return;
6922 } else if (VT == MVT::v16i8) {
6923 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv16b);
6924 return;
6925 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6926 VT == MVT::v4bf16) {
6927 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4h);
6928 return;
6929 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6930 VT == MVT::v8bf16) {
6931 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8h);
6932 return;
6933 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6934 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2s);
6935 return;
6936 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6937 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4s);
6938 return;
6939 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6940 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2d);
6941 return;
6942 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6943 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d);
6944 return;
6945 }
6946 break;
6947 }
6948 case Intrinsic::aarch64_neon_st2: {
6949 if (VT == MVT::v8i8) {
6950 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8b);
6951 return;
6952 } else if (VT == MVT::v16i8) {
6953 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov16b);
6954 return;
6955 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6956 VT == MVT::v4bf16) {
6957 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4h);
6958 return;
6959 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6960 VT == MVT::v8bf16) {
6961 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8h);
6962 return;
6963 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6964 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2s);
6965 return;
6966 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6967 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4s);
6968 return;
6969 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6970 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2d);
6971 return;
6972 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6973 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d);
6974 return;
6975 }
6976 break;
6977 }
6978 case Intrinsic::aarch64_neon_st3: {
6979 if (VT == MVT::v8i8) {
6980 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8b);
6981 return;
6982 } else if (VT == MVT::v16i8) {
6983 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev16b);
6984 return;
6985 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6986 VT == MVT::v4bf16) {
6987 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4h);
6988 return;
6989 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6990 VT == MVT::v8bf16) {
6991 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8h);
6992 return;
6993 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6994 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2s);
6995 return;
6996 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6997 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4s);
6998 return;
6999 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7000 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2d);
7001 return;
7002 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7003 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d);
7004 return;
7005 }
7006 break;
7007 }
7008 case Intrinsic::aarch64_neon_st4: {
7009 if (VT == MVT::v8i8) {
7010 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8b);
7011 return;
7012 } else if (VT == MVT::v16i8) {
7013 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv16b);
7014 return;
7015 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
7016 VT == MVT::v4bf16) {
7017 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4h);
7018 return;
7019 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7020 VT == MVT::v8bf16) {
7021 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8h);
7022 return;
7023 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7024 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2s);
7025 return;
7026 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7027 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4s);
7028 return;
7029 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7030 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2d);
7031 return;
7032 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7033 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d);
7034 return;
7035 }
7036 break;
7037 }
7038 case Intrinsic::aarch64_neon_st2lane: {
7039 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7040 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i8);
7041 return;
7042 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7043 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7044 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i16);
7045 return;
7046 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7047 VT == MVT::v2f32) {
7048 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i32);
7049 return;
7050 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7051 VT == MVT::v1f64) {
7052 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i64);
7053 return;
7054 }
7055 break;
7056 }
7057 case Intrinsic::aarch64_neon_st3lane: {
7058 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7059 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i8);
7060 return;
7061 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7062 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7063 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i16);
7064 return;
7065 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7066 VT == MVT::v2f32) {
7067 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i32);
7068 return;
7069 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7070 VT == MVT::v1f64) {
7071 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i64);
7072 return;
7073 }
7074 break;
7075 }
7076 case Intrinsic::aarch64_neon_st4lane: {
7077 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7078 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i8);
7079 return;
7080 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7081 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7082 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i16);
7083 return;
7084 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7085 VT == MVT::v2f32) {
7086 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i32);
7087 return;
7088 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7089 VT == MVT::v1f64) {
7090 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i64);
7091 return;
7092 }
7093 break;
7094 }
7095 case Intrinsic::aarch64_sve_st2q: {
7096 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 4, Opc_rr: AArch64::ST2Q, Opc_ri: AArch64::ST2Q_IMM);
7097 return;
7098 }
7099 case Intrinsic::aarch64_sve_st3q: {
7100 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 4, Opc_rr: AArch64::ST3Q, Opc_ri: AArch64::ST3Q_IMM);
7101 return;
7102 }
7103 case Intrinsic::aarch64_sve_st4q: {
7104 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 4, Opc_rr: AArch64::ST4Q, Opc_ri: AArch64::ST4Q_IMM);
7105 return;
7106 }
7107 case Intrinsic::aarch64_sve_st2: {
7108 if (VT == MVT::nxv16i8) {
7109 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 0, Opc_rr: AArch64::ST2B, Opc_ri: AArch64::ST2B_IMM);
7110 return;
7111 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7112 VT == MVT::nxv8bf16) {
7113 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 1, Opc_rr: AArch64::ST2H, Opc_ri: AArch64::ST2H_IMM);
7114 return;
7115 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7116 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 2, Opc_rr: AArch64::ST2W, Opc_ri: AArch64::ST2W_IMM);
7117 return;
7118 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7119 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 3, Opc_rr: AArch64::ST2D, Opc_ri: AArch64::ST2D_IMM);
7120 return;
7121 }
7122 break;
7123 }
7124 case Intrinsic::aarch64_sve_st3: {
7125 if (VT == MVT::nxv16i8) {
7126 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 0, Opc_rr: AArch64::ST3B, Opc_ri: AArch64::ST3B_IMM);
7127 return;
7128 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7129 VT == MVT::nxv8bf16) {
7130 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 1, Opc_rr: AArch64::ST3H, Opc_ri: AArch64::ST3H_IMM);
7131 return;
7132 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7133 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 2, Opc_rr: AArch64::ST3W, Opc_ri: AArch64::ST3W_IMM);
7134 return;
7135 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7136 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 3, Opc_rr: AArch64::ST3D, Opc_ri: AArch64::ST3D_IMM);
7137 return;
7138 }
7139 break;
7140 }
7141 case Intrinsic::aarch64_sve_st4: {
7142 if (VT == MVT::nxv16i8) {
7143 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 0, Opc_rr: AArch64::ST4B, Opc_ri: AArch64::ST4B_IMM);
7144 return;
7145 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7146 VT == MVT::nxv8bf16) {
7147 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 1, Opc_rr: AArch64::ST4H, Opc_ri: AArch64::ST4H_IMM);
7148 return;
7149 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7150 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 2, Opc_rr: AArch64::ST4W, Opc_ri: AArch64::ST4W_IMM);
7151 return;
7152 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7153 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 3, Opc_rr: AArch64::ST4D, Opc_ri: AArch64::ST4D_IMM);
7154 return;
7155 }
7156 break;
7157 }
7158 }
7159 break;
7160 }
7161 case AArch64ISD::LD2post: {
7162 if (VT == MVT::v8i8) {
7163 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8b_POST, SubRegIdx: AArch64::dsub0);
7164 return;
7165 } else if (VT == MVT::v16i8) {
7166 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov16b_POST, SubRegIdx: AArch64::qsub0);
7167 return;
7168 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7169 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4h_POST, SubRegIdx: AArch64::dsub0);
7170 return;
7171 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7172 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8h_POST, SubRegIdx: AArch64::qsub0);
7173 return;
7174 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7175 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2s_POST, SubRegIdx: AArch64::dsub0);
7176 return;
7177 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7178 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4s_POST, SubRegIdx: AArch64::qsub0);
7179 return;
7180 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7181 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d_POST, SubRegIdx: AArch64::dsub0);
7182 return;
7183 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7184 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2d_POST, SubRegIdx: AArch64::qsub0);
7185 return;
7186 }
7187 break;
7188 }
7189 case AArch64ISD::LD3post: {
7190 if (VT == MVT::v8i8) {
7191 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8b_POST, SubRegIdx: AArch64::dsub0);
7192 return;
7193 } else if (VT == MVT::v16i8) {
7194 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev16b_POST, SubRegIdx: AArch64::qsub0);
7195 return;
7196 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7197 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4h_POST, SubRegIdx: AArch64::dsub0);
7198 return;
7199 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7200 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8h_POST, SubRegIdx: AArch64::qsub0);
7201 return;
7202 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7203 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2s_POST, SubRegIdx: AArch64::dsub0);
7204 return;
7205 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7206 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4s_POST, SubRegIdx: AArch64::qsub0);
7207 return;
7208 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7209 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d_POST, SubRegIdx: AArch64::dsub0);
7210 return;
7211 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7212 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2d_POST, SubRegIdx: AArch64::qsub0);
7213 return;
7214 }
7215 break;
7216 }
7217 case AArch64ISD::LD4post: {
7218 if (VT == MVT::v8i8) {
7219 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8b_POST, SubRegIdx: AArch64::dsub0);
7220 return;
7221 } else if (VT == MVT::v16i8) {
7222 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv16b_POST, SubRegIdx: AArch64::qsub0);
7223 return;
7224 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7225 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4h_POST, SubRegIdx: AArch64::dsub0);
7226 return;
7227 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7228 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8h_POST, SubRegIdx: AArch64::qsub0);
7229 return;
7230 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7231 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2s_POST, SubRegIdx: AArch64::dsub0);
7232 return;
7233 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7234 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4s_POST, SubRegIdx: AArch64::qsub0);
7235 return;
7236 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7237 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d_POST, SubRegIdx: AArch64::dsub0);
7238 return;
7239 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7240 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2d_POST, SubRegIdx: AArch64::qsub0);
7241 return;
7242 }
7243 break;
7244 }
7245 case AArch64ISD::LD1x2post: {
7246 if (VT == MVT::v8i8) {
7247 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8b_POST, SubRegIdx: AArch64::dsub0);
7248 return;
7249 } else if (VT == MVT::v16i8) {
7250 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov16b_POST, SubRegIdx: AArch64::qsub0);
7251 return;
7252 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7253 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4h_POST, SubRegIdx: AArch64::dsub0);
7254 return;
7255 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7256 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8h_POST, SubRegIdx: AArch64::qsub0);
7257 return;
7258 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7259 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2s_POST, SubRegIdx: AArch64::dsub0);
7260 return;
7261 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7262 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4s_POST, SubRegIdx: AArch64::qsub0);
7263 return;
7264 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7265 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d_POST, SubRegIdx: AArch64::dsub0);
7266 return;
7267 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7268 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2d_POST, SubRegIdx: AArch64::qsub0);
7269 return;
7270 }
7271 break;
7272 }
7273 case AArch64ISD::LD1x3post: {
7274 if (VT == MVT::v8i8) {
7275 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8b_POST, SubRegIdx: AArch64::dsub0);
7276 return;
7277 } else if (VT == MVT::v16i8) {
7278 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev16b_POST, SubRegIdx: AArch64::qsub0);
7279 return;
7280 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7281 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4h_POST, SubRegIdx: AArch64::dsub0);
7282 return;
7283 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7284 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8h_POST, SubRegIdx: AArch64::qsub0);
7285 return;
7286 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7287 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2s_POST, SubRegIdx: AArch64::dsub0);
7288 return;
7289 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7290 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4s_POST, SubRegIdx: AArch64::qsub0);
7291 return;
7292 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7293 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d_POST, SubRegIdx: AArch64::dsub0);
7294 return;
7295 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7296 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2d_POST, SubRegIdx: AArch64::qsub0);
7297 return;
7298 }
7299 break;
7300 }
7301 case AArch64ISD::LD1x4post: {
7302 if (VT == MVT::v8i8) {
7303 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8b_POST, SubRegIdx: AArch64::dsub0);
7304 return;
7305 } else if (VT == MVT::v16i8) {
7306 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv16b_POST, SubRegIdx: AArch64::qsub0);
7307 return;
7308 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7309 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4h_POST, SubRegIdx: AArch64::dsub0);
7310 return;
7311 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7312 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8h_POST, SubRegIdx: AArch64::qsub0);
7313 return;
7314 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7315 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2s_POST, SubRegIdx: AArch64::dsub0);
7316 return;
7317 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7318 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4s_POST, SubRegIdx: AArch64::qsub0);
7319 return;
7320 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7321 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d_POST, SubRegIdx: AArch64::dsub0);
7322 return;
7323 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7324 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2d_POST, SubRegIdx: AArch64::qsub0);
7325 return;
7326 }
7327 break;
7328 }
7329 case AArch64ISD::LD1DUPpost: {
7330 if (VT == MVT::v8i8) {
7331 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv8b_POST, SubRegIdx: AArch64::dsub0);
7332 return;
7333 } else if (VT == MVT::v16i8) {
7334 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv16b_POST, SubRegIdx: AArch64::qsub0);
7335 return;
7336 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7337 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv4h_POST, SubRegIdx: AArch64::dsub0);
7338 return;
7339 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7340 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv8h_POST, SubRegIdx: AArch64::qsub0);
7341 return;
7342 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7343 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv2s_POST, SubRegIdx: AArch64::dsub0);
7344 return;
7345 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7346 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv4s_POST, SubRegIdx: AArch64::qsub0);
7347 return;
7348 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7349 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv1d_POST, SubRegIdx: AArch64::dsub0);
7350 return;
7351 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7352 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv2d_POST, SubRegIdx: AArch64::qsub0);
7353 return;
7354 }
7355 break;
7356 }
7357 case AArch64ISD::LD2DUPpost: {
7358 if (VT == MVT::v8i8) {
7359 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8b_POST, SubRegIdx: AArch64::dsub0);
7360 return;
7361 } else if (VT == MVT::v16i8) {
7362 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv16b_POST, SubRegIdx: AArch64::qsub0);
7363 return;
7364 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7365 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4h_POST, SubRegIdx: AArch64::dsub0);
7366 return;
7367 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7368 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8h_POST, SubRegIdx: AArch64::qsub0);
7369 return;
7370 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7371 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2s_POST, SubRegIdx: AArch64::dsub0);
7372 return;
7373 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7374 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4s_POST, SubRegIdx: AArch64::qsub0);
7375 return;
7376 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7377 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv1d_POST, SubRegIdx: AArch64::dsub0);
7378 return;
7379 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7380 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2d_POST, SubRegIdx: AArch64::qsub0);
7381 return;
7382 }
7383 break;
7384 }
7385 case AArch64ISD::LD3DUPpost: {
7386 if (VT == MVT::v8i8) {
7387 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8b_POST, SubRegIdx: AArch64::dsub0);
7388 return;
7389 } else if (VT == MVT::v16i8) {
7390 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv16b_POST, SubRegIdx: AArch64::qsub0);
7391 return;
7392 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7393 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4h_POST, SubRegIdx: AArch64::dsub0);
7394 return;
7395 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7396 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8h_POST, SubRegIdx: AArch64::qsub0);
7397 return;
7398 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7399 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2s_POST, SubRegIdx: AArch64::dsub0);
7400 return;
7401 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7402 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4s_POST, SubRegIdx: AArch64::qsub0);
7403 return;
7404 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7405 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv1d_POST, SubRegIdx: AArch64::dsub0);
7406 return;
7407 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7408 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2d_POST, SubRegIdx: AArch64::qsub0);
7409 return;
7410 }
7411 break;
7412 }
7413 case AArch64ISD::LD4DUPpost: {
7414 if (VT == MVT::v8i8) {
7415 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8b_POST, SubRegIdx: AArch64::dsub0);
7416 return;
7417 } else if (VT == MVT::v16i8) {
7418 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv16b_POST, SubRegIdx: AArch64::qsub0);
7419 return;
7420 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7421 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4h_POST, SubRegIdx: AArch64::dsub0);
7422 return;
7423 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7424 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8h_POST, SubRegIdx: AArch64::qsub0);
7425 return;
7426 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7427 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2s_POST, SubRegIdx: AArch64::dsub0);
7428 return;
7429 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7430 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4s_POST, SubRegIdx: AArch64::qsub0);
7431 return;
7432 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7433 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv1d_POST, SubRegIdx: AArch64::dsub0);
7434 return;
7435 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7436 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2d_POST, SubRegIdx: AArch64::qsub0);
7437 return;
7438 }
7439 break;
7440 }
7441 case AArch64ISD::LD1LANEpost: {
7442 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7443 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i8_POST);
7444 return;
7445 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7446 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7447 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i16_POST);
7448 return;
7449 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7450 VT == MVT::v2f32) {
7451 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i32_POST);
7452 return;
7453 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7454 VT == MVT::v1f64) {
7455 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i64_POST);
7456 return;
7457 }
7458 break;
7459 }
7460 case AArch64ISD::LD2LANEpost: {
7461 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7462 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i8_POST);
7463 return;
7464 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7465 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7466 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i16_POST);
7467 return;
7468 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7469 VT == MVT::v2f32) {
7470 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i32_POST);
7471 return;
7472 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7473 VT == MVT::v1f64) {
7474 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i64_POST);
7475 return;
7476 }
7477 break;
7478 }
7479 case AArch64ISD::LD3LANEpost: {
7480 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7481 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i8_POST);
7482 return;
7483 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7484 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7485 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i16_POST);
7486 return;
7487 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7488 VT == MVT::v2f32) {
7489 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i32_POST);
7490 return;
7491 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7492 VT == MVT::v1f64) {
7493 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i64_POST);
7494 return;
7495 }
7496 break;
7497 }
7498 case AArch64ISD::LD4LANEpost: {
7499 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7500 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i8_POST);
7501 return;
7502 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7503 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7504 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i16_POST);
7505 return;
7506 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7507 VT == MVT::v2f32) {
7508 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i32_POST);
7509 return;
7510 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7511 VT == MVT::v1f64) {
7512 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i64_POST);
7513 return;
7514 }
7515 break;
7516 }
7517 case AArch64ISD::ST2post: {
7518 VT = Node->getOperand(Num: 1).getValueType();
7519 if (VT == MVT::v8i8) {
7520 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8b_POST);
7521 return;
7522 } else if (VT == MVT::v16i8) {
7523 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov16b_POST);
7524 return;
7525 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7526 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4h_POST);
7527 return;
7528 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7529 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8h_POST);
7530 return;
7531 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7532 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2s_POST);
7533 return;
7534 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7535 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4s_POST);
7536 return;
7537 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7538 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2d_POST);
7539 return;
7540 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7541 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d_POST);
7542 return;
7543 }
7544 break;
7545 }
7546 case AArch64ISD::ST3post: {
7547 VT = Node->getOperand(Num: 1).getValueType();
7548 if (VT == MVT::v8i8) {
7549 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8b_POST);
7550 return;
7551 } else if (VT == MVT::v16i8) {
7552 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev16b_POST);
7553 return;
7554 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7555 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4h_POST);
7556 return;
7557 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7558 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8h_POST);
7559 return;
7560 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7561 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2s_POST);
7562 return;
7563 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7564 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4s_POST);
7565 return;
7566 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7567 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2d_POST);
7568 return;
7569 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7570 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d_POST);
7571 return;
7572 }
7573 break;
7574 }
7575 case AArch64ISD::ST4post: {
7576 VT = Node->getOperand(Num: 1).getValueType();
7577 if (VT == MVT::v8i8) {
7578 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8b_POST);
7579 return;
7580 } else if (VT == MVT::v16i8) {
7581 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv16b_POST);
7582 return;
7583 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7584 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4h_POST);
7585 return;
7586 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7587 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8h_POST);
7588 return;
7589 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7590 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2s_POST);
7591 return;
7592 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7593 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4s_POST);
7594 return;
7595 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7596 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2d_POST);
7597 return;
7598 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7599 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d_POST);
7600 return;
7601 }
7602 break;
7603 }
7604 case AArch64ISD::ST1x2post: {
7605 VT = Node->getOperand(Num: 1).getValueType();
7606 if (VT == MVT::v8i8) {
7607 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8b_POST);
7608 return;
7609 } else if (VT == MVT::v16i8) {
7610 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov16b_POST);
7611 return;
7612 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7613 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4h_POST);
7614 return;
7615 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7616 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8h_POST);
7617 return;
7618 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7619 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2s_POST);
7620 return;
7621 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7622 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4s_POST);
7623 return;
7624 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7625 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d_POST);
7626 return;
7627 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7628 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2d_POST);
7629 return;
7630 }
7631 break;
7632 }
7633 case AArch64ISD::ST1x3post: {
7634 VT = Node->getOperand(Num: 1).getValueType();
7635 if (VT == MVT::v8i8) {
7636 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8b_POST);
7637 return;
7638 } else if (VT == MVT::v16i8) {
7639 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev16b_POST);
7640 return;
7641 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7642 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4h_POST);
7643 return;
7644 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16 ) {
7645 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8h_POST);
7646 return;
7647 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7648 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2s_POST);
7649 return;
7650 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7651 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4s_POST);
7652 return;
7653 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7654 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d_POST);
7655 return;
7656 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7657 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2d_POST);
7658 return;
7659 }
7660 break;
7661 }
7662 case AArch64ISD::ST1x4post: {
7663 VT = Node->getOperand(Num: 1).getValueType();
7664 if (VT == MVT::v8i8) {
7665 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8b_POST);
7666 return;
7667 } else if (VT == MVT::v16i8) {
7668 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv16b_POST);
7669 return;
7670 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7671 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4h_POST);
7672 return;
7673 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7674 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8h_POST);
7675 return;
7676 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7677 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2s_POST);
7678 return;
7679 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7680 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4s_POST);
7681 return;
7682 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7683 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d_POST);
7684 return;
7685 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7686 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2d_POST);
7687 return;
7688 }
7689 break;
7690 }
7691 case AArch64ISD::ST2LANEpost: {
7692 VT = Node->getOperand(Num: 1).getValueType();
7693 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7694 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i8_POST);
7695 return;
7696 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7697 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7698 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i16_POST);
7699 return;
7700 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7701 VT == MVT::v2f32) {
7702 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i32_POST);
7703 return;
7704 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7705 VT == MVT::v1f64) {
7706 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i64_POST);
7707 return;
7708 }
7709 break;
7710 }
7711 case AArch64ISD::ST3LANEpost: {
7712 VT = Node->getOperand(Num: 1).getValueType();
7713 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7714 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i8_POST);
7715 return;
7716 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7717 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7718 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i16_POST);
7719 return;
7720 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7721 VT == MVT::v2f32) {
7722 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i32_POST);
7723 return;
7724 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7725 VT == MVT::v1f64) {
7726 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i64_POST);
7727 return;
7728 }
7729 break;
7730 }
7731 case AArch64ISD::ST4LANEpost: {
7732 VT = Node->getOperand(Num: 1).getValueType();
7733 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7734 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i8_POST);
7735 return;
7736 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7737 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7738 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i16_POST);
7739 return;
7740 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7741 VT == MVT::v2f32) {
7742 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i32_POST);
7743 return;
7744 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7745 VT == MVT::v1f64) {
7746 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i64_POST);
7747 return;
7748 }
7749 break;
7750 }
7751 }
7752
7753 // Select the default instruction
7754 SelectCode(N: Node);
7755}
7756
7757/// createAArch64ISelDag - This pass converts a legalized DAG into a
7758/// AArch64-specific DAG, ready for instruction scheduling.
7759FunctionPass *llvm::createAArch64ISelDag(AArch64TargetMachine &TM,
7760 CodeGenOptLevel OptLevel) {
7761 return new AArch64DAGToDAGISelLegacy(TM, OptLevel);
7762}
7763
7764/// When \p PredVT is a scalable vector predicate in the form
7765/// MVT::nx<M>xi1, it builds the correspondent scalable vector of
7766/// integers MVT::nx<M>xi<bits> s.t. M x bits = 128. When targeting
7767/// structured vectors (NumVec >1), the output data type is
7768/// MVT::nx<M*NumVec>xi<bits> s.t. M x bits = 128. If the input
7769/// PredVT is not in the form MVT::nx<M>xi1, it returns an invalid
7770/// EVT.
7771static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT,
7772 unsigned NumVec) {
7773 assert(NumVec > 0 && NumVec < 5 && "Invalid number of vectors.");
7774 if (!PredVT.isScalableVectorOf(EltVT: MVT::i1))
7775 return EVT();
7776
7777 if (PredVT != MVT::nxv16i1 && PredVT != MVT::nxv8i1 &&
7778 PredVT != MVT::nxv4i1 && PredVT != MVT::nxv2i1)
7779 return EVT();
7780
7781 ElementCount EC = PredVT.getVectorElementCount();
7782 EVT ScalarVT =
7783 EVT::getIntegerVT(Context&: Ctx, BitWidth: AArch64::SVEBitsPerBlock / EC.getKnownMinValue());
7784 EVT MemVT = EVT::getVectorVT(Context&: Ctx, VT: ScalarVT, EC: EC * NumVec);
7785
7786 return MemVT;
7787}
7788
7789/// Builds an integer vector type large enough to hold \p NumVec instances
7790/// of \p VecVT.
7791static EVT getMultipleVectorType(LLVMContext &Ctx, EVT VecVT, unsigned NumVec) {
7792 return EVT::getVectorVT(Context&: Ctx, VT: VecVT.getScalarType().changeTypeToInteger(),
7793 EC: VecVT.getVectorElementCount() * NumVec);
7794}
7795
7796/// Return the EVT of the data associated to a memory operation in \p
7797/// Root. If such EVT cannot be retrieved, it returns an invalid EVT.
7798static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root) {
7799 if (auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(Val: Root))
7800 return MemIntr->getMemoryVT();
7801
7802 if (isa<MemSDNode>(Val: Root)) {
7803 EVT MemVT = cast<MemSDNode>(Val: Root)->getMemoryVT();
7804
7805 EVT DataVT;
7806 if (auto *Load = dyn_cast<LoadSDNode>(Val: Root))
7807 DataVT = Load->getValueType(ResNo: 0);
7808 else if (auto *Load = dyn_cast<MaskedLoadSDNode>(Val: Root))
7809 DataVT = Load->getValueType(ResNo: 0);
7810 else if (auto *Store = dyn_cast<StoreSDNode>(Val: Root))
7811 DataVT = Store->getValue().getValueType();
7812 else if (auto *Store = dyn_cast<MaskedStoreSDNode>(Val: Root))
7813 DataVT = Store->getValue().getValueType();
7814 else
7815 llvm_unreachable("Unexpected MemSDNode!");
7816
7817 return DataVT.changeVectorElementType(Context&: Ctx, EltVT: MemVT.getVectorElementType());
7818 }
7819
7820 const unsigned Opcode = Root->getOpcode();
7821 // For custom ISD nodes, we have to look at them individually to extract the
7822 // type of the data moved to/from memory.
7823 switch (Opcode) {
7824 case AArch64ISD::LD1_MERGE_ZERO:
7825 case AArch64ISD::LD1S_MERGE_ZERO:
7826 case AArch64ISD::LDNF1_MERGE_ZERO:
7827 case AArch64ISD::LDNF1S_MERGE_ZERO:
7828 return cast<VTSDNode>(Val: Root->getOperand(Num: 3))->getVT();
7829 case AArch64ISD::ST1_PRED:
7830 return cast<VTSDNode>(Val: Root->getOperand(Num: 4))->getVT();
7831 default:
7832 break;
7833 }
7834
7835 if (Opcode != ISD::INTRINSIC_VOID && Opcode != ISD::INTRINSIC_W_CHAIN)
7836 return EVT();
7837
7838 switch (Root->getConstantOperandVal(Num: 1)) {
7839 default:
7840 return EVT();
7841 case Intrinsic::aarch64_sme_ldr:
7842 case Intrinsic::aarch64_sme_str:
7843 return MVT::nxv16i8;
7844 case Intrinsic::aarch64_sve_prf:
7845 // We are using an SVE prefetch intrinsic. Type must be inferred from the
7846 // width of the predicate.
7847 return getPackedVectorTypeFromPredicateType(
7848 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/1);
7849 case Intrinsic::aarch64_sve_ld2_sret:
7850 case Intrinsic::aarch64_sve_ld2q_sret:
7851 return getPackedVectorTypeFromPredicateType(
7852 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/2);
7853 case Intrinsic::aarch64_sve_st2q:
7854 return getPackedVectorTypeFromPredicateType(
7855 Ctx, PredVT: Root->getOperand(Num: 4)->getValueType(ResNo: 0), /*NumVec=*/2);
7856 case Intrinsic::aarch64_sve_ld3_sret:
7857 case Intrinsic::aarch64_sve_ld3q_sret:
7858 return getPackedVectorTypeFromPredicateType(
7859 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/3);
7860 case Intrinsic::aarch64_sve_st3q:
7861 return getPackedVectorTypeFromPredicateType(
7862 Ctx, PredVT: Root->getOperand(Num: 5)->getValueType(ResNo: 0), /*NumVec=*/3);
7863 case Intrinsic::aarch64_sve_ld4_sret:
7864 case Intrinsic::aarch64_sve_ld4q_sret:
7865 return getPackedVectorTypeFromPredicateType(
7866 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/4);
7867 case Intrinsic::aarch64_sve_st4q:
7868 return getPackedVectorTypeFromPredicateType(
7869 Ctx, PredVT: Root->getOperand(Num: 6)->getValueType(ResNo: 0), /*NumVec=*/4);
7870 case Intrinsic::aarch64_sve_ld1_pn_x2:
7871 case Intrinsic::aarch64_sve_ldnt1_pn_x2:
7872 return getMultipleVectorType(Ctx, VecVT: Root->getValueType(ResNo: 0),
7873 /*NumVec=*/2);
7874 case Intrinsic::aarch64_sve_ld1_pn_x4:
7875 case Intrinsic::aarch64_sve_ldnt1_pn_x4:
7876 return getMultipleVectorType(Ctx, VecVT: Root->getValueType(ResNo: 0),
7877 /*NumVec=*/4);
7878 case Intrinsic::aarch64_sve_st1_pn_x2:
7879 case Intrinsic::aarch64_sve_stnt1_pn_x2:
7880 return getMultipleVectorType(Ctx, VecVT: Root->getOperand(Num: 2).getValueType(),
7881 /*NumVec=*/2);
7882 case Intrinsic::aarch64_sve_st1_pn_x4:
7883 case Intrinsic::aarch64_sve_stnt1_pn_x4:
7884 return getMultipleVectorType(Ctx, VecVT: Root->getOperand(Num: 2).getValueType(),
7885 /*NumVec=*/4);
7886 case Intrinsic::aarch64_sve_ld1udq:
7887 case Intrinsic::aarch64_sve_st1dq:
7888 return EVT(MVT::nxv1i64);
7889 case Intrinsic::aarch64_sve_ld1uwq:
7890 case Intrinsic::aarch64_sve_st1wq:
7891 return EVT(MVT::nxv1i32);
7892 }
7893}
7894
7895/// SelectAddrModeIndexedSVE - Attempt selection of the addressing mode:
7896/// Base + OffImm * sizeof(MemVT) for Min >= OffImm <= Max
7897/// where Root is the memory access using N for its address.
7898template <int64_t Min, int64_t Max>
7899bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(SDNode *Root, SDValue N,
7900 SDValue &Base,
7901 SDValue &OffImm) {
7902 const EVT MemVT = getMemVTFromNode(Ctx&: *(CurDAG->getContext()), Root);
7903 const DataLayout &DL = CurDAG->getDataLayout();
7904 const MachineFrameInfo &MFI = MF->getFrameInfo();
7905
7906 if (N.getOpcode() == ISD::FrameIndex) {
7907 int FI = cast<FrameIndexSDNode>(Val&: N)->getIndex();
7908 // We can only encode VL scaled offsets, so only fold in frame indexes
7909 // referencing SVE objects.
7910 if (MFI.hasScalableStackID(ObjectIdx: FI)) {
7911 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
7912 OffImm = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i64);
7913 return true;
7914 }
7915
7916 return false;
7917 }
7918
7919 if (MemVT == EVT())
7920 return false;
7921
7922 if (N.getOpcode() != ISD::ADD)
7923 return false;
7924
7925 SDValue VScale = N.getOperand(i: 1);
7926 int64_t MulImm = std::numeric_limits<int64_t>::max();
7927 if (VScale.getOpcode() == ISD::VSCALE) {
7928 MulImm = cast<ConstantSDNode>(Val: VScale.getOperand(i: 0))->getSExtValue();
7929 } else if (auto C = dyn_cast<ConstantSDNode>(Val&: VScale)) {
7930 int64_t ByteOffset = C->getSExtValue();
7931 const auto KnownVScale =
7932 Subtarget->getSVEVectorSizeInBits() / AArch64::SVEBitsPerBlock;
7933
7934 if (!KnownVScale || ByteOffset % KnownVScale != 0)
7935 return false;
7936
7937 MulImm = ByteOffset / KnownVScale;
7938 } else
7939 return false;
7940
7941 TypeSize TS = MemVT.getSizeInBits();
7942 int64_t MemWidthBytes = static_cast<int64_t>(TS.getKnownMinValue()) / 8;
7943
7944 if ((MulImm % MemWidthBytes) != 0)
7945 return false;
7946
7947 int64_t Offset = MulImm / MemWidthBytes;
7948 if (Offset < Min || Offset > Max)
7949 return false;
7950
7951 Base = N.getOperand(i: 0);
7952 if (Base.getOpcode() == ISD::FrameIndex) {
7953 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
7954 // We can only encode VL scaled offsets, so only fold in frame indexes
7955 // referencing SVE objects.
7956 if (MFI.hasScalableStackID(ObjectIdx: FI))
7957 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
7958 }
7959
7960 OffImm = CurDAG->getTargetConstant(Val: Offset, DL: SDLoc(N), VT: MVT::i64);
7961 return true;
7962}
7963
7964/// Select register plus register addressing mode for SVE, with scaled
7965/// offset.
7966bool AArch64DAGToDAGISel::SelectSVERegRegAddrMode(SDValue N, unsigned Scale,
7967 SDValue &Base,
7968 SDValue &Offset) {
7969 if (N.getOpcode() != ISD::ADD)
7970 return false;
7971
7972 // Process an ADD node.
7973 const SDValue LHS = N.getOperand(i: 0);
7974 const SDValue RHS = N.getOperand(i: 1);
7975
7976 // 8 bit data does not come with the SHL node, so it is treated
7977 // separately.
7978 if (Scale == 0) {
7979 Base = LHS;
7980 Offset = RHS;
7981 return true;
7982 }
7983
7984 if (auto C = dyn_cast<ConstantSDNode>(Val: RHS)) {
7985 int64_t ImmOff = C->getSExtValue();
7986 unsigned Size = 1 << Scale;
7987
7988 // To use the reg+reg addressing mode, the immediate must be a multiple of
7989 // the vector element's byte size.
7990 if (ImmOff % Size)
7991 return false;
7992
7993 SDLoc DL(N);
7994 Base = LHS;
7995 Offset = CurDAG->getTargetConstant(Val: ImmOff >> Scale, DL, VT: MVT::i64);
7996 SDValue Ops[] = {Offset};
7997 SDNode *MI = CurDAG->getMachineNode(Opcode: AArch64::MOVi64imm, dl: DL, VT: MVT::i64, Ops);
7998 Offset = SDValue(MI, 0);
7999 return true;
8000 }
8001
8002 // Check if the RHS is a shift node with a constant.
8003 if (RHS.getOpcode() != ISD::SHL)
8004 return false;
8005
8006 const SDValue ShiftRHS = RHS.getOperand(i: 1);
8007 if (auto *C = dyn_cast<ConstantSDNode>(Val: ShiftRHS))
8008 if (C->getZExtValue() == Scale) {
8009 Base = LHS;
8010 Offset = RHS.getOperand(i: 0);
8011 return true;
8012 }
8013
8014 return false;
8015}
8016
8017bool AArch64DAGToDAGISel::SelectAllActivePredicate(SDValue N) {
8018 const AArch64TargetLowering *TLI =
8019 static_cast<const AArch64TargetLowering *>(getTargetLowering());
8020
8021 return TLI->isAllActivePredicate(DAG: *CurDAG, N);
8022}
8023
8024bool AArch64DAGToDAGISel::SelectAnyPredicate(SDValue N) {
8025 return N.getValueType().isScalableVectorOf(EltVT: MVT::i1);
8026}
8027
8028bool AArch64DAGToDAGISel::SelectSMETileSlice(SDValue N, unsigned MaxSize,
8029 SDValue &Base, SDValue &Offset,
8030 unsigned Scale) {
8031 auto MatchConstantOffset = [&](SDValue CN) -> SDValue {
8032 if (auto *C = dyn_cast<ConstantSDNode>(Val&: CN)) {
8033 int64_t ImmOff = C->getSExtValue();
8034 if ((ImmOff > 0 && ImmOff <= MaxSize && (ImmOff % Scale == 0)))
8035 return CurDAG->getTargetConstant(Val: ImmOff / Scale, DL: SDLoc(N), VT: MVT::i64);
8036 }
8037 return SDValue();
8038 };
8039
8040 if (SDValue C = MatchConstantOffset(N)) {
8041 Base = getZeroRegister(DAG&: *CurDAG, DL: SDLoc(N), VT: MVT::i32);
8042 Offset = C;
8043 return true;
8044 }
8045
8046 // Try to untangle an ADD node into a 'reg + offset'
8047 if (CurDAG->isBaseWithConstantOffset(Op: N)) {
8048 if (SDValue C = MatchConstantOffset(N.getOperand(i: 1))) {
8049 Base = N.getOperand(i: 0);
8050 Offset = C;
8051 return true;
8052 }
8053 }
8054
8055 // By default, just match reg + 0.
8056 Base = N;
8057 Offset = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i64);
8058 return true;
8059}
8060
8061bool AArch64DAGToDAGISel::SelectCmpBranchUImm6Operand(SDNode *P, SDValue N,
8062 SDValue &Imm) {
8063 AArch64CC::CondCode CC =
8064 static_cast<AArch64CC::CondCode>(P->getConstantOperandVal(Num: 1));
8065 if (auto *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
8066 // Check conservatively if the immediate fits the valid range [0, 64).
8067 // Immediate variants for GE and HS definitely need to be decremented
8068 // when lowering the pseudos later, so an immediate of 1 would become 0.
8069 // For the inverse conditions LT and LO we don't know for sure if they
8070 // will need a decrement but should the decision be made to reverse the
8071 // branch condition, we again end up with the need to decrement.
8072 // The same argument holds for LE, LS, GT and HI and possibly
8073 // incremented immediates. This can lead to slightly less optimal
8074 // codegen, e.g. we never codegen the legal case
8075 // cblt w0, #63, A
8076 // because we could end up with the illegal case
8077 // cbge w0, #64, B
8078 // should the decision to reverse the branch direction be made. For the
8079 // lower bound cases this is no problem since we can express comparisons
8080 // against 0 with either tbz/tnbz or using wzr/xzr.
8081 uint64_t LowerBound = 0, UpperBound = 64;
8082 switch (CC) {
8083 case AArch64CC::GE:
8084 case AArch64CC::HS:
8085 case AArch64CC::LT:
8086 case AArch64CC::LO:
8087 LowerBound = 1;
8088 break;
8089 case AArch64CC::LE:
8090 case AArch64CC::LS:
8091 case AArch64CC::GT:
8092 case AArch64CC::HI:
8093 UpperBound = 63;
8094 break;
8095 default:
8096 break;
8097 }
8098
8099 if (CN->getAPIntValue().uge(RHS: LowerBound) &&
8100 CN->getAPIntValue().ult(RHS: UpperBound)) {
8101 SDLoc DL(N);
8102 Imm = CurDAG->getTargetConstant(Val: CN->getZExtValue(), DL, VT: N.getValueType());
8103 return true;
8104 }
8105 }
8106
8107 return false;
8108}
8109
8110template <bool MatchCBB>
8111bool AArch64DAGToDAGISel::SelectCmpBranchExtOperand(SDValue N, SDValue &Reg,
8112 SDValue &ExtType) {
8113
8114 // Use an invalid shift-extend value to indicate we don't need to extend later
8115 if (N.getOpcode() == ISD::AssertZext || N.getOpcode() == ISD::AssertSext) {
8116 EVT Ty = cast<VTSDNode>(Val: N.getOperand(i: 1))->getVT();
8117 if (Ty != (MatchCBB ? MVT::i8 : MVT::i16))
8118 return false;
8119 Reg = N.getOperand(i: 0);
8120 ExtType = CurDAG->getSignedTargetConstant(Val: AArch64_AM::InvalidShiftExtend,
8121 DL: SDLoc(N), VT: MVT::i32);
8122 return true;
8123 }
8124
8125 AArch64_AM::ShiftExtendType ET = getExtendTypeForNode(N);
8126
8127 if ((MatchCBB && (ET == AArch64_AM::UXTB || ET == AArch64_AM::SXTB)) ||
8128 (!MatchCBB && (ET == AArch64_AM::UXTH || ET == AArch64_AM::SXTH))) {
8129 Reg = N.getOperand(i: 0);
8130 ExtType =
8131 CurDAG->getTargetConstant(Val: getExtendEncoding(ET), DL: SDLoc(N), VT: MVT::i32);
8132 return true;
8133 }
8134
8135 return false;
8136}
8137
8138/// Try to fold AArch64 CSEL/FCMP patterns to FMAXNM/FMINNM.
8139///
8140/// This is intentionally done in PreprocessISelDAG rather than DAGCombine:
8141/// doing this earlier based on the defining operation of X can be invalidated
8142/// by later DAG combines. At this point the DAG is being prepared for
8143/// instruction selection, so the use of isKnownNeverSNaN(X) applies to the
8144/// final SDValue being selected.
8145/// Only handles FCMP(X, C) with scalar FP types, where C is a non-NaN constant.
8146/// The nsz requirement is needed only when C is zero, to avoid signed-zero
8147/// mismatches. The never-sNaN check is required because AArch64 FMAXNM/FMINNM
8148/// differ from fcmp+fcsel for signaling NaN inputs.
8149SDValue AArch64DAGToDAGISel::tryFoldCselToFMaxMin(SDNode &N) {
8150 EVT VT = N.getValueType(ResNo: 0);
8151
8152 // Scalar FP only.
8153 if (!VT.isFloatingPoint() || VT.isVector())
8154 return SDValue();
8155
8156 SDValue TVal = N.getOperand(Num: 0);
8157 SDValue FVal = N.getOperand(Num: 1);
8158 SDValue CCVal = N.getOperand(Num: 2);
8159 SDValue Cmp = N.getOperand(Num: 3);
8160
8161 if (Cmp.getOpcode() != AArch64ISD::FCMP)
8162 return SDValue();
8163
8164 auto *CC = dyn_cast<ConstantSDNode>(Val&: CCVal);
8165 if (!CC)
8166 return SDValue();
8167
8168 SDValue CmpLHS = Cmp.getOperand(i: 0);
8169 SDValue CmpRHS = Cmp.getOperand(i: 1);
8170 unsigned CondCode = CC->getZExtValue();
8171
8172 // Map VT and operation (max/min) to machine opcode.
8173 auto getOpc = [](EVT VT, bool isMax) -> unsigned {
8174 if (VT == MVT::f16)
8175 return isMax ? AArch64::FMAXNMHrr : AArch64::FMINNMHrr;
8176 else if (VT == MVT::f32)
8177 return isMax ? AArch64::FMAXNMSrr : AArch64::FMINNMSrr;
8178 else if (VT == MVT::f64)
8179 return isMax ? AArch64::FMAXNMDrr : AArch64::FMINNMDrr;
8180 else
8181 return 0; // unsupported
8182 };
8183
8184 // Determine whether to use max or min based on condition code and operands.
8185 bool isMax;
8186 if (CondCode == AArch64CC::GT || CondCode == AArch64CC::GE) {
8187 if (TVal == CmpLHS && FVal == CmpRHS)
8188 isMax = true;
8189 else
8190 return SDValue();
8191 } else if (CondCode == AArch64CC::MI || CondCode == AArch64CC::LS) {
8192 if (TVal == CmpLHS && FVal == CmpRHS)
8193 isMax = false;
8194 else
8195 return SDValue();
8196 } else {
8197 return SDValue();
8198 }
8199
8200 // Get the machine opcode for this VT and operation.
8201 unsigned Opc = getOpc(VT, isMax);
8202 if (!Opc)
8203 return SDValue();
8204
8205 // Constant must be non-NaN.
8206 auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: CmpRHS);
8207 if (!CFP || CFP->getValueAPF().isNaN())
8208 return SDValue();
8209
8210 // nsz flag required only when constant is zero: fmaxnm(+0,-0)=+0 differs from
8211 // fcmp+select's -0. For non-zero constants, semantics are identical.
8212 if (CFP->isZero() && !N.getFlags().hasNoSignedZeros())
8213 return SDValue();
8214
8215 // Only fold if variable operand is never sNaN.
8216 // This runs after DAG combines, so later combines cannot remove a defining
8217 // operation used by isKnownNeverSNaN().
8218 if (!CurDAG->isKnownNeverSNaN(Op: CmpLHS))
8219 return SDValue();
8220
8221 SDLoc DL(&N);
8222
8223 // Directly emit the machine node
8224 return SDValue(CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT, Op1: CmpLHS, Op2: CmpRHS), 0);
8225}
8226
8227void AArch64DAGToDAGISel::PreprocessISelDAG() {
8228 bool MadeChange = false;
8229 for (SDNode &N : llvm::make_early_inc_range(Range: CurDAG->allnodes())) {
8230 if (N.use_empty())
8231 continue;
8232
8233 SDValue Result;
8234 switch (N.getOpcode()) {
8235 case ISD::SCALAR_TO_VECTOR: {
8236 EVT ScalarTy = N.getValueType(ResNo: 0).getVectorElementType();
8237 if ((ScalarTy == MVT::i32 || ScalarTy == MVT::i64) &&
8238 ScalarTy == N.getOperand(Num: 0).getValueType())
8239 Result = addBitcastHints(DAG&: *CurDAG, N);
8240
8241 break;
8242 }
8243 case AArch64ISD::CSEL:
8244 Result = tryFoldCselToFMaxMin(N);
8245 break;
8246 default:
8247 break;
8248 }
8249
8250 if (Result) {
8251 LLVM_DEBUG(dbgs() << "AArch64 DAG preprocessing replacing:\nOld: ");
8252 LLVM_DEBUG(N.dump(CurDAG));
8253 LLVM_DEBUG(dbgs() << "\nNew: ");
8254 LLVM_DEBUG(Result.dump(CurDAG));
8255 LLVM_DEBUG(dbgs() << "\n");
8256
8257 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(&N, 0), To: Result);
8258 MadeChange = true;
8259 }
8260 }
8261
8262 if (MadeChange)
8263 CurDAG->RemoveDeadNodes();
8264
8265 SelectionDAGISel::PreprocessISelDAG();
8266}
8267