1//=== AArch64PostLegalizerLowering.cpp --------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// Post-legalization lowering for instructions.
11///
12/// This is used to offload pattern matching from the selector.
13///
14/// For example, this combiner will notice that a G_SHUFFLE_VECTOR is actually
15/// a G_ZIP, G_UZP, etc.
16///
17/// General optimization combines should be handled by either the
18/// AArch64PostLegalizerCombiner or the AArch64PreLegalizerCombiner.
19///
20//===----------------------------------------------------------------------===//
21
22#include "AArch64.h"
23#include "AArch64ExpandImm.h"
24#include "AArch64GlobalISelUtils.h"
25#include "AArch64PerfectShuffle.h"
26#include "AArch64Subtarget.h"
27#include "GISel/AArch64LegalizerInfo.h"
28#include "MCTargetDesc/AArch64MCTargetDesc.h"
29#include "Utils/AArch64BaseInfo.h"
30#include "llvm/CodeGen/GlobalISel/Combiner.h"
31#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
32#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
33#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
34#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
35#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
36#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
37#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
38#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
39#include "llvm/CodeGen/GlobalISel/Utils.h"
40#include "llvm/CodeGen/MachineFrameInfo.h"
41#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
42#include "llvm/CodeGen/MachineFunctionPass.h"
43#include "llvm/CodeGen/MachineInstrBuilder.h"
44#include "llvm/CodeGen/MachinePassManager.h"
45#include "llvm/CodeGen/MachineRegisterInfo.h"
46#include "llvm/CodeGen/TargetOpcodes.h"
47#include "llvm/IR/InstrTypes.h"
48#include "llvm/Support/ErrorHandling.h"
49#include <optional>
50
51#define GET_GICOMBINER_DEPS
52#include "AArch64GenPostLegalizeGILowering.inc"
53#undef GET_GICOMBINER_DEPS
54
55#define DEBUG_TYPE "aarch64-postlegalizer-lowering"
56
57using namespace llvm;
58using namespace MIPatternMatch;
59using namespace AArch64GISelUtils;
60
61#define GET_GICOMBINER_TYPES
62#include "AArch64GenPostLegalizeGILowering.inc"
63#undef GET_GICOMBINER_TYPES
64
65namespace {
66
67/// Represents a pseudo instruction which replaces a G_SHUFFLE_VECTOR.
68///
69/// Used for matching target-supported shuffles before codegen.
70struct ShuffleVectorPseudo {
71 unsigned Opc; ///< Opcode for the instruction. (E.g. G_ZIP1)
72 Register Dst; ///< Destination register.
73 SmallVector<SrcOp, 2> SrcOps; ///< Source registers.
74 ShuffleVectorPseudo(unsigned Opc, Register Dst,
75 std::initializer_list<SrcOp> SrcOps)
76 : Opc(Opc), Dst(Dst), SrcOps(SrcOps){};
77 ShuffleVectorPseudo() = default;
78};
79
80/// Check if a G_EXT instruction can handle a shuffle mask \p M when the vector
81/// sources of the shuffle are different.
82std::optional<std::pair<bool, uint64_t>> getExtMask(ArrayRef<int> M,
83 unsigned NumElts) {
84 // Look for the first non-undef element.
85 auto FirstRealElt = find_if(Range&: M, P: [](int Elt) { return Elt >= 0; });
86 if (FirstRealElt == M.end())
87 return std::nullopt;
88
89 // Use APInt to handle overflow when calculating expected element.
90 unsigned MaskBits = APInt(32, NumElts * 2).logBase2();
91 APInt ExpectedElt = APInt(MaskBits, *FirstRealElt + 1, false, true);
92
93 // The following shuffle indices must be the successive elements after the
94 // first real element.
95 if (any_of(
96 Range: make_range(x: std::next(x: FirstRealElt), y: M.end()),
97 P: [&ExpectedElt](int Elt) { return Elt != ExpectedElt++ && Elt >= 0; }))
98 return std::nullopt;
99
100 // The index of an EXT is the first element if it is not UNDEF.
101 // Watch out for the beginning UNDEFs. The EXT index should be the expected
102 // value of the first element. E.g.
103 // <-1, -1, 3, ...> is treated as <1, 2, 3, ...>.
104 // <-1, -1, 0, 1, ...> is treated as <2*NumElts-2, 2*NumElts-1, 0, 1, ...>.
105 // ExpectedElt is the last mask index plus 1.
106 uint64_t Imm = ExpectedElt.getZExtValue();
107 bool ReverseExt = false;
108
109 // There are two difference cases requiring to reverse input vectors.
110 // For example, for vector <4 x i32> we have the following cases,
111 // Case 1: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, -1, 0>)
112 // Case 2: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, 7, 0>)
113 // For both cases, we finally use mask <5, 6, 7, 0>, which requires
114 // to reverse two input vectors.
115 if (Imm < NumElts)
116 ReverseExt = true;
117 else
118 Imm -= NumElts;
119 return std::make_pair(x&: ReverseExt, y&: Imm);
120}
121
122/// Helper function for matchINS.
123///
124/// \returns a value when \p M is an ins mask for \p NumInputElements.
125///
126/// First element of the returned pair is true when the produced
127/// G_INSERT_VECTOR_ELT destination should be the LHS of the G_SHUFFLE_VECTOR.
128///
129/// Second element is the destination lane for the G_INSERT_VECTOR_ELT.
130std::optional<std::pair<bool, int>> isINSMask(ArrayRef<int> M,
131 int NumInputElements) {
132 if (M.size() != static_cast<size_t>(NumInputElements))
133 return std::nullopt;
134 int NumLHSMatch = 0, NumRHSMatch = 0;
135 int LastLHSMismatch = -1, LastRHSMismatch = -1;
136 for (int Idx = 0; Idx < NumInputElements; ++Idx) {
137 if (M[Idx] == -1) {
138 ++NumLHSMatch;
139 ++NumRHSMatch;
140 continue;
141 }
142 M[Idx] == Idx ? ++NumLHSMatch : LastLHSMismatch = Idx;
143 M[Idx] == Idx + NumInputElements ? ++NumRHSMatch : LastRHSMismatch = Idx;
144 }
145 const int NumNeededToMatch = NumInputElements - 1;
146 if (NumLHSMatch == NumNeededToMatch)
147 return std::make_pair(x: true, y&: LastLHSMismatch);
148 if (NumRHSMatch == NumNeededToMatch)
149 return std::make_pair(x: false, y&: LastRHSMismatch);
150 return std::nullopt;
151}
152
153/// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with a
154/// G_REV instruction. Returns the appropriate G_REV opcode in \p Opc.
155bool matchREV(MachineInstr &MI, MachineRegisterInfo &MRI,
156 ShuffleVectorPseudo &MatchInfo) {
157 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
158 ArrayRef<int> ShuffleMask = MI.getOperand(i: 3).getShuffleMask();
159 Register Dst = MI.getOperand(i: 0).getReg();
160 Register Src = MI.getOperand(i: 1).getReg();
161 LLT Ty = MRI.getType(Reg: Dst);
162 unsigned EltSize = Ty.getScalarSizeInBits();
163
164 // Element size for a rev cannot be 64.
165 if (EltSize == 64)
166 return false;
167
168 unsigned NumElts = Ty.getNumElements();
169
170 // Try to produce a G_REV instruction
171 for (unsigned LaneSize : {64U, 32U, 16U}) {
172 if (isREVMask(M: ShuffleMask, EltSize, NumElts, BlockSize: LaneSize)) {
173 unsigned Opcode;
174 if (LaneSize == 64U)
175 Opcode = AArch64::G_REV64;
176 else if (LaneSize == 32U)
177 Opcode = AArch64::G_REV32;
178 else
179 Opcode = AArch64::G_BSWAP;
180
181 MatchInfo = ShuffleVectorPseudo(Opcode, Dst, {Src});
182 return true;
183 }
184 }
185
186 return false;
187}
188
189/// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with
190/// a G_TRN1 or G_TRN2 instruction.
191bool matchTRN(MachineInstr &MI, MachineRegisterInfo &MRI,
192 ShuffleVectorPseudo &MatchInfo) {
193 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
194 unsigned WhichResult;
195 unsigned OperandOrder = 0;
196 ArrayRef<int> ShuffleMask = MI.getOperand(i: 3).getShuffleMask();
197 Register Dst = MI.getOperand(i: 0).getReg();
198 unsigned NumElts = MRI.getType(Reg: Dst).getNumElements();
199 bool TRNMask = isTRNMask(M: ShuffleMask, NumElts, WhichResultOut&: WhichResult, OperandOrderOut&: OperandOrder);
200 if (!TRNMask && !isTRN_v_undef_Mask(M: ShuffleMask, NumElts, WhichResult))
201 return false;
202 unsigned Opc = (WhichResult == 0) ? AArch64::G_TRN1 : AArch64::G_TRN2;
203 Register V1 = MI.getOperand(i: OperandOrder == 0 ? 1 : 2).getReg();
204 Register V2 = MI.getOperand(i: OperandOrder == 0 && TRNMask ? 2 : 1).getReg();
205 MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2});
206 return true;
207}
208
209/// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with
210/// a G_UZP1 or G_UZP2 instruction.
211///
212/// \param [in] MI - The shuffle vector instruction.
213/// \param [out] MatchInfo - Either G_UZP1 or G_UZP2 on success.
214bool matchUZP(MachineInstr &MI, MachineRegisterInfo &MRI,
215 ShuffleVectorPseudo &MatchInfo) {
216 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
217 unsigned WhichResult;
218 ArrayRef<int> ShuffleMask = MI.getOperand(i: 3).getShuffleMask();
219 Register Dst = MI.getOperand(i: 0).getReg();
220 unsigned NumElts = MRI.getType(Reg: Dst).getNumElements();
221 bool UZPMask = isUZPMask(M: ShuffleMask, NumElts, WhichResultOut&: WhichResult);
222 if (!UZPMask && !isUZP_v_undef_Mask(M: ShuffleMask, NumElts, WhichResult))
223 return false;
224 unsigned Opc = (WhichResult == 0) ? AArch64::G_UZP1 : AArch64::G_UZP2;
225 Register V1 = MI.getOperand(i: 1).getReg();
226 Register V2 = MI.getOperand(i: UZPMask ? 2 : 1).getReg();
227 MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2});
228 return true;
229}
230
231bool matchZip(MachineInstr &MI, MachineRegisterInfo &MRI,
232 ShuffleVectorPseudo &MatchInfo) {
233 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
234 unsigned WhichResult;
235 unsigned OperandOrder = 0;
236 ArrayRef<int> ShuffleMask = MI.getOperand(i: 3).getShuffleMask();
237 Register Dst = MI.getOperand(i: 0).getReg();
238 unsigned NumElts = MRI.getType(Reg: Dst).getNumElements();
239 bool ZIPMask = isZIPMask(M: ShuffleMask, NumElts, WhichResultOut&: WhichResult, OperandOrderOut&: OperandOrder);
240 if (!ZIPMask && !isZIP_v_undef_Mask(M: ShuffleMask, NumElts, WhichResult))
241 return false;
242 unsigned Opc = (WhichResult == 0) ? AArch64::G_ZIP1 : AArch64::G_ZIP2;
243 Register V1 = MI.getOperand(i: OperandOrder == 0 ? 1 : 2).getReg();
244 Register V2 = MI.getOperand(i: OperandOrder == 0 && ZIPMask ? 2 : 1).getReg();
245 MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2});
246 return true;
247}
248
249/// Helper function for matchDup.
250bool matchDupFromInsertVectorElt(int Lane, MachineInstr &MI,
251 MachineRegisterInfo &MRI,
252 ShuffleVectorPseudo &MatchInfo) {
253 if (Lane != 0)
254 return false;
255
256 // Try to match a vector splat operation into a dup instruction.
257 // We're looking for this pattern:
258 //
259 // %scalar:gpr(s64) = COPY $x0
260 // %undef:fpr(<2 x s64>) = G_IMPLICIT_DEF
261 // %cst0:gpr(s32) = G_CONSTANT i32 0
262 // %zerovec:fpr(<2 x s32>) = G_BUILD_VECTOR %cst0(s32), %cst0(s32)
263 // %ins:fpr(<2 x s64>) = G_INSERT_VECTOR_ELT %undef, %scalar(s64), %cst0(s32)
264 // %splat:fpr(<2 x s64>) = G_SHUFFLE_VECTOR %ins(<2 x s64>), %undef,
265 // %zerovec(<2 x s32>)
266 //
267 // ...into:
268 // %splat = G_DUP %scalar
269
270 // Begin matching the insert.
271 auto *InsMI = getOpcodeDef(Opcode: TargetOpcode::G_INSERT_VECTOR_ELT,
272 Reg: MI.getOperand(i: 1).getReg(), MRI);
273 if (!InsMI)
274 return false;
275 // Match the undef vector operand.
276 if (!getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: InsMI->getOperand(i: 1).getReg(),
277 MRI))
278 return false;
279
280 // Match the index constant 0.
281 if (!mi_match(R: InsMI->getOperand(i: 3).getReg(), MRI, P: m_ZeroInt()))
282 return false;
283
284 MatchInfo = ShuffleVectorPseudo(AArch64::G_DUP, MI.getOperand(i: 0).getReg(),
285 {InsMI->getOperand(i: 2).getReg()});
286 return true;
287}
288
289/// Helper function for matchDup.
290bool matchDupFromBuildVector(int Lane, MachineInstr &MI,
291 MachineRegisterInfo &MRI,
292 ShuffleVectorPseudo &MatchInfo) {
293 assert(Lane >= 0 && "Expected positive lane?");
294 int NumElements = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getNumElements();
295 // Test if the LHS is a BUILD_VECTOR. If it is, then we can just reference the
296 // lane's definition directly.
297 auto *BuildVecMI =
298 getOpcodeDef(Opcode: TargetOpcode::G_BUILD_VECTOR,
299 Reg: MI.getOperand(i: Lane < NumElements ? 1 : 2).getReg(), MRI);
300 // If Lane >= NumElements then it is point to RHS, just check from RHS
301 if (NumElements <= Lane)
302 Lane -= NumElements;
303
304 if (!BuildVecMI)
305 return false;
306 Register Reg = BuildVecMI->getOperand(i: Lane + 1).getReg();
307 MatchInfo =
308 ShuffleVectorPseudo(AArch64::G_DUP, MI.getOperand(i: 0).getReg(), {Reg});
309 return true;
310}
311
312bool matchDup(MachineInstr &MI, MachineRegisterInfo &MRI,
313 ShuffleVectorPseudo &MatchInfo) {
314 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
315 auto MaybeLane = getSplatIndex(MI);
316 if (!MaybeLane)
317 return false;
318 int Lane = *MaybeLane;
319 // If this is undef splat, generate it via "just" vdup, if possible.
320 if (Lane < 0)
321 Lane = 0;
322 if (matchDupFromInsertVectorElt(Lane, MI, MRI, MatchInfo))
323 return true;
324 if (matchDupFromBuildVector(Lane, MI, MRI, MatchInfo))
325 return true;
326 return false;
327}
328
329// Check if an EXT instruction can handle the shuffle mask when the vector
330// sources of the shuffle are the same.
331bool isSingletonExtMask(ArrayRef<int> M, LLT Ty) {
332 unsigned NumElts = Ty.getNumElements();
333
334 // Assume that the first shuffle index is not UNDEF. Fail if it is.
335 if (M[0] < 0)
336 return false;
337
338 // If this is a VEXT shuffle, the immediate value is the index of the first
339 // element. The other shuffle indices must be the successive elements after
340 // the first one.
341 unsigned ExpectedElt = M[0];
342 for (unsigned I = 1; I < NumElts; ++I) {
343 // Increment the expected index. If it wraps around, just follow it
344 // back to index zero and keep going.
345 ++ExpectedElt;
346 if (ExpectedElt == NumElts)
347 ExpectedElt = 0;
348
349 if (M[I] < 0)
350 continue; // Ignore UNDEF indices.
351 if (ExpectedElt != static_cast<unsigned>(M[I]))
352 return false;
353 }
354
355 return true;
356}
357
358bool matchEXT(MachineInstr &MI, MachineRegisterInfo &MRI,
359 ShuffleVectorPseudo &MatchInfo) {
360 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
361 Register Dst = MI.getOperand(i: 0).getReg();
362 LLT DstTy = MRI.getType(Reg: Dst);
363 Register V1 = MI.getOperand(i: 1).getReg();
364 Register V2 = MI.getOperand(i: 2).getReg();
365 auto Mask = MI.getOperand(i: 3).getShuffleMask();
366 uint64_t Imm;
367 auto ExtInfo = getExtMask(M: Mask, NumElts: DstTy.getNumElements());
368 uint64_t ExtFactor = MRI.getType(Reg: V1).getScalarSizeInBits() / 8;
369
370 if (!ExtInfo) {
371 if (!getOpcodeDef<GImplicitDef>(Reg: V2, MRI) ||
372 !isSingletonExtMask(M: Mask, Ty: DstTy))
373 return false;
374
375 Imm = Mask[0] * ExtFactor;
376 MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst, {V1, V1, Imm});
377 return true;
378 }
379 bool ReverseExt;
380 std::tie(args&: ReverseExt, args&: Imm) = *ExtInfo;
381 if (ReverseExt)
382 std::swap(a&: V1, b&: V2);
383 Imm *= ExtFactor;
384 MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst, {V1, V2, Imm});
385 return true;
386}
387
388/// Replace a G_SHUFFLE_VECTOR instruction with a pseudo.
389/// \p Opc is the opcode to use. \p MI is the G_SHUFFLE_VECTOR.
390void applyShuffleVectorPseudo(MachineInstr &MI, MachineRegisterInfo &MRI,
391 ShuffleVectorPseudo &MatchInfo) {
392 MachineIRBuilder MIRBuilder(MI);
393 if (MatchInfo.Opc == TargetOpcode::G_BSWAP) {
394 assert(MatchInfo.SrcOps.size() == 1);
395 LLT DstTy = MRI.getType(Reg: MatchInfo.Dst);
396 assert(DstTy == LLT::fixed_vector(8, 8) ||
397 DstTy == LLT::fixed_vector(16, 8));
398 LLT BSTy = DstTy == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8)
399 ? LLT::fixed_vector(NumElements: 4, ScalarTy: LLT::integer(SizeInBits: 16))
400 : LLT::fixed_vector(NumElements: 8, ScalarTy: LLT::integer(SizeInBits: 16));
401 // FIXME: NVCAST
402 auto BS1 = MIRBuilder.buildInstr(Opc: TargetOpcode::G_BITCAST, DstOps: {BSTy},
403 SrcOps: MatchInfo.SrcOps[0]);
404 auto BS2 = MIRBuilder.buildInstr(Opc: MatchInfo.Opc, DstOps: {BSTy}, SrcOps: {BS1});
405 MIRBuilder.buildInstr(Opc: TargetOpcode::G_BITCAST, DstOps: {MatchInfo.Dst}, SrcOps: {BS2});
406 } else
407 MIRBuilder.buildInstr(Opc: MatchInfo.Opc, DstOps: {MatchInfo.Dst}, SrcOps: MatchInfo.SrcOps);
408 MI.eraseFromParent();
409}
410
411/// Replace a G_SHUFFLE_VECTOR instruction with G_EXT.
412/// Special-cased because the constant operand must be emitted as a G_CONSTANT
413/// for the imported tablegen patterns to work.
414void applyEXT(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) {
415 MachineIRBuilder MIRBuilder(MI);
416 if (MatchInfo.SrcOps[2].getImm() == 0)
417 MIRBuilder.buildCopy(Res: MatchInfo.Dst, Op: MatchInfo.SrcOps[0]);
418 else {
419 // Tablegen patterns expect an i32 G_CONSTANT as the final op.
420 auto Cst = MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: 32),
421 Val: MatchInfo.SrcOps[2].getImm());
422 MIRBuilder.buildInstr(Opc: MatchInfo.Opc, DstOps: {MatchInfo.Dst},
423 SrcOps: {MatchInfo.SrcOps[0], MatchInfo.SrcOps[1], Cst});
424 }
425 MI.eraseFromParent();
426}
427
428void applyFullRev(MachineInstr &MI, MachineRegisterInfo &MRI) {
429 Register Dst = MI.getOperand(i: 0).getReg();
430 Register Src = MI.getOperand(i: 1).getReg();
431 LLT DstTy = MRI.getType(Reg: Dst);
432 assert(DstTy.getSizeInBits() == 128 &&
433 "Expected 128bit vector in applyFullRev");
434 MachineIRBuilder MIRBuilder(MI);
435 auto Cst = MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: 32), Val: 8);
436 auto Rev = MIRBuilder.buildInstr(Opc: AArch64::G_REV64, DstOps: {DstTy}, SrcOps: {Src});
437 MIRBuilder.buildInstr(Opc: AArch64::G_EXT, DstOps: {Dst}, SrcOps: {Rev, Rev, Cst});
438 MI.eraseFromParent();
439}
440
441bool matchNonConstInsert(MachineInstr &MI, MachineRegisterInfo &MRI) {
442 assert(MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT);
443
444 auto ValAndVReg =
445 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 3).getReg(), MRI);
446 return !ValAndVReg;
447}
448
449void applyNonConstInsert(MachineInstr &MI, MachineRegisterInfo &MRI,
450 MachineIRBuilder &Builder) {
451 auto &Insert = cast<GInsertVectorElement>(Val&: MI);
452 Builder.setInstrAndDebugLoc(Insert);
453
454 Register Offset = Insert.getIndexReg();
455 LLT VecTy = MRI.getType(Reg: Insert.getReg(Idx: 0));
456 LLT EltTy = MRI.getType(Reg: Insert.getElementReg());
457 LLT IdxTy = MRI.getType(Reg: Insert.getIndexReg());
458
459 if (VecTy.isScalableVector())
460 return;
461
462 // Create a stack slot and store the vector into it
463 MachineFunction &MF = Builder.getMF();
464 Align Alignment(
465 std::min<uint64_t>(a: VecTy.getSizeInBytes().getKnownMinValue(), b: 16));
466 int FrameIdx = MF.getFrameInfo().CreateStackObject(Size: VecTy.getSizeInBytes(),
467 Alignment, isSpillSlot: false);
468 LLT FramePtrTy = LLT::pointer(AddressSpace: 0, SizeInBits: 64);
469 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIdx);
470 auto StackTemp = Builder.buildFrameIndex(Res: FramePtrTy, Idx: FrameIdx);
471
472 Builder.buildStore(Val: Insert.getOperand(i: 1), Addr: StackTemp, PtrInfo, Alignment: Align(8));
473
474 // Get the pointer to the element, and be sure not to hit undefined behavior
475 // if the index is out of bounds.
476 assert(isPowerOf2_64(VecTy.getNumElements()) &&
477 "Expected a power-2 vector size");
478 auto Mask = Builder.buildConstant(Res: IdxTy, Val: VecTy.getNumElements() - 1);
479 Register And = Builder.buildAnd(Dst: IdxTy, Src0: Offset, Src1: Mask).getReg(Idx: 0);
480 auto EltSize = Builder.buildConstant(Res: IdxTy, Val: EltTy.getSizeInBytes());
481 Register Mul = Builder.buildMul(Dst: IdxTy, Src0: And, Src1: EltSize).getReg(Idx: 0);
482 Register EltPtr =
483 Builder.buildPtrAdd(Res: MRI.getType(Reg: StackTemp.getReg(Idx: 0)), Op0: StackTemp, Op1: Mul)
484 .getReg(Idx: 0);
485
486 // Write the inserted element
487 Builder.buildStore(Val: Insert.getElementReg(), Addr: EltPtr, PtrInfo, Alignment: Align(1));
488 // Reload the whole vector.
489 Builder.buildLoad(Res: Insert.getReg(Idx: 0), Addr: StackTemp, PtrInfo, Alignment: Align(8));
490 Insert.eraseFromParent();
491}
492
493/// Match a G_SHUFFLE_VECTOR with a mask which corresponds to a
494/// G_INSERT_VECTOR_ELT and G_EXTRACT_VECTOR_ELT pair.
495///
496/// e.g.
497/// %shuf = G_SHUFFLE_VECTOR %left, %right, shufflemask(0, 0)
498///
499/// Can be represented as
500///
501/// %extract = G_EXTRACT_VECTOR_ELT %left, 0
502/// %ins = G_INSERT_VECTOR_ELT %left, %extract, 1
503///
504bool matchINS(MachineInstr &MI, MachineRegisterInfo &MRI,
505 std::tuple<Register, int, Register, int> &MatchInfo) {
506 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
507 ArrayRef<int> ShuffleMask = MI.getOperand(i: 3).getShuffleMask();
508 Register Dst = MI.getOperand(i: 0).getReg();
509 int NumElts = MRI.getType(Reg: Dst).getNumElements();
510 auto DstIsLeftAndDstLane = isINSMask(M: ShuffleMask, NumInputElements: NumElts);
511 if (!DstIsLeftAndDstLane)
512 return false;
513 bool DstIsLeft;
514 int DstLane;
515 std::tie(args&: DstIsLeft, args&: DstLane) = *DstIsLeftAndDstLane;
516 Register Left = MI.getOperand(i: 1).getReg();
517 Register Right = MI.getOperand(i: 2).getReg();
518 Register DstVec = DstIsLeft ? Left : Right;
519 Register SrcVec = Left;
520
521 int SrcLane = ShuffleMask[DstLane];
522 if (SrcLane >= NumElts) {
523 SrcVec = Right;
524 SrcLane -= NumElts;
525 }
526
527 MatchInfo = std::make_tuple(args&: DstVec, args&: DstLane, args&: SrcVec, args&: SrcLane);
528 return true;
529}
530
531void applyINS(MachineInstr &MI, MachineRegisterInfo &MRI,
532 MachineIRBuilder &Builder,
533 std::tuple<Register, int, Register, int> &MatchInfo) {
534 Builder.setInstrAndDebugLoc(MI);
535 Register Dst = MI.getOperand(i: 0).getReg();
536 auto ScalarTy = MRI.getType(Reg: Dst).getElementType();
537 Register DstVec, SrcVec;
538 int DstLane, SrcLane;
539 std::tie(args&: DstVec, args&: DstLane, args&: SrcVec, args&: SrcLane) = MatchInfo;
540 auto SrcCst = Builder.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: SrcLane);
541 auto Extract = Builder.buildExtractVectorElement(Res: ScalarTy, Val: SrcVec, Idx: SrcCst);
542 auto DstCst = Builder.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: DstLane);
543 Builder.buildInsertVectorElement(Res: Dst, Val: DstVec, Elt: Extract, Idx: DstCst);
544 MI.eraseFromParent();
545}
546
547/// isVShiftRImm - Check if this is a valid vector for the immediate
548/// operand of a vector shift right operation. The value must be in the range:
549/// 1 <= Value <= ElementBits for a right shift.
550bool isVShiftRImm(Register Reg, MachineRegisterInfo &MRI, LLT Ty,
551 int64_t &Cnt) {
552 assert(Ty.isVector() && "vector shift count is not a vector type");
553 MachineInstr *MI = MRI.getVRegDef(Reg);
554 auto Cst = getAArch64VectorSplatScalar(MI: *MI, MRI);
555 if (!Cst)
556 return false;
557 Cnt = *Cst;
558 int64_t ElementBits = Ty.getScalarSizeInBits();
559 return Cnt >= 1 && Cnt <= ElementBits;
560}
561
562/// Match a vector G_ASHR or G_LSHR with a valid immediate shift.
563bool matchVAshrLshrImm(MachineInstr &MI, MachineRegisterInfo &MRI,
564 int64_t &Imm) {
565 assert(MI.getOpcode() == TargetOpcode::G_ASHR ||
566 MI.getOpcode() == TargetOpcode::G_LSHR);
567 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
568 if (!Ty.isVector())
569 return false;
570 return isVShiftRImm(Reg: MI.getOperand(i: 2).getReg(), MRI, Ty, Cnt&: Imm);
571}
572
573void applyVAshrLshrImm(MachineInstr &MI, MachineRegisterInfo &MRI,
574 int64_t &Imm) {
575 unsigned Opc = MI.getOpcode();
576 assert(Opc == TargetOpcode::G_ASHR || Opc == TargetOpcode::G_LSHR);
577 unsigned NewOpc =
578 Opc == TargetOpcode::G_ASHR ? AArch64::G_VASHR : AArch64::G_VLSHR;
579 MachineIRBuilder MIB(MI);
580 MIB.buildInstr(Opc: NewOpc, DstOps: {MI.getOperand(i: 0)}, SrcOps: {MI.getOperand(i: 1)}).addImm(Val: Imm);
581 MI.eraseFromParent();
582}
583
584/// Determine whether an integer G_ICMP against 1 or -1 can compare
585/// against 0 instead.
586///
587/// AArch64 can fold a compare-with-zero more cheaply than some non-arithmetic
588/// immediates (SUBS/ADDS, or TST when the LHS is an AND). When the predicate
589/// can be adjusted without changing semantics, the RHS may become 0.
590///
591/// Supported transforms (signed predicates only):
592/// (and X, Y) slt 1 => (and X, Y) sle 0
593/// (and X, Y) sge 1 => (and X, Y) sgt 0
594/// X sle -1 => X slt 0
595/// X sgt -1 => X sge 0
596///
597/// The compare-against-1 cases require the LHS to be G_AND because the
598/// compare-with-zero path enables ANDS (TST) selection, and ANDS flags are
599/// only reliable for those signed comparisons. This mirrors SelectionDAG
600/// emitComparison().
601///
602/// For compare-against--1 on a non-AND LHS, \p LHS must have a single
603/// non-debug use so other users are not left with a different immediate.
604///
605/// \param LHS The compare LHS register.
606/// \param C The constant RHS (only 1 or all-ones are considered).
607/// \param P In/out predicate; updated when a transform applies.
608/// \param MRI Used to inspect the LHS definition and use count.
609/// \returns true if \p P was updated and comparing against 0 is equivalent.
610static bool shouldBeAdjustedToZero(Register LHS, const APInt &C,
611 CmpInst::Predicate &P,
612 const MachineRegisterInfo &MRI) {
613 const bool IsAndLHS = getOpcodeDef<GAnd>(Reg: LHS, MRI) != nullptr;
614
615 if (C.isOne() && (P == CmpInst::ICMP_SLT || P == CmpInst::ICMP_SGE) &&
616 IsAndLHS) {
617 P = (P == CmpInst::ICMP_SLT) ? CmpInst::ICMP_SLE : CmpInst::ICMP_SGT;
618 return true;
619 }
620
621 if (!IsAndLHS && !MRI.hasOneNonDBGUse(RegNo: LHS))
622 return false;
623
624 if (C.isAllOnes() && (P == CmpInst::ICMP_SLE || P == CmpInst::ICMP_SGT)) {
625 P = (P == CmpInst::ICMP_SLE) ? CmpInst::ICMP_SLT : CmpInst::ICMP_SGE;
626 return true;
627 }
628 return false;
629}
630
631/// Determine if it is possible to modify the \p RHS and predicate \p P of a
632/// G_ICMP instruction such that the right-hand side is an arithmetic immediate.
633///
634/// \returns A pair containing the updated immediate and predicate which may
635/// be used to optimize the instruction.
636///
637/// \note This assumes that the comparison has been legalized.
638std::optional<std::pair<uint64_t, CmpInst::Predicate>>
639tryAdjustICmpImmAndPred(Register LHS, Register RHS, CmpInst::Predicate P,
640 const MachineRegisterInfo &MRI) {
641 const auto &Ty = MRI.getType(Reg: RHS);
642 if (Ty.isVector())
643 return std::nullopt;
644 assert((Ty.getSizeInBits() == 32 || Ty.getSizeInBits() == 64) &&
645 "Expected 32 or 64 bit compare only?");
646
647 // If the RHS is not a constant, or the RHS is already a valid arithmetic
648 // immediate, then there is nothing to change.
649 auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: RHS, MRI);
650 if (!ValAndVReg)
651 return std::nullopt;
652 APInt C = ValAndVReg->Value;
653 if (shouldBeAdjustedToZero(LHS, C, P, MRI))
654 return {{0, P}};
655
656 if (AArch64_AM::isLegalCmpImmed(C))
657 return std::nullopt;
658
659 uint64_t OriginalC = C.getZExtValue();
660
661 // We have a non-arithmetic immediate. Check if adjusting the immediate and
662 // adjusting the predicate will result in a legal arithmetic immediate.
663 switch (P) {
664 default:
665 return std::nullopt;
666 case CmpInst::ICMP_SLT:
667 case CmpInst::ICMP_SGE:
668 // Check for
669 //
670 // x slt c => x sle c - 1
671 // x sge c => x sgt c - 1
672 //
673 // When c is not the smallest possible negative number.
674 if (C.isMinSignedValue())
675 return std::nullopt;
676 P = (P == CmpInst::ICMP_SLT) ? CmpInst::ICMP_SLE : CmpInst::ICMP_SGT;
677 C = C - 1;
678 break;
679 case CmpInst::ICMP_ULT:
680 case CmpInst::ICMP_UGE:
681 // Check for
682 //
683 // x ult c => x ule c - 1
684 // x uge c => x ugt c - 1
685 //
686 // When c is not zero.
687 assert(!C.isZero() && "C should not be zero here!");
688 P = (P == CmpInst::ICMP_ULT) ? CmpInst::ICMP_ULE : CmpInst::ICMP_UGT;
689 C = C - 1;
690 break;
691 case CmpInst::ICMP_SLE:
692 case CmpInst::ICMP_SGT:
693 // Check for
694 //
695 // x sle c => x slt c + 1
696 // x sgt c => s sge c + 1
697 //
698 // When c is not the largest possible signed integer.
699 if (C.isMaxSignedValue())
700 return std::nullopt;
701 P = (P == CmpInst::ICMP_SLE) ? CmpInst::ICMP_SLT : CmpInst::ICMP_SGE;
702 C = C + 1;
703 break;
704 case CmpInst::ICMP_ULE:
705 case CmpInst::ICMP_UGT:
706 // Check for
707 //
708 // x ule c => x ult c + 1
709 // x ugt c => s uge c + 1
710 //
711 // When c is not the largest possible unsigned integer.
712 if (C.isAllOnes())
713 return std::nullopt;
714 P = (P == CmpInst::ICMP_ULE) ? CmpInst::ICMP_ULT : CmpInst::ICMP_UGE;
715 C = C + 1;
716 break;
717 }
718
719 // Check if the new constant is valid, and return the updated constant and
720 // predicate if it is.
721 uint64_t NewC = C.getZExtValue();
722 if (AArch64_AM::isLegalCmpImmed(C))
723 return {{NewC, P}};
724
725 auto NumberOfInstrToLoadImm = [=](uint64_t Imm) {
726 SmallVector<AArch64_IMM::ImmInsnModel> Insn;
727 AArch64_IMM::expandMOVImm(Imm, BitSize: 32, Insn);
728 return Insn.size();
729 };
730
731 if (NumberOfInstrToLoadImm(OriginalC) > NumberOfInstrToLoadImm(NewC))
732 return {{NewC, P}};
733
734 return std::nullopt;
735}
736
737/// Determine whether or not it is possible to update the RHS and predicate of
738/// a G_ICMP instruction such that the RHS will be selected as an arithmetic
739/// immediate.
740///
741/// \p MI - The G_ICMP instruction
742/// \p MatchInfo - The new RHS immediate and predicate on success
743///
744/// See tryAdjustICmpImmAndPred for valid transformations.
745bool matchAdjustICmpImmAndPred(
746 MachineInstr &MI, const MachineRegisterInfo &MRI,
747 std::pair<uint64_t, CmpInst::Predicate> &MatchInfo) {
748 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
749 Register LHS = MI.getOperand(i: 2).getReg();
750 Register RHS = MI.getOperand(i: 3).getReg();
751 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
752 if (auto MaybeNewImmAndPred = tryAdjustICmpImmAndPred(LHS, RHS, P: Pred, MRI)) {
753 MatchInfo = *MaybeNewImmAndPred;
754 return true;
755 }
756 return false;
757}
758
759void applyAdjustICmpImmAndPred(
760 MachineInstr &MI, std::pair<uint64_t, CmpInst::Predicate> &MatchInfo,
761 MachineIRBuilder &MIB, GISelChangeObserver &Observer) {
762 MIB.setInstrAndDebugLoc(MI);
763 MachineOperand &RHS = MI.getOperand(i: 3);
764 MachineRegisterInfo &MRI = *MIB.getMRI();
765 auto Cst = MIB.buildConstant(Res: MRI.cloneVirtualRegister(VReg: RHS.getReg()),
766 Val: MatchInfo.first);
767 Observer.changingInstr(MI);
768 RHS.setReg(Cst->getOperand(i: 0).getReg());
769 MI.getOperand(i: 1).setPredicate(MatchInfo.second);
770 Observer.changedInstr(MI);
771}
772
773bool matchDupLane(MachineInstr &MI, MachineRegisterInfo &MRI,
774 std::pair<unsigned, int> &MatchInfo) {
775 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
776 Register Src1Reg = MI.getOperand(i: 1).getReg();
777 const LLT SrcTy = MRI.getType(Reg: Src1Reg);
778 const LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
779
780 auto LaneIdx = getSplatIndex(MI);
781 if (!LaneIdx)
782 return false;
783
784 // The lane idx should be within the first source vector.
785 if (*LaneIdx >= SrcTy.getNumElements())
786 return false;
787
788 if (DstTy != SrcTy)
789 return false;
790
791 LLT ScalarTy = SrcTy.getElementType();
792 unsigned ScalarSize = ScalarTy.getSizeInBits();
793
794 unsigned Opc = 0;
795 switch (SrcTy.getNumElements()) {
796 case 2:
797 if (ScalarSize == 64)
798 Opc = AArch64::G_DUPLANE64;
799 else if (ScalarSize == 32)
800 Opc = AArch64::G_DUPLANE32;
801 break;
802 case 4:
803 if (ScalarSize == 32)
804 Opc = AArch64::G_DUPLANE32;
805 else if (ScalarSize == 16)
806 Opc = AArch64::G_DUPLANE16;
807 break;
808 case 8:
809 if (ScalarSize == 8)
810 Opc = AArch64::G_DUPLANE8;
811 else if (ScalarSize == 16)
812 Opc = AArch64::G_DUPLANE16;
813 break;
814 case 16:
815 if (ScalarSize == 8)
816 Opc = AArch64::G_DUPLANE8;
817 break;
818 default:
819 break;
820 }
821 if (!Opc)
822 return false;
823
824 MatchInfo.first = Opc;
825 MatchInfo.second = *LaneIdx;
826 return true;
827}
828
829void applyDupLane(MachineInstr &MI, MachineRegisterInfo &MRI,
830 MachineIRBuilder &B, std::pair<unsigned, int> &MatchInfo) {
831 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
832 Register Src1Reg = MI.getOperand(i: 1).getReg();
833 const LLT SrcTy = MRI.getType(Reg: Src1Reg);
834
835 B.setInstrAndDebugLoc(MI);
836 auto Lane = B.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: MatchInfo.second);
837
838 Register DupSrc = MI.getOperand(i: 1).getReg();
839 // For types like <2 x s32>, we can use G_DUPLANE32, with a <4 x s32> source.
840 // To do this, we can use a G_CONCAT_VECTORS to do the widening.
841 if (SrcTy.getSizeInBits() == 64) {
842 auto Undef = B.buildUndef(Res: SrcTy);
843 DupSrc = B.buildConcatVectors(Res: SrcTy.multiplyElements(Factor: 2),
844 Ops: {Src1Reg, Undef.getReg(Idx: 0)})
845 .getReg(Idx: 0);
846 }
847 B.buildInstr(Opc: MatchInfo.first, DstOps: {MI.getOperand(i: 0).getReg()}, SrcOps: {DupSrc, Lane});
848 MI.eraseFromParent();
849}
850
851bool matchScalarizeVectorUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI) {
852 auto &Unmerge = cast<GUnmerge>(Val&: MI);
853 Register Src1Reg = Unmerge.getReg(Idx: Unmerge.getNumOperands() - 1);
854 const LLT SrcTy = MRI.getType(Reg: Src1Reg);
855 if (SrcTy.getSizeInBits() != 128 && SrcTy.getSizeInBits() != 64)
856 return false;
857 return SrcTy.isVector() && !SrcTy.isScalable() &&
858 (Unmerge.getNumOperands() == (unsigned)SrcTy.getNumElements() + 1 ||
859 (Unmerge.getNumDefs() == 2 && SrcTy.getSizeInBits() == 128 &&
860 MRI.getType(Reg: Unmerge.getReg(Idx: 0)).getSizeInBits() == 64));
861}
862
863void applyScalarizeVectorUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
864 MachineIRBuilder &B) {
865 auto &Unmerge = cast<GUnmerge>(Val&: MI);
866 Register Src1Reg = Unmerge.getReg(Idx: Unmerge.getNumOperands() - 1);
867 const LLT SrcTy = MRI.getType(Reg: Src1Reg);
868 const LLT DstTy = MRI.getType(Reg: Unmerge.getReg(Idx: 0));
869 assert((SrcTy.isVector() && !SrcTy.isScalable()) &&
870 "Expected a fixed length vector");
871
872 if (DstTy.isVector()) {
873 assert(Unmerge.getNumDefs() == 2);
874 if (!MRI.use_nodbg_empty(RegNo: Unmerge.getReg(Idx: 0)))
875 B.buildExtractSubvector(Res: Unmerge.getReg(Idx: 0), Src: Src1Reg, Index: 0);
876 if (!MRI.use_nodbg_empty(RegNo: Unmerge.getReg(Idx: 1)))
877 B.buildExtractSubvector(Res: Unmerge.getReg(Idx: 1), Src: Src1Reg,
878 Index: SrcTy.getNumElements() / 2);
879 } else {
880 for (int I = 0; I < SrcTy.getNumElements(); ++I)
881 if (!MRI.use_nodbg_empty(RegNo: Unmerge.getReg(Idx: I)))
882 B.buildExtractVectorElementConstant(Res: Unmerge.getReg(Idx: I), Val: Src1Reg, Idx: I);
883 }
884 MI.eraseFromParent();
885}
886
887bool matchBuildVectorToDup(MachineInstr &MI, Register &Src,
888 MachineRegisterInfo &MRI) {
889 assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
890
891 // Later, during selection, we'll try to match imported patterns using
892 // immAllOnesV and immAllZerosV. These require G_BUILD_VECTOR. Don't lower
893 // G_BUILD_VECTORs which could match those patterns.
894 if (isBuildVectorAllZeros(MI, MRI) || isBuildVectorAllOnes(MI, MRI))
895 return false;
896
897 // Find buildvector which always uses the same register or undef. Return true
898 // so long as at least 2 registers were found (not all-undef or only 1
899 // non-undef entry).
900 Register Reg = 0;
901 unsigned NumNonUndef = 0;
902 for (const MachineOperand &Op : drop_begin(RangeOrContainer: MI.operands())) {
903 if (getOpcodeDef<GImplicitDef>(Reg: Op.getReg(), MRI))
904 continue;
905
906 if (!Reg)
907 Reg = Op.getReg();
908 else if (Op.getReg() != Reg)
909 return false;
910 NumNonUndef++;
911 }
912
913 Src = Reg;
914 return Reg && NumNonUndef > 1;
915}
916
917void applyBuildVectorToDup(MachineInstr &MI, Register Src,
918 MachineRegisterInfo &MRI, MachineIRBuilder &B) {
919 B.setInstrAndDebugLoc(MI);
920 B.buildInstr(Opc: AArch64::G_DUP, DstOps: {MI.getOperand(i: 0).getReg()}, SrcOps: {Src});
921 MI.eraseFromParent();
922}
923
924/// \returns how many instructions would be saved by folding a G_ICMP's shift
925/// and/or extension operations.
926static unsigned getCmpOperandFoldingProfit(Register CmpOp,
927 MachineRegisterInfo &MRI) {
928 // FIXME: This is duplicated with the selector. (See: selectShiftedRegister)
929 auto IsSupportedExtend = [&](const MachineInstr &MI) {
930 if (MI.getOpcode() == TargetOpcode::G_SEXT_INREG)
931 return true;
932 if (MI.getOpcode() == TargetOpcode::G_AND) {
933 auto ValAndVReg =
934 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI);
935 if (ValAndVReg) {
936 uint64_t Mask = ValAndVReg->Value.getZExtValue();
937 return (Mask == 0xFF || Mask == 0xFFFF || Mask == 0xFFFFFFFF);
938 }
939 }
940 return false;
941 };
942
943 // No instructions to save if there's more than one use or no uses.
944 if (!MRI.hasOneNonDBGUse(RegNo: CmpOp))
945 return 0;
946
947 MachineInstr *Def = getDefIgnoringCopies(Reg: CmpOp, MRI);
948 if (IsSupportedExtend(*Def))
949 return 1;
950
951 unsigned Opc = Def->getOpcode();
952 if (Opc == TargetOpcode::G_SHL || Opc == TargetOpcode::G_LSHR ||
953 Opc == TargetOpcode::G_ASHR) {
954 auto MaybeShiftAmt =
955 getIConstantVRegValWithLookThrough(VReg: Def->getOperand(i: 2).getReg(), MRI);
956 if (MaybeShiftAmt) {
957 uint64_t ShiftAmt = MaybeShiftAmt->Value.getZExtValue();
958 MachineInstr *ShiftLHS =
959 getDefIgnoringCopies(Reg: Def->getOperand(i: 1).getReg(), MRI);
960 if (IsSupportedExtend(*ShiftLHS))
961 return (ShiftAmt <= 4) ? 2 : 1;
962 LLT Ty = MRI.getType(Reg: Def->getOperand(i: 0).getReg());
963 if (Ty.isVector())
964 return 0;
965 unsigned ShiftSize = Ty.getSizeInBits();
966 if ((ShiftSize == 32 && ShiftAmt <= 31) ||
967 (ShiftSize == 64 && ShiftAmt <= 63))
968 return 1;
969 }
970 }
971
972 return 0;
973}
974
975/// \returns true if it would be profitable to swap the LHS and RHS of a G_ICMP
976/// instruction \p MI.
977bool trySwapICmpOperands(MachineInstr &MI, MachineRegisterInfo &MRI) {
978 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
979 // Swap the operands if it would introduce a profitable folding opportunity.
980 // (e.g. a shift + extend).
981 //
982 // For example:
983 // lsl w13, w11, #1
984 // cmp w13, w12
985 // can be turned into:
986 // cmp w12, w11, lsl #1
987
988 // Don't swap if there's a constant on the RHS and it is a legal compare
989 // immediate, because we know we can fold that.
990 Register RHS = MI.getOperand(i: 3).getReg();
991 auto RHSCst = getIConstantVRegValWithLookThrough(VReg: RHS, MRI);
992 if (RHSCst && AArch64_AM::isLegalCmpImmed(C: RHSCst->Value))
993 return false;
994
995 Register LHS = MI.getOperand(i: 2).getReg();
996 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
997 auto GetRegForProfit = [&](Register Reg) {
998 MachineInstr *Def = getDefIgnoringCopies(Reg, MRI);
999 return isCMN(MaybeSub: Def, Pred, MRI) ? Def->getOperand(i: 2).getReg() : Reg;
1000 };
1001
1002 // Don't have a constant on the RHS. If we swap the LHS and RHS of the
1003 // compare, would we be able to fold more instructions?
1004 Register TheLHS = GetRegForProfit(LHS);
1005 Register TheRHS = GetRegForProfit(RHS);
1006
1007 // If the LHS is more likely to give us a folding opportunity, then swap the
1008 // LHS and RHS.
1009 return (getCmpOperandFoldingProfit(CmpOp: TheLHS, MRI) >
1010 getCmpOperandFoldingProfit(CmpOp: TheRHS, MRI));
1011}
1012
1013void applySwapICmpOperands(MachineInstr &MI, GISelChangeObserver &Observer) {
1014 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
1015 Register LHS = MI.getOperand(i: 2).getReg();
1016 Register RHS = MI.getOperand(i: 3).getReg();
1017 Observer.changingInstr(MI);
1018 MI.getOperand(i: 1).setPredicate(CmpInst::getSwappedPredicate(pred: Pred));
1019 MI.getOperand(i: 2).setReg(RHS);
1020 MI.getOperand(i: 3).setReg(LHS);
1021 Observer.changedInstr(MI);
1022}
1023
1024/// \returns a function which builds a vector floating point compare instruction
1025/// for a condition code \p CC.
1026/// \param [in] NoNans - True if the instruction has nnan flag.
1027std::function<Register(MachineIRBuilder &)>
1028getVectorFCMP(AArch64CC::CondCode CC, Register LHS, Register RHS, bool NoNans,
1029 MachineRegisterInfo &MRI) {
1030 LLT OldTy = MRI.getType(Reg: LHS);
1031 LLT DstTy = LLT::fixed_vector(NumElements: OldTy.getNumElements(),
1032 ScalarTy: LLT::integer(SizeInBits: OldTy.getScalarSizeInBits()));
1033 assert(DstTy.isVector() && "Expected vector types only?");
1034 switch (CC) {
1035 default:
1036 llvm_unreachable("Unexpected condition code!");
1037 case AArch64CC::NE:
1038 return [LHS, RHS, DstTy](MachineIRBuilder &MIB) {
1039 auto FCmp = MIB.buildInstr(Opc: AArch64::G_FCMEQ, DstOps: {DstTy}, SrcOps: {LHS, RHS});
1040 return MIB.buildNot(Dst: DstTy, Src0: FCmp).getReg(Idx: 0);
1041 };
1042 case AArch64CC::EQ:
1043 return [LHS, RHS, DstTy](MachineIRBuilder &MIB) {
1044 return MIB.buildInstr(Opc: AArch64::G_FCMEQ, DstOps: {DstTy}, SrcOps: {LHS, RHS}).getReg(Idx: 0);
1045 };
1046 case AArch64CC::GE:
1047 return [LHS, RHS, DstTy](MachineIRBuilder &MIB) {
1048 return MIB.buildInstr(Opc: AArch64::G_FCMGE, DstOps: {DstTy}, SrcOps: {LHS, RHS}).getReg(Idx: 0);
1049 };
1050 case AArch64CC::GT:
1051 return [LHS, RHS, DstTy](MachineIRBuilder &MIB) {
1052 return MIB.buildInstr(Opc: AArch64::G_FCMGT, DstOps: {DstTy}, SrcOps: {LHS, RHS}).getReg(Idx: 0);
1053 };
1054 case AArch64CC::LS:
1055 return [LHS, RHS, DstTy](MachineIRBuilder &MIB) {
1056 return MIB.buildInstr(Opc: AArch64::G_FCMGE, DstOps: {DstTy}, SrcOps: {RHS, LHS}).getReg(Idx: 0);
1057 };
1058 case AArch64CC::MI:
1059 return [LHS, RHS, DstTy](MachineIRBuilder &MIB) {
1060 return MIB.buildInstr(Opc: AArch64::G_FCMGT, DstOps: {DstTy}, SrcOps: {RHS, LHS}).getReg(Idx: 0);
1061 };
1062 }
1063}
1064
1065/// Try to lower a vector G_FCMP \p MI into an AArch64-specific pseudo.
1066bool matchLowerVectorFCMP(MachineInstr &MI, MachineRegisterInfo &MRI,
1067 MachineIRBuilder &MIB) {
1068 assert(MI.getOpcode() == TargetOpcode::G_FCMP);
1069 const auto &ST = MI.getMF()->getSubtarget<AArch64Subtarget>();
1070
1071 Register Dst = MI.getOperand(i: 0).getReg();
1072 LLT DstTy = MRI.getType(Reg: Dst);
1073 if (!DstTy.isVector() || !ST.hasNEON())
1074 return false;
1075 Register LHS = MI.getOperand(i: 2).getReg();
1076 unsigned EltSize = MRI.getType(Reg: LHS).getScalarSizeInBits();
1077 if (EltSize == 16 && !ST.hasFullFP16())
1078 return false;
1079 if (EltSize != 16 && EltSize != 32 && EltSize != 64)
1080 return false;
1081
1082 return true;
1083}
1084
1085/// Try to lower a vector G_FCMP \p MI into an AArch64-specific pseudo.
1086void applyLowerVectorFCMP(MachineInstr &MI, MachineRegisterInfo &MRI,
1087 MachineIRBuilder &MIB) {
1088 assert(MI.getOpcode() == TargetOpcode::G_FCMP);
1089
1090 const auto &CmpMI = cast<GFCmp>(Val&: MI);
1091
1092 Register Dst = CmpMI.getReg(Idx: 0);
1093 CmpInst::Predicate Pred = CmpMI.getCond();
1094 Register LHS = CmpMI.getLHSReg();
1095 Register RHS = CmpMI.getRHSReg();
1096
1097 LLT DstTy = MRI.getType(Reg: Dst);
1098
1099 bool Invert = false;
1100 AArch64CC::CondCode CC, CC2 = AArch64CC::AL;
1101 if ((Pred == CmpInst::Predicate::FCMP_ORD ||
1102 Pred == CmpInst::Predicate::FCMP_UNO) &&
1103 isBuildVectorAllZeros(MI: *MRI.getVRegDef(Reg: RHS), MRI)) {
1104 // The special case "fcmp ord %a, 0" is the canonical check that LHS isn't
1105 // NaN, so equivalent to a == a and doesn't need the two comparisons an
1106 // "ord" normally would.
1107 // Similarly, "fcmp uno %a, 0" is the canonical check that LHS is NaN and is
1108 // thus equivalent to a != a.
1109 RHS = LHS;
1110 CC = Pred == CmpInst::Predicate::FCMP_ORD ? AArch64CC::EQ : AArch64CC::NE;
1111 } else
1112 changeVectorFCMPPredToAArch64CC(P: Pred, CondCode&: CC, CondCode2&: CC2, Invert);
1113
1114 // Instead of having an apply function, just build here to simplify things.
1115 MIB.setInstrAndDebugLoc(MI);
1116
1117 // TODO: Also consider GISelValueTracking result if eligible.
1118 const bool NoNans = MI.getFlag(Flag: MachineInstr::FmNoNans);
1119
1120 auto Cmp = getVectorFCMP(CC, LHS, RHS, NoNans, MRI);
1121 Register CmpRes;
1122 if (CC2 == AArch64CC::AL)
1123 CmpRes = Cmp(MIB);
1124 else {
1125 auto Cmp2 = getVectorFCMP(CC: CC2, LHS, RHS, NoNans, MRI);
1126 auto Cmp2Dst = Cmp2(MIB);
1127 auto Cmp1Dst = Cmp(MIB);
1128 CmpRes = MIB.buildOr(Dst: DstTy, Src0: Cmp1Dst, Src1: Cmp2Dst).getReg(Idx: 0);
1129 }
1130 if (Invert)
1131 CmpRes = MIB.buildNot(Dst: DstTy, Src0: CmpRes).getReg(Idx: 0);
1132 MRI.replaceRegWith(FromReg: Dst, ToReg: CmpRes);
1133 MI.eraseFromParent();
1134}
1135
1136// Matches G_BUILD_VECTOR where at least one source operand is not a constant
1137bool matchLowerBuildToInsertVecElt(MachineInstr &MI, MachineRegisterInfo &MRI) {
1138 auto *GBuildVec = cast<GBuildVector>(Val: &MI);
1139
1140 // Check if the values are all constants
1141 for (unsigned I = 0; I < GBuildVec->getNumSources(); ++I) {
1142 auto ConstVal =
1143 getAnyConstantVRegValWithLookThrough(VReg: GBuildVec->getSourceReg(I), MRI);
1144
1145 if (!ConstVal.has_value())
1146 return true;
1147 }
1148
1149 return false;
1150}
1151
1152void applyLowerBuildToInsertVecElt(MachineInstr &MI, MachineRegisterInfo &MRI,
1153 MachineIRBuilder &B) {
1154 auto *GBuildVec = cast<GBuildVector>(Val: &MI);
1155 LLT DstTy = MRI.getType(Reg: GBuildVec->getReg(Idx: 0));
1156 Register DstReg = B.buildUndef(Res: DstTy).getReg(Idx: 0);
1157
1158 for (unsigned I = 0; I < GBuildVec->getNumSources(); ++I) {
1159 Register SrcReg = GBuildVec->getSourceReg(I);
1160 if (mi_match(R: SrcReg, MRI, P: m_GImplicitDef()))
1161 continue;
1162 auto IdxReg = B.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: I);
1163 DstReg =
1164 B.buildInsertVectorElement(Res: DstTy, Val: DstReg, Elt: SrcReg, Idx: IdxReg).getReg(Idx: 0);
1165 }
1166 B.buildCopy(Res: GBuildVec->getReg(Idx: 0), Op: DstReg);
1167 GBuildVec->eraseFromParent();
1168}
1169
1170bool matchFormTruncstore(MachineInstr &MI, MachineRegisterInfo &MRI,
1171 Register &SrcReg) {
1172 assert(MI.getOpcode() == TargetOpcode::G_STORE);
1173 Register DstReg = MI.getOperand(i: 0).getReg();
1174 if (cast<GLoadStore>(Val&: MI).isAtomic())
1175 return false;
1176 if (MRI.getType(Reg: DstReg).isVector())
1177 return false;
1178 // Match a store of a truncate.
1179 if (!mi_match(R: DstReg, MRI, P: m_GTrunc(Src: m_Reg(R&: SrcReg))))
1180 return false;
1181 // Only form truncstores for value types of max 64b.
1182 return MRI.getType(Reg: SrcReg).getSizeInBits() <= 64;
1183}
1184
1185void applyFormTruncstore(MachineInstr &MI, MachineRegisterInfo &MRI,
1186 MachineIRBuilder &B, GISelChangeObserver &Observer,
1187 Register &SrcReg) {
1188 assert(MI.getOpcode() == TargetOpcode::G_STORE);
1189 Observer.changingInstr(MI);
1190 MI.getOperand(i: 0).setReg(SrcReg);
1191 Observer.changedInstr(MI);
1192}
1193
1194// Lower vector G_SEXT_INREG back to shifts for selection. We allowed them to
1195// form in the first place for combine opportunities, so any remaining ones
1196// at this stage need be lowered back.
1197bool matchVectorSextInReg(MachineInstr &MI, MachineRegisterInfo &MRI) {
1198 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1199 Register DstReg = MI.getOperand(i: 0).getReg();
1200 LLT DstTy = MRI.getType(Reg: DstReg);
1201 return DstTy.isVector();
1202}
1203
1204void applyVectorSextInReg(MachineInstr &MI, MachineRegisterInfo &MRI,
1205 MachineIRBuilder &B, GISelChangeObserver &Observer) {
1206 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1207 B.setInstrAndDebugLoc(MI);
1208 LegalizerHelper Helper(*MI.getMF(), Observer, B);
1209 Helper.lower(MI, TypeIdx: 0, /* Unused hint type */ Ty: LLT());
1210}
1211
1212/// Combine <N x t>, unused = unmerge(G_EXT <2*N x t> v, undef, N)
1213/// => unused, <N x t> = unmerge v
1214bool matchUnmergeExtToUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
1215 Register &MatchInfo) {
1216 auto &Unmerge = cast<GUnmerge>(Val&: MI);
1217 if (Unmerge.getNumDefs() != 2)
1218 return false;
1219 if (!MRI.use_nodbg_empty(RegNo: Unmerge.getReg(Idx: 1)))
1220 return false;
1221
1222 LLT DstTy = MRI.getType(Reg: Unmerge.getReg(Idx: 0));
1223 if (!DstTy.isVector())
1224 return false;
1225
1226 MachineInstr *Ext = getOpcodeDef(Opcode: AArch64::G_EXT, Reg: Unmerge.getSourceReg(), MRI);
1227 if (!Ext)
1228 return false;
1229
1230 Register ExtSrc1 = Ext->getOperand(i: 1).getReg();
1231 Register ExtSrc2 = Ext->getOperand(i: 2).getReg();
1232 auto LowestVal =
1233 getIConstantVRegValWithLookThrough(VReg: Ext->getOperand(i: 3).getReg(), MRI);
1234 if (!LowestVal || LowestVal->Value.getZExtValue() != DstTy.getSizeInBytes())
1235 return false;
1236
1237 if (!getOpcodeDef<GImplicitDef>(Reg: ExtSrc2, MRI))
1238 return false;
1239
1240 MatchInfo = ExtSrc1;
1241 return true;
1242}
1243
1244void applyUnmergeExtToUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
1245 MachineIRBuilder &B,
1246 GISelChangeObserver &Observer, Register &SrcReg) {
1247 Observer.changingInstr(MI);
1248 // Swap dst registers.
1249 Register Dst1 = MI.getOperand(i: 0).getReg();
1250 MI.getOperand(i: 0).setReg(MI.getOperand(i: 1).getReg());
1251 MI.getOperand(i: 1).setReg(Dst1);
1252 MI.getOperand(i: 2).setReg(SrcReg);
1253 Observer.changedInstr(MI);
1254}
1255
1256// Match mul({z/s}ext , {z/s}ext) => {u/s}mull OR
1257// Match v2s64 mul instructions, which will then be scalarised later on
1258// Doing these two matches in one function to ensure that the order of matching
1259// will always be the same.
1260// Try lowering MUL to MULL before trying to scalarize if needed.
1261bool matchMulv2s64(MachineInstr &MI, MachineRegisterInfo &MRI) {
1262 // Get the instructions that defined the source operand
1263 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1264 return DstTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64);
1265}
1266
1267void applyMulv2s64(MachineInstr &MI, MachineRegisterInfo &MRI,
1268 MachineIRBuilder &B, GISelChangeObserver &Observer) {
1269 assert(MI.getOpcode() == TargetOpcode::G_MUL &&
1270 "Expected a G_MUL instruction");
1271
1272 // Get the instructions that defined the source operand
1273 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1274 assert(DstTy == LLT::fixed_vector(2, 64) && "Expected v2s64 Mul");
1275 LegalizerHelper Helper(*MI.getMF(), Observer, B);
1276 Helper.fewerElementsVector(
1277 MI, TypeIdx: 0,
1278 NarrowTy: DstTy.changeElementCount(EC: DstTy.getElementCount().divideCoefficientBy(RHS: 2)));
1279}
1280
1281class AArch64PostLegalizerLoweringImpl : public Combiner {
1282protected:
1283 const CombinerHelper Helper;
1284 const AArch64PostLegalizerLoweringImplRuleConfig &RuleConfig;
1285 const AArch64Subtarget &STI;
1286
1287public:
1288 AArch64PostLegalizerLoweringImpl(
1289 MachineFunction &MF, CombinerInfo &CInfo, GISelCSEInfo *CSEInfo,
1290 const AArch64PostLegalizerLoweringImplRuleConfig &RuleConfig,
1291 const AArch64Subtarget &STI);
1292
1293 static const char *getName() { return "AArch6400PreLegalizerCombiner"; }
1294
1295 bool tryCombineAll(MachineInstr &I) const override;
1296
1297private:
1298#define GET_GICOMBINER_CLASS_MEMBERS
1299#include "AArch64GenPostLegalizeGILowering.inc"
1300#undef GET_GICOMBINER_CLASS_MEMBERS
1301};
1302
1303#define GET_GICOMBINER_IMPL
1304#include "AArch64GenPostLegalizeGILowering.inc"
1305#undef GET_GICOMBINER_IMPL
1306
1307AArch64PostLegalizerLoweringImpl::AArch64PostLegalizerLoweringImpl(
1308 MachineFunction &MF, CombinerInfo &CInfo, GISelCSEInfo *CSEInfo,
1309 const AArch64PostLegalizerLoweringImplRuleConfig &RuleConfig,
1310 const AArch64Subtarget &STI)
1311 : Combiner(MF, CInfo, /*VT*/ nullptr, CSEInfo),
1312 Helper(Observer, B, /*IsPreLegalize*/ true), RuleConfig(RuleConfig),
1313 STI(STI),
1314#define GET_GICOMBINER_CONSTRUCTOR_INITS
1315#include "AArch64GenPostLegalizeGILowering.inc"
1316#undef GET_GICOMBINER_CONSTRUCTOR_INITS
1317{
1318}
1319
1320bool runPostLegalizerLowering(
1321 MachineFunction &MF,
1322 const AArch64PostLegalizerLoweringImplRuleConfig &RuleConfig) {
1323 if (MF.getProperties().hasFailedISel())
1324 return false;
1325 const Function &F = MF.getFunction();
1326
1327 const AArch64Subtarget &ST = MF.getSubtarget<AArch64Subtarget>();
1328 CombinerInfo CInfo(/*AllowIllegalOps=*/true, /*ShouldLegalizeIllegal=*/false,
1329 /*LegalizerInfo=*/nullptr, /*OptEnabled=*/true,
1330 F.hasOptSize(), F.hasMinSize());
1331 // Disable fixed-point iteration to reduce compile-time
1332 CInfo.MaxIterations = 1;
1333 CInfo.ObserverLvl = CombinerInfo::ObserverLevel::SinglePass;
1334 // PostLegalizerCombiner performs DCE, so a full DCE pass is unnecessary.
1335 CInfo.EnableFullDCE = false;
1336 AArch64PostLegalizerLoweringImpl Impl(MF, CInfo, /*CSEInfo=*/nullptr,
1337 RuleConfig, ST);
1338 return Impl.combineMachineInstrs();
1339}
1340
1341class AArch64PostLegalizerLoweringLegacy : public MachineFunctionPass {
1342public:
1343 static char ID;
1344
1345 AArch64PostLegalizerLoweringLegacy();
1346
1347 StringRef getPassName() const override {
1348 return "AArch64PostLegalizerLowering";
1349 }
1350
1351 bool runOnMachineFunction(MachineFunction &MF) override;
1352 void getAnalysisUsage(AnalysisUsage &AU) const override;
1353
1354private:
1355 AArch64PostLegalizerLoweringImplRuleConfig RuleConfig;
1356};
1357} // end anonymous namespace
1358
1359void AArch64PostLegalizerLoweringLegacy::getAnalysisUsage(
1360 AnalysisUsage &AU) const {
1361 AU.setPreservesCFG();
1362 getSelectionDAGFallbackAnalysisUsage(AU);
1363 MachineFunctionPass::getAnalysisUsage(AU);
1364}
1365
1366AArch64PostLegalizerLoweringLegacy::AArch64PostLegalizerLoweringLegacy()
1367 : MachineFunctionPass(ID) {
1368 if (!RuleConfig.parseCommandLineOption())
1369 report_fatal_error(reason: "Invalid rule identifier");
1370}
1371
1372bool AArch64PostLegalizerLoweringLegacy::runOnMachineFunction(
1373 MachineFunction &MF) {
1374 assert(MF.getProperties().hasLegalized() && "Expected a legalized function?");
1375 return runPostLegalizerLowering(MF, RuleConfig);
1376}
1377
1378char AArch64PostLegalizerLoweringLegacy::ID = 0;
1379INITIALIZE_PASS_BEGIN(AArch64PostLegalizerLoweringLegacy, DEBUG_TYPE,
1380 "Lower AArch64 MachineInstrs after legalization", false,
1381 false)
1382INITIALIZE_PASS_END(AArch64PostLegalizerLoweringLegacy, DEBUG_TYPE,
1383 "Lower AArch64 MachineInstrs after legalization", false,
1384 false)
1385
1386AArch64PostLegalizerLoweringPass::AArch64PostLegalizerLoweringPass()
1387 : RuleConfig(
1388 std::make_unique<AArch64PostLegalizerLoweringImplRuleConfig>()) {
1389 if (!RuleConfig->parseCommandLineOption())
1390 reportFatalUsageError(reason: "invalid rule identifier");
1391}
1392
1393AArch64PostLegalizerLoweringPass::AArch64PostLegalizerLoweringPass(
1394 AArch64PostLegalizerLoweringPass &&) = default;
1395
1396AArch64PostLegalizerLoweringPass::~AArch64PostLegalizerLoweringPass() = default;
1397
1398PreservedAnalyses
1399AArch64PostLegalizerLoweringPass::run(MachineFunction &MF,
1400 MachineFunctionAnalysisManager &MFAM) {
1401 MFPropsModifier _(*this, MF);
1402 const bool Changed = runPostLegalizerLowering(MF, RuleConfig: *RuleConfig);
1403
1404 if (!Changed)
1405 return PreservedAnalyses::all();
1406
1407 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
1408 PA.preserveSet<CFGAnalyses>();
1409 return PA;
1410}
1411
1412namespace llvm {
1413FunctionPass *createAArch64PostLegalizerLowering() {
1414 return new AArch64PostLegalizerLoweringLegacy();
1415}
1416} // end namespace llvm
1417