1//=== AArch64PostLegalizerLowering.cpp --------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// Post-legalization lowering for instructions.
11///
12/// This is used to offload pattern matching from the selector.
13///
14/// For example, this combiner will notice that a G_SHUFFLE_VECTOR is actually
15/// a G_ZIP, G_UZP, etc.
16///
17/// General optimization combines should be handled by either the
18/// AArch64PostLegalizerCombiner or the AArch64PreLegalizerCombiner.
19///
20//===----------------------------------------------------------------------===//
21
22#include "AArch64.h"
23#include "AArch64ExpandImm.h"
24#include "AArch64GlobalISelUtils.h"
25#include "AArch64PerfectShuffle.h"
26#include "AArch64Subtarget.h"
27#include "GISel/AArch64LegalizerInfo.h"
28#include "MCTargetDesc/AArch64MCTargetDesc.h"
29#include "Utils/AArch64BaseInfo.h"
30#include "llvm/CodeGen/GlobalISel/Combiner.h"
31#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
32#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
33#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
34#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
35#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
36#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
37#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
38#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
39#include "llvm/CodeGen/GlobalISel/Utils.h"
40#include "llvm/CodeGen/MachineFrameInfo.h"
41#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
42#include "llvm/CodeGen/MachineFunctionPass.h"
43#include "llvm/CodeGen/MachineInstrBuilder.h"
44#include "llvm/CodeGen/MachinePassManager.h"
45#include "llvm/CodeGen/MachineRegisterInfo.h"
46#include "llvm/CodeGen/TargetOpcodes.h"
47#include "llvm/IR/InstrTypes.h"
48#include "llvm/Support/ErrorHandling.h"
49#include <optional>
50
51#define GET_GICOMBINER_DEPS
52#include "AArch64GenPostLegalizeGILowering.inc"
53#undef GET_GICOMBINER_DEPS
54
55#define DEBUG_TYPE "aarch64-postlegalizer-lowering"
56
57using namespace llvm;
58using namespace MIPatternMatch;
59using namespace AArch64GISelUtils;
60
61#define GET_GICOMBINER_TYPES
62#include "AArch64GenPostLegalizeGILowering.inc"
63#undef GET_GICOMBINER_TYPES
64
65namespace {
66
67/// Represents a pseudo instruction which replaces a G_SHUFFLE_VECTOR.
68///
69/// Used for matching target-supported shuffles before codegen.
70struct ShuffleVectorPseudo {
71 unsigned Opc; ///< Opcode for the instruction. (E.g. G_ZIP1)
72 Register Dst; ///< Destination register.
73 SmallVector<SrcOp, 2> SrcOps; ///< Source registers.
74 ShuffleVectorPseudo(unsigned Opc, Register Dst,
75 std::initializer_list<SrcOp> SrcOps)
76 : Opc(Opc), Dst(Dst), SrcOps(SrcOps){};
77 ShuffleVectorPseudo() = default;
78};
79
80/// Helper function for matchINS.
81///
82/// \returns a value when \p M is an ins mask for \p NumInputElements.
83///
84/// First element of the returned pair is true when the produced
85/// G_INSERT_VECTOR_ELT destination should be the LHS of the G_SHUFFLE_VECTOR.
86///
87/// Second element is the destination lane for the G_INSERT_VECTOR_ELT.
88std::optional<std::pair<bool, int>> isINSMask(ArrayRef<int> M,
89 int NumInputElements) {
90 if (M.size() != static_cast<size_t>(NumInputElements))
91 return std::nullopt;
92 int NumLHSMatch = 0, NumRHSMatch = 0;
93 int LastLHSMismatch = -1, LastRHSMismatch = -1;
94 for (int Idx = 0; Idx < NumInputElements; ++Idx) {
95 if (M[Idx] == -1) {
96 ++NumLHSMatch;
97 ++NumRHSMatch;
98 continue;
99 }
100 M[Idx] == Idx ? ++NumLHSMatch : LastLHSMismatch = Idx;
101 M[Idx] == Idx + NumInputElements ? ++NumRHSMatch : LastRHSMismatch = Idx;
102 }
103 const int NumNeededToMatch = NumInputElements - 1;
104 if (NumLHSMatch == NumNeededToMatch)
105 return std::make_pair(x: true, y&: LastLHSMismatch);
106 if (NumRHSMatch == NumNeededToMatch)
107 return std::make_pair(x: false, y&: LastRHSMismatch);
108 return std::nullopt;
109}
110
111/// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with a
112/// G_REV instruction. Returns the appropriate G_REV opcode in \p Opc.
113bool matchREV(MachineInstr &MI, MachineRegisterInfo &MRI,
114 ShuffleVectorPseudo &MatchInfo) {
115 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
116 ArrayRef<int> ShuffleMask = MI.getOperand(i: 3).getShuffleMask();
117 Register Dst = MI.getOperand(i: 0).getReg();
118 Register Src = MI.getOperand(i: 1).getReg();
119 LLT Ty = MRI.getType(Reg: Dst);
120 unsigned EltSize = Ty.getScalarSizeInBits();
121
122 // Element size for a rev cannot be 64.
123 if (EltSize == 64)
124 return false;
125
126 unsigned NumElts = Ty.getNumElements();
127
128 // Try to produce a G_REV instruction
129 for (unsigned LaneSize : {64U, 32U, 16U}) {
130 if (isREVMask(M: ShuffleMask, EltSize, NumElts, BlockSize: LaneSize)) {
131 unsigned Opcode;
132 if (LaneSize == 64U)
133 Opcode = AArch64::G_REV64;
134 else if (LaneSize == 32U)
135 Opcode = AArch64::G_REV32;
136 else
137 Opcode = AArch64::G_BSWAP;
138
139 MatchInfo = ShuffleVectorPseudo(Opcode, Dst, {Src});
140 return true;
141 }
142 }
143
144 return false;
145}
146
147/// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with
148/// a G_TRN1 or G_TRN2 instruction.
149bool matchTRN(MachineInstr &MI, MachineRegisterInfo &MRI,
150 ShuffleVectorPseudo &MatchInfo) {
151 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
152 unsigned WhichResult;
153 unsigned OperandOrder = 0;
154 ArrayRef<int> ShuffleMask = MI.getOperand(i: 3).getShuffleMask();
155 Register Dst = MI.getOperand(i: 0).getReg();
156 unsigned NumElts = MRI.getType(Reg: Dst).getNumElements();
157 bool TRNMask = isTRNMask(M: ShuffleMask, NumElts, WhichResultOut&: WhichResult, OperandOrderOut&: OperandOrder);
158 if (!TRNMask && !isTRN_v_undef_Mask(M: ShuffleMask, NumElts, WhichResult))
159 return false;
160 unsigned Opc = (WhichResult == 0) ? AArch64::G_TRN1 : AArch64::G_TRN2;
161 Register V1 = MI.getOperand(i: OperandOrder == 0 ? 1 : 2).getReg();
162 Register V2 = MI.getOperand(i: OperandOrder == 0 && TRNMask ? 2 : 1).getReg();
163 MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2});
164 return true;
165}
166
167/// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with
168/// a G_UZP1 or G_UZP2 instruction.
169///
170/// \param [in] MI - The shuffle vector instruction.
171/// \param [out] MatchInfo - Either G_UZP1 or G_UZP2 on success.
172bool matchUZP(MachineInstr &MI, MachineRegisterInfo &MRI,
173 ShuffleVectorPseudo &MatchInfo) {
174 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
175 unsigned WhichResult;
176 ArrayRef<int> ShuffleMask = MI.getOperand(i: 3).getShuffleMask();
177 Register Dst = MI.getOperand(i: 0).getReg();
178 unsigned NumElts = MRI.getType(Reg: Dst).getNumElements();
179 bool UZPMask = isUZPMask(M: ShuffleMask, NumElts, WhichResultOut&: WhichResult);
180 if (!UZPMask && !isUZP_v_undef_Mask(M: ShuffleMask, NumElts, WhichResult))
181 return false;
182 unsigned Opc = (WhichResult == 0) ? AArch64::G_UZP1 : AArch64::G_UZP2;
183 Register V1 = MI.getOperand(i: 1).getReg();
184 Register V2 = MI.getOperand(i: UZPMask ? 2 : 1).getReg();
185 MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2});
186 return true;
187}
188
189bool matchZip(MachineInstr &MI, MachineRegisterInfo &MRI,
190 ShuffleVectorPseudo &MatchInfo) {
191 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
192 unsigned WhichResult;
193 unsigned OperandOrder = 0;
194 ArrayRef<int> ShuffleMask = MI.getOperand(i: 3).getShuffleMask();
195 Register Dst = MI.getOperand(i: 0).getReg();
196 unsigned NumElts = MRI.getType(Reg: Dst).getNumElements();
197 bool ZIPMask = isZIPMask(M: ShuffleMask, NumElts, WhichResultOut&: WhichResult, OperandOrderOut&: OperandOrder);
198 if (!ZIPMask && !isZIP_v_undef_Mask(M: ShuffleMask, NumElts, WhichResult))
199 return false;
200 unsigned Opc = (WhichResult == 0) ? AArch64::G_ZIP1 : AArch64::G_ZIP2;
201 Register V1 = MI.getOperand(i: OperandOrder == 0 ? 1 : 2).getReg();
202 Register V2 = MI.getOperand(i: OperandOrder == 0 && ZIPMask ? 2 : 1).getReg();
203 MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2});
204 return true;
205}
206
207/// Helper function for matchDup.
208bool matchDupFromInsertVectorElt(int Lane, MachineInstr &MI,
209 MachineRegisterInfo &MRI,
210 ShuffleVectorPseudo &MatchInfo) {
211 if (Lane != 0)
212 return false;
213
214 // Try to match a vector splat operation into a dup instruction.
215 // We're looking for this pattern:
216 //
217 // %scalar:gpr(s64) = COPY $x0
218 // %undef:fpr(<2 x s64>) = G_IMPLICIT_DEF
219 // %cst0:gpr(s32) = G_CONSTANT i32 0
220 // %zerovec:fpr(<2 x s32>) = G_BUILD_VECTOR %cst0(s32), %cst0(s32)
221 // %ins:fpr(<2 x s64>) = G_INSERT_VECTOR_ELT %undef, %scalar(s64), %cst0(s32)
222 // %splat:fpr(<2 x s64>) = G_SHUFFLE_VECTOR %ins(<2 x s64>), %undef,
223 // %zerovec(<2 x s32>)
224 //
225 // ...into:
226 // %splat = G_SPLAT_VECTOR %scalar
227
228 // Begin matching the insert.
229 auto *InsMI = getOpcodeDef(Opcode: TargetOpcode::G_INSERT_VECTOR_ELT,
230 Reg: MI.getOperand(i: 1).getReg(), MRI);
231 if (!InsMI)
232 return false;
233 // Match the undef vector operand.
234 if (!getOpcodeDef(Opcode: TargetOpcode::G_IMPLICIT_DEF, Reg: InsMI->getOperand(i: 1).getReg(),
235 MRI))
236 return false;
237
238 // Match the index constant 0.
239 if (!mi_match(R: InsMI->getOperand(i: 3).getReg(), MRI, P: m_ZeroInt()))
240 return false;
241
242 MatchInfo = ShuffleVectorPseudo(TargetOpcode::G_SPLAT_VECTOR,
243 MI.getOperand(i: 0).getReg(),
244 {InsMI->getOperand(i: 2).getReg()});
245 return true;
246}
247
248/// Helper function for matchDup.
249bool matchDupFromBuildVector(int Lane, MachineInstr &MI,
250 MachineRegisterInfo &MRI,
251 ShuffleVectorPseudo &MatchInfo) {
252 assert(Lane >= 0 && "Expected positive lane?");
253 int NumElements = MRI.getType(Reg: MI.getOperand(i: 1).getReg()).getNumElements();
254 // Test if the LHS is a BUILD_VECTOR. If it is, then we can just reference the
255 // lane's definition directly.
256 auto *BuildVecMI =
257 getOpcodeDef(Opcode: TargetOpcode::G_BUILD_VECTOR,
258 Reg: MI.getOperand(i: Lane < NumElements ? 1 : 2).getReg(), MRI);
259 // If Lane >= NumElements then it is point to RHS, just check from RHS
260 if (NumElements <= Lane)
261 Lane -= NumElements;
262
263 if (!BuildVecMI)
264 return false;
265 Register Reg = BuildVecMI->getOperand(i: Lane + 1).getReg();
266 MatchInfo = ShuffleVectorPseudo(TargetOpcode::G_SPLAT_VECTOR,
267 MI.getOperand(i: 0).getReg(), {Reg});
268 return true;
269}
270
271bool matchDup(MachineInstr &MI, MachineRegisterInfo &MRI,
272 ShuffleVectorPseudo &MatchInfo) {
273 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
274 auto MaybeLane = getSplatIndex(MI);
275 if (!MaybeLane)
276 return false;
277 int Lane = *MaybeLane;
278 // If this is undef splat, generate it via "just" vdup, if possible.
279 if (Lane < 0)
280 Lane = 0;
281 if (matchDupFromInsertVectorElt(Lane, MI, MRI, MatchInfo))
282 return true;
283 if (matchDupFromBuildVector(Lane, MI, MRI, MatchInfo))
284 return true;
285 return false;
286}
287
288bool matchEXT(MachineInstr &MI, MachineRegisterInfo &MRI,
289 ShuffleVectorPseudo &MatchInfo) {
290 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
291 Register Dst = MI.getOperand(i: 0).getReg();
292 LLT DstTy = MRI.getType(Reg: Dst);
293 Register V1 = MI.getOperand(i: 1).getReg();
294 Register V2 = MI.getOperand(i: 2).getReg();
295 auto Mask = MI.getOperand(i: 3).getShuffleMask();
296 uint64_t ExtFactor = MRI.getType(Reg: V1).getScalarSizeInBits() / 8;
297
298 unsigned Imm;
299 bool ReverseExt;
300 if (isEXTMask(M: Mask, NumElts: DstTy.getNumElements(), ReverseEXT&: ReverseExt, Imm)) {
301 if (ReverseExt)
302 std::swap(a&: V1, b&: V2);
303 Imm *= ExtFactor;
304 MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst,
305 {V1, V2, static_cast<uint64_t>(Imm)});
306 return true;
307 }
308
309 if (!getOpcodeDef<GImplicitDef>(Reg: V2, MRI) ||
310 !isSingletonEXTMask(M: Mask, NumElts: DstTy.getNumElements(), Imm))
311 return false;
312
313 Imm = Mask[0] * ExtFactor;
314 MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst,
315 {V1, V1, static_cast<uint64_t>(Imm)});
316 return true;
317}
318
319/// Replace a G_SHUFFLE_VECTOR instruction with a pseudo.
320/// \p Opc is the opcode to use. \p MI is the G_SHUFFLE_VECTOR.
321void applyShuffleVectorPseudo(MachineInstr &MI, MachineRegisterInfo &MRI,
322 ShuffleVectorPseudo &MatchInfo) {
323 MachineIRBuilder MIRBuilder(MI);
324 if (MatchInfo.Opc == TargetOpcode::G_BSWAP) {
325 assert(MatchInfo.SrcOps.size() == 1);
326 LLT DstTy = MRI.getType(Reg: MatchInfo.Dst);
327 assert(DstTy == LLT::fixed_vector(8, 8) ||
328 DstTy == LLT::fixed_vector(16, 8));
329 LLT BSTy = DstTy == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8)
330 ? LLT::fixed_vector(NumElements: 4, ScalarTy: LLT::integer(SizeInBits: 16))
331 : LLT::fixed_vector(NumElements: 8, ScalarTy: LLT::integer(SizeInBits: 16));
332 // FIXME: NVCAST
333 auto BS1 = MIRBuilder.buildInstr(Opc: TargetOpcode::G_BITCAST, DstOps: {BSTy},
334 SrcOps: MatchInfo.SrcOps[0]);
335 auto BS2 = MIRBuilder.buildInstr(Opc: MatchInfo.Opc, DstOps: {BSTy}, SrcOps: {BS1});
336 MIRBuilder.buildInstr(Opc: TargetOpcode::G_BITCAST, DstOps: {MatchInfo.Dst}, SrcOps: {BS2});
337 } else
338 MIRBuilder.buildInstr(Opc: MatchInfo.Opc, DstOps: {MatchInfo.Dst}, SrcOps: MatchInfo.SrcOps);
339 MI.eraseFromParent();
340}
341
342/// Replace a G_SHUFFLE_VECTOR instruction with G_EXT.
343/// Special-cased because the constant operand must be emitted as a G_CONSTANT
344/// for the imported tablegen patterns to work.
345void applyEXT(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) {
346 MachineIRBuilder MIRBuilder(MI);
347 if (MatchInfo.SrcOps[2].getImm() == 0)
348 MIRBuilder.buildCopy(Res: MatchInfo.Dst, Op: MatchInfo.SrcOps[0]);
349 else {
350 // Tablegen patterns expect an i32 G_CONSTANT as the final op.
351 auto Cst = MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: 32),
352 Val: MatchInfo.SrcOps[2].getImm());
353 MIRBuilder.buildInstr(Opc: MatchInfo.Opc, DstOps: {MatchInfo.Dst},
354 SrcOps: {MatchInfo.SrcOps[0], MatchInfo.SrcOps[1], Cst});
355 }
356 MI.eraseFromParent();
357}
358
359static bool preferTBLToMultiInstrShuffle(MachineBasicBlock *MBB) {
360 if (MBB->getParent()->getFunction().hasOptSize())
361 return false;
362 // Look for a simple single block loop.
363 return is_contained(Range: successors(BB: MBB), Element: MBB);
364}
365
366bool matchFullRev(MachineInstr &MI, MachineRegisterInfo &MRI) {
367 if (preferTBLToMultiInstrShuffle(MBB: MI.getParent()))
368 return false;
369
370 auto &Shuf = cast<GShuffleVector>(Val&: MI);
371 ArrayRef<int> Mask = Shuf.getMask();
372 return mi_match(R: Shuf.getSrc2Reg(), MRI, P: m_GImplicitDef()) &&
373 ShuffleVectorInst::isReverseMask(Mask, NumSrcElts: Mask.size());
374}
375
376void applyFullRev(MachineInstr &MI, MachineRegisterInfo &MRI) {
377 Register Dst = MI.getOperand(i: 0).getReg();
378 Register Src = MI.getOperand(i: 1).getReg();
379 LLT DstTy = MRI.getType(Reg: Dst);
380 assert(DstTy.getSizeInBits() == 128 &&
381 "Expected 128bit vector in applyFullRev");
382 MachineIRBuilder MIRBuilder(MI);
383 auto Cst = MIRBuilder.buildConstant(Res: LLT::integer(SizeInBits: 32), Val: 8);
384 auto Rev = MIRBuilder.buildInstr(Opc: AArch64::G_REV64, DstOps: {DstTy}, SrcOps: {Src});
385 MIRBuilder.buildInstr(Opc: AArch64::G_EXT, DstOps: {Dst}, SrcOps: {Rev, Rev, Cst});
386 MI.eraseFromParent();
387}
388
389bool matchNonConstInsert(MachineInstr &MI, MachineRegisterInfo &MRI) {
390 assert(MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT);
391
392 auto ValAndVReg =
393 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 3).getReg(), MRI);
394 return !ValAndVReg;
395}
396
397void applyNonConstInsert(MachineInstr &MI, MachineRegisterInfo &MRI,
398 MachineIRBuilder &Builder) {
399 auto &Insert = cast<GInsertVectorElement>(Val&: MI);
400 Builder.setInstrAndDebugLoc(Insert);
401
402 Register Offset = Insert.getIndexReg();
403 LLT VecTy = MRI.getType(Reg: Insert.getReg(Idx: 0));
404 LLT EltTy = MRI.getType(Reg: Insert.getElementReg());
405 LLT IdxTy = MRI.getType(Reg: Insert.getIndexReg());
406
407 if (VecTy.isScalableVector())
408 return;
409
410 // Create a stack slot and store the vector into it
411 MachineFunction &MF = Builder.getMF();
412 Align Alignment(
413 std::min<uint64_t>(a: VecTy.getSizeInBytes().getKnownMinValue(), b: 16));
414 int FrameIdx = MF.getFrameInfo().CreateStackObject(Size: VecTy.getSizeInBytes(),
415 Alignment, isSpillSlot: false);
416 LLT FramePtrTy = LLT::pointer(AddressSpace: 0, SizeInBits: 64);
417 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIdx);
418 auto StackTemp = Builder.buildFrameIndex(Res: FramePtrTy, Idx: FrameIdx);
419
420 Builder.buildStore(Val: Insert.getOperand(i: 1), Addr: StackTemp, PtrInfo, Alignment: Align(8));
421
422 // Get the pointer to the element, and be sure not to hit undefined behavior
423 // if the index is out of bounds.
424 assert(isPowerOf2_64(VecTy.getNumElements()) &&
425 "Expected a power-2 vector size");
426 auto Mask = Builder.buildConstant(Res: IdxTy, Val: VecTy.getNumElements() - 1);
427 Register And = Builder.buildAnd(Dst: IdxTy, Src0: Offset, Src1: Mask).getReg(Idx: 0);
428 auto EltSize = Builder.buildConstant(Res: IdxTy, Val: EltTy.getSizeInBytes());
429 Register Mul = Builder.buildMul(Dst: IdxTy, Src0: And, Src1: EltSize).getReg(Idx: 0);
430 Register EltPtr =
431 Builder.buildPtrAdd(Res: MRI.getType(Reg: StackTemp.getReg(Idx: 0)), Op0: StackTemp, Op1: Mul)
432 .getReg(Idx: 0);
433
434 // Write the inserted element
435 Builder.buildStore(Val: Insert.getElementReg(), Addr: EltPtr, PtrInfo, Alignment: Align(1));
436 // Reload the whole vector.
437 Builder.buildLoad(Res: Insert.getReg(Idx: 0), Addr: StackTemp, PtrInfo, Alignment: Align(8));
438 Insert.eraseFromParent();
439}
440
441/// Match a G_SHUFFLE_VECTOR with a mask which corresponds to a
442/// G_INSERT_VECTOR_ELT and G_EXTRACT_VECTOR_ELT pair.
443///
444/// e.g.
445/// %shuf = G_SHUFFLE_VECTOR %left, %right, shufflemask(0, 0)
446///
447/// Can be represented as
448///
449/// %extract = G_EXTRACT_VECTOR_ELT %left, 0
450/// %ins = G_INSERT_VECTOR_ELT %left, %extract, 1
451///
452bool matchINS(MachineInstr &MI, MachineRegisterInfo &MRI,
453 std::tuple<Register, int, Register, int> &MatchInfo) {
454 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
455 ArrayRef<int> ShuffleMask = MI.getOperand(i: 3).getShuffleMask();
456 Register Dst = MI.getOperand(i: 0).getReg();
457 int NumElts = MRI.getType(Reg: Dst).getNumElements();
458 auto DstIsLeftAndDstLane = isINSMask(M: ShuffleMask, NumInputElements: NumElts);
459 if (!DstIsLeftAndDstLane)
460 return false;
461 bool DstIsLeft;
462 int DstLane;
463 std::tie(args&: DstIsLeft, args&: DstLane) = *DstIsLeftAndDstLane;
464 Register Left = MI.getOperand(i: 1).getReg();
465 Register Right = MI.getOperand(i: 2).getReg();
466 Register DstVec = DstIsLeft ? Left : Right;
467 Register SrcVec = Left;
468
469 int SrcLane = ShuffleMask[DstLane];
470 if (SrcLane >= NumElts) {
471 SrcVec = Right;
472 SrcLane -= NumElts;
473 }
474
475 MatchInfo = std::make_tuple(args&: DstVec, args&: DstLane, args&: SrcVec, args&: SrcLane);
476 return true;
477}
478
479void applyINS(MachineInstr &MI, MachineRegisterInfo &MRI,
480 MachineIRBuilder &Builder,
481 std::tuple<Register, int, Register, int> &MatchInfo) {
482 Builder.setInstrAndDebugLoc(MI);
483 Register Dst = MI.getOperand(i: 0).getReg();
484 auto ScalarTy = MRI.getType(Reg: Dst).getElementType();
485 Register DstVec, SrcVec;
486 int DstLane, SrcLane;
487 std::tie(args&: DstVec, args&: DstLane, args&: SrcVec, args&: SrcLane) = MatchInfo;
488 auto SrcCst = Builder.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: SrcLane);
489 auto Extract = Builder.buildExtractVectorElement(Res: ScalarTy, Val: SrcVec, Idx: SrcCst);
490 auto DstCst = Builder.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: DstLane);
491 Builder.buildInsertVectorElement(Res: Dst, Val: DstVec, Elt: Extract, Idx: DstCst);
492 MI.eraseFromParent();
493}
494
495/// Match perfect shuffle entries
496bool matchPerfectShuffle(MachineInstr &MI, MachineRegisterInfo &MRI,
497 SmallVector<ShuffleEntry> &Entries) {
498 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
499 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
500 assert(Ty.getSizeInBits() == 64 || Ty.getSizeInBits() == 128);
501 if (Ty.getNumElements() != 4)
502 return false;
503 if (!generatePerfectShuffle(Mask: MI.getOperand(i: 3).getShuffleMask(), NumElts: 4, Entries))
504 return false;
505 return Entries.size() == 1 || !preferTBLToMultiInstrShuffle(MBB: MI.getParent());
506}
507
508void applyPerfectShuffle(MachineInstr &MI, MachineRegisterInfo &MRI,
509 MachineIRBuilder &MIB,
510 SmallVector<ShuffleEntry> &Entries) {
511 SmallVector<Register> Vals;
512 auto getValue = [](unsigned Idx, Register LHS, Register RHS,
513 SmallVector<Register> &Vals) {
514 if (Idx == ShuffleEntry::LHS)
515 return LHS;
516 if (Idx == ShuffleEntry::RHS)
517 return RHS;
518 assert(Idx < Vals.size());
519 return Vals[Idx];
520 };
521
522 Register V1 = MI.getOperand(i: 1).getReg();
523 Register V2 = MI.getOperand(i: 2).getReg();
524 LLT VT = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
525
526 for (const ShuffleEntry &Entry : Entries) {
527 Register OpLHS = getValue(Entry.LHSID, V1, V2, Vals);
528
529 switch (Entry.Op) {
530 case ShuffleEntry::OP_COPY:
531 case ShuffleEntry::OP_MOVLANE:
532 llvm_unreachable("Did not expect a OP_COPY or OP_MOVLANE");
533 case ShuffleEntry::OP_MOVLANE64: {
534 unsigned ExtLane = (Entry.RHSID >> 8) & 0xff;
535 unsigned ToLane = Entry.RHSID & 0xff;
536 bool Input2 = Entry.RHSID >> 16;
537
538 Register Input = Input2 ? V2 : V1;
539 assert(
540 (VT.getScalarSizeInBits() == 16 || VT.getScalarSizeInBits() == 32) &&
541 "Expected 16 or 32 bit shuffle elements");
542 LLT ExtTy = VT.getScalarSizeInBits() == 16
543 ? LLT::fixed_vector(NumElements: 2, ScalarTy: LLT::float32())
544 : LLT::fixed_vector(NumElements: 2, ScalarTy: LLT::float64());
545
546 Input = MIB.buildBitcast(Dst: ExtTy, Src: Input).getReg(Idx: 0);
547 OpLHS = MIB.buildBitcast(Dst: ExtTy, Src: OpLHS).getReg(Idx: 0);
548 auto Ext = MIB.buildExtractVectorElementConstant(Res: ExtTy.getScalarType(),
549 Val: Input, Idx: ExtLane);
550 auto Ins = MIB.buildInsertVectorElement(
551 Res: ExtTy, Val: OpLHS, Elt: Ext, Idx: MIB.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: ToLane));
552 Vals.push_back(Elt: MIB.buildBitcast(Dst: VT, Src: Ins).getReg(Idx: 0));
553 break;
554 }
555 case ShuffleEntry::OP_MOVLANE32: {
556 unsigned ExtLane = (Entry.RHSID >> 8) & 0xff;
557 unsigned ToLane = Entry.RHSID & 0xff;
558 bool Input2 = Entry.RHSID >> 16;
559
560 Register Input = Input2 ? V2 : V1;
561 auto Ext = MIB.buildExtractVectorElementConstant(Res: VT.getScalarType(),
562 Val: Input, Idx: ExtLane);
563 Vals.push_back(
564 Elt: MIB.buildInsertVectorElement(
565 Res: VT, Val: OpLHS, Elt: Ext, Idx: MIB.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: ToLane))
566 .getReg(Idx: 0));
567 break;
568 }
569 case ShuffleEntry::OP_VREV: {
570 // VREV divides the vector in half and swaps within the half.
571 assert(VT.getScalarSizeInBits() == 32 || VT.getScalarSizeInBits() == 16);
572 if (VT.getScalarSizeInBits() == 32) {
573 Vals.push_back(
574 Elt: MIB.buildInstr(Opc: AArch64::G_REV64, DstOps: {VT}, SrcOps: {OpLHS}).getReg(Idx: 0));
575 } else {
576 Vals.push_back(
577 Elt: MIB.buildInstr(Opc: AArch64::G_REV32, DstOps: {VT}, SrcOps: {OpLHS}).getReg(Idx: 0));
578 }
579 break;
580 }
581 case ShuffleEntry::OP_VDUP0:
582 case ShuffleEntry::OP_VDUP1:
583 case ShuffleEntry::OP_VDUP2:
584 case ShuffleEntry::OP_VDUP3: {
585 unsigned Opcode;
586 if (VT.getScalarSizeInBits() == 8)
587 Opcode = AArch64::G_DUPLANE8;
588 else if (VT.getScalarSizeInBits() == 16)
589 Opcode = AArch64::G_DUPLANE16;
590 else if (VT.getScalarSizeInBits() == 32)
591 Opcode = AArch64::G_DUPLANE32;
592 else if (VT.getScalarSizeInBits() == 64)
593 Opcode = AArch64::G_DUPLANE64;
594 else
595 llvm_unreachable("Invalid vector element type?");
596
597 if (VT.getSizeInBits() == 64) {
598 auto Undef = MIB.buildUndef(Res: VT);
599 OpLHS = MIB.buildConcatVectors(Res: VT.multiplyElements(Factor: 2),
600 Ops: {OpLHS, Undef.getReg(Idx: 0)})
601 .getReg(Idx: 0);
602 }
603 auto Lane = MIB.buildConstant(Res: LLT::integer(SizeInBits: 64),
604 Val: Entry.Op - ShuffleEntry::OP_VDUP0);
605 Vals.push_back(Elt: MIB.buildInstr(Opc: Opcode, DstOps: {VT}, SrcOps: {OpLHS, Lane}).getReg(Idx: 0));
606 break;
607 }
608 case ShuffleEntry::OP_VEXT1:
609 case ShuffleEntry::OP_VEXT2:
610 case ShuffleEntry::OP_VEXT3: {
611 unsigned ExtFactor = VT.getScalarSizeInBits() / 8;
612 unsigned Imm = (Entry.Op - ShuffleEntry::OP_VEXT1 + 1) * ExtFactor;
613 Vals.push_back(Elt: MIB.buildInstr(Opc: AArch64::G_EXT, DstOps: {VT},
614 SrcOps: {OpLHS, getValue(Entry.RHSID, V1, V2, Vals),
615 MIB.buildConstant(Res: LLT::integer(SizeInBits: 32), Val: Imm)})
616 .getReg(Idx: 0));
617 break;
618 }
619 case ShuffleEntry::OP_VUZPL:
620 Vals.push_back(
621 Elt: MIB.buildInstr(Opc: AArch64::G_UZP1, DstOps: {VT},
622 SrcOps: {OpLHS, getValue(Entry.RHSID, V1, V2, Vals)})
623 .getReg(Idx: 0));
624 break;
625 case ShuffleEntry::OP_VUZPR:
626 Vals.push_back(
627 Elt: MIB.buildInstr(Opc: AArch64::G_UZP2, DstOps: {VT},
628 SrcOps: {OpLHS, getValue(Entry.RHSID, V1, V2, Vals)})
629 .getReg(Idx: 0));
630 break;
631 case ShuffleEntry::OP_VZIPL:
632 Vals.push_back(
633 Elt: MIB.buildInstr(Opc: AArch64::G_ZIP1, DstOps: {VT},
634 SrcOps: {OpLHS, getValue(Entry.RHSID, V1, V2, Vals)})
635 .getReg(Idx: 0));
636 break;
637 case ShuffleEntry::OP_VZIPR:
638 Vals.push_back(
639 Elt: MIB.buildInstr(Opc: AArch64::G_ZIP2, DstOps: {VT},
640 SrcOps: {OpLHS, getValue(Entry.RHSID, V1, V2, Vals)})
641 .getReg(Idx: 0));
642 break;
643 case ShuffleEntry::OP_VTRNL:
644 Vals.push_back(
645 Elt: MIB.buildInstr(Opc: AArch64::G_TRN1, DstOps: {VT},
646 SrcOps: {OpLHS, getValue(Entry.RHSID, V1, V2, Vals)})
647 .getReg(Idx: 0));
648 break;
649 case ShuffleEntry::OP_VTRNR:
650 Vals.push_back(
651 Elt: MIB.buildInstr(Opc: AArch64::G_TRN2, DstOps: {VT},
652 SrcOps: {OpLHS, getValue(Entry.RHSID, V1, V2, Vals)})
653 .getReg(Idx: 0));
654 break;
655 }
656 }
657 assert(Vals.size() == Entries.size());
658 MRI.replaceRegWith(FromReg: MI.getOperand(i: 0).getReg(), ToReg: Vals.back());
659 MI.eraseFromParent();
660}
661
662/// isVShiftRImm - Check if this is a valid vector for the immediate
663/// operand of a vector shift right operation. The value must be in the range:
664/// 1 <= Value <= ElementBits for a right shift.
665bool isVShiftRImm(Register Reg, MachineRegisterInfo &MRI, LLT Ty,
666 int64_t &Cnt) {
667 assert(Ty.isVector() && "vector shift count is not a vector type");
668 int64_t ElementBits = Ty.getScalarSizeInBits();
669 assert(ElementBits <= 64 && "expected <= 64bit shift type");
670 MachineInstr *MI = MRI.getVRegDef(Reg);
671 std::optional<Register> Splat = getVectorSplat(MI: *MI, MRI);
672 if (!Splat)
673 return false;
674 std::optional<APInt> C = getIConstantVRegVal(VReg: *Splat, MRI);
675 if (!C)
676 return false;
677 Cnt = C->getSExtValue();
678 return Cnt >= 1 && Cnt <= ElementBits;
679}
680
681/// Match a vector G_ASHR or G_LSHR with a valid immediate shift.
682bool matchVAshrLshrImm(MachineInstr &MI, MachineRegisterInfo &MRI,
683 int64_t &Imm) {
684 assert(MI.getOpcode() == TargetOpcode::G_ASHR ||
685 MI.getOpcode() == TargetOpcode::G_LSHR);
686 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 1).getReg());
687 if (!Ty.isVector())
688 return false;
689 return isVShiftRImm(Reg: MI.getOperand(i: 2).getReg(), MRI, Ty, Cnt&: Imm);
690}
691
692void applyVAshrLshrImm(MachineInstr &MI, MachineRegisterInfo &MRI,
693 int64_t &Imm) {
694 unsigned Opc = MI.getOpcode();
695 assert(Opc == TargetOpcode::G_ASHR || Opc == TargetOpcode::G_LSHR);
696 unsigned NewOpc =
697 Opc == TargetOpcode::G_ASHR ? AArch64::G_VASHR : AArch64::G_VLSHR;
698 MachineIRBuilder MIB(MI);
699 MIB.buildInstr(Opc: NewOpc, DstOps: {MI.getOperand(i: 0)}, SrcOps: {MI.getOperand(i: 1)}).addImm(Val: Imm);
700 MI.eraseFromParent();
701}
702
703/// Determine whether an integer G_ICMP against 1 or -1 can compare
704/// against 0 instead.
705///
706/// AArch64 can fold a compare-with-zero more cheaply than some non-arithmetic
707/// immediates (SUBS/ADDS, or TST when the LHS is an AND). When the predicate
708/// can be adjusted without changing semantics, the RHS may become 0.
709///
710/// Supported transforms (signed predicates only):
711/// (and X, Y) slt 1 => (and X, Y) sle 0
712/// (and X, Y) sge 1 => (and X, Y) sgt 0
713/// X sle -1 => X slt 0
714/// X sgt -1 => X sge 0
715///
716/// The compare-against-1 cases require the LHS to be G_AND because the
717/// compare-with-zero path enables ANDS (TST) selection, and ANDS flags are
718/// only reliable for those signed comparisons. This mirrors SelectionDAG
719/// emitComparison().
720///
721/// For compare-against--1 on a non-AND LHS, \p LHS must have a single
722/// non-debug use so other users are not left with a different immediate.
723///
724/// \param LHS The compare LHS register.
725/// \param C The constant RHS (only 1 or all-ones are considered).
726/// \param P In/out predicate; updated when a transform applies.
727/// \param MRI Used to inspect the LHS definition and use count.
728/// \returns true if \p P was updated and comparing against 0 is equivalent.
729static bool shouldBeAdjustedToZero(Register LHS, const APInt &C,
730 CmpInst::Predicate &P,
731 const MachineRegisterInfo &MRI) {
732 const bool IsAndLHS = getOpcodeDef<GAnd>(Reg: LHS, MRI) != nullptr;
733
734 if (C.isOne() && (P == CmpInst::ICMP_SLT || P == CmpInst::ICMP_SGE) &&
735 IsAndLHS) {
736 P = (P == CmpInst::ICMP_SLT) ? CmpInst::ICMP_SLE : CmpInst::ICMP_SGT;
737 return true;
738 }
739
740 if (!IsAndLHS && !MRI.hasOneNonDBGUse(RegNo: LHS))
741 return false;
742
743 if (C.isAllOnes() && (P == CmpInst::ICMP_SLE || P == CmpInst::ICMP_SGT)) {
744 P = (P == CmpInst::ICMP_SLE) ? CmpInst::ICMP_SLT : CmpInst::ICMP_SGE;
745 return true;
746 }
747 return false;
748}
749
750/// Determine if it is possible to modify the \p RHS and predicate \p P of a
751/// G_ICMP instruction such that the right-hand side is an arithmetic immediate.
752///
753/// \returns A pair containing the updated immediate and predicate which may
754/// be used to optimize the instruction.
755///
756/// \note This assumes that the comparison has been legalized.
757std::optional<std::pair<uint64_t, CmpInst::Predicate>>
758tryAdjustICmpImmAndPred(Register LHS, Register RHS, CmpInst::Predicate P,
759 const MachineRegisterInfo &MRI) {
760 const auto &Ty = MRI.getType(Reg: RHS);
761 if (Ty.isVector())
762 return std::nullopt;
763 assert((Ty.getSizeInBits() == 32 || Ty.getSizeInBits() == 64) &&
764 "Expected 32 or 64 bit compare only?");
765
766 // If the RHS is not a constant, or the RHS is already a valid arithmetic
767 // immediate, then there is nothing to change.
768 auto ValAndVReg = getIConstantVRegValWithLookThrough(VReg: RHS, MRI);
769 if (!ValAndVReg)
770 return std::nullopt;
771 APInt C = ValAndVReg->Value;
772 if (shouldBeAdjustedToZero(LHS, C, P, MRI))
773 return {{0, P}};
774
775 if (AArch64_AM::isLegalCmpImmed(C))
776 return std::nullopt;
777
778 uint64_t OriginalC = C.getZExtValue();
779
780 // We have a non-arithmetic immediate. Check if adjusting the immediate and
781 // adjusting the predicate will result in a legal arithmetic immediate.
782 switch (P) {
783 default:
784 return std::nullopt;
785 case CmpInst::ICMP_SLT:
786 case CmpInst::ICMP_SGE:
787 // Check for
788 //
789 // x slt c => x sle c - 1
790 // x sge c => x sgt c - 1
791 //
792 // When c is not the smallest possible negative number.
793 if (C.isMinSignedValue())
794 return std::nullopt;
795 P = (P == CmpInst::ICMP_SLT) ? CmpInst::ICMP_SLE : CmpInst::ICMP_SGT;
796 C = C - 1;
797 break;
798 case CmpInst::ICMP_ULT:
799 case CmpInst::ICMP_UGE:
800 // Check for
801 //
802 // x ult c => x ule c - 1
803 // x uge c => x ugt c - 1
804 //
805 // When c is not zero.
806 assert(!C.isZero() && "C should not be zero here!");
807 P = (P == CmpInst::ICMP_ULT) ? CmpInst::ICMP_ULE : CmpInst::ICMP_UGT;
808 C = C - 1;
809 break;
810 case CmpInst::ICMP_SLE:
811 case CmpInst::ICMP_SGT:
812 // Check for
813 //
814 // x sle c => x slt c + 1
815 // x sgt c => s sge c + 1
816 //
817 // When c is not the largest possible signed integer.
818 if (C.isMaxSignedValue())
819 return std::nullopt;
820 P = (P == CmpInst::ICMP_SLE) ? CmpInst::ICMP_SLT : CmpInst::ICMP_SGE;
821 C = C + 1;
822 break;
823 case CmpInst::ICMP_ULE:
824 case CmpInst::ICMP_UGT:
825 // Check for
826 //
827 // x ule c => x ult c + 1
828 // x ugt c => s uge c + 1
829 //
830 // When c is not the largest possible unsigned integer.
831 if (C.isAllOnes())
832 return std::nullopt;
833 P = (P == CmpInst::ICMP_ULE) ? CmpInst::ICMP_ULT : CmpInst::ICMP_UGE;
834 C = C + 1;
835 break;
836 }
837
838 // Check if the new constant is valid, and return the updated constant and
839 // predicate if it is.
840 uint64_t NewC = C.getZExtValue();
841 if (AArch64_AM::isLegalCmpImmed(C))
842 return {{NewC, P}};
843
844 auto NumberOfInstrToLoadImm = [=](uint64_t Imm) {
845 SmallVector<AArch64_IMM::ImmInsnModel> Insn;
846 AArch64_IMM::expandMOVImm(Imm, BitSize: 32, Insn);
847 return Insn.size();
848 };
849
850 if (NumberOfInstrToLoadImm(OriginalC) > NumberOfInstrToLoadImm(NewC))
851 return {{NewC, P}};
852
853 return std::nullopt;
854}
855
856/// Determine whether or not it is possible to update the RHS and predicate of
857/// a G_ICMP instruction such that the RHS will be selected as an arithmetic
858/// immediate.
859///
860/// \p MI - The G_ICMP instruction
861/// \p MatchInfo - The new RHS immediate and predicate on success
862///
863/// See tryAdjustICmpImmAndPred for valid transformations.
864bool matchAdjustICmpImmAndPred(
865 MachineInstr &MI, const MachineRegisterInfo &MRI,
866 std::pair<uint64_t, CmpInst::Predicate> &MatchInfo) {
867 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
868 Register LHS = MI.getOperand(i: 2).getReg();
869 Register RHS = MI.getOperand(i: 3).getReg();
870 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
871 if (auto MaybeNewImmAndPred = tryAdjustICmpImmAndPred(LHS, RHS, P: Pred, MRI)) {
872 MatchInfo = *MaybeNewImmAndPred;
873 return true;
874 }
875 return false;
876}
877
878void applyAdjustICmpImmAndPred(
879 MachineInstr &MI, std::pair<uint64_t, CmpInst::Predicate> &MatchInfo,
880 MachineIRBuilder &MIB, GISelChangeObserver &Observer) {
881 MIB.setInstrAndDebugLoc(MI);
882 MachineOperand &RHS = MI.getOperand(i: 3);
883 MachineRegisterInfo &MRI = *MIB.getMRI();
884 auto Cst = MIB.buildConstant(Res: MRI.cloneVirtualRegister(VReg: RHS.getReg()),
885 Val: MatchInfo.first);
886 Observer.changingInstr(MI);
887 RHS.setReg(Cst->getOperand(i: 0).getReg());
888 MI.getOperand(i: 1).setPredicate(MatchInfo.second);
889 Observer.changedInstr(MI);
890}
891
892bool matchDupLane(MachineInstr &MI, MachineRegisterInfo &MRI,
893 std::pair<unsigned, int> &MatchInfo) {
894 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
895 Register Src1Reg = MI.getOperand(i: 1).getReg();
896 const LLT SrcTy = MRI.getType(Reg: Src1Reg);
897 const LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
898
899 auto LaneIdx = getSplatIndex(MI);
900 if (!LaneIdx)
901 return false;
902
903 // The lane idx should be within the first source vector.
904 if (*LaneIdx >= SrcTy.getNumElements())
905 return false;
906
907 if (DstTy != SrcTy)
908 return false;
909
910 LLT ScalarTy = SrcTy.getElementType();
911 unsigned ScalarSize = ScalarTy.getSizeInBits();
912
913 unsigned Opc = 0;
914 switch (SrcTy.getNumElements()) {
915 case 2:
916 if (ScalarSize == 64)
917 Opc = AArch64::G_DUPLANE64;
918 else if (ScalarSize == 32)
919 Opc = AArch64::G_DUPLANE32;
920 break;
921 case 4:
922 if (ScalarSize == 32)
923 Opc = AArch64::G_DUPLANE32;
924 else if (ScalarSize == 16)
925 Opc = AArch64::G_DUPLANE16;
926 break;
927 case 8:
928 if (ScalarSize == 8)
929 Opc = AArch64::G_DUPLANE8;
930 else if (ScalarSize == 16)
931 Opc = AArch64::G_DUPLANE16;
932 break;
933 case 16:
934 if (ScalarSize == 8)
935 Opc = AArch64::G_DUPLANE8;
936 break;
937 default:
938 break;
939 }
940 if (!Opc)
941 return false;
942
943 MatchInfo.first = Opc;
944 MatchInfo.second = *LaneIdx;
945 return true;
946}
947
948void applyDupLane(MachineInstr &MI, MachineRegisterInfo &MRI,
949 MachineIRBuilder &B, std::pair<unsigned, int> &MatchInfo) {
950 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
951 Register Src1Reg = MI.getOperand(i: 1).getReg();
952 const LLT SrcTy = MRI.getType(Reg: Src1Reg);
953
954 B.setInstrAndDebugLoc(MI);
955 auto Lane = B.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: MatchInfo.second);
956
957 Register DupSrc = MI.getOperand(i: 1).getReg();
958 // For types like <2 x s32>, we can use G_DUPLANE32, with a <4 x s32> source.
959 // To do this, we can use a G_CONCAT_VECTORS to do the widening.
960 if (SrcTy.getSizeInBits() == 64) {
961 auto Undef = B.buildUndef(Res: SrcTy);
962 DupSrc = B.buildConcatVectors(Res: SrcTy.multiplyElements(Factor: 2),
963 Ops: {Src1Reg, Undef.getReg(Idx: 0)})
964 .getReg(Idx: 0);
965 }
966 B.buildInstr(Opc: MatchInfo.first, DstOps: {MI.getOperand(i: 0).getReg()}, SrcOps: {DupSrc, Lane});
967 MI.eraseFromParent();
968}
969
970bool matchScalarizeVectorUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI) {
971 auto &Unmerge = cast<GUnmerge>(Val&: MI);
972 Register Src1Reg = Unmerge.getReg(Idx: Unmerge.getNumOperands() - 1);
973 const LLT SrcTy = MRI.getType(Reg: Src1Reg);
974 if (SrcTy.getSizeInBits() != 128 && SrcTy.getSizeInBits() != 64)
975 return false;
976 return SrcTy.isVector() && !SrcTy.isScalable() &&
977 (Unmerge.getNumOperands() == (unsigned)SrcTy.getNumElements() + 1 ||
978 (Unmerge.getNumDefs() == 2 && SrcTy.getSizeInBits() == 128 &&
979 MRI.getType(Reg: Unmerge.getReg(Idx: 0)).getSizeInBits() == 64));
980}
981
982void applyScalarizeVectorUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
983 MachineIRBuilder &B) {
984 auto &Unmerge = cast<GUnmerge>(Val&: MI);
985 Register Src1Reg = Unmerge.getReg(Idx: Unmerge.getNumOperands() - 1);
986 const LLT SrcTy = MRI.getType(Reg: Src1Reg);
987 const LLT DstTy = MRI.getType(Reg: Unmerge.getReg(Idx: 0));
988 assert((SrcTy.isVector() && !SrcTy.isScalable()) &&
989 "Expected a fixed length vector");
990
991 if (DstTy.isVector()) {
992 assert(Unmerge.getNumDefs() == 2);
993 if (!MRI.use_nodbg_empty(RegNo: Unmerge.getReg(Idx: 0)))
994 B.buildExtractSubvector(Res: Unmerge.getReg(Idx: 0), Src: Src1Reg, Index: 0);
995 if (!MRI.use_nodbg_empty(RegNo: Unmerge.getReg(Idx: 1)))
996 B.buildExtractSubvector(Res: Unmerge.getReg(Idx: 1), Src: Src1Reg,
997 Index: SrcTy.getNumElements() / 2);
998 } else {
999 for (int I = 0; I < SrcTy.getNumElements(); ++I)
1000 if (!MRI.use_nodbg_empty(RegNo: Unmerge.getReg(Idx: I)))
1001 B.buildExtractVectorElementConstant(Res: Unmerge.getReg(Idx: I), Val: Src1Reg, Idx: I);
1002 }
1003 MI.eraseFromParent();
1004}
1005
1006bool matchBuildVectorToDup(MachineInstr &MI, Register &Src,
1007 MachineRegisterInfo &MRI) {
1008 assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
1009
1010 // Later, during selection, we'll try to match imported patterns using
1011 // immAllOnesV and immAllZerosV. These require G_BUILD_VECTOR. Don't lower
1012 // G_BUILD_VECTORs which could match those patterns.
1013 if (isBuildVectorAllZeros(MI, MRI) || isBuildVectorAllOnes(MI, MRI))
1014 return false;
1015
1016 // Find buildvector which always uses the same register or undef. Return true
1017 // so long as at least 2 registers were found (not all-undef or only 1
1018 // non-undef entry).
1019 Register Reg = 0;
1020 unsigned NumNonUndef = 0;
1021 for (const MachineOperand &Op : drop_begin(RangeOrContainer: MI.operands())) {
1022 if (getOpcodeDef<GImplicitDef>(Reg: Op.getReg(), MRI))
1023 continue;
1024
1025 if (!Reg)
1026 Reg = Op.getReg();
1027 else if (Op.getReg() != Reg)
1028 return false;
1029 NumNonUndef++;
1030 }
1031
1032 Src = Reg;
1033 return Reg && NumNonUndef > 1;
1034}
1035
1036void applyBuildVectorToDup(MachineInstr &MI, Register Src,
1037 MachineRegisterInfo &MRI, MachineIRBuilder &B) {
1038 B.setInstrAndDebugLoc(MI);
1039 B.buildInstr(Opc: TargetOpcode::G_SPLAT_VECTOR, DstOps: {MI.getOperand(i: 0).getReg()},
1040 SrcOps: {Src});
1041 MI.eraseFromParent();
1042}
1043
1044bool matchBuildVectorToHalfDup(MachineInstr &MI,
1045 std::pair<Register, Register> &Srcs,
1046 MachineRegisterInfo &MRI) {
1047 assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
1048 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1049 if (Ty.getSizeInBits() != 128 || isConstantOrConstantVector(MI, MRI))
1050 return false;
1051
1052 // Later, during selection, we'll try to match imported patterns using
1053 // immAllOnesV and immAllZerosV. These require G_BUILD_VECTOR. Don't lower
1054 // G_BUILD_VECTORs which could match those patterns.
1055 if (isBuildVectorAllZeros(MI, MRI) || isBuildVectorAllOnes(MI, MRI))
1056 return false;
1057
1058 // Find buildvector which always uses the same register or undef in the first
1059 // and second halves of the register.
1060 Register Reg1;
1061 Register Reg2;
1062 unsigned NumOps = MI.getNumOperands() - 1;
1063 if (NumOps <= 2 || NumOps % 2 != 0)
1064 return false;
1065
1066 unsigned NumNonUndef1 = 0, NumNonUndef2 = 0;
1067 for (unsigned I = 0; I < NumOps / 2; I++) {
1068 Register Src1 = MI.getOperand(i: I + 1).getReg();
1069 if (!getOpcodeDef<GImplicitDef>(Reg: Src1, MRI)) {
1070 if (!Reg1)
1071 Reg1 = Src1;
1072 else if (Src1 != Reg1)
1073 return false;
1074 NumNonUndef1++;
1075 }
1076
1077 Register Src2 = MI.getOperand(i: NumOps / 2 + I + 1).getReg();
1078 if (!getOpcodeDef<GImplicitDef>(Reg: Src2, MRI)) {
1079 if (!Reg2)
1080 Reg2 = Src2;
1081 else if (Src2 != Reg2)
1082 return false;
1083 NumNonUndef2++;
1084 }
1085 }
1086
1087 Srcs = {Reg1, Reg2};
1088 return NumNonUndef1 >= 1 && NumNonUndef2 >= 1 &&
1089 NumNonUndef1 + NumNonUndef2 >= 3 && Reg1 != Reg2;
1090}
1091
1092void applyBuildVectorToHalfDup(MachineInstr &MI,
1093 std::pair<Register, Register> &Srcs,
1094 MachineRegisterInfo &MRI, MachineIRBuilder &B) {
1095 B.setInstrAndDebugLoc(MI);
1096 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1097 LLT HalfTy = Ty.divide(Factor: 2);
1098 auto D1 = Srcs.first
1099 ? B.buildInstr(Opc: AArch64::G_SPLAT_VECTOR, DstOps: {HalfTy}, SrcOps: {Srcs.first})
1100 : B.buildUndef(Res: HalfTy);
1101 auto D2 = Srcs.second
1102 ? B.buildInstr(Opc: AArch64::G_SPLAT_VECTOR, DstOps: {HalfTy}, SrcOps: {Srcs.second})
1103 : B.buildUndef(Res: HalfTy);
1104 B.buildConcatVectors(Res: MI.getOperand(i: 0), Ops: {D1.getReg(Idx: 0), D2.getReg(Idx: 0)});
1105 MI.eraseFromParent();
1106}
1107
1108/// \returns how many instructions would be saved by folding a G_ICMP's shift
1109/// and/or extension operations.
1110static unsigned getCmpOperandFoldingProfit(Register CmpOp,
1111 MachineRegisterInfo &MRI) {
1112 // FIXME: This is duplicated with the selector. (See: selectShiftedRegister)
1113 auto IsSupportedExtend = [&](const MachineInstr &MI) {
1114 if (MI.getOpcode() == TargetOpcode::G_SEXT_INREG)
1115 return true;
1116 if (MI.getOpcode() == TargetOpcode::G_AND) {
1117 auto ValAndVReg =
1118 getIConstantVRegValWithLookThrough(VReg: MI.getOperand(i: 2).getReg(), MRI);
1119 if (ValAndVReg) {
1120 uint64_t Mask = ValAndVReg->Value.getZExtValue();
1121 return (Mask == 0xFF || Mask == 0xFFFF || Mask == 0xFFFFFFFF);
1122 }
1123 }
1124 return false;
1125 };
1126
1127 // No instructions to save if there's more than one use or no uses.
1128 if (!MRI.hasOneNonDBGUse(RegNo: CmpOp))
1129 return 0;
1130
1131 MachineInstr *Def = getDefIgnoringCopies(Reg: CmpOp, MRI);
1132 if (IsSupportedExtend(*Def))
1133 return 1;
1134
1135 unsigned Opc = Def->getOpcode();
1136 if (Opc == TargetOpcode::G_SHL || Opc == TargetOpcode::G_LSHR ||
1137 Opc == TargetOpcode::G_ASHR) {
1138 auto MaybeShiftAmt =
1139 getIConstantVRegValWithLookThrough(VReg: Def->getOperand(i: 2).getReg(), MRI);
1140 if (MaybeShiftAmt) {
1141 uint64_t ShiftAmt = MaybeShiftAmt->Value.getZExtValue();
1142 MachineInstr *ShiftLHS =
1143 getDefIgnoringCopies(Reg: Def->getOperand(i: 1).getReg(), MRI);
1144 if (IsSupportedExtend(*ShiftLHS))
1145 return (ShiftAmt <= 4) ? 2 : 1;
1146 LLT Ty = MRI.getType(Reg: Def->getOperand(i: 0).getReg());
1147 if (Ty.isVector())
1148 return 0;
1149 unsigned ShiftSize = Ty.getSizeInBits();
1150 if ((ShiftSize == 32 && ShiftAmt <= 31) ||
1151 (ShiftSize == 64 && ShiftAmt <= 63))
1152 return 1;
1153 }
1154 }
1155
1156 return 0;
1157}
1158
1159/// \returns true if it would be profitable to swap the LHS and RHS of a G_ICMP
1160/// instruction \p MI.
1161bool trySwapICmpOperands(MachineInstr &MI, MachineRegisterInfo &MRI) {
1162 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
1163 // Swap the operands if it would introduce a profitable folding opportunity.
1164 // (e.g. a shift + extend).
1165 //
1166 // For example:
1167 // lsl w13, w11, #1
1168 // cmp w13, w12
1169 // can be turned into:
1170 // cmp w12, w11, lsl #1
1171
1172 // Don't swap if there's a constant on the RHS and it is a legal compare
1173 // immediate, because we know we can fold that.
1174 Register RHS = MI.getOperand(i: 3).getReg();
1175 auto RHSCst = getIConstantVRegValWithLookThrough(VReg: RHS, MRI);
1176 if (RHSCst && AArch64_AM::isLegalCmpImmed(C: RHSCst->Value))
1177 return false;
1178
1179 Register LHS = MI.getOperand(i: 2).getReg();
1180 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
1181 auto GetRegForProfit = [&](Register Reg) {
1182 MachineInstr *Def = getDefIgnoringCopies(Reg, MRI);
1183 return isCMN(MaybeSub: Def, Pred, MRI) ? Def->getOperand(i: 2).getReg() : Reg;
1184 };
1185
1186 // Don't have a constant on the RHS. If we swap the LHS and RHS of the
1187 // compare, would we be able to fold more instructions?
1188 Register TheLHS = GetRegForProfit(LHS);
1189 Register TheRHS = GetRegForProfit(RHS);
1190
1191 // If the LHS is more likely to give us a folding opportunity, then swap the
1192 // LHS and RHS.
1193 return (getCmpOperandFoldingProfit(CmpOp: TheLHS, MRI) >
1194 getCmpOperandFoldingProfit(CmpOp: TheRHS, MRI));
1195}
1196
1197void applySwapICmpOperands(MachineInstr &MI, GISelChangeObserver &Observer) {
1198 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
1199 Register LHS = MI.getOperand(i: 2).getReg();
1200 Register RHS = MI.getOperand(i: 3).getReg();
1201 Observer.changingInstr(MI);
1202 MI.getOperand(i: 1).setPredicate(CmpInst::getSwappedPredicate(pred: Pred));
1203 MI.getOperand(i: 2).setReg(RHS);
1204 MI.getOperand(i: 3).setReg(LHS);
1205 Observer.changedInstr(MI);
1206}
1207
1208/// \returns a function which builds a vector floating point compare instruction
1209/// for a condition code \p CC.
1210/// \param [in] NoNans - True if the instruction has nnan flag.
1211std::function<Register(MachineIRBuilder &)>
1212getVectorFCMP(AArch64CC::CondCode CC, Register LHS, Register RHS, bool NoNans,
1213 MachineRegisterInfo &MRI) {
1214 LLT OldTy = MRI.getType(Reg: LHS);
1215 LLT DstTy = LLT::fixed_vector(NumElements: OldTy.getNumElements(),
1216 ScalarTy: LLT::integer(SizeInBits: OldTy.getScalarSizeInBits()));
1217 assert(DstTy.isVector() && "Expected vector types only?");
1218 switch (CC) {
1219 default:
1220 llvm_unreachable("Unexpected condition code!");
1221 case AArch64CC::NE:
1222 return [LHS, RHS, DstTy](MachineIRBuilder &MIB) {
1223 auto FCmp = MIB.buildInstr(Opc: AArch64::G_FCMEQ, DstOps: {DstTy}, SrcOps: {LHS, RHS});
1224 return MIB.buildNot(Dst: DstTy, Src0: FCmp).getReg(Idx: 0);
1225 };
1226 case AArch64CC::EQ:
1227 return [LHS, RHS, DstTy](MachineIRBuilder &MIB) {
1228 return MIB.buildInstr(Opc: AArch64::G_FCMEQ, DstOps: {DstTy}, SrcOps: {LHS, RHS}).getReg(Idx: 0);
1229 };
1230 case AArch64CC::GE:
1231 return [LHS, RHS, DstTy](MachineIRBuilder &MIB) {
1232 return MIB.buildInstr(Opc: AArch64::G_FCMGE, DstOps: {DstTy}, SrcOps: {LHS, RHS}).getReg(Idx: 0);
1233 };
1234 case AArch64CC::GT:
1235 return [LHS, RHS, DstTy](MachineIRBuilder &MIB) {
1236 return MIB.buildInstr(Opc: AArch64::G_FCMGT, DstOps: {DstTy}, SrcOps: {LHS, RHS}).getReg(Idx: 0);
1237 };
1238 case AArch64CC::LS:
1239 return [LHS, RHS, DstTy](MachineIRBuilder &MIB) {
1240 return MIB.buildInstr(Opc: AArch64::G_FCMGE, DstOps: {DstTy}, SrcOps: {RHS, LHS}).getReg(Idx: 0);
1241 };
1242 case AArch64CC::MI:
1243 return [LHS, RHS, DstTy](MachineIRBuilder &MIB) {
1244 return MIB.buildInstr(Opc: AArch64::G_FCMGT, DstOps: {DstTy}, SrcOps: {RHS, LHS}).getReg(Idx: 0);
1245 };
1246 }
1247}
1248
1249/// Try to lower a vector G_FCMP \p MI into an AArch64-specific pseudo.
1250bool matchLowerVectorFCMP(MachineInstr &MI, MachineRegisterInfo &MRI,
1251 MachineIRBuilder &MIB) {
1252 assert(MI.getOpcode() == TargetOpcode::G_FCMP);
1253 const auto &ST = MI.getMF()->getSubtarget<AArch64Subtarget>();
1254
1255 Register Dst = MI.getOperand(i: 0).getReg();
1256 LLT DstTy = MRI.getType(Reg: Dst);
1257 if (!DstTy.isVector() || !ST.hasNEON())
1258 return false;
1259 Register LHS = MI.getOperand(i: 2).getReg();
1260 unsigned EltSize = MRI.getType(Reg: LHS).getScalarSizeInBits();
1261 if (EltSize == 16 && !ST.hasFullFP16())
1262 return false;
1263 if (EltSize != 16 && EltSize != 32 && EltSize != 64)
1264 return false;
1265
1266 return true;
1267}
1268
1269/// Try to lower a vector G_FCMP \p MI into an AArch64-specific pseudo.
1270void applyLowerVectorFCMP(MachineInstr &MI, MachineRegisterInfo &MRI,
1271 MachineIRBuilder &MIB) {
1272 assert(MI.getOpcode() == TargetOpcode::G_FCMP);
1273
1274 const auto &CmpMI = cast<GFCmp>(Val&: MI);
1275
1276 Register Dst = CmpMI.getReg(Idx: 0);
1277 CmpInst::Predicate Pred = CmpMI.getCond();
1278 Register LHS = CmpMI.getLHSReg();
1279 Register RHS = CmpMI.getRHSReg();
1280
1281 LLT DstTy = MRI.getType(Reg: Dst);
1282
1283 bool Invert = false;
1284 AArch64CC::CondCode CC, CC2 = AArch64CC::AL;
1285 if ((Pred == CmpInst::Predicate::FCMP_ORD ||
1286 Pred == CmpInst::Predicate::FCMP_UNO) &&
1287 isBuildVectorAllZeros(MI: *MRI.getVRegDef(Reg: RHS), MRI)) {
1288 // The special case "fcmp ord %a, 0" is the canonical check that LHS isn't
1289 // NaN, so equivalent to a == a and doesn't need the two comparisons an
1290 // "ord" normally would.
1291 // Similarly, "fcmp uno %a, 0" is the canonical check that LHS is NaN and is
1292 // thus equivalent to a != a.
1293 RHS = LHS;
1294 CC = Pred == CmpInst::Predicate::FCMP_ORD ? AArch64CC::EQ : AArch64CC::NE;
1295 } else
1296 changeVectorFCMPPredToAArch64CC(P: Pred, CondCode&: CC, CondCode2&: CC2, Invert);
1297
1298 // Instead of having an apply function, just build here to simplify things.
1299 MIB.setInstrAndDebugLoc(MI);
1300
1301 // TODO: Also consider GISelValueTracking result if eligible.
1302 const bool NoNans = MI.getFlag(Flag: MachineInstr::FmNoNans);
1303
1304 auto Cmp = getVectorFCMP(CC, LHS, RHS, NoNans, MRI);
1305 Register CmpRes;
1306 if (CC2 == AArch64CC::AL)
1307 CmpRes = Cmp(MIB);
1308 else {
1309 auto Cmp2 = getVectorFCMP(CC: CC2, LHS, RHS, NoNans, MRI);
1310 auto Cmp2Dst = Cmp2(MIB);
1311 auto Cmp1Dst = Cmp(MIB);
1312 CmpRes = MIB.buildOr(Dst: DstTy, Src0: Cmp1Dst, Src1: Cmp2Dst).getReg(Idx: 0);
1313 }
1314 if (Invert)
1315 CmpRes = MIB.buildNot(Dst: DstTy, Src0: CmpRes).getReg(Idx: 0);
1316 MRI.replaceRegWith(FromReg: Dst, ToReg: CmpRes);
1317 MI.eraseFromParent();
1318}
1319
1320// Matches G_BUILD_VECTOR where at least one source operand is not a constant
1321bool matchLowerBuildToInsertVecElt(MachineInstr &MI, MachineRegisterInfo &MRI) {
1322 auto *GBuildVec = cast<GBuildVector>(Val: &MI);
1323
1324 // Check if the values are all constants
1325 for (unsigned I = 0; I < GBuildVec->getNumSources(); ++I) {
1326 auto ConstVal =
1327 getAnyConstantVRegValWithLookThrough(VReg: GBuildVec->getSourceReg(I), MRI);
1328
1329 if (!ConstVal.has_value())
1330 return true;
1331 }
1332
1333 return false;
1334}
1335
1336void applyLowerBuildToInsertVecElt(MachineInstr &MI, MachineRegisterInfo &MRI,
1337 MachineIRBuilder &B) {
1338 auto *GBuildVec = cast<GBuildVector>(Val: &MI);
1339 LLT DstTy = MRI.getType(Reg: GBuildVec->getReg(Idx: 0));
1340 Register DstReg = B.buildUndef(Res: DstTy).getReg(Idx: 0);
1341
1342 for (unsigned I = 0; I < GBuildVec->getNumSources(); ++I) {
1343 Register SrcReg = GBuildVec->getSourceReg(I);
1344 if (mi_match(R: SrcReg, MRI, P: m_GImplicitDef()))
1345 continue;
1346 auto IdxReg = B.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: I);
1347 DstReg =
1348 B.buildInsertVectorElement(Res: DstTy, Val: DstReg, Elt: SrcReg, Idx: IdxReg).getReg(Idx: 0);
1349 }
1350 B.buildCopy(Res: GBuildVec->getReg(Idx: 0), Op: DstReg);
1351 GBuildVec->eraseFromParent();
1352}
1353
1354// Detect vector constants that cannot be natively represent, but can be
1355// represented as fneg(constant). For example the constant
1356// <0x800000000000000, 0x800000000000000> can be represented as fneg(<0, 0>).
1357bool matchConstBuildVectorToFNeg(MachineInstr &MI, MachineRegisterInfo &MRI,
1358 const AArch64Subtarget &ST,
1359 std::pair<APInt, unsigned> &MatchInfo) {
1360 assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
1361 if (!ST.isNeonAvailable())
1362 return false;
1363 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1364 // Limit to fneg of v2f32, v4f32, f64, v2f64 or v4f16 and v8f16 with fullfp16.
1365 if (Ty.getSizeInBits() != 64 && Ty.getSizeInBits() != 128)
1366 return false;
1367 unsigned EltSize = Ty.getScalarSizeInBits();
1368
1369 APInt Imm(Ty.getSizeInBits(), 0);
1370 for (unsigned Idx = 1; Idx < MI.getNumOperands(); Idx++) {
1371 auto C =
1372 getAnyConstantVRegValWithLookThrough(VReg: MI.getOperand(i: Idx).getReg(), MRI);
1373 if (!C)
1374 return false;
1375 Imm |= C->Value.trunc(width: EltSize)
1376 .zext(width: Ty.getSizeInBits())
1377 .shl(shiftAmt: (Idx - 1) * EltSize);
1378 }
1379
1380 SmallVector<AArch64_IMM::ImmInsnModel> Insns;
1381 if (AArch64_IMM::expandVectorMOVImm(Imm, Insn&: Insns, ST))
1382 return false;
1383
1384 auto Check = [&](APInt &Imm, unsigned Size) {
1385 APInt NImm = Imm ^ APInt::getSplat(NewLen: Imm.getBitWidth(),
1386 V: APInt::getHighBitsSet(numBits: Size, hiBitsSet: 1));
1387 SmallVector<AArch64_IMM::ImmInsnModel> Insns;
1388 MatchInfo.first = NImm;
1389 MatchInfo.second = Size;
1390 return AArch64_IMM::expandVectorMOVImm(Imm: NImm, Insn&: Insns, ST);
1391 };
1392
1393 return Check(Imm, 64) || Check(Imm, 32) ||
1394 (ST.hasFullFP16() && Check(Imm, 16));
1395}
1396
1397void applyConstBuildVectorToFNeg(MachineInstr &MI, MachineRegisterInfo &MRI,
1398 MachineIRBuilder &B,
1399 std::pair<APInt, unsigned> &MatchInfo) {
1400 B.setInstrAndDebugLoc(MI);
1401 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1402 // Extract the Immediate into chunks of size EltSize
1403 unsigned EltSize = MatchInfo.second;
1404 unsigned NumLanes = Ty.getSizeInBits() / EltSize;
1405 LLT VTy = NumLanes == 1
1406 ? LLT::floatIEEE(SizeInBits: EltSize)
1407 : LLT::fixed_vector(NumElements: NumLanes, ScalarTy: LLT::floatIEEE(SizeInBits: EltSize));
1408 SmallVector<APInt> Imms;
1409 for (unsigned I = 0; I < NumLanes; I++)
1410 Imms.push_back(Elt: MatchInfo.first.extractBits(numBits: EltSize, bitPosition: I * EltSize));
1411 // Build the new BV and FNeg of it.
1412 auto BV = NumLanes == 1 ? B.buildFConstant(Res: VTy, Val: APFloat(APFloat::IEEEdouble(),
1413 MatchInfo.first))
1414 : B.buildBuildVectorConstant(Res: VTy, Ops: Imms);
1415 auto FNeg = B.buildFNeg(Dst: Ty == VTy ? DstOp(MI.getOperand(i: 0)) : DstOp(VTy), Src0: BV);
1416 if (Ty != VTy)
1417 B.buildBitcast(Dst: MI.getOperand(i: 0), Src: FNeg);
1418 MI.eraseFromParent();
1419}
1420
1421bool matchFormTruncstore(MachineInstr &MI, MachineRegisterInfo &MRI,
1422 Register &SrcReg) {
1423 assert(MI.getOpcode() == TargetOpcode::G_STORE);
1424 Register DstReg = MI.getOperand(i: 0).getReg();
1425 if (cast<GLoadStore>(Val&: MI).isAtomic())
1426 return false;
1427 if (MRI.getType(Reg: DstReg).isVector())
1428 return false;
1429 // Match a store of a truncate.
1430 if (!mi_match(R: DstReg, MRI, P: m_GTrunc(Src: m_Reg(R&: SrcReg))))
1431 return false;
1432 // Only form truncstores for value types of max 64b.
1433 return MRI.getType(Reg: SrcReg).getSizeInBits() <= 64;
1434}
1435
1436void applyFormTruncstore(MachineInstr &MI, MachineRegisterInfo &MRI,
1437 MachineIRBuilder &B, GISelChangeObserver &Observer,
1438 Register &SrcReg) {
1439 assert(MI.getOpcode() == TargetOpcode::G_STORE);
1440 Observer.changingInstr(MI);
1441 MI.getOperand(i: 0).setReg(SrcReg);
1442 Observer.changedInstr(MI);
1443}
1444
1445// Lower vector G_SEXT_INREG back to shifts for selection. We allowed them to
1446// form in the first place for combine opportunities, so any remaining ones
1447// at this stage need be lowered back.
1448bool matchVectorSextInReg(MachineInstr &MI, MachineRegisterInfo &MRI) {
1449 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1450 Register DstReg = MI.getOperand(i: 0).getReg();
1451 LLT DstTy = MRI.getType(Reg: DstReg);
1452 return DstTy.isVector();
1453}
1454
1455void applyVectorSextInReg(MachineInstr &MI, MachineRegisterInfo &MRI,
1456 MachineIRBuilder &B, GISelChangeObserver &Observer) {
1457 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1458 B.setInstrAndDebugLoc(MI);
1459 LegalizerHelper Helper(*MI.getMF(), Observer, B);
1460 Helper.lower(MI, TypeIdx: 0, /* Unused hint type */ Ty: LLT());
1461}
1462
1463/// Combine <N x t>, unused = unmerge(G_EXT <2*N x t> v, undef, N)
1464/// => unused, <N x t> = unmerge v
1465bool matchUnmergeExtToUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
1466 Register &MatchInfo) {
1467 auto &Unmerge = cast<GUnmerge>(Val&: MI);
1468 if (Unmerge.getNumDefs() != 2)
1469 return false;
1470 if (!MRI.use_nodbg_empty(RegNo: Unmerge.getReg(Idx: 1)))
1471 return false;
1472
1473 LLT DstTy = MRI.getType(Reg: Unmerge.getReg(Idx: 0));
1474 if (!DstTy.isVector())
1475 return false;
1476
1477 MachineInstr *Ext = getOpcodeDef(Opcode: AArch64::G_EXT, Reg: Unmerge.getSourceReg(), MRI);
1478 if (!Ext)
1479 return false;
1480
1481 Register ExtSrc1 = Ext->getOperand(i: 1).getReg();
1482 Register ExtSrc2 = Ext->getOperand(i: 2).getReg();
1483 auto LowestVal =
1484 getIConstantVRegValWithLookThrough(VReg: Ext->getOperand(i: 3).getReg(), MRI);
1485 if (!LowestVal || LowestVal->Value.getZExtValue() != DstTy.getSizeInBytes())
1486 return false;
1487
1488 if (!getOpcodeDef<GImplicitDef>(Reg: ExtSrc2, MRI))
1489 return false;
1490
1491 MatchInfo = ExtSrc1;
1492 return true;
1493}
1494
1495void applyUnmergeExtToUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
1496 MachineIRBuilder &B,
1497 GISelChangeObserver &Observer, Register &SrcReg) {
1498 Observer.changingInstr(MI);
1499 // Swap dst registers.
1500 Register Dst1 = MI.getOperand(i: 0).getReg();
1501 MI.getOperand(i: 0).setReg(MI.getOperand(i: 1).getReg());
1502 MI.getOperand(i: 1).setReg(Dst1);
1503 MI.getOperand(i: 2).setReg(SrcReg);
1504 Observer.changedInstr(MI);
1505}
1506
1507// Match mul({z/s}ext , {z/s}ext) => {u/s}mull OR
1508// Match v2s64 mul instructions, which will then be scalarised later on
1509// Doing these two matches in one function to ensure that the order of matching
1510// will always be the same.
1511// Try lowering MUL to MULL before trying to scalarize if needed.
1512bool matchMulv2s64(MachineInstr &MI, MachineRegisterInfo &MRI) {
1513 // Get the instructions that defined the source operand
1514 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1515 return DstTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64);
1516}
1517
1518void applyMulv2s64(MachineInstr &MI, MachineRegisterInfo &MRI,
1519 MachineIRBuilder &B, GISelChangeObserver &Observer) {
1520 assert(MI.getOpcode() == TargetOpcode::G_MUL &&
1521 "Expected a G_MUL instruction");
1522
1523 // Get the instructions that defined the source operand
1524 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
1525 assert(DstTy == LLT::fixed_vector(2, 64) && "Expected v2s64 Mul");
1526 LegalizerHelper Helper(*MI.getMF(), Observer, B);
1527 Helper.fewerElementsVector(
1528 MI, TypeIdx: 0,
1529 NarrowTy: DstTy.changeElementCount(EC: DstTy.getElementCount().divideCoefficientBy(RHS: 2)));
1530}
1531
1532class AArch64PostLegalizerLoweringImpl : public Combiner {
1533protected:
1534 const CombinerHelper Helper;
1535 const AArch64PostLegalizerLoweringImplRuleConfig &RuleConfig;
1536 const AArch64Subtarget &STI;
1537
1538public:
1539 AArch64PostLegalizerLoweringImpl(
1540 MachineFunction &MF, CombinerInfo &CInfo, GISelCSEInfo *CSEInfo,
1541 const AArch64PostLegalizerLoweringImplRuleConfig &RuleConfig,
1542 const AArch64Subtarget &STI);
1543
1544 static const char *getName() { return "AArch6400PreLegalizerCombiner"; }
1545
1546 bool tryCombineAll(MachineInstr &I) const override;
1547
1548private:
1549#define GET_GICOMBINER_CLASS_MEMBERS
1550#include "AArch64GenPostLegalizeGILowering.inc"
1551#undef GET_GICOMBINER_CLASS_MEMBERS
1552};
1553
1554#define GET_GICOMBINER_IMPL
1555#include "AArch64GenPostLegalizeGILowering.inc"
1556#undef GET_GICOMBINER_IMPL
1557
1558AArch64PostLegalizerLoweringImpl::AArch64PostLegalizerLoweringImpl(
1559 MachineFunction &MF, CombinerInfo &CInfo, GISelCSEInfo *CSEInfo,
1560 const AArch64PostLegalizerLoweringImplRuleConfig &RuleConfig,
1561 const AArch64Subtarget &STI)
1562 : Combiner(MF, CInfo, /*VT*/ nullptr, CSEInfo),
1563 Helper(Observer, B, /*IsPreLegalize*/ true), RuleConfig(RuleConfig),
1564 STI(STI),
1565#define GET_GICOMBINER_CONSTRUCTOR_INITS
1566#include "AArch64GenPostLegalizeGILowering.inc"
1567#undef GET_GICOMBINER_CONSTRUCTOR_INITS
1568{
1569}
1570
1571bool runPostLegalizerLowering(
1572 MachineFunction &MF,
1573 const AArch64PostLegalizerLoweringImplRuleConfig &RuleConfig) {
1574 if (MF.getProperties().hasFailedISel())
1575 return false;
1576 const Function &F = MF.getFunction();
1577
1578 const AArch64Subtarget &ST = MF.getSubtarget<AArch64Subtarget>();
1579 CombinerInfo CInfo(/*AllowIllegalOps=*/true, /*ShouldLegalizeIllegal=*/false,
1580 /*LegalizerInfo=*/nullptr, /*OptEnabled=*/true,
1581 F.hasOptSize(), F.hasMinSize());
1582 // Disable fixed-point iteration to reduce compile-time
1583 CInfo.MaxIterations = 1;
1584 CInfo.ObserverLvl = CombinerInfo::ObserverLevel::SinglePass;
1585 // PostLegalizerCombiner performs DCE, so a full DCE pass is unnecessary.
1586 CInfo.EnableFullDCE = false;
1587 AArch64PostLegalizerLoweringImpl Impl(MF, CInfo, /*CSEInfo=*/nullptr,
1588 RuleConfig, ST);
1589 return Impl.combineMachineInstrs();
1590}
1591
1592class AArch64PostLegalizerLoweringLegacy : public MachineFunctionPass {
1593public:
1594 static char ID;
1595
1596 AArch64PostLegalizerLoweringLegacy();
1597
1598 StringRef getPassName() const override {
1599 return "AArch64PostLegalizerLowering";
1600 }
1601
1602 bool runOnMachineFunction(MachineFunction &MF) override;
1603 void getAnalysisUsage(AnalysisUsage &AU) const override;
1604
1605private:
1606 AArch64PostLegalizerLoweringImplRuleConfig RuleConfig;
1607};
1608} // end anonymous namespace
1609
1610void AArch64PostLegalizerLoweringLegacy::getAnalysisUsage(
1611 AnalysisUsage &AU) const {
1612 AU.setPreservesCFG();
1613 getSelectionDAGFallbackAnalysisUsage(AU);
1614 MachineFunctionPass::getAnalysisUsage(AU);
1615}
1616
1617AArch64PostLegalizerLoweringLegacy::AArch64PostLegalizerLoweringLegacy()
1618 : MachineFunctionPass(ID) {
1619 if (!RuleConfig.parseCommandLineOption())
1620 report_fatal_error(reason: "Invalid rule identifier");
1621}
1622
1623bool AArch64PostLegalizerLoweringLegacy::runOnMachineFunction(
1624 MachineFunction &MF) {
1625 assert(MF.getProperties().hasLegalized() && "Expected a legalized function?");
1626 return runPostLegalizerLowering(MF, RuleConfig);
1627}
1628
1629char AArch64PostLegalizerLoweringLegacy::ID = 0;
1630INITIALIZE_PASS_BEGIN(AArch64PostLegalizerLoweringLegacy, DEBUG_TYPE,
1631 "Lower AArch64 MachineInstrs after legalization", false,
1632 false)
1633INITIALIZE_PASS_END(AArch64PostLegalizerLoweringLegacy, DEBUG_TYPE,
1634 "Lower AArch64 MachineInstrs after legalization", false,
1635 false)
1636
1637AArch64PostLegalizerLoweringPass::AArch64PostLegalizerLoweringPass()
1638 : RuleConfig(
1639 std::make_unique<AArch64PostLegalizerLoweringImplRuleConfig>()) {
1640 if (!RuleConfig->parseCommandLineOption())
1641 reportFatalUsageError(reason: "invalid rule identifier");
1642}
1643
1644AArch64PostLegalizerLoweringPass::AArch64PostLegalizerLoweringPass(
1645 AArch64PostLegalizerLoweringPass &&) = default;
1646
1647AArch64PostLegalizerLoweringPass::~AArch64PostLegalizerLoweringPass() = default;
1648
1649PreservedAnalyses
1650AArch64PostLegalizerLoweringPass::run(MachineFunction &MF,
1651 MachineFunctionAnalysisManager &MFAM) {
1652 MFPropsModifier _(*this, MF);
1653 const bool Changed = runPostLegalizerLowering(MF, RuleConfig: *RuleConfig);
1654
1655 if (!Changed)
1656 return PreservedAnalyses::all();
1657
1658 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
1659 PA.preserveSet<CFGAnalyses>();
1660 return PA;
1661}
1662
1663namespace llvm {
1664FunctionPass *createAArch64PostLegalizerLowering() {
1665 return new AArch64PostLegalizerLoweringLegacy();
1666}
1667} // end namespace llvm
1668