1//=== lib/CodeGen/GlobalISel/AArch64PreLegalizerCombiner.cpp --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass does combining of machine instructions at the generic MI level,
10// before the legalizer.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64.h"
15#include "AArch64GlobalISelUtils.h"
16#include "AArch64TargetMachine.h"
17#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
18#include "llvm/CodeGen/GlobalISel/Combiner.h"
19#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
20#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
21#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
22#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
23#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
24#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
25#include "llvm/CodeGen/GlobalISel/Utils.h"
26#include "llvm/CodeGen/LibcallLoweringInfo.h"
27#include "llvm/CodeGen/MachineDominators.h"
28#include "llvm/CodeGen/MachineFunction.h"
29#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
30#include "llvm/CodeGen/MachineFunctionPass.h"
31#include "llvm/CodeGen/MachinePassManager.h"
32#include "llvm/CodeGen/MachineRegisterInfo.h"
33#include "llvm/IR/Instructions.h"
34#include "llvm/IR/IntrinsicsAArch64.h"
35#include <memory>
36
37#define GET_GICOMBINER_DEPS
38#include "AArch64GenPreLegalizeGICombiner.inc"
39#undef GET_GICOMBINER_DEPS
40
41#define DEBUG_TYPE "aarch64-prelegalizer-combiner"
42
43using namespace llvm;
44using namespace MIPatternMatch;
45
46#define GET_GICOMBINER_TYPES
47#include "AArch64GenPreLegalizeGICombiner.inc"
48#undef GET_GICOMBINER_TYPES
49
50namespace {
51
52/// Try to match a G_ICMP of a G_TRUNC with zero, in which the truncated bits
53/// are sign bits. In this case, we can transform the G_ICMP to directly compare
54/// the wide value with a zero.
55bool matchICmpRedundantTrunc(MachineInstr &MI, MachineRegisterInfo &MRI,
56 GISelValueTracking *VT, Register &MatchInfo) {
57 assert(MI.getOpcode() == TargetOpcode::G_ICMP && VT);
58
59 auto Pred = (CmpInst::Predicate)MI.getOperand(i: 1).getPredicate();
60 if (!ICmpInst::isEquality(P: Pred))
61 return false;
62
63 Register LHS = MI.getOperand(i: 2).getReg();
64 LLT LHSTy = MRI.getType(Reg: LHS);
65 if (!LHSTy.isScalar())
66 return false;
67
68 Register RHS = MI.getOperand(i: 3).getReg();
69 Register WideReg;
70
71 if (!mi_match(R: LHS, MRI, P: m_GTrunc(Src: m_Reg(R&: WideReg))) ||
72 !mi_match(R: RHS, MRI, P: m_SpecificICst(RequestedValue: 0)))
73 return false;
74
75 LLT WideTy = MRI.getType(Reg: WideReg);
76 if (VT->computeNumSignBits(R: WideReg) <=
77 WideTy.getSizeInBits() - LHSTy.getSizeInBits())
78 return false;
79
80 MatchInfo = WideReg;
81 return true;
82}
83
84void applyICmpRedundantTrunc(MachineInstr &MI, MachineRegisterInfo &MRI,
85 MachineIRBuilder &Builder,
86 GISelChangeObserver &Observer, Register &WideReg) {
87 assert(MI.getOpcode() == TargetOpcode::G_ICMP);
88
89 LLT WideTy = MRI.getType(Reg: WideReg);
90 // We're going to directly use the wide register as the LHS, and then use an
91 // equivalent size zero for RHS.
92 Builder.setInstrAndDebugLoc(MI);
93 auto WideZero = Builder.buildConstant(Res: WideTy, Val: 0);
94 Observer.changingInstr(MI);
95 MI.getOperand(i: 2).setReg(WideReg);
96 MI.getOperand(i: 3).setReg(WideZero.getReg(Idx: 0));
97 Observer.changedInstr(MI);
98}
99
100/// \returns true if it is possible to fold a constant into a G_GLOBAL_VALUE.
101///
102/// e.g.
103///
104/// %g = G_GLOBAL_VALUE @x -> %g = G_GLOBAL_VALUE @x + cst
105bool matchFoldGlobalOffset(MachineInstr &MI, MachineRegisterInfo &MRI,
106 std::pair<uint64_t, uint64_t> &MatchInfo) {
107 assert(MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
108 MachineFunction &MF = *MI.getMF();
109 auto &GlobalOp = MI.getOperand(i: 1);
110 auto *GV = GlobalOp.getGlobal();
111 if (GV->isThreadLocal())
112 return false;
113
114 // Don't allow anything that could represent offsets etc.
115 if (MF.getSubtarget<AArch64Subtarget>().ClassifyGlobalReference(
116 GV, TM: MF.getTarget()) != AArch64II::MO_NO_FLAG)
117 return false;
118
119 // Look for a G_GLOBAL_VALUE only used by G_PTR_ADDs against constants:
120 //
121 // %g = G_GLOBAL_VALUE @x
122 // %ptr1 = G_PTR_ADD %g, cst1
123 // %ptr2 = G_PTR_ADD %g, cst2
124 // ...
125 // %ptrN = G_PTR_ADD %g, cstN
126 //
127 // Identify the *smallest* constant. We want to be able to form this:
128 //
129 // %offset_g = G_GLOBAL_VALUE @x + min_cst
130 // %g = G_PTR_ADD %offset_g, -min_cst
131 // %ptr1 = G_PTR_ADD %g, cst1
132 // ...
133 Register Dst = MI.getOperand(i: 0).getReg();
134 uint64_t MinOffset = -1ull;
135 for (auto &UseInstr : MRI.use_nodbg_instructions(Reg: Dst)) {
136 if (UseInstr.getOpcode() != TargetOpcode::G_PTR_ADD)
137 return false;
138 auto Cst = getIConstantVRegValWithLookThrough(
139 VReg: UseInstr.getOperand(i: 2).getReg(), MRI);
140 if (!Cst)
141 return false;
142 MinOffset = std::min(a: MinOffset, b: Cst->Value.getZExtValue());
143 }
144
145 // Require that the new offset is larger than the existing one to avoid
146 // infinite loops.
147 uint64_t CurrOffset = GlobalOp.getOffset();
148 uint64_t NewOffset = MinOffset + CurrOffset;
149 if (NewOffset <= CurrOffset)
150 return false;
151
152 // Check whether folding this offset is legal. It must not go out of bounds of
153 // the referenced object to avoid violating the code model, and must be
154 // smaller than 2^20 because this is the largest offset expressible in all
155 // object formats. (The IMAGE_REL_ARM64_PAGEBASE_REL21 relocation in COFF
156 // stores an immediate signed 21 bit offset.)
157 //
158 // This check also prevents us from folding negative offsets, which will end
159 // up being treated in the same way as large positive ones. They could also
160 // cause code model violations, and aren't really common enough to matter.
161 if (NewOffset >= (1 << 20))
162 return false;
163
164 Type *T = GV->getValueType();
165 if (!T->isSized() ||
166 NewOffset > GV->getDataLayout().getTypeAllocSize(Ty: T))
167 return false;
168 MatchInfo = std::make_pair(x&: NewOffset, y&: MinOffset);
169 return true;
170}
171
172void applyFoldGlobalOffset(MachineInstr &MI, MachineRegisterInfo &MRI,
173 MachineIRBuilder &B, GISelChangeObserver &Observer,
174 std::pair<uint64_t, uint64_t> &MatchInfo) {
175 // Change:
176 //
177 // %g = G_GLOBAL_VALUE @x
178 // %ptr1 = G_PTR_ADD %g, cst1
179 // %ptr2 = G_PTR_ADD %g, cst2
180 // ...
181 // %ptrN = G_PTR_ADD %g, cstN
182 //
183 // To:
184 //
185 // %offset_g = G_GLOBAL_VALUE @x + min_cst
186 // %g = G_PTR_ADD %offset_g, -min_cst
187 // %ptr1 = G_PTR_ADD %g, cst1
188 // ...
189 // %ptrN = G_PTR_ADD %g, cstN
190 //
191 // Then, the original G_PTR_ADDs should be folded later on so that they look
192 // like this:
193 //
194 // %ptrN = G_PTR_ADD %offset_g, cstN - min_cst
195 uint64_t Offset, MinOffset;
196 std::tie(args&: Offset, args&: MinOffset) = MatchInfo;
197 B.setInstrAndDebugLoc(*std::next(x: MI.getIterator()));
198 Observer.changingInstr(MI);
199 auto &GlobalOp = MI.getOperand(i: 1);
200 auto *GV = GlobalOp.getGlobal();
201 GlobalOp.ChangeToGA(GV, Offset, TargetFlags: GlobalOp.getTargetFlags());
202 Register Dst = MI.getOperand(i: 0).getReg();
203 Register NewGVDst = MRI.cloneVirtualRegister(VReg: Dst);
204 MI.getOperand(i: 0).setReg(NewGVDst);
205 Observer.changedInstr(MI);
206 B.buildPtrAdd(
207 Res: Dst, Op0: NewGVDst,
208 Op1: B.buildConstant(Res: LLT::scalar(SizeInBits: 64), Val: -static_cast<int64_t>(MinOffset)));
209}
210
211// Combines vecreduce_add(mul(ext(x), ext(y))) -> vecreduce_add([us]dot(x, y))
212// Or vecreduce_add(ext(mul(ext(x), ext(y)))) -> vecreduce_add([us]dot(x, y))
213// Or vecreduce_add(ext(x)) -> vecreduce_add([us]dot(x, 1))
214// Similar to performVecReduceAddCombine in SelectionDAG
215bool matchExtAddvToDotAddv(MachineInstr &MI, MachineRegisterInfo &MRI,
216 const AArch64Subtarget &STI,
217 std::tuple<Register, Register, bool> &MatchInfo) {
218 assert(MI.getOpcode() == TargetOpcode::G_VECREDUCE_ADD &&
219 "Expected a G_VECREDUCE_ADD instruction");
220 assert(STI.hasDotProd() && "Target should have Dot Product feature");
221
222 MachineInstr *I1 = getDefIgnoringCopies(Reg: MI.getOperand(i: 1).getReg(), MRI);
223 Register DstReg = MI.getOperand(i: 0).getReg();
224 Register MidReg = I1->getOperand(i: 0).getReg();
225 LLT DstTy = MRI.getType(Reg: DstReg);
226 LLT MidTy = MRI.getType(Reg: MidReg);
227 if (DstTy.getScalarSizeInBits() != 32 || MidTy.getScalarSizeInBits() != 32)
228 return false;
229
230 // Detect mul(ext, ext) with symmetric ext's. If I1Opc is G_ZEXT or G_SEXT
231 // then the ext's must match the same opcode. It is set to the ext opcode on
232 // output.
233 auto tryMatchingMulOfExt = [&MRI](MachineInstr *MI, Register &Out1,
234 Register &Out2, unsigned &I1Opc) {
235 // If result of this has more than 1 use, then there is no point in creating
236 // a dot instruction
237 if (!MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
238 return false;
239
240 MachineInstr *ExtMI1 =
241 getDefIgnoringCopies(Reg: MI->getOperand(i: 1).getReg(), MRI);
242 MachineInstr *ExtMI2 =
243 getDefIgnoringCopies(Reg: MI->getOperand(i: 2).getReg(), MRI);
244 LLT Ext1DstTy = MRI.getType(Reg: ExtMI1->getOperand(i: 0).getReg());
245 LLT Ext2DstTy = MRI.getType(Reg: ExtMI2->getOperand(i: 0).getReg());
246
247 if (ExtMI1->getOpcode() != ExtMI2->getOpcode() || Ext1DstTy != Ext2DstTy)
248 return false;
249 if ((I1Opc == TargetOpcode::G_ZEXT || I1Opc == TargetOpcode::G_SEXT) &&
250 I1Opc != ExtMI1->getOpcode())
251 return false;
252 Out1 = ExtMI1->getOperand(i: 1).getReg();
253 Out2 = ExtMI2->getOperand(i: 1).getReg();
254 I1Opc = ExtMI1->getOpcode();
255 return true;
256 };
257
258 LLT SrcTy;
259 unsigned I1Opc = I1->getOpcode();
260 if (I1Opc == TargetOpcode::G_MUL) {
261 Register Out1, Out2;
262 if (!tryMatchingMulOfExt(I1, Out1, Out2, I1Opc))
263 return false;
264 SrcTy = MRI.getType(Reg: Out1);
265 std::get<0>(t&: MatchInfo) = Out1;
266 std::get<1>(t&: MatchInfo) = Out2;
267 } else if (I1Opc == TargetOpcode::G_ZEXT || I1Opc == TargetOpcode::G_SEXT) {
268 Register I1Op = I1->getOperand(i: 1).getReg();
269 MachineInstr *M = getDefIgnoringCopies(Reg: I1Op, MRI);
270 Register Out1, Out2;
271 if (M->getOpcode() == TargetOpcode::G_MUL &&
272 tryMatchingMulOfExt(M, Out1, Out2, I1Opc)) {
273 SrcTy = MRI.getType(Reg: Out1);
274 std::get<0>(t&: MatchInfo) = Out1;
275 std::get<1>(t&: MatchInfo) = Out2;
276 } else {
277 SrcTy = MRI.getType(Reg: I1Op);
278 std::get<0>(t&: MatchInfo) = I1Op;
279 std::get<1>(t&: MatchInfo) = 0;
280 }
281 } else {
282 return false;
283 }
284
285 if (I1Opc == TargetOpcode::G_ZEXT)
286 std::get<2>(t&: MatchInfo) = 0;
287 else if (I1Opc == TargetOpcode::G_SEXT)
288 std::get<2>(t&: MatchInfo) = 1;
289 else
290 return false;
291
292 if (SrcTy.getScalarSizeInBits() != 8 || SrcTy.getNumElements() % 8 != 0)
293 return false;
294
295 return true;
296}
297
298void applyExtAddvToDotAddv(MachineInstr &MI, MachineRegisterInfo &MRI,
299 MachineIRBuilder &Builder,
300 GISelChangeObserver &Observer,
301 const AArch64Subtarget &STI,
302 std::tuple<Register, Register, bool> &MatchInfo) {
303 assert(MI.getOpcode() == TargetOpcode::G_VECREDUCE_ADD &&
304 "Expected a G_VECREDUCE_ADD instruction");
305 assert(STI.hasDotProd() && "Target should have Dot Product feature");
306
307 // Initialise the variables
308 unsigned DotOpcode =
309 std::get<2>(t&: MatchInfo) ? AArch64::G_SDOT : AArch64::G_UDOT;
310 Register Ext1SrcReg = std::get<0>(t&: MatchInfo);
311
312 // If there is one source register, create a vector of 0s as the second
313 // source register
314 Register Ext2SrcReg;
315 if (std::get<1>(t&: MatchInfo) == 0)
316 Ext2SrcReg = Builder.buildConstant(Res: MRI.getType(Reg: Ext1SrcReg), Val: 1)
317 ->getOperand(i: 0)
318 .getReg();
319 else
320 Ext2SrcReg = std::get<1>(t&: MatchInfo);
321
322 // Find out how many DOT instructions are needed
323 LLT SrcTy = MRI.getType(Reg: Ext1SrcReg);
324 LLT MidTy;
325 unsigned NumOfDotMI;
326 if (SrcTy.getNumElements() % 16 == 0) {
327 NumOfDotMI = SrcTy.getNumElements() / 16;
328 MidTy = LLT::fixed_vector(NumElements: 4, ScalarTy: LLT::integer(SizeInBits: 32));
329 } else if (SrcTy.getNumElements() % 8 == 0) {
330 NumOfDotMI = SrcTy.getNumElements() / 8;
331 MidTy = LLT::fixed_vector(NumElements: 2, ScalarTy: LLT::integer(SizeInBits: 32));
332 } else {
333 llvm_unreachable("Source type number of elements is not multiple of 8");
334 }
335
336 // Handle case where one DOT instruction is needed
337 if (NumOfDotMI == 1) {
338 auto Zeroes = Builder.buildConstant(Res: MidTy, Val: 0)->getOperand(i: 0).getReg();
339 auto Dot = Builder.buildInstr(Opc: DotOpcode, DstOps: {MidTy},
340 SrcOps: {Zeroes, Ext1SrcReg, Ext2SrcReg});
341 Builder.buildVecReduceAdd(Dst: MI.getOperand(i: 0), Src: Dot->getOperand(i: 0));
342 } else {
343 // If not pad the last v8 element with 0s to a v16
344 SmallVector<Register, 4> Ext1UnmergeReg;
345 SmallVector<Register, 4> Ext2UnmergeReg;
346 if (SrcTy.getNumElements() % 16 != 0) {
347 SmallVector<Register> Leftover1;
348 SmallVector<Register> Leftover2;
349
350 // Split the elements into v16i8 and v8i8
351 LLT MainTy = LLT::fixed_vector(NumElements: 16, ScalarTy: LLT::integer(SizeInBits: 8));
352 LLT LeftoverTy1, LeftoverTy2;
353 if (!extractParts(Reg: Ext1SrcReg, RegTy: MRI.getType(Reg: Ext1SrcReg), MainTy,
354 LeftoverTy&: LeftoverTy1, VRegs&: Ext1UnmergeReg, LeftoverVRegs&: Leftover1, MIRBuilder&: Builder, MRI) ||
355 !extractParts(Reg: Ext2SrcReg, RegTy: MRI.getType(Reg: Ext2SrcReg), MainTy,
356 LeftoverTy&: LeftoverTy2, VRegs&: Ext2UnmergeReg, LeftoverVRegs&: Leftover2, MIRBuilder&: Builder, MRI)) {
357 llvm_unreachable("Unable to split this vector properly");
358 }
359
360 // Pad the leftover v8i8 vector with register of 0s of type v8i8
361 auto v8Zeroes =
362 Builder.buildConstant(Res: LLT::fixed_vector(NumElements: 8, ScalarTy: LLT::integer(SizeInBits: 8)), Val: 0);
363
364 Ext1UnmergeReg.push_back(
365 Elt: Builder
366 .buildMergeLikeInstr(Res: LLT::fixed_vector(NumElements: 16, ScalarTy: LLT::integer(SizeInBits: 8)),
367 Ops: {Leftover1[0], v8Zeroes})
368 .getReg(Idx: 0));
369 Ext2UnmergeReg.push_back(
370 Elt: Builder
371 .buildMergeLikeInstr(Res: LLT::fixed_vector(NumElements: 16, ScalarTy: LLT::integer(SizeInBits: 8)),
372 Ops: {Leftover2[0], v8Zeroes})
373 .getReg(Idx: 0));
374
375 } else {
376 // Unmerge the source vectors to v16i8
377 unsigned SrcNumElts = SrcTy.getNumElements();
378 extractParts(Reg: Ext1SrcReg, Ty: LLT::fixed_vector(NumElements: 16, ScalarTy: LLT::integer(SizeInBits: 8)),
379 NumParts: SrcNumElts / 16, VRegs&: Ext1UnmergeReg, MIRBuilder&: Builder, MRI);
380 extractParts(Reg: Ext2SrcReg, Ty: LLT::fixed_vector(NumElements: 16, ScalarTy: LLT::integer(SizeInBits: 8)),
381 NumParts: SrcNumElts / 16, VRegs&: Ext2UnmergeReg, MIRBuilder&: Builder, MRI);
382 }
383
384 // Build the UDOT instructions
385 SmallVector<Register, 2> DotReg;
386 unsigned NumElements = 0;
387 for (unsigned i = 0; i < Ext1UnmergeReg.size(); i++) {
388 LLT ZeroesLLT;
389 // Check if it is 16 or 8 elements. Set Zeroes to the according size
390 if (MRI.getType(Reg: Ext1UnmergeReg[i]).getNumElements() == 16) {
391 ZeroesLLT = LLT::fixed_vector(NumElements: 4, ScalarTy: LLT::integer(SizeInBits: 32));
392 NumElements += 4;
393 } else {
394 ZeroesLLT = LLT::fixed_vector(NumElements: 2, ScalarTy: LLT::integer(SizeInBits: 32));
395 NumElements += 2;
396 }
397 auto Zeroes = Builder.buildConstant(Res: ZeroesLLT, Val: 0);
398 DotReg.push_back(
399 Elt: Builder
400 .buildInstr(Opc: DotOpcode, DstOps: {ZeroesLLT},
401 SrcOps: {Zeroes, Ext1UnmergeReg[i], Ext2UnmergeReg[i]})
402 .getReg(Idx: 0));
403 }
404
405 // Merge the output
406 auto ConcatMI = Builder.buildConcatVectors(
407 Res: LLT::fixed_vector(NumElements, ScalarTy: LLT::integer(SizeInBits: 32)), Ops: DotReg);
408
409 // Put it through a vector reduction
410 Builder.buildVecReduceAdd(Dst: MI.getOperand(i: 0).getReg(),
411 Src: ConcatMI->getOperand(i: 0).getReg());
412 }
413
414 // Erase the dead instructions
415 MI.eraseFromParent();
416}
417
418// Matches {U/S}ADDV(ext(x)) => {U/S}ADDLV(x)
419// Ensure that the type coming from the extend instruction is the right size
420bool matchExtUaddvToUaddlv(MachineInstr &MI, MachineRegisterInfo &MRI,
421 std::pair<Register, bool> &MatchInfo) {
422 assert(MI.getOpcode() == TargetOpcode::G_VECREDUCE_ADD &&
423 "Expected G_VECREDUCE_ADD Opcode");
424
425 // Check if the last instruction is an extend
426 MachineInstr *ExtMI = getDefIgnoringCopies(Reg: MI.getOperand(i: 1).getReg(), MRI);
427 auto ExtOpc = ExtMI->getOpcode();
428
429 if (ExtOpc == TargetOpcode::G_ZEXT)
430 std::get<1>(in&: MatchInfo) = 0;
431 else if (ExtOpc == TargetOpcode::G_SEXT)
432 std::get<1>(in&: MatchInfo) = 1;
433 else
434 return false;
435
436 // Check if the source register is a valid type
437 Register ExtSrcReg = ExtMI->getOperand(i: 1).getReg();
438 LLT ExtSrcTy = MRI.getType(Reg: ExtSrcReg);
439 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
440 if (ExtSrcTy.getScalarSizeInBits() * 2 > DstTy.getScalarSizeInBits())
441 return false;
442 if ((DstTy.getScalarSizeInBits() == 16 &&
443 ExtSrcTy.getNumElements() % 8 == 0 && ExtSrcTy.getNumElements() < 256) ||
444 (DstTy.getScalarSizeInBits() == 32 &&
445 ExtSrcTy.getNumElements() % 4 == 0) ||
446 (DstTy.getScalarSizeInBits() == 64 &&
447 ExtSrcTy.getNumElements() % 4 == 0)) {
448 std::get<0>(in&: MatchInfo) = ExtSrcReg;
449 return true;
450 }
451 return false;
452}
453
454void applyExtUaddvToUaddlv(MachineInstr &MI, MachineRegisterInfo &MRI,
455 MachineIRBuilder &B, GISelChangeObserver &Observer,
456 std::pair<Register, bool> &MatchInfo) {
457 assert(MI.getOpcode() == TargetOpcode::G_VECREDUCE_ADD &&
458 "Expected G_VECREDUCE_ADD Opcode");
459
460 unsigned Opc = std::get<1>(in&: MatchInfo) ? AArch64::G_SADDLV : AArch64::G_UADDLV;
461 Register SrcReg = std::get<0>(in&: MatchInfo);
462 Register DstReg = MI.getOperand(i: 0).getReg();
463 LLT SrcTy = MRI.getType(Reg: SrcReg);
464 LLT DstTy = MRI.getType(Reg: DstReg);
465
466 // If SrcTy has more elements than expected, split them into multiple
467 // instructions and sum the results
468 LLT MainTy;
469 SmallVector<Register, 1> WorkingRegisters;
470 unsigned SrcScalSize = SrcTy.getScalarSizeInBits();
471 unsigned SrcNumElem = SrcTy.getNumElements();
472 if ((SrcScalSize == 8 && SrcNumElem > 16) ||
473 (SrcScalSize == 16 && SrcNumElem > 8) ||
474 (SrcScalSize == 32 && SrcNumElem > 4)) {
475
476 LLT LeftoverTy;
477 SmallVector<Register, 4> LeftoverRegs;
478 if (SrcScalSize == 8)
479 MainTy = LLT::fixed_vector(NumElements: 16, ScalarTy: LLT::integer(SizeInBits: 8));
480 else if (SrcScalSize == 16)
481 MainTy = LLT::fixed_vector(NumElements: 8, ScalarTy: LLT::integer(SizeInBits: 16));
482 else if (SrcScalSize == 32)
483 MainTy = LLT::fixed_vector(NumElements: 4, ScalarTy: LLT::integer(SizeInBits: 32));
484 else
485 llvm_unreachable("Source's Scalar Size not supported");
486
487 // Extract the parts and put each extracted sources through U/SADDLV and put
488 // the values inside a small vec
489 extractParts(Reg: SrcReg, RegTy: SrcTy, MainTy, LeftoverTy, VRegs&: WorkingRegisters,
490 LeftoverVRegs&: LeftoverRegs, MIRBuilder&: B, MRI);
491 llvm::append_range(C&: WorkingRegisters, R&: LeftoverRegs);
492 } else {
493 WorkingRegisters.push_back(Elt: SrcReg);
494 MainTy = SrcTy;
495 }
496
497 unsigned MidScalarSize = MainTy.getScalarSizeInBits() * 2;
498 LLT MidScalarLLT = LLT::integer(SizeInBits: MidScalarSize);
499 Register ZeroReg = B.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: 0).getReg(Idx: 0);
500 for (unsigned I = 0; I < WorkingRegisters.size(); I++) {
501 // If the number of elements is too small to build an instruction, extend
502 // its size before applying addlv
503 LLT WorkingRegTy = MRI.getType(Reg: WorkingRegisters[I]);
504 if ((WorkingRegTy.getScalarSizeInBits() == 8) &&
505 (WorkingRegTy.getNumElements() == 4)) {
506 WorkingRegisters[I] =
507 B.buildInstr(Opc: std::get<1>(in&: MatchInfo) ? TargetOpcode::G_SEXT
508 : TargetOpcode::G_ZEXT,
509 DstOps: {LLT::fixed_vector(NumElements: 4, ScalarTy: LLT::integer(SizeInBits: 16))},
510 SrcOps: {WorkingRegisters[I]})
511 .getReg(Idx: 0);
512 }
513
514 // Generate the {U/S}ADDLV instruction, whose output is always double of the
515 // Src's Scalar size
516 LLT AddlvTy = MidScalarSize <= 32 ? LLT::fixed_vector(NumElements: 4, ScalarTy: LLT::integer(SizeInBits: 32))
517 : LLT::fixed_vector(NumElements: 2, ScalarTy: LLT::integer(SizeInBits: 64));
518 Register AddlvReg =
519 B.buildInstr(Opc, DstOps: {AddlvTy}, SrcOps: {WorkingRegisters[I]}).getReg(Idx: 0);
520
521 // The output from {U/S}ADDLV gets placed in the lowest lane of a v4i32 or
522 // v2i64 register.
523 // i16, i32 results uses v4i32 registers
524 // i64 results uses v2i64 registers
525 // Therefore we have to extract/truncate the the value to the right type
526 if (MidScalarSize == 32 || MidScalarSize == 64) {
527 WorkingRegisters[I] = B.buildInstr(Opc: AArch64::G_EXTRACT_VECTOR_ELT,
528 DstOps: {MidScalarLLT}, SrcOps: {AddlvReg, ZeroReg})
529 .getReg(Idx: 0);
530 } else {
531 Register ExtractReg =
532 B.buildInstr(Opc: AArch64::G_EXTRACT_VECTOR_ELT, DstOps: {LLT::integer(SizeInBits: 32)},
533 SrcOps: {AddlvReg, ZeroReg})
534 .getReg(Idx: 0);
535 WorkingRegisters[I] =
536 B.buildTrunc(Res: {MidScalarLLT}, Op: {ExtractReg}).getReg(Idx: 0);
537 }
538 }
539
540 Register OutReg;
541 if (WorkingRegisters.size() > 1) {
542 OutReg = B.buildAdd(Dst: MidScalarLLT, Src0: WorkingRegisters[0], Src1: WorkingRegisters[1])
543 .getReg(Idx: 0);
544 for (unsigned I = 2; I < WorkingRegisters.size(); I++) {
545 OutReg = B.buildAdd(Dst: MidScalarLLT, Src0: OutReg, Src1: WorkingRegisters[I]).getReg(Idx: 0);
546 }
547 } else {
548 OutReg = WorkingRegisters[0];
549 }
550
551 if (DstTy.getScalarSizeInBits() > MidScalarSize) {
552 // Handle the scalar value if the DstTy's Scalar Size is more than double
553 // Src's ScalarType
554 B.buildInstr(Opc: std::get<1>(in&: MatchInfo) ? TargetOpcode::G_SEXT
555 : TargetOpcode::G_ZEXT,
556 DstOps: {DstReg}, SrcOps: {OutReg});
557 } else {
558 B.buildCopy(Res: DstReg, Op: OutReg);
559 }
560
561 MI.eraseFromParent();
562}
563
564// Pushes ADD/SUB/MUL through extend instructions to decrease the number of
565// extend instruction at the end by allowing selection of {s|u}addl sooner
566// i32 add(i32 ext i8, i32 ext i8) => i32 ext(i16 add(i16 ext i8, i16 ext i8))
567bool matchPushAddSubExt(MachineInstr &MI, MachineRegisterInfo &MRI,
568 Register DstReg, Register SrcReg1, Register SrcReg2) {
569 assert((MI.getOpcode() == TargetOpcode::G_ADD ||
570 MI.getOpcode() == TargetOpcode::G_SUB ||
571 MI.getOpcode() == TargetOpcode::G_MUL) &&
572 "Expected a G_ADD, G_SUB or G_MUL instruction\n");
573
574 // Deal with vector types only
575 LLT DstTy = MRI.getType(Reg: DstReg);
576 if (!DstTy.isVector())
577 return false;
578
579 // Return true if G_{S|Z}EXT instruction is more than 2* source
580 Register ExtDstReg = MI.getOperand(i: 1).getReg();
581 LLT Ext1SrcTy = MRI.getType(Reg: SrcReg1);
582 LLT Ext2SrcTy = MRI.getType(Reg: SrcReg2);
583 unsigned ExtDstScal = MRI.getType(Reg: ExtDstReg).getScalarSizeInBits();
584 unsigned Ext1SrcScal = Ext1SrcTy.getScalarSizeInBits();
585 if (((Ext1SrcScal == 8 && ExtDstScal == 32) ||
586 ((Ext1SrcScal == 8 || Ext1SrcScal == 16) && ExtDstScal == 64)) &&
587 Ext1SrcTy == Ext2SrcTy)
588 return true;
589
590 return false;
591}
592
593void applyPushAddSubExt(MachineInstr &MI, MachineRegisterInfo &MRI,
594 MachineIRBuilder &B, bool isSExt, Register DstReg,
595 Register SrcReg1, Register SrcReg2) {
596 LLT SrcTy = MRI.getType(Reg: SrcReg1);
597 LLT MidTy = SrcTy.changeElementSize(NewEltSize: SrcTy.getScalarSizeInBits() * 2);
598 unsigned Opc = isSExt ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
599 Register Ext1Reg = B.buildInstr(Opc, DstOps: {MidTy}, SrcOps: {SrcReg1}).getReg(Idx: 0);
600 Register Ext2Reg = B.buildInstr(Opc, DstOps: {MidTy}, SrcOps: {SrcReg2}).getReg(Idx: 0);
601 Register AddReg =
602 B.buildInstr(Opc: MI.getOpcode(), DstOps: {MidTy}, SrcOps: {Ext1Reg, Ext2Reg}).getReg(Idx: 0);
603
604 // G_SUB has to sign-extend the result.
605 // G_ADD needs to sext from sext and can sext or zext from zext, and G_MUL
606 // needs to use the original opcode so the original opcode is used for both.
607 if (MI.getOpcode() == TargetOpcode::G_ADD ||
608 MI.getOpcode() == TargetOpcode::G_MUL)
609 B.buildInstr(Opc, DstOps: {DstReg}, SrcOps: {AddReg});
610 else
611 B.buildSExt(Res: DstReg, Op: AddReg);
612
613 MI.eraseFromParent();
614}
615
616bool matchSimplifyUADDO(MachineInstr &MI, MachineRegisterInfo &MRI,
617 std::pair<Register, Register> &MatchInfo) {
618 // Try simplify G_UADDO with 8 or 16 bit operands to wide G_ADD and TBNZ if
619 // result is only used in the no-overflow case. It is restricted to cases
620 // where we know that the high-bits of the operands are 0. If there's an
621 // overflow, then the 9th or 17th bit must be set, which can be checked
622 // using TBNZ.
623 //
624 // Change (for UADDOs on 8 and 16 bits):
625 //
626 // %z0 = G_ASSERT_ZEXT _
627 // %op0 = G_TRUNC %z0
628 // %z1 = G_ASSERT_ZEXT _
629 // %op1 = G_TRUNC %z1
630 // %val, %cond = G_UADDO %op0, %op1
631 // G_BRCOND %cond, %error.bb
632 //
633 // error.bb:
634 // (no successors and no uses of %val)
635 //
636 // To:
637 //
638 // %z0 = G_ASSERT_ZEXT _
639 // %z1 = G_ASSERT_ZEXT _
640 // %add = G_ADD %z0, %z1
641 // %val = G_TRUNC %add
642 // %bit = G_AND %add, 1 << scalar-size-in-bits(%op1)
643 // %cond = G_ICMP NE, %bit, 0
644 // G_BRCOND %cond, %error.bb
645
646 MachineOperand *DefOp0 = MRI.getOneDef(Reg: MI.getOperand(i: 2).getReg());
647 MachineOperand *DefOp1 = MRI.getOneDef(Reg: MI.getOperand(i: 3).getReg());
648 Register Op0Wide;
649 Register Op1Wide;
650 if (!mi_match(R: DefOp0->getParent(), MRI, P: m_GTrunc(Src: m_Reg(R&: Op0Wide))) ||
651 !mi_match(R: DefOp1->getParent(), MRI, P: m_GTrunc(Src: m_Reg(R&: Op1Wide))))
652 return false;
653 LLT WideTy0 = MRI.getType(Reg: Op0Wide);
654 LLT WideTy1 = MRI.getType(Reg: Op1Wide);
655 Register ResVal = MI.getOperand(i: 0).getReg();
656 LLT OpTy = MRI.getType(Reg: ResVal);
657 unsigned OpTySize = OpTy.getScalarSizeInBits();
658 // First check that the G_TRUNC feeding the G_UADDO are no-ops, because the
659 // inputs have been zero-extended.
660 if (!mi_match(R: Op0Wide, MRI,
661 P: m_GAssertZext(Src: m_Reg(), Imm: m_SpecificImm(RequestedValue: OpTySize))) ||
662 !mi_match(R: Op1Wide, MRI, P: m_GAssertZext(Src: m_Reg(), Imm: m_SpecificImm(RequestedValue: OpTySize))))
663 return false;
664
665 // Only scalar UADDO with either 8 or 16 bit operands are handled.
666 if (!WideTy0.isScalar() || !WideTy1.isScalar() || WideTy0 != WideTy1 ||
667 OpTySize >= WideTy0.getScalarSizeInBits() ||
668 (OpTySize != 8 && OpTySize != 16))
669 return false;
670
671 // The overflow-status result must be used by a branch only.
672 Register ResStatus = MI.getOperand(i: 1).getReg();
673 if (!MRI.hasOneNonDBGUse(RegNo: ResStatus))
674 return false;
675 MachineInstr *CondUser = &*MRI.use_instr_nodbg_begin(RegNo: ResStatus);
676 if (CondUser->getOpcode() != TargetOpcode::G_BRCOND)
677 return false;
678
679 // Make sure the computed result is only used in the no-overflow blocks.
680 MachineBasicBlock *CurrentMBB = MI.getParent();
681 MachineBasicBlock *FailMBB = CondUser->getOperand(i: 1).getMBB();
682 if (!FailMBB->succ_empty() || CondUser->getParent() != CurrentMBB)
683 return false;
684 if (any_of(Range: MRI.use_nodbg_instructions(Reg: ResVal),
685 P: [&MI, FailMBB, CurrentMBB](MachineInstr &I) {
686 return &MI != &I &&
687 (I.getParent() == FailMBB || I.getParent() == CurrentMBB);
688 }))
689 return false;
690
691 MatchInfo = {Op0Wide, Op1Wide};
692 return true;
693}
694
695void applySimplifyUADDO(MachineInstr &MI, MachineRegisterInfo &MRI,
696 MachineIRBuilder &B, GISelChangeObserver &Observer,
697 const CombinerHelper &Helper,
698 const std::pair<Register, Register> &MatchInfo) {
699 Register Op0Wide = MatchInfo.first;
700 Register Op1Wide = MatchInfo.second;
701 Register ResVal = MI.getOperand(i: 0).getReg();
702 Register ResStatus = MI.getOperand(i: 1).getReg();
703 unsigned OpTySize = MRI.getType(Reg: ResVal).getScalarSizeInBits();
704
705 // Remove G_UADDO.
706 B.setInstrAndDebugLoc(*MI.getNextNode());
707 MI.eraseFromParent();
708
709 // Emit wide add.
710 Register AddDst = MRI.cloneVirtualRegister(VReg: Op0Wide);
711 B.buildInstr(Opc: TargetOpcode::G_ADD, DstOps: {AddDst}, SrcOps: {Op0Wide, Op1Wide});
712
713 // Emit check of the 9th or 17th bit and update users (the branch). This will
714 // later be folded to TBNZ.
715 Register CondBit = MRI.cloneVirtualRegister(VReg: Op0Wide);
716 B.buildAnd(
717 Dst: CondBit, Src0: AddDst,
718 Src1: B.buildConstant(Res: LLT::integer(SizeInBits: 32), Val: OpTySize == 8 ? 1 << 8 : 1 << 16));
719 B.buildICmp(Pred: CmpInst::ICMP_NE, Res: ResStatus, Op0: CondBit,
720 Op1: B.buildConstant(Res: LLT::integer(SizeInBits: 32), Val: 0));
721
722 // Update ZEXts users of the result value. Because all uses are in the
723 // no-overflow case, we know that the top bits are 0 and we can ignore ZExts.
724 B.buildZExtOrTrunc(Res: ResVal, Op: AddDst);
725 for (MachineOperand &U : make_early_inc_range(Range: MRI.use_operands(Reg: ResVal))) {
726 Register WideReg;
727 if (mi_match(R: U.getParent(), MRI, P: m_GZExt(Src: m_Reg(R&: WideReg)))) {
728 auto OldR = U.getParent()->getOperand(i: 0).getReg();
729 Observer.erasingInstr(MI&: *U.getParent());
730 U.getParent()->eraseFromParent();
731 Helper.replaceRegWith(MRI, FromReg: OldR, ToReg: AddDst);
732 }
733 }
734}
735
736// Look for load + shuffles that can be converted to LD2/LD3/LD4
737bool matchDeinterleavingLoad(MachineInstr &MI, MachineRegisterInfo &MRI,
738 SmallVector<Register> &Srcs) {
739 GLoad &Load = cast<GLoad>(Val&: MI);
740 LLT Ty = MRI.getType(Reg: Load.getDstReg());
741 if (!Ty.isVector() || !Load.isSimple() ||
742 Load.getMemSizeInBits() != Ty.getSizeInBits())
743 return false;
744
745 switch (Ty.getScalarSizeInBits()) {
746 case 8:
747 case 16:
748 case 32:
749 case 64:
750 break;
751 default:
752 return false;
753 }
754
755 // Find shuffle uses that deinterleave the result. We are generally looking
756 // for Factor deinterleaves of each value, but some might be missing.
757 unsigned Factor = 0;
758 for (MachineInstr &Use : MRI.use_nodbg_instructions(Reg: Load.getDstReg())) {
759 auto *Shuf = dyn_cast<GShuffleVector>(Val: &Use);
760 if (!Shuf || Shuf->getSrc1Reg() != Load.getDstReg())
761 return false;
762 ArrayRef<int> Mask = Shuf->getMask();
763
764 if (!Factor) {
765 if (ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 2))
766 Factor = 2;
767 else if (ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 3))
768 Factor = 3;
769 else if (ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 4))
770 Factor = 4;
771 else
772 return false;
773
774 Srcs.resize(N: Factor);
775 }
776
777 LLT SubTy = MRI.getType(Reg: Shuf->getReg(Idx: 0));
778 if (Ty.getSizeInBits() != SubTy.getSizeInBits() * Factor ||
779 SubTy.getSizeInBits() % 64 != 0)
780 return false;
781
782 unsigned Index;
783 if (!ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor, Index) ||
784 Index >= Factor || Srcs[Index].isValid())
785 return false;
786 Srcs[Index] = Shuf->getReg(Idx: 0);
787 }
788
789 return Factor != 0;
790}
791
792void applyDeinterleavingLoad(MachineInstr &MI, MachineRegisterInfo &MRI,
793 MachineIRBuilder &B, SmallVector<Register> &Srcs) {
794 unsigned Factor = Srcs.size();
795 unsigned IID;
796 switch (Factor) {
797 case 2:
798 IID = Intrinsic::aarch64_neon_ld2;
799 break;
800 case 3:
801 IID = Intrinsic::aarch64_neon_ld3;
802 break;
803 case 4:
804 IID = Intrinsic::aarch64_neon_ld4;
805 break;
806 default:
807 llvm_unreachable("Unexpected deinterleave size!");
808 }
809
810 Register Pointer = MI.getOperand(i: 1).getReg();
811 assert(MI.getNumMemOperands() == 1);
812 MachineMemOperand *MMO = *MI.memoperands_begin();
813 MachineFunction &MF = *MI.getParent()->getParent();
814
815 // Generate a LD2/3/4 per vector part into Srcs registers. Blank/noreg
816 // elements will be filled with a valid register.
817 auto generateLoad = [&](SmallVector<Register> &Srcs, LLT Ty,
818 unsigned Offset) {
819 auto PtrOffset = Pointer;
820 if (Offset != 0)
821 PtrOffset =
822 B.buildObjectPtrOffset(Res: MRI.getType(Reg: Pointer), Op0: Pointer,
823 Op1: B.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: Offset))
824 .getReg(Idx: 0);
825 for (Register &S : Srcs)
826 if (!S.isValid())
827 S = MRI.createGenericVirtualRegister(Ty);
828 auto LD = B.buildIntrinsic(ID: IID, Res: Srcs);
829 LD.addReg(RegNo: PtrOffset);
830 LD.addMemOperand(
831 MMO: MF.getMachineMemOperand(MMO, Offset, Ty: Ty.multiplyElements(Factor)));
832 };
833
834 // Erase the old instructions
835 for (Register Src : Srcs)
836 if (Src.isValid())
837 MRI.getVRegDef(Reg: Src)->eraseFromParent();
838
839 // Generate loads split into 128 / 64bit chunks concat together.
840 LLT Ty = MRI.getType(Reg: *find_if(Range&: Srcs, P: [](Register R) { return R.isValid(); }));
841 LLT MainTy =
842 LLT::fixed_vector(NumElements: 128 / Ty.getScalarSizeInBits(), ScalarTy: Ty.getScalarType());
843 if (Ty == MainTy || Ty == MainTy.divide(Factor: 2)) {
844 // For 128bit and 64bit vectors we build the ld2/ld3/ld4 inplace
845 generateLoad(Srcs, Ty, 0);
846 } else {
847 // For larger vectors we split into more smaller chunks of size 128bit with
848 // a possible 64bit remainder.
849 unsigned NumVecs = Ty.getSizeInBits() / 128;
850 SmallVector<SmallVector<Register>> Regs;
851 for (unsigned I = 0; I < NumVecs; I++) {
852 Regs.push_back(Elt: SmallVector<Register>(Factor));
853 generateLoad(Regs.back(), MainTy, I * 16 * Factor);
854 }
855 if (Ty.getSizeInBits() % 128 == 0) {
856 // Merge all Regs into the final result vectors.
857 for (unsigned R = 0; R < Factor; R++) {
858 if (!Srcs[R].isValid())
859 continue;
860 SmallVector<Register> Ops;
861 for (unsigned I = 0; I < NumVecs; I++)
862 Ops.push_back(Elt: Regs[I][R]);
863 B.buildMergeLikeInstr(Res: Srcs[R], Ops);
864 }
865 } else {
866 LLT LeftOverTy = MainTy.divide(Factor: 2);
867 SmallVector<Register> LeftOverRegs(Factor);
868 generateLoad(LeftOverRegs, LeftOverTy, NumVecs * 16 * Factor);
869
870 // Split up Regs to LeftOverTy and combine back to larger vectors.
871 for (unsigned R = 0; R < Factor; R++) {
872 if (!Srcs[R].isValid())
873 continue;
874 SmallVector<Register> Ops;
875 for (unsigned I = 0; I < NumVecs; I++) {
876 auto Unmerge = B.buildUnmerge(Res: LeftOverTy, Op: Regs[I][R]);
877 Ops.push_back(Elt: Unmerge.getReg(Idx: 0));
878 Ops.push_back(Elt: Unmerge.getReg(Idx: 1));
879 }
880 Ops.push_back(Elt: LeftOverRegs[R]);
881 B.buildMergeLikeInstr(Res: Srcs[R], Ops);
882 }
883 }
884 }
885
886 MI.eraseFromParent();
887}
888
889// Look for store + shuffles that can be converted to ST2/ST3/ST4
890bool matchInterleavingStore(MachineInstr &MI, MachineRegisterInfo &MRI,
891 SmallVector<Register> &Srcs) {
892 GStore &Store = cast<GStore>(Val&: MI);
893 LLT Ty = MRI.getType(Reg: Store.getValueReg());
894 if (!Ty.isVector() || !Store.isSimple() ||
895 Store.getMemSizeInBits() != Ty.getSizeInBits() ||
896 !MRI.hasOneNonDBGUse(RegNo: Store.getValueReg()))
897 return false;
898
899 switch (Ty.getScalarSizeInBits()) {
900 case 8:
901 case 16:
902 case 32:
903 case 64:
904 break;
905 default:
906 return false;
907 }
908
909 MachineInstr *Value = MRI.getVRegDef(Reg: Store.getValueReg());
910 if (auto *Shuf = dyn_cast<GShuffleVector>(Val: Value)) {
911 ArrayRef<int> Mask = Shuf->getMask();
912 if (Ty.getSizeInBits() % 128 == 0 &&
913 ShuffleVectorInst::isInterleaveMask(Mask, Factor: 2, NumInputElts: Mask.size())) {
914 if (MRI.getType(Reg: Shuf->getOperand(i: 1).getReg()).getNumElements() ==
915 Mask.size() / 2) {
916 Srcs.push_back(Elt: Shuf->getOperand(i: 1).getReg());
917 Srcs.push_back(Elt: Shuf->getOperand(i: 2).getReg());
918 return true;
919 }
920
921 MachineInstr *C1 = MRI.getVRegDef(Reg: Shuf->getOperand(i: 1).getReg());
922 if (MRI.getType(Reg: Shuf->getOperand(i: 1).getReg()).getNumElements() !=
923 Mask.size() ||
924 C1->getOpcode() != TargetOpcode::G_CONCAT_VECTORS ||
925 C1->getNumOperands() != 3)
926 return false;
927 Srcs.push_back(Elt: C1->getOperand(i: 1).getReg());
928 Srcs.push_back(Elt: C1->getOperand(i: 2).getReg());
929 return true;
930 }
931 if (Ty.getSizeInBits() % 192 == 0 &&
932 ShuffleVectorInst::isInterleaveMask(Mask, Factor: 3, NumInputElts: Mask.size())) {
933 MachineInstr *C1 = MRI.getVRegDef(Reg: Shuf->getOperand(i: 1).getReg());
934 MachineInstr *C2 = MRI.getVRegDef(Reg: Shuf->getOperand(i: 2).getReg());
935 if (C1->getOpcode() != TargetOpcode::G_CONCAT_VECTORS ||
936 C1->getNumOperands() != 3 ||
937 C2->getOpcode() != TargetOpcode::G_CONCAT_VECTORS ||
938 C2->getNumOperands() != 3)
939 return false;
940 Srcs.push_back(Elt: C1->getOperand(i: 1).getReg());
941 Srcs.push_back(Elt: C1->getOperand(i: 2).getReg());
942 Srcs.push_back(Elt: C2->getOperand(i: 1).getReg());
943 return true;
944 }
945 if (Ty.getSizeInBits() % 256 == 0 &&
946 ShuffleVectorInst::isInterleaveMask(Mask, Factor: 4, NumInputElts: Mask.size())) {
947 MachineInstr *C1 = MRI.getVRegDef(Reg: Shuf->getOperand(i: 1).getReg());
948 MachineInstr *C2 = MRI.getVRegDef(Reg: Shuf->getOperand(i: 2).getReg());
949 if (C1->getOpcode() != TargetOpcode::G_CONCAT_VECTORS ||
950 C1->getNumOperands() != 3 ||
951 C2->getOpcode() != TargetOpcode::G_CONCAT_VECTORS ||
952 C2->getNumOperands() != 3)
953 return false;
954 Srcs.push_back(Elt: C1->getOperand(i: 1).getReg());
955 Srcs.push_back(Elt: C1->getOperand(i: 2).getReg());
956 Srcs.push_back(Elt: C2->getOperand(i: 1).getReg());
957 Srcs.push_back(Elt: C2->getOperand(i: 2).getReg());
958 return true;
959 }
960 }
961
962 return false;
963}
964
965void applyInterleavingStore(MachineInstr &MI, MachineRegisterInfo &MRI,
966 MachineIRBuilder &B, SmallVector<Register> &Srcs) {
967 unsigned IID;
968 switch (Srcs.size()) {
969 case 2:
970 IID = Intrinsic::aarch64_neon_st2;
971 break;
972 case 3:
973 IID = Intrinsic::aarch64_neon_st3;
974 break;
975 case 4:
976 IID = Intrinsic::aarch64_neon_st4;
977 break;
978 default:
979 llvm_unreachable("Unexpected deinterleave size!");
980 }
981
982 // Break down larger vectors into 128 blocks and a possible 64bit remainder
983 LLT Ty = MRI.getType(Reg: Srcs[0]);
984 LLT MainTy =
985 LLT::fixed_vector(NumElements: 128 / Ty.getScalarSizeInBits(), ScalarTy: Ty.getScalarType());
986 LLT LeftOverTy;
987 SmallVector<SmallVector<Register>> Regs(Srcs.size()), LeftOver(Srcs.size());
988 if (Ty == MainTy) {
989 for (unsigned I = 0; I < Srcs.size(); I++)
990 Regs[I].push_back(Elt: Srcs[I]);
991 } else if (Ty == MainTy.divide(Factor: 2)) {
992 for (unsigned I = 0; I < Srcs.size(); I++)
993 LeftOver[I].push_back(Elt: Srcs[I]);
994 LeftOverTy = MainTy.divide(Factor: 2);
995 } else {
996 for (unsigned I = 0; I < Srcs.size(); I++)
997 extractParts(Reg: Srcs[I], RegTy: Ty, MainTy, LeftoverTy&: LeftOverTy, VRegs&: Regs[I], LeftoverVRegs&: LeftOver[I], MIRBuilder&: B,
998 MRI);
999 assert(LeftOver[0].size() == 0 ||
1000 (LeftOver[0].size() == 1 && LeftOverTy == MainTy.divide(2)));
1001 }
1002
1003 Register Pointer = MI.getOperand(i: 1).getReg();
1004 assert(MI.getNumMemOperands() == 1);
1005 MachineMemOperand *MMO = *MI.memoperands_begin();
1006 MachineFunction &MF = *MI.getParent()->getParent();
1007
1008 // Generate a ST2/3/4 per vector part
1009 auto generateStore = [&](SmallVector<Register> Srcs, LLT Ty,
1010 unsigned Offset) {
1011 auto PtrOffset = Pointer;
1012 if (Offset != 0)
1013 PtrOffset =
1014 B.buildObjectPtrOffset(Res: MRI.getType(Reg: Pointer), Op0: Pointer,
1015 Op1: B.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: Offset))
1016 .getReg(Idx: 0);
1017 auto ST = B.buildIntrinsic(ID: IID, Res: ArrayRef<Register>());
1018 for (Register S : Srcs)
1019 ST.addReg(RegNo: S);
1020 ST.addReg(RegNo: PtrOffset);
1021 ST.addMemOperand(
1022 MMO: MF.getMachineMemOperand(MMO, Offset, Ty: Ty.multiplyElements(Factor: Srcs.size())));
1023 };
1024 for (unsigned I = 0; I < Regs[0].size(); I++) {
1025 for (unsigned S = 0; S < Srcs.size(); S++)
1026 Srcs[S] = Regs[S][I];
1027 generateStore(Srcs, MainTy, I * 16 * Srcs.size());
1028 }
1029 if (!LeftOver[0].empty()) {
1030 for (unsigned S = 0; S < Srcs.size(); S++)
1031 Srcs[S] = LeftOver[S][0];
1032 generateStore(Srcs, LeftOverTy, Regs[0].size() * 16 * Srcs.size());
1033 }
1034 MI.eraseFromParent();
1035}
1036
1037class AArch64PreLegalizerCombinerImpl : public Combiner {
1038protected:
1039 const CombinerHelper Helper;
1040 const AArch64PreLegalizerCombinerImplRuleConfig &RuleConfig;
1041 const AArch64Subtarget &STI;
1042 const LibcallLoweringInfo &Libcalls;
1043
1044public:
1045 AArch64PreLegalizerCombinerImpl(
1046 MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &VT,
1047 GISelCSEInfo *CSEInfo,
1048 const AArch64PreLegalizerCombinerImplRuleConfig &RuleConfig,
1049 const AArch64Subtarget &STI, const LibcallLoweringInfo &Libcalls,
1050 MachineDominatorTree *MDT, const LegalizerInfo *LI);
1051
1052 static const char *getName() { return "AArch6400PreLegalizerCombiner"; }
1053
1054 bool tryCombineAll(MachineInstr &I) const override;
1055
1056 bool tryCombineAllImpl(MachineInstr &I) const;
1057
1058private:
1059#define GET_GICOMBINER_CLASS_MEMBERS
1060#include "AArch64GenPreLegalizeGICombiner.inc"
1061#undef GET_GICOMBINER_CLASS_MEMBERS
1062};
1063
1064#define GET_GICOMBINER_IMPL
1065#include "AArch64GenPreLegalizeGICombiner.inc"
1066#undef GET_GICOMBINER_IMPL
1067
1068AArch64PreLegalizerCombinerImpl::AArch64PreLegalizerCombinerImpl(
1069 MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &VT,
1070 GISelCSEInfo *CSEInfo,
1071 const AArch64PreLegalizerCombinerImplRuleConfig &RuleConfig,
1072 const AArch64Subtarget &STI, const LibcallLoweringInfo &Libcalls,
1073 MachineDominatorTree *MDT, const LegalizerInfo *LI)
1074 : Combiner(MF, CInfo, &VT, CSEInfo),
1075 Helper(Observer, B, /*IsPreLegalize*/ true, &VT, MDT, LI),
1076 RuleConfig(RuleConfig), STI(STI), Libcalls(Libcalls),
1077#define GET_GICOMBINER_CONSTRUCTOR_INITS
1078#include "AArch64GenPreLegalizeGICombiner.inc"
1079#undef GET_GICOMBINER_CONSTRUCTOR_INITS
1080{
1081}
1082
1083bool AArch64PreLegalizerCombinerImpl::tryCombineAll(MachineInstr &MI) const {
1084 if (tryCombineAllImpl(I&: MI))
1085 return true;
1086
1087 return false;
1088}
1089
1090bool runCombiner(MachineFunction &MF, GISelCSEInfo *CSEInfo,
1091 GISelValueTracking *VT, MachineDominatorTree *MDT,
1092 const LibcallLoweringInfo &Libcalls,
1093 const AArch64PreLegalizerCombinerImplRuleConfig &RuleConfig,
1094 bool EnableOpt) {
1095 const AArch64Subtarget &ST = MF.getSubtarget<AArch64Subtarget>();
1096 const auto *LI = ST.getLegalizerInfo();
1097
1098 const Function &F = MF.getFunction();
1099
1100 CombinerInfo CInfo(/*AllowIllegalOps=*/true, /*ShouldLegalizeIllegal=*/false,
1101 /*LegalizerInfo=*/nullptr, EnableOpt, F.hasOptSize(),
1102 F.hasMinSize());
1103 // Disable fixed-point iteration to reduce compile-time
1104 CInfo.MaxIterations = 1;
1105 CInfo.ObserverLvl = CombinerInfo::ObserverLevel::SinglePass;
1106 // This is the first Combiner, so the input IR might contain dead
1107 // instructions.
1108 CInfo.EnableFullDCE = true;
1109 AArch64PreLegalizerCombinerImpl Impl(MF, CInfo, *VT, CSEInfo, RuleConfig, ST,
1110 Libcalls, MDT, LI);
1111 return Impl.combineMachineInstrs();
1112}
1113
1114// Pass boilerplate
1115// ================
1116
1117class AArch64PreLegalizerCombinerLegacy : public MachineFunctionPass {
1118public:
1119 static char ID;
1120
1121 AArch64PreLegalizerCombinerLegacy();
1122
1123 StringRef getPassName() const override {
1124 return "AArch64PreLegalizerCombiner";
1125 }
1126
1127 bool runOnMachineFunction(MachineFunction &MF) override;
1128
1129 void getAnalysisUsage(AnalysisUsage &AU) const override;
1130
1131private:
1132 AArch64PreLegalizerCombinerImplRuleConfig RuleConfig;
1133};
1134} // end anonymous namespace
1135
1136void AArch64PreLegalizerCombinerLegacy::getAnalysisUsage(
1137 AnalysisUsage &AU) const {
1138 AU.setPreservesCFG();
1139 getSelectionDAGFallbackAnalysisUsage(AU);
1140 AU.addRequired<GISelValueTrackingAnalysisLegacy>();
1141 AU.addPreserved<GISelValueTrackingAnalysisLegacy>();
1142 AU.addRequired<MachineDominatorTreeWrapperPass>();
1143 AU.addRequired<GISelCSEAnalysisWrapperPass>();
1144 AU.addPreserved<GISelCSEAnalysisWrapperPass>();
1145 AU.addRequired<LibcallLoweringInfoWrapper>();
1146 MachineFunctionPass::getAnalysisUsage(AU);
1147}
1148
1149AArch64PreLegalizerCombinerLegacy::AArch64PreLegalizerCombinerLegacy()
1150 : MachineFunctionPass(ID) {
1151 if (!RuleConfig.parseCommandLineOption())
1152 report_fatal_error(reason: "Invalid rule identifier");
1153}
1154
1155bool AArch64PreLegalizerCombinerLegacy::runOnMachineFunction(
1156 MachineFunction &MF) {
1157 if (MF.getProperties().hasFailedISel())
1158 return false;
1159 // Enable CSE.
1160 GISelCSEAnalysisWrapper &Wrapper =
1161 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
1162 auto *CSEInfo =
1163 &Wrapper.get(CSEOpt: getStandardCSEConfigForOpt(Level: MF.getTarget().getOptLevel()));
1164
1165 const AArch64Subtarget &ST = MF.getSubtarget<AArch64Subtarget>();
1166 const LibcallLoweringInfo &Libcalls =
1167 getAnalysis<LibcallLoweringInfoWrapper>().getLibcallLowering(
1168 M: *MF.getFunction().getParent(), Subtarget: ST);
1169
1170 GISelValueTracking *VT =
1171 &getAnalysis<GISelValueTrackingAnalysisLegacy>().get(MF);
1172 MachineDominatorTree *MDT =
1173 &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
1174 bool EnableOpt = MF.getTarget().getOptLevel() != CodeGenOptLevel::None &&
1175 !skipFunction(F: MF.getFunction());
1176 return runCombiner(MF, CSEInfo, VT, MDT, Libcalls, RuleConfig, EnableOpt);
1177}
1178
1179char AArch64PreLegalizerCombinerLegacy::ID = 0;
1180INITIALIZE_PASS_BEGIN(AArch64PreLegalizerCombinerLegacy, DEBUG_TYPE,
1181 "Combine AArch64 machine instrs before legalization",
1182 false, false)
1183INITIALIZE_PASS_DEPENDENCY(GISelValueTrackingAnalysisLegacy)
1184INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
1185INITIALIZE_PASS_DEPENDENCY(LibcallLoweringInfoWrapper)
1186INITIALIZE_PASS_END(AArch64PreLegalizerCombinerLegacy, DEBUG_TYPE,
1187 "Combine AArch64 machine instrs before legalization", false,
1188 false)
1189
1190AArch64PreLegalizerCombinerPass::AArch64PreLegalizerCombinerPass()
1191 : RuleConfig(
1192 std::make_unique<AArch64PreLegalizerCombinerImplRuleConfig>()) {
1193 if (!RuleConfig->parseCommandLineOption())
1194 reportFatalUsageError(reason: "invalid rule identifier");
1195}
1196
1197AArch64PreLegalizerCombinerPass::AArch64PreLegalizerCombinerPass(
1198 AArch64PreLegalizerCombinerPass &&) = default;
1199
1200AArch64PreLegalizerCombinerPass::~AArch64PreLegalizerCombinerPass() = default;
1201
1202PreservedAnalyses
1203AArch64PreLegalizerCombinerPass::run(MachineFunction &MF,
1204 MachineFunctionAnalysisManager &MFAM) {
1205 if (MF.getProperties().hasFailedISel())
1206 return PreservedAnalyses::all();
1207
1208 auto *CSEInfo = MFAM.getResult<GISelCSEAnalysis>(IR&: MF).get();
1209 GISelValueTracking &VT = MFAM.getResult<GISelValueTrackingAnalysis>(IR&: MF);
1210 MachineDominatorTree &MDT = MFAM.getResult<MachineDominatorTreeAnalysis>(IR&: MF);
1211
1212 const AArch64Subtarget &ST = MF.getSubtarget<AArch64Subtarget>();
1213 auto &MAMProxy =
1214 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF);
1215 const ModuleLibcallLoweringInfo *LibcallResult =
1216 MAMProxy.getCachedResult<LibcallLoweringModuleAnalysis>(
1217 IR&: *MF.getFunction().getParent());
1218 if (!LibcallResult)
1219 reportFatalUsageError(reason: "LibcallLoweringModuleAnalysis result not available");
1220
1221 const LibcallLoweringInfo &Libcalls = getLibcallLowering(ModuleInfo: *LibcallResult, Subtarget: ST);
1222
1223 bool EnableOpt = MF.getTarget().getOptLevel() != CodeGenOptLevel::None;
1224
1225 if (!runCombiner(MF, CSEInfo, VT: &VT, MDT: &MDT, Libcalls, RuleConfig: *RuleConfig, EnableOpt))
1226 return PreservedAnalyses::all();
1227
1228 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
1229 PA.preserveSet<CFGAnalyses>();
1230 PA.preserve<GISelValueTrackingAnalysis>();
1231 PA.preserve<GISelCSEAnalysis>();
1232 return PA;
1233}
1234
1235namespace llvm {
1236FunctionPass *createAArch64PreLegalizerCombiner() {
1237 return new AArch64PreLegalizerCombinerLegacy();
1238}
1239} // end namespace llvm
1240