1//=== AArch64PostLegalizerCombiner.cpp --------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// Post-legalization combines on generic MachineInstrs.
11///
12/// The combines here must preserve instruction legality.
13///
14/// Lowering combines (e.g. pseudo matching) should be handled by
15/// AArch64PostLegalizerLowering.
16///
17/// Combines which don't rely on instruction legality should go in the
18/// AArch64PreLegalizerCombiner.
19///
20//===----------------------------------------------------------------------===//
21
22#include "AArch64.h"
23#include "AArch64TargetMachine.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
26#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
27#include "llvm/CodeGen/GlobalISel/Combiner.h"
28#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
29#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
30#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
31#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
32#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
33#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
34#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
35#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
36#include "llvm/CodeGen/GlobalISel/Utils.h"
37#include "llvm/CodeGen/MachineDominators.h"
38#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
39#include "llvm/CodeGen/MachineFunctionPass.h"
40#include "llvm/CodeGen/MachinePassManager.h"
41#include "llvm/CodeGen/MachineRegisterInfo.h"
42#include "llvm/CodeGen/TargetOpcodes.h"
43#include "llvm/Support/Debug.h"
44
45#define GET_GICOMBINER_DEPS
46#include "AArch64GenPostLegalizeGICombiner.inc"
47#undef GET_GICOMBINER_DEPS
48
49#define DEBUG_TYPE "aarch64-postlegalizer-combiner"
50
51using namespace llvm;
52using namespace MIPatternMatch;
53
54#define GET_GICOMBINER_TYPES
55#include "AArch64GenPostLegalizeGICombiner.inc"
56#undef GET_GICOMBINER_TYPES
57
58namespace {
59
60/// This combine tries do what performExtractVectorEltCombine does in SDAG.
61/// Rewrite for pairwise fadd pattern
62/// (s32 (g_extract_vector_elt
63/// (g_fadd (vXs32 Other)
64/// (g_vector_shuffle (vXs32 Other) undef <1,X,...> )) 0))
65/// ->
66/// (s32 (g_fadd (g_extract_vector_elt (vXs32 Other) 0)
67/// (g_extract_vector_elt (vXs32 Other) 1))
68bool matchExtractVecEltPairwiseAdd(
69 MachineInstr &MI, MachineRegisterInfo &MRI,
70 std::tuple<unsigned, LLT, Register> &MatchInfo) {
71 Register Src1 = MI.getOperand(i: 1).getReg();
72 Register Src2 = MI.getOperand(i: 2).getReg();
73 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
74
75 auto Cst = getIConstantVRegValWithLookThrough(VReg: Src2, MRI);
76 if (!Cst || Cst->Value != 0)
77 return false;
78 // SDAG also checks for FullFP16, but this looks to be beneficial anyway.
79
80 // Now check for an fadd operation. TODO: expand this for integer add?
81 auto *FAddMI = getOpcodeDef(Opcode: TargetOpcode::G_FADD, Reg: Src1, MRI);
82 if (!FAddMI)
83 return false;
84
85 // If we add support for integer add, must restrict these types to just s64.
86 unsigned DstSize = DstTy.getSizeInBits();
87 if (DstSize != 16 && DstSize != 32 && DstSize != 64)
88 return false;
89
90 Register Src1Op1 = FAddMI->getOperand(i: 1).getReg();
91 Register Src1Op2 = FAddMI->getOperand(i: 2).getReg();
92 MachineInstr *Shuffle =
93 getOpcodeDef(Opcode: TargetOpcode::G_SHUFFLE_VECTOR, Reg: Src1Op2, MRI);
94 MachineInstr *Other = MRI.getVRegDef(Reg: Src1Op1);
95 if (!Shuffle) {
96 Shuffle = getOpcodeDef(Opcode: TargetOpcode::G_SHUFFLE_VECTOR, Reg: Src1Op1, MRI);
97 Other = MRI.getVRegDef(Reg: Src1Op2);
98 }
99
100 // We're looking for a shuffle that moves the second element to index 0.
101 if (Shuffle && Shuffle->getOperand(i: 3).getShuffleMask()[0] == 1 &&
102 Other == MRI.getVRegDef(Reg: Shuffle->getOperand(i: 1).getReg())) {
103 std::get<0>(t&: MatchInfo) = TargetOpcode::G_FADD;
104 std::get<1>(t&: MatchInfo) = DstTy;
105 std::get<2>(t&: MatchInfo) = Other->getOperand(i: 0).getReg();
106 return true;
107 }
108 return false;
109}
110
111void applyExtractVecEltPairwiseAdd(
112 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B,
113 std::tuple<unsigned, LLT, Register> &MatchInfo) {
114 unsigned Opc = std::get<0>(t&: MatchInfo);
115 assert(Opc == TargetOpcode::G_FADD && "Unexpected opcode!");
116 // We want to generate two extracts of elements 0 and 1, and add them.
117 LLT Ty = std::get<1>(t&: MatchInfo);
118 Register Src = std::get<2>(t&: MatchInfo);
119 LLT s64 = LLT::integer(SizeInBits: 64);
120 B.setInstrAndDebugLoc(MI);
121 auto Elt0 = B.buildExtractVectorElement(Res: Ty, Val: Src, Idx: B.buildConstant(Res: s64, Val: 0));
122 auto Elt1 = B.buildExtractVectorElement(Res: Ty, Val: Src, Idx: B.buildConstant(Res: s64, Val: 1));
123 B.buildInstr(Opc, DstOps: {MI.getOperand(i: 0).getReg()}, SrcOps: {Elt0, Elt1});
124 MI.eraseFromParent();
125}
126
127bool isSignExtended(Register R, MachineRegisterInfo &MRI) {
128 // TODO: check if extended build vector as well.
129 return mi_match(R, MRI, P: m_GSExt(Src: m_Reg())) ||
130 mi_match(R, MRI, P: m_GSExtInReg(Src: m_Reg()));
131}
132
133bool isZeroExtended(Register R, MachineRegisterInfo &MRI) {
134 // TODO: check if extended build vector as well.
135 return mi_match(R, MRI, P: m_GZExt(Src: m_Reg()));
136}
137
138bool matchAArch64MulConstCombine(
139 MachineInstr &MI, MachineRegisterInfo &MRI,
140 std::function<void(MachineIRBuilder &B, Register DstReg)> &ApplyFn) {
141 assert(MI.getOpcode() == TargetOpcode::G_MUL);
142 Register LHS = MI.getOperand(i: 1).getReg();
143 Register RHS = MI.getOperand(i: 2).getReg();
144 Register Dst = MI.getOperand(i: 0).getReg();
145 const LLT Ty = MRI.getType(Reg: LHS);
146
147 // The below optimizations require a constant RHS.
148 auto Const = getIConstantVRegValWithLookThrough(VReg: RHS, MRI);
149 if (!Const)
150 return false;
151
152 APInt ConstValue = Const->Value.sext(width: Ty.getSizeInBits());
153 // The following code is ported from AArch64ISelLowering.
154 // Multiplication of a power of two plus/minus one can be done more
155 // cheaply as shift+add/sub. For now, this is true unilaterally. If
156 // future CPUs have a cheaper MADD instruction, this may need to be
157 // gated on a subtarget feature. For Cyclone, 32-bit MADD is 4 cycles and
158 // 64-bit is 5 cycles, so this is always a win.
159 // More aggressively, some multiplications N0 * C can be lowered to
160 // shift+add+shift if the constant C = A * B where A = 2^N + 1 and B = 2^M,
161 // e.g. 6=3*2=(2+1)*2.
162 // TODO: consider lowering more cases, e.g. C = 14, -6, -14 or even 45
163 // which equals to (1+2)*16-(1+2).
164 // TrailingZeroes is used to test if the mul can be lowered to
165 // shift+add+shift.
166 unsigned TrailingZeroes = ConstValue.countr_zero();
167 if (TrailingZeroes) {
168 // Conservatively do not lower to shift+add+shift if the mul might be
169 // folded into smul or umul.
170 if (MRI.hasOneNonDBGUse(RegNo: LHS) &&
171 (isSignExtended(R: LHS, MRI) || isZeroExtended(R: LHS, MRI)))
172 return false;
173 // Conservatively do not lower to shift+add+shift if the mul might be
174 // folded into madd or msub.
175 if (MRI.hasOneNonDBGUse(RegNo: Dst)) {
176 MachineInstr &UseMI = *MRI.use_instr_begin(RegNo: Dst);
177 unsigned UseOpc = UseMI.getOpcode();
178 if (UseOpc == TargetOpcode::G_ADD || UseOpc == TargetOpcode::G_PTR_ADD ||
179 UseOpc == TargetOpcode::G_SUB)
180 return false;
181 }
182 }
183 // Use ShiftedConstValue instead of ConstValue to support both shift+add/sub
184 // and shift+add+shift.
185 APInt ShiftedConstValue = ConstValue.ashr(ShiftAmt: TrailingZeroes);
186
187 unsigned ShiftAmt, AddSubOpc;
188 // Is the shifted value the LHS operand of the add/sub?
189 bool ShiftValUseIsLHS = true;
190 // Do we need to negate the result?
191 bool NegateResult = false;
192
193 if (ConstValue.isNonNegative()) {
194 // (mul x, 2^N + 1) => (add (shl x, N), x)
195 // (mul x, 2^N - 1) => (sub (shl x, N), x)
196 // (mul x, (2^N + 1) * 2^M) => (shl (add (shl x, N), x), M)
197 APInt SCVMinus1 = ShiftedConstValue - 1;
198 APInt CVPlus1 = ConstValue + 1;
199 if (SCVMinus1.isPowerOf2()) {
200 ShiftAmt = SCVMinus1.logBase2();
201 AddSubOpc = TargetOpcode::G_ADD;
202 } else if (CVPlus1.isPowerOf2()) {
203 ShiftAmt = CVPlus1.logBase2();
204 AddSubOpc = TargetOpcode::G_SUB;
205 } else
206 return false;
207 } else {
208 // (mul x, -(2^N - 1)) => (sub x, (shl x, N))
209 // (mul x, -(2^N + 1)) => - (add (shl x, N), x)
210 APInt CVNegPlus1 = -ConstValue + 1;
211 APInt CVNegMinus1 = -ConstValue - 1;
212 if (CVNegPlus1.isPowerOf2()) {
213 ShiftAmt = CVNegPlus1.logBase2();
214 AddSubOpc = TargetOpcode::G_SUB;
215 ShiftValUseIsLHS = false;
216 } else if (CVNegMinus1.isPowerOf2()) {
217 ShiftAmt = CVNegMinus1.logBase2();
218 AddSubOpc = TargetOpcode::G_ADD;
219 NegateResult = true;
220 } else
221 return false;
222 }
223
224 if (NegateResult && TrailingZeroes)
225 return false;
226
227 ApplyFn = [=](MachineIRBuilder &B, Register DstReg) {
228 auto Shift = B.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: ShiftAmt);
229 auto ShiftedVal = B.buildShl(Dst: Ty, Src0: LHS, Src1: Shift);
230
231 Register AddSubLHS = ShiftValUseIsLHS ? ShiftedVal.getReg(Idx: 0) : LHS;
232 Register AddSubRHS = ShiftValUseIsLHS ? LHS : ShiftedVal.getReg(Idx: 0);
233 auto Res = B.buildInstr(Opc: AddSubOpc, DstOps: {Ty}, SrcOps: {AddSubLHS, AddSubRHS});
234 assert(!(NegateResult && TrailingZeroes) &&
235 "NegateResult and TrailingZeroes cannot both be true for now.");
236 // Negate the result.
237 if (NegateResult) {
238 B.buildSub(Dst: DstReg, Src0: B.buildConstant(Res: Ty, Val: 0), Src1: Res);
239 return;
240 }
241 // Shift the result.
242 if (TrailingZeroes) {
243 B.buildShl(Dst: DstReg, Src0: Res,
244 Src1: B.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: TrailingZeroes));
245 return;
246 }
247 B.buildCopy(Res: DstReg, Op: Res.getReg(Idx: 0));
248 };
249 return true;
250}
251
252void applyAArch64MulConstCombine(
253 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B,
254 std::function<void(MachineIRBuilder &B, Register DstReg)> &ApplyFn) {
255 B.setInstrAndDebugLoc(MI);
256 ApplyFn(B, MI.getOperand(i: 0).getReg());
257 MI.eraseFromParent();
258}
259
260/// Try to fold a G_MERGE_VALUES of 2 s32 sources, where the second source
261/// is a zero, into a G_ZEXT of the first.
262bool matchFoldMergeToZext(MachineInstr &MI, MachineRegisterInfo &MRI) {
263 auto &Merge = cast<GMerge>(Val&: MI);
264 LLT SrcTy = MRI.getType(Reg: Merge.getSourceReg(I: 0));
265 if (SrcTy != LLT::scalar(SizeInBits: 32) || Merge.getNumSources() != 2)
266 return false;
267 return mi_match(R: Merge.getSourceReg(I: 1), MRI, P: m_SpecificICst(RequestedValue: 0));
268}
269
270void applyFoldMergeToZext(MachineInstr &MI, MachineRegisterInfo &MRI,
271 MachineIRBuilder &B, GISelChangeObserver &Observer) {
272 // Mutate %d(s64) = G_MERGE_VALUES %a(s32), 0(s32)
273 // ->
274 // %d(s64) = G_ZEXT %a(s32)
275 Observer.changingInstr(MI);
276 MI.setDesc(B.getTII().get(Opcode: TargetOpcode::G_ZEXT));
277 MI.removeOperand(OpNo: 2);
278 Observer.changedInstr(MI);
279}
280
281/// \returns True if a G_ANYEXT instruction \p MI should be mutated to a G_ZEXT
282/// instruction.
283bool matchMutateAnyExtToZExt(MachineInstr &MI, MachineRegisterInfo &MRI) {
284 // If this is coming from a scalar compare then we can use a G_ZEXT instead of
285 // a G_ANYEXT:
286 //
287 // %cmp:_(s32) = G_[I|F]CMP ... <-- produces 0/1.
288 // %ext:_(s64) = G_ANYEXT %cmp(s32)
289 //
290 // By doing this, we can leverage more KnownBits combines.
291 assert(MI.getOpcode() == TargetOpcode::G_ANYEXT);
292 Register Dst = MI.getOperand(i: 0).getReg();
293 Register Src = MI.getOperand(i: 1).getReg();
294 return MRI.getType(Reg: Dst).isScalar() &&
295 mi_match(R: Src, MRI,
296 P: m_any_of(preds: m_GICmp(P: m_Pred(), L: m_Reg(), R: m_Reg()),
297 preds: m_GFCmp(P: m_Pred(), L: m_Reg(), R: m_Reg())));
298}
299
300void applyMutateAnyExtToZExt(MachineInstr &MI, MachineRegisterInfo &MRI,
301 MachineIRBuilder &B,
302 GISelChangeObserver &Observer) {
303 Observer.changingInstr(MI);
304 MI.setDesc(B.getTII().get(Opcode: TargetOpcode::G_ZEXT));
305 Observer.changedInstr(MI);
306}
307
308/// Match a 128b store of zero and split it into two 64 bit stores, for
309/// size/performance reasons.
310bool matchSplitStoreZero128(MachineInstr &MI, MachineRegisterInfo &MRI) {
311 GStore &Store = cast<GStore>(Val&: MI);
312 if (!Store.isSimple())
313 return false;
314 LLT ValTy = MRI.getType(Reg: Store.getValueReg());
315 if (ValTy.isScalableVector())
316 return false;
317 if (!ValTy.isVector() || ValTy.getSizeInBits() != 128)
318 return false;
319 if (Store.getMemSizeInBits() != ValTy.getSizeInBits())
320 return false; // Don't split truncating stores.
321 if (!MRI.hasOneNonDBGUse(RegNo: Store.getValueReg()))
322 return false;
323 auto MaybeCst = isConstantOrConstantSplatVector(Def: Store.getValueReg(), MRI);
324 return MaybeCst && MaybeCst->isZero();
325}
326
327void applySplitStoreZero128(MachineInstr &MI, MachineRegisterInfo &MRI,
328 MachineIRBuilder &B,
329 GISelChangeObserver &Observer) {
330 B.setInstrAndDebugLoc(MI);
331 GStore &Store = cast<GStore>(Val&: MI);
332 assert(MRI.getType(Store.getValueReg()).isVector() &&
333 "Expected a vector store value");
334 LLT NewTy = LLT::integer(SizeInBits: 64);
335 Register PtrReg = Store.getPointerReg();
336 auto Zero = B.buildConstant(Res: NewTy, Val: 0);
337 auto HighPtr =
338 B.buildPtrAdd(Res: MRI.getType(Reg: PtrReg), Op0: PtrReg, Op1: B.buildConstant(Res: NewTy, Val: 8));
339 auto &MF = *MI.getMF();
340 auto *LowMMO = MF.getMachineMemOperand(MMO: &Store.getMMO(), Offset: 0, Ty: NewTy);
341 auto *HighMMO = MF.getMachineMemOperand(MMO: &Store.getMMO(), Offset: 8, Ty: NewTy);
342 B.buildStore(Val: Zero, Addr: PtrReg, MMO&: *LowMMO);
343 B.buildStore(Val: Zero, Addr: HighPtr, MMO&: *HighMMO);
344 Store.eraseFromParent();
345}
346
347bool matchOrToBSP(MachineInstr &MI, MachineRegisterInfo &MRI,
348 std::tuple<Register, Register, Register> &MatchInfo) {
349 const LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
350 if (!DstTy.isVector())
351 return false;
352
353 Register AO1, AO2, BVO1, BVO2;
354 if (!mi_match(MI, MRI,
355 P: m_GOr(L: m_GAnd(L: m_Reg(R&: AO1), R: m_Reg(R&: BVO1)),
356 R: m_GAnd(L: m_Reg(R&: AO2), R: m_Reg(R&: BVO2)))))
357 return false;
358
359 auto *BV1 = getOpcodeDef<GBuildVector>(Reg: BVO1, MRI);
360 auto *BV2 = getOpcodeDef<GBuildVector>(Reg: BVO2, MRI);
361 if (!BV1 || !BV2)
362 return false;
363
364 for (int I = 0, E = DstTy.getNumElements(); I < E; I++) {
365 auto ValAndVReg1 =
366 getIConstantVRegValWithLookThrough(VReg: BV1->getSourceReg(I), MRI);
367 auto ValAndVReg2 =
368 getIConstantVRegValWithLookThrough(VReg: BV2->getSourceReg(I), MRI);
369 if (!ValAndVReg1 || !ValAndVReg2 ||
370 ValAndVReg1->Value != ~ValAndVReg2->Value)
371 return false;
372 }
373
374 MatchInfo = {AO1, AO2, BVO1};
375 return true;
376}
377
378void applyOrToBSP(MachineInstr &MI, MachineRegisterInfo &MRI,
379 MachineIRBuilder &B,
380 std::tuple<Register, Register, Register> &MatchInfo) {
381 B.setInstrAndDebugLoc(MI);
382 B.buildInstr(
383 Opc: AArch64::G_BSP, DstOps: {MI.getOperand(i: 0).getReg()},
384 SrcOps: {std::get<2>(t&: MatchInfo), std::get<0>(t&: MatchInfo), std::get<1>(t&: MatchInfo)});
385 MI.eraseFromParent();
386}
387
388/// Match G_TRUNC (G_OR X, Y) => G_ADDHN X, Y when both inputs are sign
389/// extended from the result element type. The high half of the addition then
390/// equals the truncation of the OR.
391bool matchTruncOrToADDHN(MachineInstr &MI, MachineRegisterInfo &MRI,
392 GISelValueTracking *VT, Register Dst, Register Or,
393 Register Src0, Register Src1) {
394 if (!MRI.hasOneUse(RegNo: Or))
395 return false;
396
397 LLT DstTy = MRI.getType(Reg: Dst);
398 LLT SrcTy = MRI.getType(Reg: Or);
399 if (!((DstTy == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8) &&
400 SrcTy == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16)) ||
401 (DstTy == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16) &&
402 SrcTy == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32)) ||
403 (DstTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32) &&
404 SrcTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64))))
405 return false;
406
407 // If the narrow result is immediately any-extended back to the original type,
408 // the G_OR is cheaper than G_ADDHN followed by a vector widen.
409 if (MRI.hasOneNonDBGUse(RegNo: Dst)) {
410 MachineInstr &UseMI = *MRI.use_nodbg_instructions(Reg: Dst).begin();
411 if (UseMI.getOpcode() == TargetOpcode::G_ANYEXT &&
412 MRI.getType(Reg: UseMI.getOperand(i: 0).getReg()) == SrcTy)
413 return false;
414 }
415
416 unsigned EltSize = SrcTy.getScalarSizeInBits();
417 if (VT->computeNumSignBits(R: Src0) != EltSize ||
418 VT->computeNumSignBits(R: Src1) != EltSize)
419 return false;
420
421 return true;
422}
423
424// Combines Mul(And(Srl(X, 15), 0x10001), 0xffff) into CMLTz
425bool matchCombineMulCMLT(MachineInstr &MI, MachineRegisterInfo &MRI,
426 Register &SrcReg) {
427 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
428
429 if (DstTy != LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64) && DstTy != LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32) &&
430 DstTy != LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32) && DstTy != LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16) &&
431 DstTy != LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16))
432 return false;
433
434 auto AndMI = getDefIgnoringCopies(Reg: MI.getOperand(i: 1).getReg(), MRI);
435 if (AndMI->getOpcode() != TargetOpcode::G_AND)
436 return false;
437 auto LShrMI = getDefIgnoringCopies(Reg: AndMI->getOperand(i: 1).getReg(), MRI);
438 if (LShrMI->getOpcode() != TargetOpcode::G_LSHR)
439 return false;
440
441 // Check the constant splat values
442 auto V1 = isConstantOrConstantSplatVector(Def: MI.getOperand(i: 2).getReg(), MRI);
443 auto V2 = isConstantOrConstantSplatVector(Def: AndMI->getOperand(i: 2).getReg(), MRI);
444 auto V3 =
445 isConstantOrConstantSplatVector(Def: LShrMI->getOperand(i: 2).getReg(), MRI);
446 if (!V1.has_value() || !V2.has_value() || !V3.has_value())
447 return false;
448 unsigned HalfSize = DstTy.getScalarSizeInBits() / 2;
449 if (!V1.value().isMask(numBits: HalfSize) || V2.value() != (1ULL | 1ULL << HalfSize) ||
450 V3 != (HalfSize - 1))
451 return false;
452
453 SrcReg = LShrMI->getOperand(i: 1).getReg();
454
455 return true;
456}
457
458void applyCombineMulCMLT(MachineInstr &MI, MachineRegisterInfo &MRI,
459 MachineIRBuilder &B, Register &SrcReg) {
460 Register DstReg = MI.getOperand(i: 0).getReg();
461 LLT DstTy = MRI.getType(Reg: DstReg);
462 LLT HalfTy =
463 DstTy.changeElementCount(EC: DstTy.getElementCount().multiplyCoefficientBy(RHS: 2))
464 .changeElementSize(NewEltSize: DstTy.getScalarSizeInBits() / 2);
465
466 Register ZeroVec = B.buildConstant(Res: HalfTy, Val: 0).getReg(Idx: 0);
467 Register CastReg =
468 B.buildInstr(Opc: TargetOpcode::G_BITCAST, DstOps: {HalfTy}, SrcOps: {SrcReg}).getReg(Idx: 0);
469 Register CMLTReg =
470 B.buildICmp(Pred: CmpInst::Predicate::ICMP_SLT, Res: HalfTy, Op0: CastReg, Op1: ZeroVec)
471 .getReg(Idx: 0);
472
473 B.buildInstr(Opc: TargetOpcode::G_BITCAST, DstOps: {DstReg}, SrcOps: {CMLTReg}).getReg(Idx: 0);
474 MI.eraseFromParent();
475}
476
477// Match mul({z/s}ext , {z/s}ext) => {u/s}mull
478bool matchExtMulToMULL(MachineInstr &MI, MachineRegisterInfo &MRI,
479 GISelValueTracking *KB,
480 std::tuple<bool, Register, Register> &MatchInfo) {
481 // Get the instructions that defined the source operand
482 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
483 MachineInstr *I1 = getDefIgnoringCopies(Reg: MI.getOperand(i: 1).getReg(), MRI);
484 MachineInstr *I2 = getDefIgnoringCopies(Reg: MI.getOperand(i: 2).getReg(), MRI);
485 unsigned I1Opc = I1->getOpcode();
486 unsigned I2Opc = I2->getOpcode();
487 unsigned EltSize = DstTy.getScalarSizeInBits();
488
489 if (!DstTy.isVector() || I1->getNumOperands() < 2 || I2->getNumOperands() < 2)
490 return false;
491
492 auto IsAtLeastDoubleExtend = [&](Register R) {
493 LLT Ty = MRI.getType(Reg: R);
494 return EltSize >= Ty.getScalarSizeInBits() * 2;
495 };
496
497 // If the source operands were EXTENDED before, then {U/S}MULL can be used
498 bool IsZExt1 =
499 I1Opc == TargetOpcode::G_ZEXT || I1Opc == TargetOpcode::G_ANYEXT;
500 bool IsZExt2 =
501 I2Opc == TargetOpcode::G_ZEXT || I2Opc == TargetOpcode::G_ANYEXT;
502 if (IsZExt1 && IsZExt2 && IsAtLeastDoubleExtend(I1->getOperand(i: 1).getReg()) &&
503 IsAtLeastDoubleExtend(I2->getOperand(i: 1).getReg())) {
504 get<0>(t&: MatchInfo) = true;
505 get<1>(t&: MatchInfo) = I1->getOperand(i: 1).getReg();
506 get<2>(t&: MatchInfo) = I2->getOperand(i: 1).getReg();
507 return true;
508 }
509
510 bool IsSExt1 =
511 I1Opc == TargetOpcode::G_SEXT || I1Opc == TargetOpcode::G_ANYEXT;
512 bool IsSExt2 =
513 I2Opc == TargetOpcode::G_SEXT || I2Opc == TargetOpcode::G_ANYEXT;
514 if (IsSExt1 && IsSExt2 && IsAtLeastDoubleExtend(I1->getOperand(i: 1).getReg()) &&
515 IsAtLeastDoubleExtend(I2->getOperand(i: 1).getReg())) {
516 get<0>(t&: MatchInfo) = false;
517 get<1>(t&: MatchInfo) = I1->getOperand(i: 1).getReg();
518 get<2>(t&: MatchInfo) = I2->getOperand(i: 1).getReg();
519 return true;
520 }
521
522 // Select UMULL if we can replace the other operand with an extend.
523 APInt Mask = APInt::getHighBitsSet(numBits: EltSize, hiBitsSet: EltSize / 2);
524 if (KB && (IsZExt1 || IsZExt2) &&
525 IsAtLeastDoubleExtend(IsZExt1 ? I1->getOperand(i: 1).getReg()
526 : I2->getOperand(i: 1).getReg())) {
527 Register ZExtOp =
528 IsZExt1 ? MI.getOperand(i: 2).getReg() : MI.getOperand(i: 1).getReg();
529 if (KB->maskedValueIsZero(Val: ZExtOp, Mask)) {
530 get<0>(t&: MatchInfo) = true;
531 get<1>(t&: MatchInfo) = IsZExt1 ? I1->getOperand(i: 1).getReg() : ZExtOp;
532 get<2>(t&: MatchInfo) = IsZExt1 ? ZExtOp : I2->getOperand(i: 1).getReg();
533 return true;
534 }
535 } else if (KB && DstTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64) &&
536 KB->maskedValueIsZero(Val: MI.getOperand(i: 1).getReg(), Mask) &&
537 KB->maskedValueIsZero(Val: MI.getOperand(i: 2).getReg(), Mask)) {
538 get<0>(t&: MatchInfo) = true;
539 get<1>(t&: MatchInfo) = MI.getOperand(i: 1).getReg();
540 get<2>(t&: MatchInfo) = MI.getOperand(i: 2).getReg();
541 return true;
542 }
543
544 if (KB && (IsSExt1 || IsSExt2) &&
545 IsAtLeastDoubleExtend(IsSExt1 ? I1->getOperand(i: 1).getReg()
546 : I2->getOperand(i: 1).getReg())) {
547 Register SExtOp =
548 IsSExt1 ? MI.getOperand(i: 2).getReg() : MI.getOperand(i: 1).getReg();
549 if (KB->computeNumSignBits(R: SExtOp) > EltSize / 2) {
550 get<0>(t&: MatchInfo) = false;
551 get<1>(t&: MatchInfo) = IsSExt1 ? I1->getOperand(i: 1).getReg() : SExtOp;
552 get<2>(t&: MatchInfo) = IsSExt1 ? SExtOp : I2->getOperand(i: 1).getReg();
553 return true;
554 }
555 } else if (KB && DstTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64) &&
556 KB->computeNumSignBits(R: MI.getOperand(i: 1).getReg()) > EltSize / 2 &&
557 KB->computeNumSignBits(R: MI.getOperand(i: 2).getReg()) > EltSize / 2) {
558 get<0>(t&: MatchInfo) = false;
559 get<1>(t&: MatchInfo) = MI.getOperand(i: 1).getReg();
560 get<2>(t&: MatchInfo) = MI.getOperand(i: 2).getReg();
561 return true;
562 }
563
564 return false;
565}
566
567void applyExtMulToMULL(MachineInstr &MI, MachineRegisterInfo &MRI,
568 MachineIRBuilder &B, GISelChangeObserver &Observer,
569 std::tuple<bool, Register, Register> &MatchInfo) {
570 assert(MI.getOpcode() == TargetOpcode::G_MUL &&
571 "Expected a G_MUL instruction");
572
573 // Get the instructions that defined the source operand
574 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
575 bool IsZExt = get<0>(t&: MatchInfo);
576 Register Src1Reg = get<1>(t&: MatchInfo);
577 Register Src2Reg = get<2>(t&: MatchInfo);
578 LLT Src1Ty = MRI.getType(Reg: Src1Reg);
579 LLT Src2Ty = MRI.getType(Reg: Src2Reg);
580 LLT HalfDstTy = DstTy.changeElementSize(NewEltSize: DstTy.getScalarSizeInBits() / 2);
581 unsigned ExtOpc = IsZExt ? TargetOpcode::G_ZEXT : TargetOpcode::G_SEXT;
582
583 if (Src1Ty.getScalarSizeInBits() * 2 != DstTy.getScalarSizeInBits())
584 Src1Reg = B.buildExtOrTrunc(ExtOpc, Res: {HalfDstTy}, Op: {Src1Reg}).getReg(Idx: 0);
585 if (Src2Ty.getScalarSizeInBits() * 2 != DstTy.getScalarSizeInBits())
586 Src2Reg = B.buildExtOrTrunc(ExtOpc, Res: {HalfDstTy}, Op: {Src2Reg}).getReg(Idx: 0);
587
588 B.buildInstr(Opc: IsZExt ? AArch64::G_UMULL : AArch64::G_SMULL,
589 DstOps: {MI.getOperand(i: 0).getReg()}, SrcOps: {Src1Reg, Src2Reg});
590 MI.eraseFromParent();
591}
592
593static bool matchSubAddMulReassoc(Register Mul1, Register Mul2, Register Sub,
594 Register Src, MachineRegisterInfo &MRI) {
595 if (!MRI.hasOneUse(RegNo: Sub))
596 return false;
597 if (getIConstantVRegValWithLookThrough(VReg: Src, MRI))
598 return false;
599 MachineInstr *M1 = getDefIgnoringCopies(Reg: Mul1, MRI);
600 if (M1->getOpcode() != AArch64::G_MUL &&
601 M1->getOpcode() != AArch64::G_SMULL &&
602 M1->getOpcode() != AArch64::G_UMULL)
603 return false;
604 MachineInstr *M2 = getDefIgnoringCopies(Reg: Mul2, MRI);
605 if (M2->getOpcode() != AArch64::G_MUL &&
606 M2->getOpcode() != AArch64::G_SMULL &&
607 M2->getOpcode() != AArch64::G_UMULL)
608 return false;
609 return true;
610}
611
612static void applySubAddMulReassoc(MachineInstr &MI, MachineInstr &Sub,
613 MachineRegisterInfo &MRI, MachineIRBuilder &B,
614 GISelChangeObserver &Observer) {
615 Register Src = MI.getOperand(i: 1).getReg();
616 Register Tmp = MI.getOperand(i: 2).getReg();
617 Register Mul1 = Sub.getOperand(i: 1).getReg();
618 Register Mul2 = Sub.getOperand(i: 2).getReg();
619 Observer.changingInstr(MI);
620 B.buildInstr(Opc: AArch64::G_SUB, DstOps: {Tmp}, SrcOps: {Src, Mul1});
621 MI.getOperand(i: 1).setReg(Tmp);
622 MI.getOperand(i: 2).setReg(Mul2);
623 Sub.eraseFromParent();
624 Observer.changedInstr(MI);
625}
626
627class AArch64PostLegalizerCombinerImpl : public Combiner {
628protected:
629 const CombinerHelper Helper;
630 const AArch64PostLegalizerCombinerImplRuleConfig &RuleConfig;
631 const AArch64Subtarget &STI;
632
633public:
634 AArch64PostLegalizerCombinerImpl(
635 MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &VT,
636 GISelCSEInfo *CSEInfo,
637 const AArch64PostLegalizerCombinerImplRuleConfig &RuleConfig,
638 const AArch64Subtarget &STI, MachineDominatorTree *MDT,
639 const LegalizerInfo *LI);
640
641 static const char *getName() { return "AArch64PostLegalizerCombiner"; }
642
643 bool tryCombineAll(MachineInstr &I) const override;
644
645private:
646#define GET_GICOMBINER_CLASS_MEMBERS
647#include "AArch64GenPostLegalizeGICombiner.inc"
648#undef GET_GICOMBINER_CLASS_MEMBERS
649};
650
651#define GET_GICOMBINER_IMPL
652#include "AArch64GenPostLegalizeGICombiner.inc"
653#undef GET_GICOMBINER_IMPL
654
655AArch64PostLegalizerCombinerImpl::AArch64PostLegalizerCombinerImpl(
656 MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &VT,
657 GISelCSEInfo *CSEInfo,
658 const AArch64PostLegalizerCombinerImplRuleConfig &RuleConfig,
659 const AArch64Subtarget &STI, MachineDominatorTree *MDT,
660 const LegalizerInfo *LI)
661 : Combiner(MF, CInfo, &VT, CSEInfo),
662 Helper(Observer, B, /*IsPreLegalize*/ false, &VT, MDT, LI),
663 RuleConfig(RuleConfig), STI(STI),
664#define GET_GICOMBINER_CONSTRUCTOR_INITS
665#include "AArch64GenPostLegalizeGICombiner.inc"
666#undef GET_GICOMBINER_CONSTRUCTOR_INITS
667{
668}
669
670struct StoreInfo {
671 GStore *St = nullptr;
672 // The G_PTR_ADD that's used by the store. We keep this to cache the
673 // MachineInstr def.
674 GPtrAdd *Ptr = nullptr;
675 // The signed offset to the Ptr instruction.
676 int64_t Offset = 0;
677 LLT StoredType;
678};
679
680static bool tryOptimizeConsecStores(SmallVectorImpl<StoreInfo> &Stores,
681 CSEMIRBuilder &MIB) {
682 if (Stores.size() <= 2)
683 return false;
684
685 // Profitabity checks:
686 int64_t BaseOffset = Stores[0].Offset;
687 unsigned NumPairsExpected = Stores.size() / 2;
688 unsigned TotalInstsExpected = NumPairsExpected + (Stores.size() % 2);
689 // Size savings will depend on whether we can fold the offset, as an
690 // immediate of an ADD.
691 auto &TLI = *MIB.getMF().getSubtarget().getTargetLowering();
692 if (!TLI.isLegalAddImmediate(BaseOffset))
693 TotalInstsExpected++;
694 int SavingsExpected = Stores.size() - TotalInstsExpected;
695 if (SavingsExpected <= 0)
696 return false;
697
698 auto &MRI = MIB.getMF().getRegInfo();
699
700 // We have a series of consecutive stores. Factor out the common base
701 // pointer and rewrite the offsets.
702 Register NewBase = Stores[0].Ptr->getReg(Idx: 0);
703 for (auto &SInfo : Stores) {
704 // Compute a new pointer with the new base ptr and adjusted offset.
705 MIB.setInstrAndDebugLoc(*SInfo.St);
706 auto NewOff =
707 MIB.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: SInfo.Offset - BaseOffset);
708 auto NewPtr = MIB.buildPtrAdd(Res: MRI.getType(Reg: SInfo.St->getPointerReg()),
709 Op0: NewBase, Op1: NewOff);
710 if (MIB.getObserver())
711 MIB.getObserver()->changingInstr(MI&: *SInfo.St);
712 SInfo.St->getOperand(i: 1).setReg(NewPtr.getReg(Idx: 0));
713 if (MIB.getObserver())
714 MIB.getObserver()->changedInstr(MI&: *SInfo.St);
715 }
716 LLVM_DEBUG(dbgs() << "Split a series of " << Stores.size()
717 << " stores into a base pointer and offsets.\n");
718 return true;
719}
720
721static cl::opt<bool>
722 EnableConsecutiveMemOpOpt("aarch64-postlegalizer-consecutive-memops",
723 cl::init(Val: true), cl::Hidden,
724 cl::desc("Enable consecutive memop optimization "
725 "in AArch64PostLegalizerCombiner"));
726
727static bool optimizeConsecutiveMemOpAddressing(MachineFunction &MF,
728 CSEMIRBuilder &MIB) {
729 // This combine needs to run after all reassociations/folds on pointer
730 // addressing have been done, specifically those that combine two G_PTR_ADDs
731 // with constant offsets into a single G_PTR_ADD with a combined offset.
732 // The goal of this optimization is to undo that combine in the case where
733 // doing so has prevented the formation of pair stores due to illegal
734 // addressing modes of STP. The reason that we do it here is because
735 // it's much easier to undo the transformation of a series consecutive
736 // mem ops, than it is to detect when doing it would be a bad idea looking
737 // at a single G_PTR_ADD in the reassociation/ptradd_immed_chain combine.
738 //
739 // An example:
740 // G_STORE %11:_(<2 x s64>), %base:_(p0) :: (store (<2 x s64>), align 1)
741 // %off1:_(s64) = G_CONSTANT i64 4128
742 // %p1:_(p0) = G_PTR_ADD %0:_, %off1:_(s64)
743 // G_STORE %11:_(<2 x s64>), %p1:_(p0) :: (store (<2 x s64>), align 1)
744 // %off2:_(s64) = G_CONSTANT i64 4144
745 // %p2:_(p0) = G_PTR_ADD %0:_, %off2:_(s64)
746 // G_STORE %11:_(<2 x s64>), %p2:_(p0) :: (store (<2 x s64>), align 1)
747 // %off3:_(s64) = G_CONSTANT i64 4160
748 // %p3:_(p0) = G_PTR_ADD %0:_, %off3:_(s64)
749 // G_STORE %11:_(<2 x s64>), %17:_(p0) :: (store (<2 x s64>), align 1)
750 bool Changed = false;
751 auto &MRI = MF.getRegInfo();
752
753 if (!EnableConsecutiveMemOpOpt)
754 return Changed;
755
756 SmallVector<StoreInfo, 8> Stores;
757 // If we see a load, then we keep track of any values defined by it.
758 // In the following example, STP formation will fail anyway because
759 // the latter store is using a load result that appears after the
760 // the prior store. In this situation if we factor out the offset then
761 // we increase code size for no benefit.
762 // G_STORE %v1:_(s64), %base:_(p0) :: (store (s64))
763 // %v2:_(s64) = G_LOAD %ldptr:_(p0) :: (load (s64))
764 // G_STORE %v2:_(s64), %base:_(p0) :: (store (s64))
765 SmallVector<Register> LoadValsSinceLastStore;
766
767 auto storeIsValid = [&](StoreInfo &Last, StoreInfo New) {
768 // Check if this store is consecutive to the last one.
769 if (Last.Ptr->getBaseReg() != New.Ptr->getBaseReg() ||
770 (Last.Offset + static_cast<int64_t>(Last.StoredType.getSizeInBytes()) !=
771 New.Offset) ||
772 Last.StoredType != New.StoredType)
773 return false;
774
775 // Check if this store is using a load result that appears after the
776 // last store. If so, bail out.
777 if (any_of(Range&: LoadValsSinceLastStore, P: [&](Register LoadVal) {
778 return New.St->getValueReg() == LoadVal;
779 }))
780 return false;
781
782 // Check if the current offset would be too large for STP.
783 // If not, then STP formation should be able to handle it, so we don't
784 // need to do anything.
785 int64_t MaxLegalOffset;
786 switch (New.StoredType.getSizeInBits()) {
787 case 32:
788 MaxLegalOffset = 252;
789 break;
790 case 64:
791 MaxLegalOffset = 504;
792 break;
793 case 128:
794 MaxLegalOffset = 1008;
795 break;
796 default:
797 llvm_unreachable("Unexpected stored type size");
798 }
799 if (New.Offset < MaxLegalOffset)
800 return false;
801
802 // If factoring it out still wouldn't help then don't bother.
803 return New.Offset - Stores[0].Offset <= MaxLegalOffset;
804 };
805
806 auto resetState = [&]() {
807 Stores.clear();
808 LoadValsSinceLastStore.clear();
809 };
810
811 for (auto &MBB : MF) {
812 // We're looking inside a single BB at a time since the memset pattern
813 // should only be in a single block.
814 resetState();
815 for (auto &MI : MBB) {
816 // Skip for scalable vectors
817 if (auto *LdSt = dyn_cast<GLoadStore>(Val: &MI);
818 LdSt && MRI.getType(Reg: LdSt->getOperand(i: 0).getReg()).isScalableVector())
819 continue;
820
821 if (auto *St = dyn_cast<GStore>(Val: &MI)) {
822 Register PtrBaseReg;
823 APInt Offset;
824 LLT StoredValTy = MRI.getType(Reg: St->getValueReg());
825 unsigned ValSize = StoredValTy.getSizeInBits();
826 if (ValSize < 32 || St->getMMO().getSizeInBits() != ValSize)
827 continue;
828
829 Register PtrReg = St->getPointerReg();
830 if (mi_match(
831 R: PtrReg, MRI,
832 P: m_OneNonDBGUse(SP: m_GPtrAdd(L: m_Reg(R&: PtrBaseReg), R: m_ICst(Cst&: Offset))))) {
833 GPtrAdd *PtrAdd = cast<GPtrAdd>(Val: MRI.getVRegDef(Reg: PtrReg));
834 StoreInfo New = {.St: St, .Ptr: PtrAdd, .Offset: Offset.getSExtValue(), .StoredType: StoredValTy};
835
836 if (Stores.empty()) {
837 Stores.push_back(Elt: New);
838 continue;
839 }
840
841 // Check if this store is a valid continuation of the sequence.
842 auto &Last = Stores.back();
843 if (storeIsValid(Last, New)) {
844 Stores.push_back(Elt: New);
845 LoadValsSinceLastStore.clear(); // Reset the load value tracking.
846 } else {
847 // The store isn't a valid to consider for the prior sequence,
848 // so try to optimize what we have so far and start a new sequence.
849 Changed |= tryOptimizeConsecStores(Stores, MIB);
850 resetState();
851 Stores.push_back(Elt: New);
852 }
853 }
854 } else if (auto *Ld = dyn_cast<GLoad>(Val: &MI)) {
855 LoadValsSinceLastStore.push_back(Elt: Ld->getDstReg());
856 }
857 }
858 Changed |= tryOptimizeConsecStores(Stores, MIB);
859 resetState();
860 }
861
862 return Changed;
863}
864
865bool runCombiner(MachineFunction &MF, GISelCSEInfo *CSEInfo,
866 GISelValueTracking *VT, MachineDominatorTree *MDT,
867 const AArch64PostLegalizerCombinerImplRuleConfig &RuleConfig,
868 bool EnableOpt, bool IsOptNone) {
869 if (MF.getProperties().hasFailedISel())
870 return false;
871 const Function &F = MF.getFunction();
872
873 const AArch64Subtarget &ST = MF.getSubtarget<AArch64Subtarget>();
874 const LegalizerInfo *LI = ST.getLegalizerInfo();
875
876 CombinerInfo CInfo(/*AllowIllegalOps=*/false, /*ShouldLegalizeIllegal=*/false,
877 /*LegalizerInfo=*/LI, EnableOpt, F.hasOptSize(),
878 F.hasMinSize());
879 // Disable fixed-point iteration to reduce compile-time
880 CInfo.MaxIterations = 1;
881 CInfo.ObserverLvl = CombinerInfo::ObserverLevel::SinglePass;
882 // Legalizer performs DCE, so a full DCE pass is unnecessary.
883 CInfo.EnableFullDCE = false;
884 AArch64PostLegalizerCombinerImpl Impl(MF, CInfo, *VT, CSEInfo, RuleConfig, ST,
885 MDT, LI);
886 bool Changed = Impl.combineMachineInstrs();
887
888 CSEMIRBuilder MIB(MF);
889 MIB.setCSEInfo(CSEInfo);
890 Changed |= optimizeConsecutiveMemOpAddressing(MF, MIB);
891 return Changed;
892}
893
894class AArch64PostLegalizerCombinerLegacy : public MachineFunctionPass {
895public:
896 static char ID;
897
898 AArch64PostLegalizerCombinerLegacy(bool IsOptNone = false);
899
900 StringRef getPassName() const override {
901 return "AArch64PostLegalizerCombiner";
902 }
903
904 bool runOnMachineFunction(MachineFunction &MF) override;
905 void getAnalysisUsage(AnalysisUsage &AU) const override;
906
907 MachineFunctionProperties getRequiredProperties() const override {
908 return MachineFunctionProperties().set(
909 MachineFunctionProperties::Property::Legalized);
910 }
911
912private:
913 bool IsOptNone;
914 AArch64PostLegalizerCombinerImplRuleConfig RuleConfig;
915};
916} // end anonymous namespace
917
918void AArch64PostLegalizerCombinerLegacy::getAnalysisUsage(
919 AnalysisUsage &AU) const {
920 AU.setPreservesCFG();
921 getSelectionDAGFallbackAnalysisUsage(AU);
922 AU.addRequired<GISelValueTrackingAnalysisLegacy>();
923 AU.addPreserved<GISelValueTrackingAnalysisLegacy>();
924 if (!IsOptNone) {
925 AU.addRequired<MachineDominatorTreeWrapperPass>();
926 AU.addRequired<GISelCSEAnalysisWrapperPass>();
927 AU.addPreserved<GISelCSEAnalysisWrapperPass>();
928 }
929 MachineFunctionPass::getAnalysisUsage(AU);
930}
931
932AArch64PostLegalizerCombinerLegacy::AArch64PostLegalizerCombinerLegacy(
933 bool IsOptNone)
934 : MachineFunctionPass(ID), IsOptNone(IsOptNone) {
935 if (!RuleConfig.parseCommandLineOption())
936 reportFatalUsageError(reason: "Invalid rule identifier");
937}
938
939bool AArch64PostLegalizerCombinerLegacy::runOnMachineFunction(
940 MachineFunction &MF) {
941 if (MF.getProperties().hasFailedISel())
942 return false;
943
944 GISelValueTracking *VT =
945 &getAnalysis<GISelValueTrackingAnalysisLegacy>().get(MF);
946 MachineDominatorTree *MDT =
947 IsOptNone ? nullptr
948 : &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
949 GISelCSEAnalysisWrapper &Wrapper =
950 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
951 auto *CSEInfo =
952 &Wrapper.get(CSEOpt: getStandardCSEConfigForOpt(Level: MF.getTarget().getOptLevel()));
953
954 bool EnableOpt = MF.getTarget().getOptLevel() != CodeGenOptLevel::None &&
955 !skipFunction(F: MF.getFunction());
956
957 return runCombiner(MF, CSEInfo, VT, MDT, RuleConfig, EnableOpt, IsOptNone);
958}
959
960char AArch64PostLegalizerCombinerLegacy::ID = 0;
961INITIALIZE_PASS_BEGIN(AArch64PostLegalizerCombinerLegacy, DEBUG_TYPE,
962 "Combine AArch64 MachineInstrs after legalization", false,
963 false)
964INITIALIZE_PASS_DEPENDENCY(GISelValueTrackingAnalysisLegacy)
965INITIALIZE_PASS_END(AArch64PostLegalizerCombinerLegacy, DEBUG_TYPE,
966 "Combine AArch64 MachineInstrs after legalization", false,
967 false)
968
969AArch64PostLegalizerCombinerPass::AArch64PostLegalizerCombinerPass(
970 const AArch64TargetMachine *TM)
971 : RuleConfig(
972 std::make_unique<AArch64PostLegalizerCombinerImplRuleConfig>()),
973 TM(TM) {
974 if (!RuleConfig->parseCommandLineOption())
975 reportFatalUsageError(reason: "invalid rule identifier");
976}
977
978AArch64PostLegalizerCombinerPass::AArch64PostLegalizerCombinerPass(
979 AArch64PostLegalizerCombinerPass &&) = default;
980
981AArch64PostLegalizerCombinerPass::~AArch64PostLegalizerCombinerPass() = default;
982
983PreservedAnalyses
984AArch64PostLegalizerCombinerPass::run(MachineFunction &MF,
985 MachineFunctionAnalysisManager &MFAM) {
986 if (MF.getProperties().hasFailedISel())
987 return PreservedAnalyses::all();
988
989 const bool IsOptNone = TM->isGlobalISelOptNone();
990 bool EnableOpt = !IsOptNone;
991
992 GISelValueTracking *VT = &MFAM.getResult<GISelValueTrackingAnalysis>(IR&: MF);
993 MachineDominatorTree *MDT =
994 IsOptNone ? nullptr : &MFAM.getResult<MachineDominatorTreeAnalysis>(IR&: MF);
995 GISelCSEInfo *CSEInfo = MFAM.getResult<GISelCSEAnalysis>(IR&: MF).get();
996
997 if (!runCombiner(MF, CSEInfo, VT, MDT, RuleConfig: *RuleConfig, EnableOpt, IsOptNone))
998 return PreservedAnalyses::all();
999
1000 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
1001 PA.preserveSet<CFGAnalyses>();
1002 PA.preserve<GISelValueTrackingAnalysis>();
1003 PA.preserve<GISelCSEAnalysis>();
1004 return PA;
1005}
1006
1007namespace llvm {
1008FunctionPass *createAArch64PostLegalizerCombinerLegacy(bool IsOptNone) {
1009 return new AArch64PostLegalizerCombinerLegacy(IsOptNone);
1010}
1011} // end namespace llvm
1012