1//=== AArch64PostLegalizerCombiner.cpp --------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// Post-legalization combines on generic MachineInstrs.
11///
12/// The combines here must preserve instruction legality.
13///
14/// Lowering combines (e.g. pseudo matching) should be handled by
15/// AArch64PostLegalizerLowering.
16///
17/// Combines which don't rely on instruction legality should go in the
18/// AArch64PreLegalizerCombiner.
19///
20//===----------------------------------------------------------------------===//
21
22#include "AArch64.h"
23#include "AArch64GlobalISelUtils.h"
24#include "AArch64TargetMachine.h"
25#include "llvm/ADT/STLExtras.h"
26#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
27#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
28#include "llvm/CodeGen/GlobalISel/Combiner.h"
29#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
30#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
31#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
32#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
33#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
34#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
35#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
36#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
37#include "llvm/CodeGen/GlobalISel/Utils.h"
38#include "llvm/CodeGen/MachineDominators.h"
39#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
40#include "llvm/CodeGen/MachineFunctionPass.h"
41#include "llvm/CodeGen/MachinePassManager.h"
42#include "llvm/CodeGen/MachineRegisterInfo.h"
43#include "llvm/CodeGen/TargetOpcodes.h"
44#include "llvm/IR/IntrinsicsAArch64.h"
45#include "llvm/Support/Debug.h"
46
47#define GET_GICOMBINER_DEPS
48#include "AArch64GenPostLegalizeGICombiner.inc"
49#undef GET_GICOMBINER_DEPS
50
51#define DEBUG_TYPE "aarch64-postlegalizer-combiner"
52
53using namespace llvm;
54using namespace MIPatternMatch;
55
56#define GET_GICOMBINER_TYPES
57#include "AArch64GenPostLegalizeGICombiner.inc"
58#undef GET_GICOMBINER_TYPES
59
60namespace {
61
62/// This combine tries do what performExtractVectorEltCombine does in SDAG.
63/// Rewrite for pairwise fadd pattern
64/// (s32 (g_extract_vector_elt
65/// (g_fadd (vXs32 Other)
66/// (g_vector_shuffle (vXs32 Other) undef <1,X,...> )) 0))
67/// ->
68/// (s32 (g_fadd (g_extract_vector_elt (vXs32 Other) 0)
69/// (g_extract_vector_elt (vXs32 Other) 1))
70bool matchExtractVecEltPairwiseAdd(
71 MachineInstr &MI, MachineRegisterInfo &MRI,
72 std::tuple<unsigned, LLT, Register> &MatchInfo) {
73 Register Src1 = MI.getOperand(i: 1).getReg();
74 Register Src2 = MI.getOperand(i: 2).getReg();
75 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
76
77 auto Cst = getIConstantVRegValWithLookThrough(VReg: Src2, MRI);
78 if (!Cst || Cst->Value != 0)
79 return false;
80 // SDAG also checks for FullFP16, but this looks to be beneficial anyway.
81
82 // Now check for an fadd operation. TODO: expand this for integer add?
83 auto *FAddMI = getOpcodeDef(Opcode: TargetOpcode::G_FADD, Reg: Src1, MRI);
84 if (!FAddMI)
85 return false;
86
87 // If we add support for integer add, must restrict these types to just s64.
88 unsigned DstSize = DstTy.getSizeInBits();
89 if (DstSize != 16 && DstSize != 32 && DstSize != 64)
90 return false;
91
92 Register Src1Op1 = FAddMI->getOperand(i: 1).getReg();
93 Register Src1Op2 = FAddMI->getOperand(i: 2).getReg();
94 MachineInstr *Shuffle =
95 getOpcodeDef(Opcode: TargetOpcode::G_SHUFFLE_VECTOR, Reg: Src1Op2, MRI);
96 MachineInstr *Other = MRI.getVRegDef(Reg: Src1Op1);
97 if (!Shuffle) {
98 Shuffle = getOpcodeDef(Opcode: TargetOpcode::G_SHUFFLE_VECTOR, Reg: Src1Op1, MRI);
99 Other = MRI.getVRegDef(Reg: Src1Op2);
100 }
101
102 // We're looking for a shuffle that moves the second element to index 0.
103 if (Shuffle && Shuffle->getOperand(i: 3).getShuffleMask()[0] == 1 &&
104 Other == MRI.getVRegDef(Reg: Shuffle->getOperand(i: 1).getReg())) {
105 std::get<0>(t&: MatchInfo) = TargetOpcode::G_FADD;
106 std::get<1>(t&: MatchInfo) = DstTy;
107 std::get<2>(t&: MatchInfo) = Other->getOperand(i: 0).getReg();
108 return true;
109 }
110 return false;
111}
112
113void applyExtractVecEltPairwiseAdd(
114 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B,
115 std::tuple<unsigned, LLT, Register> &MatchInfo) {
116 unsigned Opc = std::get<0>(t&: MatchInfo);
117 assert(Opc == TargetOpcode::G_FADD && "Unexpected opcode!");
118 // We want to generate two extracts of elements 0 and 1, and add them.
119 LLT Ty = std::get<1>(t&: MatchInfo);
120 Register Src = std::get<2>(t&: MatchInfo);
121 LLT s64 = LLT::integer(SizeInBits: 64);
122 B.setInstrAndDebugLoc(MI);
123 auto Elt0 = B.buildExtractVectorElement(Res: Ty, Val: Src, Idx: B.buildConstant(Res: s64, Val: 0));
124 auto Elt1 = B.buildExtractVectorElement(Res: Ty, Val: Src, Idx: B.buildConstant(Res: s64, Val: 1));
125 B.buildInstr(Opc, DstOps: {MI.getOperand(i: 0).getReg()}, SrcOps: {Elt0, Elt1});
126 MI.eraseFromParent();
127}
128
129bool isSignExtended(Register R, MachineRegisterInfo &MRI) {
130 // TODO: check if extended build vector as well.
131 return mi_match(R, MRI, P: m_GSExt(Src: m_Reg())) ||
132 mi_match(R, MRI, P: m_GSExtInReg(Src: m_Reg()));
133}
134
135bool isZeroExtended(Register R, MachineRegisterInfo &MRI) {
136 // TODO: check if extended build vector as well.
137 return mi_match(R, MRI, P: m_GZExt(Src: m_Reg()));
138}
139
140bool matchAArch64MulConstCombine(
141 MachineInstr &MI, MachineRegisterInfo &MRI,
142 std::function<void(MachineIRBuilder &B, Register DstReg)> &ApplyFn) {
143 assert(MI.getOpcode() == TargetOpcode::G_MUL);
144 Register LHS = MI.getOperand(i: 1).getReg();
145 Register RHS = MI.getOperand(i: 2).getReg();
146 Register Dst = MI.getOperand(i: 0).getReg();
147 const LLT Ty = MRI.getType(Reg: LHS);
148
149 // The below optimizations require a constant RHS.
150 auto Const = getIConstantVRegValWithLookThrough(VReg: RHS, MRI);
151 if (!Const)
152 return false;
153
154 APInt ConstValue = Const->Value.sext(width: Ty.getSizeInBits());
155 // The following code is ported from AArch64ISelLowering.
156 // Multiplication of a power of two plus/minus one can be done more
157 // cheaply as shift+add/sub. For now, this is true unilaterally. If
158 // future CPUs have a cheaper MADD instruction, this may need to be
159 // gated on a subtarget feature. For Cyclone, 32-bit MADD is 4 cycles and
160 // 64-bit is 5 cycles, so this is always a win.
161 // More aggressively, some multiplications N0 * C can be lowered to
162 // shift+add+shift if the constant C = A * B where A = 2^N + 1 and B = 2^M,
163 // e.g. 6=3*2=(2+1)*2.
164 // TODO: consider lowering more cases, e.g. C = 14, -6, -14 or even 45
165 // which equals to (1+2)*16-(1+2).
166 // TrailingZeroes is used to test if the mul can be lowered to
167 // shift+add+shift.
168 unsigned TrailingZeroes = ConstValue.countr_zero();
169 if (TrailingZeroes) {
170 // Conservatively do not lower to shift+add+shift if the mul might be
171 // folded into smul or umul.
172 if (MRI.hasOneNonDBGUse(RegNo: LHS) &&
173 (isSignExtended(R: LHS, MRI) || isZeroExtended(R: LHS, MRI)))
174 return false;
175 // Conservatively do not lower to shift+add+shift if the mul might be
176 // folded into madd or msub.
177 if (MRI.hasOneNonDBGUse(RegNo: Dst)) {
178 MachineInstr &UseMI = *MRI.use_instr_nodbg_begin(RegNo: Dst);
179 unsigned UseOpc = UseMI.getOpcode();
180 if (UseOpc == TargetOpcode::G_ADD || UseOpc == TargetOpcode::G_PTR_ADD ||
181 UseOpc == TargetOpcode::G_SUB)
182 return false;
183 }
184 }
185 // Use ShiftedConstValue instead of ConstValue to support both shift+add/sub
186 // and shift+add+shift.
187 APInt ShiftedConstValue = ConstValue.ashr(ShiftAmt: TrailingZeroes);
188
189 unsigned ShiftAmt, AddSubOpc;
190 // Is the shifted value the LHS operand of the add/sub?
191 bool ShiftValUseIsLHS = true;
192 // Do we need to negate the result?
193 bool NegateResult = false;
194
195 if (ConstValue.isNonNegative()) {
196 // (mul x, 2^N + 1) => (add (shl x, N), x)
197 // (mul x, 2^N - 1) => (sub (shl x, N), x)
198 // (mul x, (2^N + 1) * 2^M) => (shl (add (shl x, N), x), M)
199 APInt SCVMinus1 = ShiftedConstValue - 1;
200 APInt CVPlus1 = ConstValue + 1;
201 if (SCVMinus1.isPowerOf2()) {
202 ShiftAmt = SCVMinus1.logBase2();
203 AddSubOpc = TargetOpcode::G_ADD;
204 } else if (CVPlus1.isPowerOf2()) {
205 ShiftAmt = CVPlus1.logBase2();
206 AddSubOpc = TargetOpcode::G_SUB;
207 } else
208 return false;
209 } else {
210 // (mul x, -(2^N - 1)) => (sub x, (shl x, N))
211 // (mul x, -(2^N + 1)) => - (add (shl x, N), x)
212 APInt CVNegPlus1 = -ConstValue + 1;
213 APInt CVNegMinus1 = -ConstValue - 1;
214 if (CVNegPlus1.isPowerOf2()) {
215 ShiftAmt = CVNegPlus1.logBase2();
216 AddSubOpc = TargetOpcode::G_SUB;
217 ShiftValUseIsLHS = false;
218 } else if (CVNegMinus1.isPowerOf2()) {
219 ShiftAmt = CVNegMinus1.logBase2();
220 AddSubOpc = TargetOpcode::G_ADD;
221 NegateResult = true;
222 } else
223 return false;
224 }
225
226 if (NegateResult && TrailingZeroes)
227 return false;
228
229 ApplyFn = [=](MachineIRBuilder &B, Register DstReg) {
230 auto Shift = B.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: ShiftAmt);
231 auto ShiftedVal = B.buildShl(Dst: Ty, Src0: LHS, Src1: Shift);
232
233 Register AddSubLHS = ShiftValUseIsLHS ? ShiftedVal.getReg(Idx: 0) : LHS;
234 Register AddSubRHS = ShiftValUseIsLHS ? LHS : ShiftedVal.getReg(Idx: 0);
235 auto Res = B.buildInstr(Opc: AddSubOpc, DstOps: {Ty}, SrcOps: {AddSubLHS, AddSubRHS});
236 assert(!(NegateResult && TrailingZeroes) &&
237 "NegateResult and TrailingZeroes cannot both be true for now.");
238 // Negate the result.
239 if (NegateResult) {
240 B.buildSub(Dst: DstReg, Src0: B.buildConstant(Res: Ty, Val: 0), Src1: Res);
241 return;
242 }
243 // Shift the result.
244 if (TrailingZeroes) {
245 B.buildShl(Dst: DstReg, Src0: Res,
246 Src1: B.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: TrailingZeroes));
247 return;
248 }
249 B.buildCopy(Res: DstReg, Op: Res.getReg(Idx: 0));
250 };
251 return true;
252}
253
254void applyAArch64MulConstCombine(
255 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B,
256 std::function<void(MachineIRBuilder &B, Register DstReg)> &ApplyFn) {
257 B.setInstrAndDebugLoc(MI);
258 ApplyFn(B, MI.getOperand(i: 0).getReg());
259 MI.eraseFromParent();
260}
261
262/// Match a 128b store of zero and split it into two 64 bit stores, for
263/// size/performance reasons.
264bool matchSplitStoreZero128(MachineInstr &MI, MachineRegisterInfo &MRI) {
265 GStore &Store = cast<GStore>(Val&: MI);
266 if (!Store.isSimple())
267 return false;
268 LLT ValTy = MRI.getType(Reg: Store.getValueReg());
269 if (ValTy.isScalableVector())
270 return false;
271 if (!ValTy.isVector() || ValTy.getSizeInBits() != 128)
272 return false;
273 if (Store.getMemSizeInBits() != ValTy.getSizeInBits())
274 return false; // Don't split truncating stores.
275 if (!MRI.hasOneNonDBGUse(RegNo: Store.getValueReg()))
276 return false;
277 auto MaybeCst = isConstantOrConstantSplatVector(Def: Store.getValueReg(), MRI);
278 return MaybeCst && MaybeCst->isZero();
279}
280
281void applySplitStoreZero128(MachineInstr &MI, MachineRegisterInfo &MRI,
282 MachineIRBuilder &B,
283 GISelChangeObserver &Observer) {
284 B.setInstrAndDebugLoc(MI);
285 GStore &Store = cast<GStore>(Val&: MI);
286 assert(MRI.getType(Store.getValueReg()).isVector() &&
287 "Expected a vector store value");
288 LLT NewTy = LLT::integer(SizeInBits: 64);
289 Register PtrReg = Store.getPointerReg();
290 auto Zero = B.buildConstant(Res: NewTy, Val: 0);
291 auto HighPtr =
292 B.buildPtrAdd(Res: MRI.getType(Reg: PtrReg), Op0: PtrReg, Op1: B.buildConstant(Res: NewTy, Val: 8));
293 auto &MF = *MI.getMF();
294 auto *LowMMO = MF.getMachineMemOperand(MMO: &Store.getMMO(), Offset: 0, Ty: NewTy);
295 auto *HighMMO = MF.getMachineMemOperand(MMO: &Store.getMMO(), Offset: 8, Ty: NewTy);
296 B.buildStore(Val: Zero, Addr: PtrReg, MMO&: *LowMMO);
297 B.buildStore(Val: Zero, Addr: HighPtr, MMO&: *HighMMO);
298 Store.eraseFromParent();
299}
300
301bool matchOrToBSP(MachineInstr &MI, MachineRegisterInfo &MRI,
302 std::tuple<Register, Register, Register> &MatchInfo) {
303 const LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
304 if (!DstTy.isVector())
305 return false;
306
307 Register AO1, AO2, BVO1, BVO2;
308 if (!mi_match(MI, MRI,
309 P: m_GOr(L: m_GAnd(L: m_Reg(R&: AO1), R: m_Reg(R&: BVO1)),
310 R: m_GAnd(L: m_Reg(R&: AO2), R: m_Reg(R&: BVO2)))))
311 return false;
312
313 auto *BV1 = getOpcodeDef<GBuildVector>(Reg: BVO1, MRI);
314 auto *BV2 = getOpcodeDef<GBuildVector>(Reg: BVO2, MRI);
315 if (!BV1 || !BV2)
316 return false;
317
318 for (int I = 0, E = DstTy.getNumElements(); I < E; I++) {
319 auto ValAndVReg1 =
320 getIConstantVRegValWithLookThrough(VReg: BV1->getSourceReg(I), MRI);
321 auto ValAndVReg2 =
322 getIConstantVRegValWithLookThrough(VReg: BV2->getSourceReg(I), MRI);
323 if (!ValAndVReg1 || !ValAndVReg2 ||
324 ValAndVReg1->Value != ~ValAndVReg2->Value)
325 return false;
326 }
327
328 MatchInfo = {AO1, AO2, BVO1};
329 return true;
330}
331
332void applyOrToBSP(MachineInstr &MI, MachineRegisterInfo &MRI,
333 MachineIRBuilder &B,
334 std::tuple<Register, Register, Register> &MatchInfo) {
335 B.setInstrAndDebugLoc(MI);
336 B.buildInstr(
337 Opc: AArch64::G_BSP, DstOps: {MI.getOperand(i: 0).getReg()},
338 SrcOps: {std::get<2>(t&: MatchInfo), std::get<0>(t&: MatchInfo), std::get<1>(t&: MatchInfo)});
339 MI.eraseFromParent();
340}
341
342/// Match G_TRUNC (G_OR X, Y) => G_ADDHN X, Y when both inputs are sign
343/// extended from the result element type. The high half of the addition then
344/// equals the truncation of the OR.
345bool matchTruncOrToADDHN(MachineInstr &MI, MachineRegisterInfo &MRI,
346 GISelValueTracking *VT, Register Dst, Register Or,
347 Register Src0, Register Src1) {
348 if (!MRI.hasOneNonDBGUse(RegNo: Or))
349 return false;
350
351 LLT DstTy = MRI.getType(Reg: Dst);
352 LLT SrcTy = MRI.getType(Reg: Or);
353 if (!((DstTy == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8) &&
354 SrcTy == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16)) ||
355 (DstTy == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16) &&
356 SrcTy == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32)) ||
357 (DstTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32) &&
358 SrcTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64))))
359 return false;
360
361 // If the narrow result is immediately any-extended back to the original type,
362 // the G_OR is cheaper than G_ADDHN followed by a vector widen.
363 if (MRI.hasOneNonDBGUse(RegNo: Dst)) {
364 MachineInstr &UseMI = *MRI.use_nodbg_instructions(Reg: Dst).begin();
365 if (UseMI.getOpcode() == TargetOpcode::G_ANYEXT &&
366 MRI.getType(Reg: UseMI.getOperand(i: 0).getReg()) == SrcTy)
367 return false;
368 }
369
370 unsigned EltSize = SrcTy.getScalarSizeInBits();
371 if (VT->computeNumSignBits(R: Src0) != EltSize ||
372 VT->computeNumSignBits(R: Src1) != EltSize)
373 return false;
374
375 return true;
376}
377
378// Combines Mul(And(Srl(X, 15), 0x10001), 0xffff) into CMLTz
379bool matchCombineMulCMLT(MachineInstr &MI, MachineRegisterInfo &MRI,
380 Register &SrcReg) {
381 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
382
383 if (DstTy != LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64) && DstTy != LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32) &&
384 DstTy != LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32) && DstTy != LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16) &&
385 DstTy != LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16))
386 return false;
387
388 auto AndMI = getDefIgnoringCopies(Reg: MI.getOperand(i: 1).getReg(), MRI);
389 if (AndMI->getOpcode() != TargetOpcode::G_AND)
390 return false;
391 auto LShrMI = getDefIgnoringCopies(Reg: AndMI->getOperand(i: 1).getReg(), MRI);
392 if (LShrMI->getOpcode() != TargetOpcode::G_LSHR)
393 return false;
394
395 // Check the constant splat values
396 auto V1 = isConstantOrConstantSplatVector(Def: MI.getOperand(i: 2).getReg(), MRI);
397 auto V2 = isConstantOrConstantSplatVector(Def: AndMI->getOperand(i: 2).getReg(), MRI);
398 auto V3 =
399 isConstantOrConstantSplatVector(Def: LShrMI->getOperand(i: 2).getReg(), MRI);
400 if (!V1.has_value() || !V2.has_value() || !V3.has_value())
401 return false;
402 unsigned HalfSize = DstTy.getScalarSizeInBits() / 2;
403 if (!V1.value().isMask(numBits: HalfSize) || V2.value() != (1ULL | 1ULL << HalfSize) ||
404 V3 != (HalfSize - 1))
405 return false;
406
407 SrcReg = LShrMI->getOperand(i: 1).getReg();
408
409 return true;
410}
411
412void applyCombineMulCMLT(MachineInstr &MI, MachineRegisterInfo &MRI,
413 MachineIRBuilder &B, Register &SrcReg) {
414 Register DstReg = MI.getOperand(i: 0).getReg();
415 LLT DstTy = MRI.getType(Reg: DstReg);
416 LLT HalfTy = DstTy.changeElementCount(EC: DstTy.getElementCount() * 2)
417 .changeElementSize(NewEltSize: DstTy.getScalarSizeInBits() / 2);
418
419 Register ZeroVec = B.buildConstant(Res: HalfTy, Val: 0).getReg(Idx: 0);
420 Register CastReg =
421 B.buildInstr(Opc: TargetOpcode::G_BITCAST, DstOps: {HalfTy}, SrcOps: {SrcReg}).getReg(Idx: 0);
422 Register CMLTReg =
423 B.buildICmp(Pred: CmpInst::Predicate::ICMP_SLT, Res: HalfTy, Op0: CastReg, Op1: ZeroVec)
424 .getReg(Idx: 0);
425
426 B.buildInstr(Opc: TargetOpcode::G_BITCAST, DstOps: {DstReg}, SrcOps: {CMLTReg}).getReg(Idx: 0);
427 MI.eraseFromParent();
428}
429
430// Match mul({z/s}ext , {z/s}ext) => {u/s}mull
431bool matchExtMulToMULL(MachineInstr &MI, MachineRegisterInfo &MRI,
432 GISelValueTracking *KB,
433 std::tuple<bool, Register, Register> &MatchInfo) {
434 // Get the instructions that defined the source operand
435 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
436 MachineInstr *I1 = getDefIgnoringCopies(Reg: MI.getOperand(i: 1).getReg(), MRI);
437 MachineInstr *I2 = getDefIgnoringCopies(Reg: MI.getOperand(i: 2).getReg(), MRI);
438 unsigned I1Opc = I1->getOpcode();
439 unsigned I2Opc = I2->getOpcode();
440 unsigned EltSize = DstTy.getScalarSizeInBits();
441
442 if (!DstTy.isVector() || I1->getNumOperands() < 2 || I2->getNumOperands() < 2)
443 return false;
444
445 auto IsAtLeastDoubleExtend = [&](Register R) {
446 LLT Ty = MRI.getType(Reg: R);
447 return EltSize >= Ty.getScalarSizeInBits() * 2;
448 };
449
450 // If the source operands were EXTENDED before, then {U/S}MULL can be used
451 bool IsZExt1 =
452 I1Opc == TargetOpcode::G_ZEXT || I1Opc == TargetOpcode::G_ANYEXT;
453 bool IsZExt2 =
454 I2Opc == TargetOpcode::G_ZEXT || I2Opc == TargetOpcode::G_ANYEXT;
455 if (IsZExt1 && IsZExt2 && IsAtLeastDoubleExtend(I1->getOperand(i: 1).getReg()) &&
456 IsAtLeastDoubleExtend(I2->getOperand(i: 1).getReg())) {
457 get<0>(t&: MatchInfo) = true;
458 get<1>(t&: MatchInfo) = I1->getOperand(i: 1).getReg();
459 get<2>(t&: MatchInfo) = I2->getOperand(i: 1).getReg();
460 return true;
461 }
462
463 bool IsSExt1 =
464 I1Opc == TargetOpcode::G_SEXT || I1Opc == TargetOpcode::G_ANYEXT;
465 bool IsSExt2 =
466 I2Opc == TargetOpcode::G_SEXT || I2Opc == TargetOpcode::G_ANYEXT;
467 if (IsSExt1 && IsSExt2 && IsAtLeastDoubleExtend(I1->getOperand(i: 1).getReg()) &&
468 IsAtLeastDoubleExtend(I2->getOperand(i: 1).getReg())) {
469 get<0>(t&: MatchInfo) = false;
470 get<1>(t&: MatchInfo) = I1->getOperand(i: 1).getReg();
471 get<2>(t&: MatchInfo) = I2->getOperand(i: 1).getReg();
472 return true;
473 }
474
475 // Select UMULL if we can replace the other operand with an extend.
476 APInt Mask = APInt::getHighBitsSet(numBits: EltSize, hiBitsSet: EltSize / 2);
477 if (KB && (IsZExt1 || IsZExt2) &&
478 IsAtLeastDoubleExtend(IsZExt1 ? I1->getOperand(i: 1).getReg()
479 : I2->getOperand(i: 1).getReg())) {
480 Register ZExtOp =
481 IsZExt1 ? MI.getOperand(i: 2).getReg() : MI.getOperand(i: 1).getReg();
482 if (KB->maskedValueIsZero(Val: ZExtOp, Mask)) {
483 get<0>(t&: MatchInfo) = true;
484 get<1>(t&: MatchInfo) = IsZExt1 ? I1->getOperand(i: 1).getReg() : ZExtOp;
485 get<2>(t&: MatchInfo) = IsZExt1 ? ZExtOp : I2->getOperand(i: 1).getReg();
486 return true;
487 }
488 } else if (KB && DstTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64) &&
489 KB->maskedValueIsZero(Val: MI.getOperand(i: 1).getReg(), Mask) &&
490 KB->maskedValueIsZero(Val: MI.getOperand(i: 2).getReg(), Mask)) {
491 get<0>(t&: MatchInfo) = true;
492 get<1>(t&: MatchInfo) = MI.getOperand(i: 1).getReg();
493 get<2>(t&: MatchInfo) = MI.getOperand(i: 2).getReg();
494 return true;
495 }
496
497 if (KB && (IsSExt1 || IsSExt2) &&
498 IsAtLeastDoubleExtend(IsSExt1 ? I1->getOperand(i: 1).getReg()
499 : I2->getOperand(i: 1).getReg())) {
500 Register SExtOp =
501 IsSExt1 ? MI.getOperand(i: 2).getReg() : MI.getOperand(i: 1).getReg();
502 if (KB->computeNumSignBits(R: SExtOp) > EltSize / 2) {
503 get<0>(t&: MatchInfo) = false;
504 get<1>(t&: MatchInfo) = IsSExt1 ? I1->getOperand(i: 1).getReg() : SExtOp;
505 get<2>(t&: MatchInfo) = IsSExt1 ? SExtOp : I2->getOperand(i: 1).getReg();
506 return true;
507 }
508 } else if (KB && DstTy == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64) &&
509 KB->computeNumSignBits(R: MI.getOperand(i: 1).getReg()) > EltSize / 2 &&
510 KB->computeNumSignBits(R: MI.getOperand(i: 2).getReg()) > EltSize / 2) {
511 get<0>(t&: MatchInfo) = false;
512 get<1>(t&: MatchInfo) = MI.getOperand(i: 1).getReg();
513 get<2>(t&: MatchInfo) = MI.getOperand(i: 2).getReg();
514 return true;
515 }
516
517 return false;
518}
519
520void applyExtMulToMULL(MachineInstr &MI, MachineRegisterInfo &MRI,
521 MachineIRBuilder &B, GISelChangeObserver &Observer,
522 std::tuple<bool, Register, Register> &MatchInfo) {
523 assert(MI.getOpcode() == TargetOpcode::G_MUL &&
524 "Expected a G_MUL instruction");
525
526 // Get the instructions that defined the source operand
527 LLT DstTy = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
528 bool IsZExt = get<0>(t&: MatchInfo);
529 Register Src1Reg = get<1>(t&: MatchInfo);
530 Register Src2Reg = get<2>(t&: MatchInfo);
531 LLT Src1Ty = MRI.getType(Reg: Src1Reg);
532 LLT Src2Ty = MRI.getType(Reg: Src2Reg);
533 LLT HalfDstTy = DstTy.changeElementSize(NewEltSize: DstTy.getScalarSizeInBits() / 2);
534 unsigned ExtOpc = IsZExt ? TargetOpcode::G_ZEXT : TargetOpcode::G_SEXT;
535
536 if (Src1Ty.getScalarSizeInBits() * 2 != DstTy.getScalarSizeInBits())
537 Src1Reg = B.buildExtOrTrunc(ExtOpc, Res: {HalfDstTy}, Op: {Src1Reg}).getReg(Idx: 0);
538 if (Src2Ty.getScalarSizeInBits() * 2 != DstTy.getScalarSizeInBits())
539 Src2Reg = B.buildExtOrTrunc(ExtOpc, Res: {HalfDstTy}, Op: {Src2Reg}).getReg(Idx: 0);
540
541 B.buildInstr(Opc: IsZExt ? AArch64::G_UMULL : AArch64::G_SMULL,
542 DstOps: {MI.getOperand(i: 0).getReg()}, SrcOps: {Src1Reg, Src2Reg});
543 MI.eraseFromParent();
544}
545
546static bool matchSubAddMulReassoc(Register Mul1, Register Mul2, Register Sub,
547 Register Src, MachineRegisterInfo &MRI) {
548 if (!MRI.hasOneNonDBGUse(RegNo: Sub))
549 return false;
550 if (getIConstantVRegValWithLookThrough(VReg: Src, MRI))
551 return false;
552 MachineInstr *M1 = getDefIgnoringCopies(Reg: Mul1, MRI);
553 if (M1->getOpcode() != AArch64::G_MUL &&
554 M1->getOpcode() != AArch64::G_SMULL &&
555 M1->getOpcode() != AArch64::G_UMULL)
556 return false;
557 MachineInstr *M2 = getDefIgnoringCopies(Reg: Mul2, MRI);
558 if (M2->getOpcode() != AArch64::G_MUL &&
559 M2->getOpcode() != AArch64::G_SMULL &&
560 M2->getOpcode() != AArch64::G_UMULL)
561 return false;
562 return true;
563}
564
565/// Match a vector G_ASHR/G_LSHR whose amount is not a valid immediate. These
566/// are lowered to SSHL/USHL of the negated amount by vashr_to_sshl and
567/// vlshr_to_ushl, so the negation can be folded into a constant amount.
568/// Splat-immediate shifts are left for vashr_vlshr_imm.
569bool matchVAshrLshrToSShl(MachineInstr &MI, MachineRegisterInfo &MRI) {
570 assert(MI.getOpcode() == TargetOpcode::G_ASHR ||
571 MI.getOpcode() == TargetOpcode::G_LSHR);
572 LLT Ty = MRI.getType(Reg: MI.getOperand(i: 0).getReg());
573 if (!Ty.isVector())
574 return false;
575 // Leave shifts by zero to the identity combines, which remove them.
576 if (isBuildVectorAllZeros(MI: *MRI.getVRegDef(Reg: MI.getOperand(i: 2).getReg()), MRI,
577 /*AllowUndef=*/true))
578 return false;
579 // Leave shifts by a valid immediate to vashr_vlshr_imm.
580 int64_t Cnt;
581 return !AArch64GISelUtils::isVShiftRImm(Reg: MI.getOperand(i: 2).getReg(), MRI, Ty,
582 Cnt);
583}
584
585class AArch64PostLegalizerCombinerImpl : public Combiner {
586protected:
587 const CombinerHelper Helper;
588 const AArch64PostLegalizerCombinerImplRuleConfig &RuleConfig;
589 const AArch64Subtarget &STI;
590
591public:
592 AArch64PostLegalizerCombinerImpl(
593 MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &VT,
594 GISelCSEInfo *CSEInfo,
595 const AArch64PostLegalizerCombinerImplRuleConfig &RuleConfig,
596 const AArch64Subtarget &STI, MachineDominatorTree *MDT,
597 const LegalizerInfo *LI);
598
599 static const char *getName() { return "AArch64PostLegalizerCombiner"; }
600
601 bool tryCombineAll(MachineInstr &I) const override;
602
603private:
604#define GET_GICOMBINER_CLASS_MEMBERS
605#include "AArch64GenPostLegalizeGICombiner.inc"
606#undef GET_GICOMBINER_CLASS_MEMBERS
607};
608
609#define GET_GICOMBINER_IMPL
610#include "AArch64GenPostLegalizeGICombiner.inc"
611#undef GET_GICOMBINER_IMPL
612
613AArch64PostLegalizerCombinerImpl::AArch64PostLegalizerCombinerImpl(
614 MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &VT,
615 GISelCSEInfo *CSEInfo,
616 const AArch64PostLegalizerCombinerImplRuleConfig &RuleConfig,
617 const AArch64Subtarget &STI, MachineDominatorTree *MDT,
618 const LegalizerInfo *LI)
619 : Combiner(MF, CInfo, &VT, CSEInfo),
620 Helper(Observer, B, /*IsPreLegalize*/ false, &VT, MDT, LI),
621 RuleConfig(RuleConfig), STI(STI),
622#define GET_GICOMBINER_CONSTRUCTOR_INITS
623#include "AArch64GenPostLegalizeGICombiner.inc"
624#undef GET_GICOMBINER_CONSTRUCTOR_INITS
625{
626}
627
628struct StoreInfo {
629 GStore *St = nullptr;
630 // The G_PTR_ADD that's used by the store. We keep this to cache the
631 // MachineInstr def.
632 GPtrAdd *Ptr = nullptr;
633 // The signed offset to the Ptr instruction.
634 int64_t Offset = 0;
635 LLT StoredType;
636};
637
638static bool tryOptimizeConsecStores(SmallVectorImpl<StoreInfo> &Stores,
639 CSEMIRBuilder &MIB) {
640 if (Stores.size() <= 2)
641 return false;
642
643 // Profitabity checks:
644 int64_t BaseOffset = Stores[0].Offset;
645 unsigned NumPairsExpected = Stores.size() / 2;
646 unsigned TotalInstsExpected = NumPairsExpected + (Stores.size() % 2);
647 // Size savings will depend on whether we can fold the offset, as an
648 // immediate of an ADD.
649 auto &TLI = *MIB.getMF().getSubtarget().getTargetLowering();
650 if (!TLI.isLegalAddImmediate(BaseOffset))
651 TotalInstsExpected++;
652 int SavingsExpected = Stores.size() - TotalInstsExpected;
653 if (SavingsExpected <= 0)
654 return false;
655
656 auto &MRI = MIB.getMF().getRegInfo();
657
658 // We have a series of consecutive stores. Factor out the common base
659 // pointer and rewrite the offsets.
660 Register NewBase = Stores[0].Ptr->getReg(Idx: 0);
661 for (auto &SInfo : Stores) {
662 // Compute a new pointer with the new base ptr and adjusted offset.
663 MIB.setInstrAndDebugLoc(*SInfo.St);
664 auto NewOff =
665 MIB.buildConstant(Res: LLT::integer(SizeInBits: 64), Val: SInfo.Offset - BaseOffset);
666 auto NewPtr = MIB.buildPtrAdd(Res: MRI.getType(Reg: SInfo.St->getPointerReg()),
667 Op0: NewBase, Op1: NewOff);
668 if (MIB.getObserver())
669 MIB.getObserver()->changingInstr(MI&: *SInfo.St);
670 SInfo.St->getOperand(i: 1).setReg(NewPtr.getReg(Idx: 0));
671 if (MIB.getObserver())
672 MIB.getObserver()->changedInstr(MI&: *SInfo.St);
673 }
674 LLVM_DEBUG(dbgs() << "Split a series of " << Stores.size()
675 << " stores into a base pointer and offsets.\n");
676 return true;
677}
678
679static bool optimizeConsecutiveMemOpAddressing(MachineFunction &MF,
680 CSEMIRBuilder &MIB) {
681 // This combine needs to run after all reassociations/folds on pointer
682 // addressing have been done, specifically those that combine two G_PTR_ADDs
683 // with constant offsets into a single G_PTR_ADD with a combined offset.
684 // The goal of this optimization is to undo that combine in the case where
685 // doing so has prevented the formation of pair stores due to illegal
686 // addressing modes of STP. The reason that we do it here is because
687 // it's much easier to undo the transformation of a series consecutive
688 // mem ops, than it is to detect when doing it would be a bad idea looking
689 // at a single G_PTR_ADD in the reassociation/ptradd_immed_chain combine.
690 //
691 // An example:
692 // G_STORE %11:_(<2 x s64>), %base:_(p0) :: (store (<2 x s64>), align 1)
693 // %off1:_(s64) = G_CONSTANT i64 4128
694 // %p1:_(p0) = G_PTR_ADD %0:_, %off1:_(s64)
695 // G_STORE %11:_(<2 x s64>), %p1:_(p0) :: (store (<2 x s64>), align 1)
696 // %off2:_(s64) = G_CONSTANT i64 4144
697 // %p2:_(p0) = G_PTR_ADD %0:_, %off2:_(s64)
698 // G_STORE %11:_(<2 x s64>), %p2:_(p0) :: (store (<2 x s64>), align 1)
699 // %off3:_(s64) = G_CONSTANT i64 4160
700 // %p3:_(p0) = G_PTR_ADD %0:_, %off3:_(s64)
701 // G_STORE %11:_(<2 x s64>), %17:_(p0) :: (store (<2 x s64>), align 1)
702 bool Changed = false;
703 auto &MRI = MF.getRegInfo();
704
705 if (!MF.getSubtarget<AArch64Subtarget>()
706 .getCLOpts()
707 .postlegalizer_consecutive_memops)
708 return Changed;
709
710 SmallVector<StoreInfo, 8> Stores;
711 // If we see a load, then we keep track of any values defined by it.
712 // In the following example, STP formation will fail anyway because
713 // the latter store is using a load result that appears after the
714 // the prior store. In this situation if we factor out the offset then
715 // we increase code size for no benefit.
716 // G_STORE %v1:_(s64), %base:_(p0) :: (store (s64))
717 // %v2:_(s64) = G_LOAD %ldptr:_(p0) :: (load (s64))
718 // G_STORE %v2:_(s64), %base:_(p0) :: (store (s64))
719 SmallVector<Register> LoadValsSinceLastStore;
720
721 auto storeIsValid = [&](StoreInfo &Last, StoreInfo New) {
722 // Check if this store is consecutive to the last one.
723 if (Last.Ptr->getBaseReg() != New.Ptr->getBaseReg() ||
724 (Last.Offset + static_cast<int64_t>(Last.StoredType.getSizeInBytes()) !=
725 New.Offset) ||
726 Last.StoredType != New.StoredType)
727 return false;
728
729 // Check if this store is using a load result that appears after the
730 // last store. If so, bail out.
731 if (any_of(Range&: LoadValsSinceLastStore, P: [&](Register LoadVal) {
732 return New.St->getValueReg() == LoadVal;
733 }))
734 return false;
735
736 // Check if the current offset would be too large for STP.
737 // If not, then STP formation should be able to handle it, so we don't
738 // need to do anything.
739 int64_t MaxLegalOffset;
740 switch (New.StoredType.getSizeInBits()) {
741 case 32:
742 MaxLegalOffset = 252;
743 break;
744 case 64:
745 MaxLegalOffset = 504;
746 break;
747 case 128:
748 MaxLegalOffset = 1008;
749 break;
750 default:
751 llvm_unreachable("Unexpected stored type size");
752 }
753 if (New.Offset < MaxLegalOffset)
754 return false;
755
756 // If factoring it out still wouldn't help then don't bother.
757 return New.Offset - Stores[0].Offset <= MaxLegalOffset;
758 };
759
760 auto resetState = [&]() {
761 Stores.clear();
762 LoadValsSinceLastStore.clear();
763 };
764
765 for (auto &MBB : MF) {
766 // We're looking inside a single BB at a time since the memset pattern
767 // should only be in a single block.
768 resetState();
769 for (auto &MI : MBB) {
770 // Skip for scalable vectors
771 if (auto *LdSt = dyn_cast<GLoadStore>(Val: &MI);
772 LdSt && MRI.getType(Reg: LdSt->getOperand(i: 0).getReg()).isScalableVector())
773 continue;
774
775 if (auto *St = dyn_cast<GStore>(Val: &MI)) {
776 Register PtrBaseReg;
777 APInt Offset;
778 LLT StoredValTy = MRI.getType(Reg: St->getValueReg());
779 unsigned ValSize = StoredValTy.getSizeInBits();
780 if (ValSize < 32 || St->getMMO().getSizeInBits() != ValSize)
781 continue;
782
783 Register PtrReg = St->getPointerReg();
784 if (mi_match(R: PtrReg, MRI,
785 P: m_OneUse(SP: m_GPtrAdd(L: m_Reg(R&: PtrBaseReg), R: m_ICst(Cst&: Offset))))) {
786 GPtrAdd *PtrAdd = cast<GPtrAdd>(Val: MRI.getVRegDef(Reg: PtrReg));
787 StoreInfo New = {.St: St, .Ptr: PtrAdd, .Offset: Offset.getSExtValue(), .StoredType: StoredValTy};
788
789 if (Stores.empty()) {
790 Stores.push_back(Elt: New);
791 continue;
792 }
793
794 // Check if this store is a valid continuation of the sequence.
795 auto &Last = Stores.back();
796 if (storeIsValid(Last, New)) {
797 Stores.push_back(Elt: New);
798 LoadValsSinceLastStore.clear(); // Reset the load value tracking.
799 } else {
800 // The store isn't a valid to consider for the prior sequence,
801 // so try to optimize what we have so far and start a new sequence.
802 Changed |= tryOptimizeConsecStores(Stores, MIB);
803 resetState();
804 Stores.push_back(Elt: New);
805 }
806 }
807 } else if (auto *Ld = dyn_cast<GLoad>(Val: &MI)) {
808 LoadValsSinceLastStore.push_back(Elt: Ld->getDstReg());
809 }
810 }
811 Changed |= tryOptimizeConsecStores(Stores, MIB);
812 resetState();
813 }
814
815 return Changed;
816}
817
818bool runCombiner(MachineFunction &MF, GISelCSEInfo *CSEInfo,
819 GISelValueTracking *VT, MachineDominatorTree *MDT,
820 const AArch64PostLegalizerCombinerImplRuleConfig &RuleConfig,
821 bool EnableOpt, bool IsOptNone) {
822 if (MF.getProperties().hasFailedISel())
823 return false;
824 const Function &F = MF.getFunction();
825
826 const AArch64Subtarget &ST = MF.getSubtarget<AArch64Subtarget>();
827 const LegalizerInfo *LI = ST.getLegalizerInfo();
828
829 CombinerInfo CInfo(/*AllowIllegalOps=*/false, /*ShouldLegalizeIllegal=*/false,
830 /*LegalizerInfo=*/LI, EnableOpt, F.hasOptSize(),
831 F.hasMinSize());
832 // Disable fixed-point iteration to reduce compile-time
833 CInfo.MaxIterations = 1;
834 CInfo.ObserverLvl = CombinerInfo::ObserverLevel::SinglePass;
835 // Legalizer performs DCE, so a full DCE pass is unnecessary.
836 CInfo.EnableFullDCE = false;
837 AArch64PostLegalizerCombinerImpl Impl(MF, CInfo, *VT, CSEInfo, RuleConfig, ST,
838 MDT, LI);
839 bool Changed = Impl.combineMachineInstrs();
840
841 CSEMIRBuilder MIB(MF);
842 MIB.setCSEInfo(CSEInfo);
843 Changed |= optimizeConsecutiveMemOpAddressing(MF, MIB);
844 return Changed;
845}
846
847class AArch64PostLegalizerCombinerLegacy : public MachineFunctionPass {
848public:
849 static char ID;
850
851 AArch64PostLegalizerCombinerLegacy(bool IsOptNone = false);
852
853 StringRef getPassName() const override {
854 return "AArch64PostLegalizerCombiner";
855 }
856
857 bool runOnMachineFunction(MachineFunction &MF) override;
858 void getAnalysisUsage(AnalysisUsage &AU) const override;
859
860 MachineFunctionProperties getRequiredProperties() const override {
861 return MachineFunctionProperties().set(
862 MachineFunctionProperties::Property::Legalized);
863 }
864
865private:
866 bool IsOptNone;
867 AArch64PostLegalizerCombinerImplRuleConfig RuleConfig;
868};
869} // end anonymous namespace
870
871void AArch64PostLegalizerCombinerLegacy::getAnalysisUsage(
872 AnalysisUsage &AU) const {
873 AU.setPreservesCFG();
874 getSelectionDAGFallbackAnalysisUsage(AU);
875 AU.addRequired<GISelValueTrackingAnalysisLegacy>();
876 AU.addPreserved<GISelValueTrackingAnalysisLegacy>();
877 if (!IsOptNone) {
878 AU.addRequired<MachineDominatorTreeWrapperPass>();
879 AU.addRequired<GISelCSEAnalysisWrapperPass>();
880 AU.addPreserved<GISelCSEAnalysisWrapperPass>();
881 }
882 MachineFunctionPass::getAnalysisUsage(AU);
883}
884
885AArch64PostLegalizerCombinerLegacy::AArch64PostLegalizerCombinerLegacy(
886 bool IsOptNone)
887 : MachineFunctionPass(ID), IsOptNone(IsOptNone) {
888 if (!RuleConfig.parseCommandLineOption())
889 reportFatalUsageError(reason: "Invalid rule identifier");
890}
891
892bool AArch64PostLegalizerCombinerLegacy::runOnMachineFunction(
893 MachineFunction &MF) {
894 if (MF.getProperties().hasFailedISel())
895 return false;
896
897 GISelValueTracking *VT =
898 &getAnalysis<GISelValueTrackingAnalysisLegacy>().get(MF);
899 MachineDominatorTree *MDT =
900 IsOptNone ? nullptr
901 : &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
902 GISelCSEAnalysisWrapper &Wrapper =
903 getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
904 auto *CSEInfo =
905 &Wrapper.get(CSEOpt: getStandardCSEConfigForOpt(Level: MF.getTarget().getOptLevel()));
906
907 bool EnableOpt = MF.getTarget().getOptLevel() != CodeGenOptLevel::None &&
908 !skipFunction(F: MF.getFunction());
909
910 return runCombiner(MF, CSEInfo, VT, MDT, RuleConfig, EnableOpt, IsOptNone);
911}
912
913char AArch64PostLegalizerCombinerLegacy::ID = 0;
914INITIALIZE_PASS_BEGIN(AArch64PostLegalizerCombinerLegacy, DEBUG_TYPE,
915 "Combine AArch64 MachineInstrs after legalization", false,
916 false)
917INITIALIZE_PASS_DEPENDENCY(GISelValueTrackingAnalysisLegacy)
918INITIALIZE_PASS_END(AArch64PostLegalizerCombinerLegacy, DEBUG_TYPE,
919 "Combine AArch64 MachineInstrs after legalization", false,
920 false)
921
922AArch64PostLegalizerCombinerPass::AArch64PostLegalizerCombinerPass(
923 const AArch64TargetMachine *TM)
924 : RuleConfig(
925 std::make_unique<AArch64PostLegalizerCombinerImplRuleConfig>()),
926 TM(TM) {
927 if (!RuleConfig->parseCommandLineOption())
928 reportFatalUsageError(reason: "invalid rule identifier");
929}
930
931AArch64PostLegalizerCombinerPass::AArch64PostLegalizerCombinerPass(
932 AArch64PostLegalizerCombinerPass &&) = default;
933
934AArch64PostLegalizerCombinerPass::~AArch64PostLegalizerCombinerPass() = default;
935
936PreservedAnalyses
937AArch64PostLegalizerCombinerPass::run(MachineFunction &MF,
938 MachineFunctionAnalysisManager &MFAM) {
939 if (MF.getProperties().hasFailedISel())
940 return PreservedAnalyses::all();
941
942 const bool IsOptNone = TM->isGlobalISelOptNone();
943 bool EnableOpt =
944 !IsOptNone && !shouldSkipOptimizationForOptBisect(IR: MF.getFunction());
945
946 GISelValueTracking *VT = &MFAM.getResult<GISelValueTrackingAnalysis>(IR&: MF);
947 MachineDominatorTree *MDT =
948 IsOptNone ? nullptr : &MFAM.getResult<MachineDominatorTreeAnalysis>(IR&: MF);
949 GISelCSEInfo *CSEInfo = MFAM.getResult<GISelCSEAnalysis>(IR&: MF).get();
950
951 if (!runCombiner(MF, CSEInfo, VT, MDT, RuleConfig: *RuleConfig, EnableOpt, IsOptNone))
952 return PreservedAnalyses::all();
953
954 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
955 PA.preserveSet<CFGAnalyses>();
956 PA.preserve<GISelValueTrackingAnalysis>();
957 PA.preserve<GISelCSEAnalysis>();
958 return PA;
959}
960
961namespace llvm {
962FunctionPass *createAArch64PostLegalizerCombinerLegacy(bool IsOptNone) {
963 return new AArch64PostLegalizerCombinerLegacy(IsOptNone);
964}
965} // end namespace llvm
966