1//===- AArch64MIPeepholeOpt.cpp - AArch64 MI peephole optimization pass ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass performs below peephole optimizations on MIR level.
10//
11// 1. MOVi32imm + (ANDS?|EOR|ORR)Wrr ==> (AND|EOR|ORR)Wri + (ANDS?|EOR|ORR)Wri
12// MOVi64imm + (ANDS?|EOR|ORR)Xrr ==> (AND|EOR|ORR)Xri + (ANDS?|EOR|ORR)Xri
13//
14// 2. MOVi32imm + ADDWrr ==> ADDWRi + ADDWRi
15// MOVi64imm + ADDXrr ==> ADDXri + ADDXri
16//
17// 3. MOVi32imm + SUBWrr ==> SUBWRi + SUBWRi
18// MOVi64imm + SUBXrr ==> SUBXri + SUBXri
19//
20// The mov pseudo instruction could be expanded to multiple mov instructions
21// later. In this case, we could try to split the constant operand of mov
22// instruction into two immediates which can be directly encoded into
23// *Wri/*Xri instructions. It makes two AND/ADD/SUB instructions instead of
24// multiple `mov` + `and/add/sub` instructions.
25//
26// 4. Remove redundant ORRWrs which is generated by zero-extend.
27//
28// %3:gpr32 = ORRWrs $wzr, %2, 0
29// %4:gpr64 = SUBREG_TO_REG %3, %subreg.sub_32
30//
31// If AArch64's 32-bit form of instruction defines the source operand of
32// ORRWrs, we can remove the ORRWrs because the upper 32 bits of the source
33// operand are set to zero.
34//
35// 5. %reg = INSERT_SUBREG %reg(tied-def 0), %subreg, subidx
36// ==> %reg:subidx = SUBREG_TO_REG %subreg, subidx
37//
38// 6. %intermediate:gpr32 = COPY %src:fpr128
39// %dst:fpr128 = INSvi32gpr %dst_vec:fpr128, dst_index, %intermediate:gpr32
40// ==> %dst:fpr128 = INSvi32lane %dst_vec:fpr128, dst_index, %src:fpr128, 0
41//
42// In cases where a source FPR is copied to a GPR in order to be copied
43// to a destination FPR, we can directly copy the values between the FPRs,
44// eliminating the use of the Integer unit. When we match a pattern of
45// INSvi[X]gpr that is preceded by a chain of COPY instructions from a FPR
46// source, we use the INSvi[X]lane to replace the COPY & INSvi[X]gpr
47// instructions.
48//
49// 7. If MI sets zero for high 64-bits implicitly, remove `mov 0` for high
50// 64-bits. For example,
51//
52// %1:fpr64 = nofpexcept FCVTNv4i16 %0:fpr128, implicit $fpcr
53// %2:fpr64 = MOVID 0
54// %4:fpr128 = IMPLICIT_DEF
55// %3:fpr128 = INSERT_SUBREG %4:fpr128(tied-def 0), %2:fpr64, %subreg.dsub
56// %6:fpr128 = IMPLICIT_DEF
57// %5:fpr128 = INSERT_SUBREG %6:fpr128(tied-def 0), %1:fpr64, %subreg.dsub
58// %7:fpr128 = INSvi64lane %5:fpr128(tied-def 0), 1, %3:fpr128, 0
59// ==>
60// %1:fpr64 = nofpexcept FCVTNv4i16 %0:fpr128, implicit $fpcr
61// %6:fpr128 = IMPLICIT_DEF
62// %7:fpr128 = INSERT_SUBREG %6:fpr128(tied-def 0), %1:fpr64, %subreg.dsub
63//
64// 8. Remove redundant CSELs that select between identical registers, by
65// replacing them with unconditional moves.
66//
67// 9. Replace UBFMXri with UBFMWri if the instruction is equivalent to a 32 bit
68// LSR or LSL alias of UBFM.
69//
70//===----------------------------------------------------------------------===//
71
72#include "AArch64ExpandImm.h"
73#include "AArch64InstrInfo.h"
74#include "MCTargetDesc/AArch64AddressingModes.h"
75#include "llvm/CodeGen/MachineDominators.h"
76#include "llvm/CodeGen/MachineLoopInfo.h"
77
78using namespace llvm;
79
80#define DEBUG_TYPE "aarch64-mi-peephole-opt"
81
82namespace {
83
84class AArch64MIPeepholeOptImpl {
85public:
86 const AArch64InstrInfo *TII;
87 const AArch64RegisterInfo *TRI;
88 MachineLoopInfo *MLI;
89 MachineRegisterInfo *MRI;
90
91 explicit AArch64MIPeepholeOptImpl(MachineLoopInfo &MLI) : MLI(&MLI) {}
92
93 bool run(MachineFunction &MF);
94
95private:
96 using OpcodePair = std::pair<unsigned, unsigned>;
97 template <typename T>
98 using SplitAndOpcFunc =
99 std::function<std::optional<OpcodePair>(T, unsigned, T &, T &)>;
100 using BuildMIFunc =
101 std::function<void(MachineInstr &, OpcodePair, unsigned, unsigned,
102 Register, Register, Register)>;
103
104 /// For instructions where an immediate operand could be split into two
105 /// separate immediate instructions, use the splitTwoPartImm two handle the
106 /// optimization.
107 ///
108 /// To implement, the following function types must be passed to
109 /// splitTwoPartImm. A SplitAndOpcFunc must be implemented that determines if
110 /// splitting the immediate is valid and returns the associated new opcode. A
111 /// BuildMIFunc must be implemented to build the two immediate instructions.
112 ///
113 /// Example Pattern (where IMM would require 2+ MOV instructions):
114 /// %dst = <Instr>rr %src IMM [...]
115 /// becomes:
116 /// %tmp = <Instr>ri %src (encode half IMM) [...]
117 /// %dst = <Instr>ri %tmp (encode half IMM) [...]
118 template <typename T>
119 bool splitTwoPartImm(MachineInstr &MI,
120 SplitAndOpcFunc<T> SplitAndOpc, BuildMIFunc BuildInstr);
121
122 bool checkMovImmInstr(MachineInstr &MI, MachineInstr *&MovMI,
123 MachineInstr *&SubregToRegMI);
124
125 template <typename T>
126 bool visitADDSUB(unsigned PosOpc, unsigned NegOpc, MachineInstr &MI);
127 template <typename T>
128 bool visitADDSSUBS(OpcodePair PosOpcs, OpcodePair NegOpcs, MachineInstr &MI);
129
130 // Strategy used to split logical immediate bitmasks.
131 enum class SplitStrategy {
132 Intersect,
133 Disjoint,
134 };
135 template <typename T>
136 bool trySplitLogicalImm(unsigned Opc, MachineInstr &MI,
137 SplitStrategy Strategy, unsigned OtherOpc = 0);
138 bool visitORR(MachineInstr &MI);
139 bool visitCSEL(MachineInstr &MI);
140 bool visitINSERT(MachineInstr &MI);
141 bool visitINSviGPR(MachineInstr &MI, unsigned Opc);
142 bool visitINSvi64lane(MachineInstr &MI);
143 bool visitFMOVDr(MachineInstr &MI);
144 bool visitUBFMXri(MachineInstr &MI);
145 bool visitCopy(MachineInstr &MI);
146};
147
148struct AArch64MIPeepholeOptLegacy : public MachineFunctionPass {
149 static char ID;
150
151 AArch64MIPeepholeOptLegacy() : MachineFunctionPass(ID) {}
152
153 bool runOnMachineFunction(MachineFunction &MF) override;
154
155 StringRef getPassName() const override {
156 return "AArch64 MI Peephole Optimization pass";
157 }
158
159 void getAnalysisUsage(AnalysisUsage &AU) const override {
160 AU.setPreservesCFG();
161 AU.addRequired<MachineLoopInfoWrapperPass>();
162 MachineFunctionPass::getAnalysisUsage(AU);
163 }
164};
165
166char AArch64MIPeepholeOptLegacy::ID = 0;
167
168} // end anonymous namespace
169
170INITIALIZE_PASS(AArch64MIPeepholeOptLegacy, "aarch64-mi-peephole-opt",
171 "AArch64 MI Peephole Optimization", false, false)
172
173template <typename T>
174static bool splitBitmaskImm(T Imm, unsigned RegSize, T &Imm1Enc, T &Imm2Enc) {
175 T UImm = static_cast<T>(Imm);
176 assert(UImm && (UImm != ~static_cast<T>(0)) && "Invalid immediate!");
177
178 // The bitmask immediate consists of consecutive ones. Let's say there is
179 // constant 0b00000000001000000000010000000000 which does not consist of
180 // consecutive ones. We can split it in to two bitmask immediate like
181 // 0b00000000001111111111110000000000 and 0b11111111111000000000011111111111.
182 // If we do AND with these two bitmask immediate, we can see original one.
183 unsigned LowestBitSet = llvm::countr_zero(UImm);
184 unsigned HighestBitSet = Log2_64(UImm);
185
186 // Create a mask which is filled with one from the position of lowest bit set
187 // to the position of highest bit set.
188 T NewImm1 = (static_cast<T>(2) << HighestBitSet) -
189 (static_cast<T>(1) << LowestBitSet);
190 // Create a mask which is filled with one outside the position of lowest bit
191 // set and the position of highest bit set.
192 T NewImm2 = UImm | ~NewImm1;
193
194 // If the split value is not valid bitmask immediate, do not split this
195 // constant.
196 if (!AArch64_AM::isLogicalImmediate(imm: NewImm2, regSize: RegSize))
197 return false;
198
199 Imm1Enc = AArch64_AM::encodeLogicalImmediate(imm: NewImm1, regSize: RegSize);
200 Imm2Enc = AArch64_AM::encodeLogicalImmediate(imm: NewImm2, regSize: RegSize);
201 return true;
202}
203
204template <typename T>
205static bool splitDisjointBitmaskImm(T Imm, unsigned RegSize, T &Imm1Enc,
206 T &Imm2Enc) {
207 assert(Imm && (Imm != ~static_cast<T>(0)) && "Invalid immediate!");
208
209 // Try to split a bitmask of the form 0b00000000011000000000011110000000 into
210 // two disjoint masks such as 0b00000000011000000000000000000000 and
211 // 0b00000000000000000000011110000000 where the inclusive/exclusive OR of the
212 // new masks match the original mask.
213 unsigned LowestBitSet = llvm::countr_zero(Imm);
214 unsigned LowestGapBitUnset =
215 LowestBitSet + llvm::countr_one(Imm >> LowestBitSet);
216
217 // Create a mask for the least significant group of consecutive ones.
218 assert(LowestGapBitUnset < sizeof(T) * CHAR_BIT && "Undefined behaviour!");
219 T NewImm1 = (static_cast<T>(1) << LowestGapBitUnset) -
220 (static_cast<T>(1) << LowestBitSet);
221 // Create a disjoint mask for the remaining ones.
222 T NewImm2 = Imm & ~NewImm1;
223
224 // Do not split if NewImm2 is not a valid bitmask immediate.
225 if (!AArch64_AM::isLogicalImmediate(imm: NewImm2, regSize: RegSize))
226 return false;
227
228 Imm1Enc = AArch64_AM::encodeLogicalImmediate(imm: NewImm1, regSize: RegSize);
229 Imm2Enc = AArch64_AM::encodeLogicalImmediate(imm: NewImm2, regSize: RegSize);
230 return true;
231}
232
233template <typename T>
234bool AArch64MIPeepholeOptImpl::trySplitLogicalImm(unsigned Opc,
235 MachineInstr &MI,
236 SplitStrategy Strategy,
237 unsigned OtherOpc) {
238 // Try below transformations.
239 //
240 // MOVi32imm + (ANDS?|EOR|ORR)Wrr ==> (AND|EOR|ORR)Wri + (ANDS?|EOR|ORR)Wri
241 // MOVi64imm + (ANDS?|EOR|ORR)Xrr ==> (AND|EOR|ORR)Xri + (ANDS?|EOR|ORR)Xri
242 //
243 // The mov pseudo instruction could be expanded to multiple mov instructions
244 // later. Let's try to split the constant operand of mov instruction into two
245 // bitmask immediates based on the given split strategy. It makes only two
246 // logical instructions instead of multiple mov + logic instructions.
247
248 return splitTwoPartImm<T>(
249 MI,
250 [Opc, Strategy, OtherOpc](T Imm, unsigned RegSize, T &Imm0,
251 T &Imm1) -> std::optional<OpcodePair> {
252 // If this immediate is already a suitable bitmask, don't split it.
253 // TODO: Should we just combine the two instructions in this case?
254 if (AArch64_AM::isLogicalImmediate(imm: Imm, regSize: RegSize))
255 return std::nullopt;
256
257 // If this immediate can be handled by one instruction, don't split it.
258 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
259 AArch64_IMM::expandMOVImm(Imm, BitSize: RegSize, Insn);
260 if (Insn.size() == 1)
261 return std::nullopt;
262
263 bool SplitSucc = false;
264 switch (Strategy) {
265 case SplitStrategy::Intersect:
266 SplitSucc = splitBitmaskImm(Imm, RegSize, Imm0, Imm1);
267 break;
268 case SplitStrategy::Disjoint:
269 SplitSucc = splitDisjointBitmaskImm(Imm, RegSize, Imm0, Imm1);
270 break;
271 }
272 if (SplitSucc)
273 return std::make_pair(x: Opc, y: !OtherOpc ? Opc : OtherOpc);
274 return std::nullopt;
275 },
276 [&TII = TII](MachineInstr &MI, OpcodePair Opcode, unsigned Imm0,
277 unsigned Imm1, Register SrcReg, Register NewTmpReg,
278 Register NewDstReg) {
279 DebugLoc DL = MI.getDebugLoc();
280 MachineBasicBlock *MBB = MI.getParent();
281 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opcode.first), DestReg: NewTmpReg)
282 .addReg(RegNo: SrcReg)
283 .addImm(Val: Imm0);
284 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opcode.second), DestReg: NewDstReg)
285 .addReg(RegNo: NewTmpReg)
286 .addImm(Val: Imm1);
287 });
288}
289
290bool AArch64MIPeepholeOptImpl::visitORR(MachineInstr &MI) {
291 // Check this ORR comes from below zero-extend pattern.
292 //
293 // def : Pat<(i64 (zext GPR32:$src)),
294 // (SUBREG_TO_REG (ORRWrs WZR, GPR32:$src, 0), sub_32)>;
295 if (MI.getOperand(i: 3).getImm() != 0)
296 return false;
297
298 if (MI.getOperand(i: 1).getReg() != AArch64::WZR)
299 return false;
300
301 if (MI.getOperand(i: 2).getSubReg())
302 return false;
303
304 MachineInstr *SrcMI = MRI->getUniqueVRegDef(Reg: MI.getOperand(i: 2).getReg());
305 if (!SrcMI)
306 return false;
307
308 // From https://developer.arm.com/documentation/dui0801/b/BABBGCAC
309 //
310 // When you use the 32-bit form of an instruction, the upper 32 bits of the
311 // source registers are ignored and the upper 32 bits of the destination
312 // register are set to zero.
313 //
314 // If AArch64's 32-bit form of instruction defines the source operand of
315 // zero-extend, we do not need the zero-extend. Let's check the MI's opcode is
316 // real AArch64 instruction and if it is not, do not process the opcode
317 // conservatively.
318 if (SrcMI->getOpcode() == TargetOpcode::COPY &&
319 SrcMI->getOperand(i: 1).getReg().isVirtual()) {
320 const TargetRegisterClass *RC =
321 MRI->getRegClass(Reg: SrcMI->getOperand(i: 1).getReg());
322
323 // A COPY from an FPR will become a FMOVSWr, so do so now so that we know
324 // that the upper bits are zero.
325 if (RC != &AArch64::FPR32RegClass &&
326 ((RC != &AArch64::FPR64RegClass && RC != &AArch64::FPR128RegClass &&
327 RC != &AArch64::ZPRRegClass) ||
328 SrcMI->getOperand(i: 1).getSubReg() != AArch64::ssub))
329 return false;
330 Register CpySrc;
331 if (SrcMI->getOperand(i: 1).getSubReg() == AArch64::ssub) {
332 CpySrc = MRI->createVirtualRegister(RegClass: &AArch64::FPR32RegClass);
333 BuildMI(BB&: *SrcMI->getParent(), I: SrcMI, MIMD: SrcMI->getDebugLoc(),
334 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: CpySrc)
335 .add(MO: SrcMI->getOperand(i: 1));
336 } else {
337 CpySrc = SrcMI->getOperand(i: 1).getReg();
338 }
339 BuildMI(BB&: *SrcMI->getParent(), I: SrcMI, MIMD: SrcMI->getDebugLoc(),
340 MCID: TII->get(Opcode: AArch64::FMOVSWr), DestReg: SrcMI->getOperand(i: 0).getReg())
341 .addReg(RegNo: CpySrc);
342 SrcMI->eraseFromParent();
343 }
344 else if (SrcMI->getOpcode() <= TargetOpcode::GENERIC_OP_END)
345 return false;
346
347 Register DefReg = MI.getOperand(i: 0).getReg();
348 Register SrcReg = MI.getOperand(i: 2).getReg();
349 MRI->replaceRegWith(FromReg: DefReg, ToReg: SrcReg);
350 MRI->clearKillFlags(Reg: SrcReg);
351 LLVM_DEBUG(dbgs() << "Removed: " << MI << "\n");
352 MI.eraseFromParent();
353
354 return true;
355}
356
357bool AArch64MIPeepholeOptImpl::visitCSEL(MachineInstr &MI) {
358 // Replace CSEL with MOV when both inputs are the same register.
359 if (MI.getOperand(i: 1).getReg() != MI.getOperand(i: 2).getReg())
360 return false;
361
362 auto ZeroReg =
363 MI.getOpcode() == AArch64::CSELXr ? AArch64::XZR : AArch64::WZR;
364 auto OrOpcode =
365 MI.getOpcode() == AArch64::CSELXr ? AArch64::ORRXrs : AArch64::ORRWrs;
366
367 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: OrOpcode))
368 .addReg(RegNo: MI.getOperand(i: 0).getReg(), Flags: RegState::Define)
369 .addReg(RegNo: ZeroReg)
370 .addReg(RegNo: MI.getOperand(i: 1).getReg())
371 .addImm(Val: 0);
372
373 MI.eraseFromParent();
374 return true;
375}
376
377bool AArch64MIPeepholeOptImpl::visitINSERT(MachineInstr &MI) {
378 // Check this INSERT_SUBREG comes from below zero-extend pattern.
379 //
380 // From %reg = INSERT_SUBREG %reg(tied-def 0), %subreg, subidx
381 // To %reg:subidx = SUBREG_TO_REG %subreg, subidx
382 //
383 // We're assuming the first operand to INSERT_SUBREG is irrelevant because a
384 // COPY would destroy the upper part of the register anyway
385 if (!MI.isRegTiedToDefOperand(UseOpIdx: 1))
386 return false;
387
388 Register DstReg = MI.getOperand(i: 0).getReg();
389 const TargetRegisterClass *RC = MRI->getRegClass(Reg: DstReg);
390 MachineInstr *SrcMI = MRI->getUniqueVRegDef(Reg: MI.getOperand(i: 2).getReg());
391 if (!SrcMI)
392 return false;
393
394 // From https://developer.arm.com/documentation/dui0801/b/BABBGCAC
395 //
396 // When you use the 32-bit form of an instruction, the upper 32 bits of the
397 // source registers are ignored and the upper 32 bits of the destination
398 // register are set to zero.
399 //
400 // If AArch64's 32-bit form of instruction defines the source operand of
401 // zero-extend, we do not need the zero-extend. Let's check the MI's opcode is
402 // real AArch64 instruction and if it is not, do not process the opcode
403 // conservatively.
404 if ((SrcMI->getOpcode() <= TargetOpcode::GENERIC_OP_END) ||
405 !AArch64::GPR64allRegClass.hasSubClassEq(RC))
406 return false;
407
408 // Build a SUBREG_TO_REG instruction
409 MachineInstr *SubregMI =
410 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
411 MCID: TII->get(Opcode: TargetOpcode::SUBREG_TO_REG), DestReg: DstReg)
412 .add(MO: MI.getOperand(i: 2))
413 .add(MO: MI.getOperand(i: 3));
414 LLVM_DEBUG(dbgs() << MI << " replace by:\n: " << *SubregMI << "\n");
415 (void)SubregMI;
416 MI.eraseFromParent();
417
418 return true;
419}
420
421template <typename T>
422static bool splitAddSubImm(T Imm, unsigned RegSize, T &Imm0, T &Imm1) {
423 // The immediate must be in the form of ((imm0 << 12) + imm1), in which both
424 // imm0 and imm1 are non-zero 12-bit unsigned int.
425 if ((Imm & 0xfff000) == 0 || (Imm & 0xfff) == 0 ||
426 (Imm & ~static_cast<T>(0xffffff)) != 0)
427 return false;
428
429 // The immediate can not be composed via a single instruction.
430 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
431 AArch64_IMM::expandMOVImm(Imm, BitSize: RegSize, Insn);
432 if (Insn.size() == 1)
433 return false;
434
435 // Split Imm into (Imm0 << 12) + Imm1;
436 Imm0 = (Imm >> 12) & 0xfff;
437 Imm1 = Imm & 0xfff;
438 return true;
439}
440
441template <typename T>
442bool AArch64MIPeepholeOptImpl::visitADDSUB(unsigned PosOpc, unsigned NegOpc,
443 MachineInstr &MI) {
444 // Try below transformation.
445 //
446 // ADDWrr X, MOVi32imm ==> ADDWri + ADDWri
447 // ADDXrr X, MOVi64imm ==> ADDXri + ADDXri
448 //
449 // SUBWrr X, MOVi32imm ==> SUBWri + SUBWri
450 // SUBXrr X, MOVi64imm ==> SUBXri + SUBXri
451 //
452 // The mov pseudo instruction could be expanded to multiple mov instructions
453 // later. Let's try to split the constant operand of mov instruction into two
454 // legal add/sub immediates. It makes only two ADD/SUB instructions instead of
455 // multiple `mov` + `and/sub` instructions.
456
457 // We can sometimes have ADDWrr WZR, MULi32imm that have not been constant
458 // folded. Make sure that we don't generate invalid instructions that use XZR
459 // in those cases.
460 if (MI.getOperand(i: 1).getReg() == AArch64::XZR ||
461 MI.getOperand(i: 1).getReg() == AArch64::WZR)
462 return false;
463
464 return splitTwoPartImm<T>(
465 MI,
466 [PosOpc, NegOpc](T Imm, unsigned RegSize, T &Imm0,
467 T &Imm1) -> std::optional<OpcodePair> {
468 if (splitAddSubImm(Imm, RegSize, Imm0, Imm1))
469 return std::make_pair(x: PosOpc, y: PosOpc);
470 if (splitAddSubImm(-Imm, RegSize, Imm0, Imm1))
471 return std::make_pair(x: NegOpc, y: NegOpc);
472 return std::nullopt;
473 },
474 [&TII = TII](MachineInstr &MI, OpcodePair Opcode, unsigned Imm0,
475 unsigned Imm1, Register SrcReg, Register NewTmpReg,
476 Register NewDstReg) {
477 DebugLoc DL = MI.getDebugLoc();
478 MachineBasicBlock *MBB = MI.getParent();
479 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opcode.first), DestReg: NewTmpReg)
480 .addReg(RegNo: SrcReg)
481 .addImm(Val: Imm0)
482 .addImm(Val: 12);
483 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opcode.second), DestReg: NewDstReg)
484 .addReg(RegNo: NewTmpReg)
485 .addImm(Val: Imm1)
486 .addImm(Val: 0);
487 });
488}
489
490template <typename T>
491bool AArch64MIPeepholeOptImpl::visitADDSSUBS(OpcodePair PosOpcs,
492 OpcodePair NegOpcs,
493 MachineInstr &MI) {
494 // Try the same transformation as ADDSUB but with additional requirement
495 // that the condition code usages are only for Equal and Not Equal
496
497 if (MI.getOperand(i: 1).getReg() == AArch64::XZR ||
498 MI.getOperand(i: 1).getReg() == AArch64::WZR)
499 return false;
500
501 return splitTwoPartImm<T>(
502 MI,
503 [PosOpcs, NegOpcs, &MI, &TRI = TRI,
504 &MRI = MRI](T Imm, unsigned RegSize, T &Imm0,
505 T &Imm1) -> std::optional<OpcodePair> {
506 OpcodePair OP;
507 if (splitAddSubImm(Imm, RegSize, Imm0, Imm1))
508 OP = PosOpcs;
509 else if (splitAddSubImm(-Imm, RegSize, Imm0, Imm1))
510 OP = NegOpcs;
511 else
512 return std::nullopt;
513 // Check conditional uses last since it is expensive for scanning
514 // proceeding instructions
515 MachineInstr *SrcMI = MRI->getVRegDef(Reg: MI.getOperand(i: 1).getReg());
516 if (!SrcMI)
517 return std::nullopt;
518 std::optional<UsedNZCV> NZCVUsed = examineCFlagsUse(MI&: *SrcMI, CmpInstr&: MI, TRI: *TRI);
519 if (!NZCVUsed || NZCVUsed->C || NZCVUsed->V)
520 return std::nullopt;
521 return OP;
522 },
523 [&TII = TII](MachineInstr &MI, OpcodePair Opcode, unsigned Imm0,
524 unsigned Imm1, Register SrcReg, Register NewTmpReg,
525 Register NewDstReg) {
526 DebugLoc DL = MI.getDebugLoc();
527 MachineBasicBlock *MBB = MI.getParent();
528 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opcode.first), DestReg: NewTmpReg)
529 .addReg(RegNo: SrcReg)
530 .addImm(Val: Imm0)
531 .addImm(Val: 12);
532 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opcode.second), DestReg: NewDstReg)
533 .addReg(RegNo: NewTmpReg)
534 .addImm(Val: Imm1)
535 .addImm(Val: 0);
536 });
537}
538
539// Checks if the corresponding MOV immediate instruction is applicable for
540// this peephole optimization.
541bool AArch64MIPeepholeOptImpl::checkMovImmInstr(MachineInstr &MI,
542 MachineInstr *&MovMI,
543 MachineInstr *&SubregToRegMI) {
544 // Check whether current MBB is in loop and the AND is loop invariant.
545 MachineBasicBlock *MBB = MI.getParent();
546 MachineLoop *L = MLI->getLoopFor(BB: MBB);
547 if (L && !L->isLoopInvariant(I&: MI))
548 return false;
549
550 // Check whether current MI's operand is MOV with immediate.
551 MovMI = MRI->getUniqueVRegDef(Reg: MI.getOperand(i: 2).getReg());
552 if (!MovMI)
553 return false;
554
555 // If it is SUBREG_TO_REG, check its operand.
556 SubregToRegMI = nullptr;
557 if (MovMI->getOpcode() == TargetOpcode::SUBREG_TO_REG) {
558 SubregToRegMI = MovMI;
559 MovMI = MRI->getUniqueVRegDef(Reg: MovMI->getOperand(i: 1).getReg());
560 if (!MovMI)
561 return false;
562 }
563
564 if (MovMI->getOpcode() != AArch64::MOVi32imm &&
565 MovMI->getOpcode() != AArch64::MOVi64imm)
566 return false;
567
568 // If the MOV has multiple uses, do not split the immediate because it causes
569 // more instructions.
570 if (!MRI->hasOneUse(RegNo: MovMI->getOperand(i: 0).getReg()))
571 return false;
572 if (SubregToRegMI && !MRI->hasOneUse(RegNo: SubregToRegMI->getOperand(i: 0).getReg()))
573 return false;
574
575 // It is OK to perform this peephole optimization.
576 return true;
577}
578
579template <typename T>
580bool AArch64MIPeepholeOptImpl::splitTwoPartImm(MachineInstr &MI,
581 SplitAndOpcFunc<T> SplitAndOpc,
582 BuildMIFunc BuildInstr) {
583 unsigned RegSize = sizeof(T) * 8;
584 assert((RegSize == 32 || RegSize == 64) &&
585 "Invalid RegSize for legal immediate peephole optimization");
586
587 // Perform several essential checks against current MI.
588 MachineInstr *MovMI, *SubregToRegMI;
589 if (!checkMovImmInstr(MI, MovMI, SubregToRegMI))
590 return false;
591
592 // Split the immediate to Imm0 and Imm1, and calculate the Opcode.
593 T Imm = static_cast<T>(MovMI->getOperand(i: 1).getImm()), Imm0, Imm1;
594 // For the 32 bit form of instruction, the upper 32 bits of the destination
595 // register are set to zero. If there is SUBREG_TO_REG, set the upper 32 bits
596 // of Imm to zero. This is essential if the Immediate value was a negative
597 // number since it was sign extended when we assign to the 64-bit Imm.
598 if (SubregToRegMI)
599 Imm &= 0xFFFFFFFF;
600 OpcodePair Opcode;
601 if (auto R = SplitAndOpc(Imm, RegSize, Imm0, Imm1))
602 Opcode = *R;
603 else
604 return false;
605
606 // Create new MIs using the first and second opcodes. Opcodes might differ for
607 // flag setting operations that should only set flags on second instruction.
608 // NewTmpReg = Opcode.first SrcReg Imm0
609 // NewDstReg = Opcode.second NewTmpReg Imm1
610
611 // Determine register classes for destinations and register operands
612 const TargetRegisterClass *FirstInstrDstRC =
613 TII->getRegClass(MCID: TII->get(Opcode: Opcode.first), OpNum: 0);
614 const TargetRegisterClass *FirstInstrOperandRC =
615 TII->getRegClass(MCID: TII->get(Opcode: Opcode.first), OpNum: 1);
616 const TargetRegisterClass *SecondInstrDstRC =
617 (Opcode.first == Opcode.second)
618 ? FirstInstrDstRC
619 : TII->getRegClass(MCID: TII->get(Opcode: Opcode.second), OpNum: 0);
620 const TargetRegisterClass *SecondInstrOperandRC =
621 (Opcode.first == Opcode.second)
622 ? FirstInstrOperandRC
623 : TII->getRegClass(MCID: TII->get(Opcode: Opcode.second), OpNum: 1);
624
625 // Get old registers destinations and new register destinations
626 Register DstReg = MI.getOperand(i: 0).getReg();
627 Register SrcReg = MI.getOperand(i: 1).getReg();
628 Register NewTmpReg = MRI->createVirtualRegister(RegClass: FirstInstrDstRC);
629 // In the situation that DstReg is not Virtual (likely WZR or XZR), we want to
630 // reuse that same destination register.
631 Register NewDstReg = DstReg.isVirtual()
632 ? MRI->createVirtualRegister(RegClass: SecondInstrDstRC)
633 : DstReg;
634
635 // Constrain registers based on their new uses
636 MRI->constrainRegClass(Reg: SrcReg, RC: FirstInstrOperandRC);
637 MRI->constrainRegClass(Reg: NewTmpReg, RC: SecondInstrOperandRC);
638 if (DstReg != NewDstReg)
639 MRI->constrainRegClass(Reg: NewDstReg, RC: MRI->getRegClass(Reg: DstReg));
640
641 // Call the delegating operation to build the instruction
642 BuildInstr(MI, Opcode, Imm0, Imm1, SrcReg, NewTmpReg, NewDstReg);
643
644 // replaceRegWith changes MI's definition register. Keep it for SSA form until
645 // deleting MI. Only if we made a new destination register.
646 if (DstReg != NewDstReg) {
647 MRI->replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
648 MI.getOperand(i: 0).setReg(DstReg);
649 }
650
651 // Record the MIs need to be removed.
652 MI.eraseFromParent();
653 if (SubregToRegMI)
654 SubregToRegMI->eraseFromParent();
655 MovMI->eraseFromParent();
656
657 return true;
658}
659
660bool AArch64MIPeepholeOptImpl::visitINSviGPR(MachineInstr &MI, unsigned Opc) {
661 // Check if this INSvi[X]gpr comes from COPY of a source FPR128
662 //
663 // From
664 // %intermediate1:gpr64 = COPY %src:fpr128
665 // %intermediate2:gpr32 = COPY %intermediate1:gpr64
666 // %dst:fpr128 = INSvi[X]gpr %dst_vec:fpr128, dst_index, %intermediate2:gpr32
667 // To
668 // %dst:fpr128 = INSvi[X]lane %dst_vec:fpr128, dst_index, %src:fpr128,
669 // src_index
670 // where src_index = 0, X = [8|16|32|64]
671
672 MachineInstr *SrcMI = MRI->getUniqueVRegDef(Reg: MI.getOperand(i: 3).getReg());
673
674 // For a chain of COPY instructions, find the initial source register
675 // and check if it's an FPR128
676 while (true) {
677 if (!SrcMI || SrcMI->getOpcode() != TargetOpcode::COPY)
678 return false;
679
680 if (!SrcMI->getOperand(i: 1).getReg().isVirtual())
681 return false;
682
683 if (MRI->getRegClass(Reg: SrcMI->getOperand(i: 1).getReg()) ==
684 &AArch64::FPR128RegClass) {
685 break;
686 }
687 SrcMI = MRI->getUniqueVRegDef(Reg: SrcMI->getOperand(i: 1).getReg());
688 }
689
690 Register DstReg = MI.getOperand(i: 0).getReg();
691 Register SrcReg = SrcMI->getOperand(i: 1).getReg();
692 MachineInstr *INSvilaneMI =
693 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc), DestReg: DstReg)
694 .add(MO: MI.getOperand(i: 1))
695 .add(MO: MI.getOperand(i: 2))
696 .addUse(RegNo: SrcReg, Flags: getRegState(RegOp: SrcMI->getOperand(i: 1)))
697 .addImm(Val: 0);
698
699 LLVM_DEBUG(dbgs() << MI << " replace by:\n: " << *INSvilaneMI << "\n");
700 (void)INSvilaneMI;
701 MI.eraseFromParent();
702 return true;
703}
704
705// All instructions that set a FPR64 will implicitly zero the top bits of the
706// register. When the def is expressed as a COPY from a GPR, turn it into an
707// explicit FMOV so it cannot be elided later in further passes.
708static bool is64bitDefwithZeroHigh64bit(MachineInstr *MI,
709 MachineRegisterInfo *MRI,
710 const AArch64InstrInfo *TII) {
711 if (!MI->getOperand(i: 0).isReg() || !MI->getOperand(i: 0).isDef())
712 return false;
713 const TargetRegisterClass *RC = MRI->getRegClass(Reg: MI->getOperand(i: 0).getReg());
714 if (RC != &AArch64::FPR64RegClass)
715 return false;
716 if (MI->getOpcode() == TargetOpcode::COPY) {
717 MachineOperand &SrcOp = MI->getOperand(i: 1);
718 if (!SrcOp.isReg())
719 return false;
720 Register SrcReg = SrcOp.getReg();
721 if (SrcOp.getSubReg()) {
722 // If operand is defined by a LD1/2/3/4 that define a D subreg tuple
723 // then upper bits of the tuple's registers are implicitly zeroed
724 // and the FMOV is unneeded.
725 if (!SrcReg.isVirtual())
726 return false;
727
728 MachineInstr *SrcDef = MRI->getUniqueVRegDef(Reg: SrcReg);
729 if (!SrcDef || SrcDef->getOpcode() <= TargetOpcode::GENERIC_OP_END ||
730 SrcDef->getDesc().isPseudo() || !SrcDef->mayLoad())
731 return false;
732
733 const TargetRegisterClass *RC = MRI->getRegClass(Reg: SrcReg);
734 return RC == &AArch64::DDRegClass || RC == &AArch64::DDDRegClass ||
735 RC == &AArch64::DDDDRegClass;
736 }
737 auto IsGPR64Like = [&]() -> bool {
738 if (SrcReg.isVirtual())
739 return AArch64::GPR64allRegClass.hasSubClassEq(
740 RC: MRI->getRegClass(Reg: SrcReg));
741 return AArch64::GPR64allRegClass.contains(Reg: SrcReg);
742 };
743 if (!IsGPR64Like())
744 return false;
745 assert(TII && "Expected InstrInfo when materializing COPYs");
746 // FMOVXDr insists on strict GPR64 operands, so fix up the COPY source.
747 MachineOperand &SrcMO = MI->getOperand(i: 1);
748 bool SrcKill = SrcMO.isKill();
749 if (SrcReg.isVirtual()) {
750 if (MRI->getRegClass(Reg: SrcReg) != &AArch64::GPR64RegClass) {
751 // Pass the value through a temporary GPR64 vreg to satisfy the
752 // verifier.
753 Register NewSrc = MRI->createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
754 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
755 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewSrc)
756 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: SrcKill));
757 SrcReg = NewSrc;
758 SrcKill = true;
759 }
760 } else if (!AArch64::GPR64RegClass.contains(Reg: SrcReg)) {
761 return false;
762 }
763 SrcMO.setReg(SrcReg);
764 SrcMO.setSubReg(0);
765 SrcMO.setIsKill(SrcKill);
766 // Replace the COPY with an explicit FMOV so the zeroing behaviour stays
767 // visible.
768 MI->setDesc(TII->get(Opcode: AArch64::FMOVXDr));
769 return true;
770 }
771 return MI->getOpcode() > TargetOpcode::GENERIC_OP_END;
772}
773
774bool AArch64MIPeepholeOptImpl::visitINSvi64lane(MachineInstr &MI) {
775 // Check the MI for low 64-bits sets zero for high 64-bits implicitly.
776 // We are expecting below case.
777 //
778 // %1:fpr64 = nofpexcept FCVTNv4i16 %0:fpr128, implicit $fpcr
779 // %6:fpr128 = IMPLICIT_DEF
780 // %5:fpr128 = INSERT_SUBREG %6:fpr128(tied-def 0), killed %1:fpr64, %subreg.dsub
781 // %7:fpr128 = INSvi64lane %5:fpr128(tied-def 0), 1, killed %3:fpr128, 0
782 MachineInstr *Low64MI = MRI->getVRegDef(Reg: MI.getOperand(i: 1).getReg());
783 if (!Low64MI || Low64MI->getOpcode() != AArch64::INSERT_SUBREG)
784 return false;
785 Low64MI = MRI->getUniqueVRegDef(Reg: Low64MI->getOperand(i: 2).getReg());
786 if (!Low64MI || !is64bitDefwithZeroHigh64bit(MI: Low64MI, MRI, TII))
787 return false;
788
789 // Check there is `mov 0` MI for high 64-bits.
790 // We are expecting below cases.
791 //
792 // %2:fpr64 = MOVID 0
793 // %4:fpr128 = IMPLICIT_DEF
794 // %3:fpr128 = INSERT_SUBREG %4:fpr128(tied-def 0), killed %2:fpr64, %subreg.dsub
795 // %7:fpr128 = INSvi64lane %5:fpr128(tied-def 0), 1, killed %3:fpr128, 0
796 // or
797 // %5:fpr128 = MOVIv2d_ns 0
798 // %6:fpr64 = COPY %5.dsub:fpr128
799 // %8:fpr128 = IMPLICIT_DEF
800 // %7:fpr128 = INSERT_SUBREG %8:fpr128(tied-def 0), killed %6:fpr64, %subreg.dsub
801 // %11:fpr128 = INSvi64lane %9:fpr128(tied-def 0), 1, killed %7:fpr128, 0
802 MachineInstr *High64MI = MRI->getUniqueVRegDef(Reg: MI.getOperand(i: 3).getReg());
803 if (!High64MI || High64MI->getOpcode() != AArch64::INSERT_SUBREG)
804 return false;
805 High64MI = MRI->getUniqueVRegDef(Reg: High64MI->getOperand(i: 2).getReg());
806 if (High64MI && High64MI->getOpcode() == TargetOpcode::COPY)
807 High64MI = MRI->getUniqueVRegDef(Reg: High64MI->getOperand(i: 1).getReg());
808 if (!High64MI || (High64MI->getOpcode() != AArch64::MOVID &&
809 High64MI->getOpcode() != AArch64::MOVIv2d_ns))
810 return false;
811 if (High64MI->getOperand(i: 1).getImm() != 0)
812 return false;
813
814 // Let's remove MIs for high 64-bits.
815 Register OldDef = MI.getOperand(i: 0).getReg();
816 Register NewDef = MI.getOperand(i: 1).getReg();
817 LLVM_DEBUG(dbgs() << "Removing: " << MI << "\n");
818 MRI->constrainRegClass(Reg: NewDef, RC: MRI->getRegClass(Reg: OldDef));
819 MRI->replaceRegWith(FromReg: OldDef, ToReg: NewDef);
820 MRI->clearKillFlags(Reg: NewDef);
821 MI.eraseFromParent();
822
823 return true;
824}
825
826bool AArch64MIPeepholeOptImpl::visitFMOVDr(MachineInstr &MI) {
827 // An FMOVDr sets the high 64-bits to zero implicitly, similar to ORR for GPR.
828 MachineInstr *Low64MI = MRI->getUniqueVRegDef(Reg: MI.getOperand(i: 1).getReg());
829 if (!Low64MI || !is64bitDefwithZeroHigh64bit(MI: Low64MI, MRI, TII))
830 return false;
831
832 // Let's remove MIs for high 64-bits.
833 Register OldDef = MI.getOperand(i: 0).getReg();
834 Register NewDef = MI.getOperand(i: 1).getReg();
835 LLVM_DEBUG(dbgs() << "Removing: " << MI << "\n");
836 MRI->clearKillFlags(Reg: OldDef);
837 MRI->clearKillFlags(Reg: NewDef);
838 MRI->constrainRegClass(Reg: NewDef, RC: MRI->getRegClass(Reg: OldDef));
839 MRI->replaceRegWith(FromReg: OldDef, ToReg: NewDef);
840 MI.eraseFromParent();
841
842 return true;
843}
844
845bool AArch64MIPeepholeOptImpl::visitUBFMXri(MachineInstr &MI) {
846 // Check if the instruction is equivalent to a 32 bit LSR or LSL alias of
847 // UBFM, and replace the UBFMXri instruction with its 32 bit variant, UBFMWri.
848 int64_t Immr = MI.getOperand(i: 2).getImm();
849 int64_t Imms = MI.getOperand(i: 3).getImm();
850
851 bool IsLSR = Imms == 31 && Immr <= Imms;
852 bool IsLSL = Immr == Imms + 33;
853 if (!IsLSR && !IsLSL)
854 return false;
855
856 if (IsLSL) {
857 Immr -= 32;
858 }
859
860 const TargetRegisterClass *DstRC64 =
861 TII->getRegClass(MCID: TII->get(Opcode: MI.getOpcode()), OpNum: 0);
862 const TargetRegisterClass *DstRC32 =
863 TRI->getSubRegisterClass(DstRC64, AArch64::sub_32);
864 assert(DstRC32 && "Destination register class of UBFMXri doesn't have a "
865 "sub_32 subregister class");
866
867 const TargetRegisterClass *SrcRC64 =
868 TII->getRegClass(MCID: TII->get(Opcode: MI.getOpcode()), OpNum: 1);
869 const TargetRegisterClass *SrcRC32 =
870 TRI->getSubRegisterClass(SrcRC64, AArch64::sub_32);
871 assert(SrcRC32 && "Source register class of UBFMXri doesn't have a sub_32 "
872 "subregister class");
873
874 Register DstReg64 = MI.getOperand(i: 0).getReg();
875 Register DstReg32 = MRI->createVirtualRegister(RegClass: DstRC32);
876 Register SrcReg64 = MI.getOperand(i: 1).getReg();
877 Register SrcReg32 = MRI->createVirtualRegister(RegClass: SrcRC32);
878
879 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::COPY),
880 DestReg: SrcReg32)
881 .addReg(RegNo: SrcReg64, Flags: {}, SubReg: AArch64::sub_32);
882 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::UBFMWri),
883 DestReg: DstReg32)
884 .addReg(RegNo: SrcReg32)
885 .addImm(Val: Immr)
886 .addImm(Val: Imms);
887 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
888 MCID: TII->get(Opcode: AArch64::SUBREG_TO_REG), DestReg: DstReg64)
889 .addReg(RegNo: DstReg32)
890 .addImm(Val: AArch64::sub_32);
891 MI.eraseFromParent();
892 return true;
893}
894
895// Across a basic-block we might have in i32 extract from a value that only
896// operates on upper bits (for example a sxtw). We can replace the COPY with a
897// new version skipping the sxtw.
898bool AArch64MIPeepholeOptImpl::visitCopy(MachineInstr &MI) {
899 Register InputReg = MI.getOperand(i: 1).getReg();
900 if (MI.getOperand(i: 1).getSubReg() != AArch64::sub_32 ||
901 !MRI->hasOneNonDBGUse(RegNo: InputReg))
902 return false;
903
904 MachineInstr *SrcMI = MRI->getUniqueVRegDef(Reg: InputReg);
905 SmallPtrSet<MachineInstr *, 4> DeadInstrs;
906 DeadInstrs.insert(Ptr: SrcMI);
907 while (SrcMI && SrcMI->isFullCopy() &&
908 MRI->hasOneNonDBGUse(RegNo: SrcMI->getOperand(i: 1).getReg())) {
909 SrcMI = MRI->getUniqueVRegDef(Reg: SrcMI->getOperand(i: 1).getReg());
910 DeadInstrs.insert(Ptr: SrcMI);
911 }
912
913 if (!SrcMI)
914 return false;
915
916 // Look for SXTW(X) and return Reg.
917 auto getSXTWSrcReg = [](MachineInstr *SrcMI) -> Register {
918 if (SrcMI->getOpcode() != AArch64::SBFMXri ||
919 SrcMI->getOperand(i: 2).getImm() != 0 ||
920 SrcMI->getOperand(i: 3).getImm() != 31)
921 return AArch64::NoRegister;
922 return SrcMI->getOperand(i: 1).getReg();
923 };
924 // Look for SUBREG_TO_REG(ORRWrr(WZR, COPY(X.sub_32)))
925 auto getUXTWSrcReg = [&](MachineInstr *SrcMI) -> Register {
926 if (SrcMI->getOpcode() != AArch64::SUBREG_TO_REG ||
927 SrcMI->getOperand(i: 2).getImm() != AArch64::sub_32 ||
928 !MRI->hasOneNonDBGUse(RegNo: SrcMI->getOperand(i: 1).getReg()))
929 return AArch64::NoRegister;
930 MachineInstr *Orr = MRI->getUniqueVRegDef(Reg: SrcMI->getOperand(i: 1).getReg());
931 if (!Orr || Orr->getOpcode() != AArch64::ORRWrr ||
932 Orr->getOperand(i: 1).getReg() != AArch64::WZR ||
933 !MRI->hasOneNonDBGUse(RegNo: Orr->getOperand(i: 2).getReg()))
934 return AArch64::NoRegister;
935 MachineInstr *Cpy = MRI->getUniqueVRegDef(Reg: Orr->getOperand(i: 2).getReg());
936 if (!Cpy || Cpy->getOpcode() != AArch64::COPY ||
937 Cpy->getOperand(i: 1).getSubReg() != AArch64::sub_32)
938 return AArch64::NoRegister;
939 DeadInstrs.insert(Ptr: Orr);
940 return Cpy->getOperand(i: 1).getReg();
941 };
942
943 Register SrcReg = getSXTWSrcReg(SrcMI);
944 if (!SrcReg)
945 SrcReg = getUXTWSrcReg(SrcMI);
946 if (!SrcReg)
947 return false;
948
949 MRI->constrainRegClass(Reg: SrcReg, RC: MRI->getRegClass(Reg: InputReg));
950 LLVM_DEBUG(dbgs() << "Optimizing: " << MI);
951 MI.getOperand(i: 1).setReg(SrcReg);
952 LLVM_DEBUG(dbgs() << " to: " << MI);
953 for (auto *DeadMI : DeadInstrs) {
954 LLVM_DEBUG(dbgs() << " Removing: " << *DeadMI);
955 DeadMI->eraseFromParent();
956 }
957 return true;
958}
959
960bool AArch64MIPeepholeOptImpl::run(MachineFunction &MF) {
961 TII = static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
962 TRI = static_cast<const AArch64RegisterInfo *>(
963 MF.getSubtarget().getRegisterInfo());
964 MRI = &MF.getRegInfo();
965
966 assert(MRI->isSSA() && "Expected to be run on SSA form!");
967
968 bool Changed = false;
969
970 for (MachineBasicBlock &MBB : MF) {
971 for (MachineInstr &MI : make_early_inc_range(Range&: MBB)) {
972 switch (MI.getOpcode()) {
973 default:
974 break;
975 case AArch64::INSERT_SUBREG:
976 Changed |= visitINSERT(MI);
977 break;
978 case AArch64::ANDWrr:
979 Changed |= trySplitLogicalImm<uint32_t>(Opc: AArch64::ANDWri, MI,
980 Strategy: SplitStrategy::Intersect);
981 break;
982 case AArch64::ANDXrr:
983 Changed |= trySplitLogicalImm<uint64_t>(Opc: AArch64::ANDXri, MI,
984 Strategy: SplitStrategy::Intersect);
985 break;
986 case AArch64::ANDSWrr:
987 Changed |= trySplitLogicalImm<uint32_t>(
988 Opc: AArch64::ANDWri, MI, Strategy: SplitStrategy::Intersect, OtherOpc: AArch64::ANDSWri);
989 break;
990 case AArch64::ANDSXrr:
991 Changed |= trySplitLogicalImm<uint64_t>(
992 Opc: AArch64::ANDXri, MI, Strategy: SplitStrategy::Intersect, OtherOpc: AArch64::ANDSXri);
993 break;
994 case AArch64::EORWrr:
995 Changed |= trySplitLogicalImm<uint32_t>(Opc: AArch64::EORWri, MI,
996 Strategy: SplitStrategy::Disjoint);
997 break;
998 case AArch64::EORXrr:
999 Changed |= trySplitLogicalImm<uint64_t>(Opc: AArch64::EORXri, MI,
1000 Strategy: SplitStrategy::Disjoint);
1001 break;
1002 case AArch64::ORRWrr:
1003 Changed |= trySplitLogicalImm<uint32_t>(Opc: AArch64::ORRWri, MI,
1004 Strategy: SplitStrategy::Disjoint);
1005 break;
1006 case AArch64::ORRXrr:
1007 Changed |= trySplitLogicalImm<uint64_t>(Opc: AArch64::ORRXri, MI,
1008 Strategy: SplitStrategy::Disjoint);
1009 break;
1010 case AArch64::ORRWrs:
1011 Changed |= visitORR(MI);
1012 break;
1013 case AArch64::ADDWrr:
1014 Changed |= visitADDSUB<uint32_t>(PosOpc: AArch64::ADDWri, NegOpc: AArch64::SUBWri, MI);
1015 break;
1016 case AArch64::SUBWrr:
1017 Changed |= visitADDSUB<uint32_t>(PosOpc: AArch64::SUBWri, NegOpc: AArch64::ADDWri, MI);
1018 break;
1019 case AArch64::ADDXrr:
1020 Changed |= visitADDSUB<uint64_t>(PosOpc: AArch64::ADDXri, NegOpc: AArch64::SUBXri, MI);
1021 break;
1022 case AArch64::SUBXrr:
1023 Changed |= visitADDSUB<uint64_t>(PosOpc: AArch64::SUBXri, NegOpc: AArch64::ADDXri, MI);
1024 break;
1025 case AArch64::ADDSWrr:
1026 Changed |=
1027 visitADDSSUBS<uint32_t>(PosOpcs: {AArch64::ADDWri, AArch64::ADDSWri},
1028 NegOpcs: {AArch64::SUBWri, AArch64::SUBSWri}, MI);
1029 break;
1030 case AArch64::SUBSWrr:
1031 Changed |=
1032 visitADDSSUBS<uint32_t>(PosOpcs: {AArch64::SUBWri, AArch64::SUBSWri},
1033 NegOpcs: {AArch64::ADDWri, AArch64::ADDSWri}, MI);
1034 break;
1035 case AArch64::ADDSXrr:
1036 Changed |=
1037 visitADDSSUBS<uint64_t>(PosOpcs: {AArch64::ADDXri, AArch64::ADDSXri},
1038 NegOpcs: {AArch64::SUBXri, AArch64::SUBSXri}, MI);
1039 break;
1040 case AArch64::SUBSXrr:
1041 Changed |=
1042 visitADDSSUBS<uint64_t>(PosOpcs: {AArch64::SUBXri, AArch64::SUBSXri},
1043 NegOpcs: {AArch64::ADDXri, AArch64::ADDSXri}, MI);
1044 break;
1045 case AArch64::CSELWr:
1046 case AArch64::CSELXr:
1047 Changed |= visitCSEL(MI);
1048 break;
1049 case AArch64::INSvi64gpr:
1050 Changed |= visitINSviGPR(MI, Opc: AArch64::INSvi64lane);
1051 break;
1052 case AArch64::INSvi32gpr:
1053 Changed |= visitINSviGPR(MI, Opc: AArch64::INSvi32lane);
1054 break;
1055 case AArch64::INSvi16gpr:
1056 Changed |= visitINSviGPR(MI, Opc: AArch64::INSvi16lane);
1057 break;
1058 case AArch64::INSvi8gpr:
1059 Changed |= visitINSviGPR(MI, Opc: AArch64::INSvi8lane);
1060 break;
1061 case AArch64::INSvi64lane:
1062 Changed |= visitINSvi64lane(MI);
1063 break;
1064 case AArch64::FMOVDr:
1065 Changed |= visitFMOVDr(MI);
1066 break;
1067 case AArch64::UBFMXri:
1068 Changed |= visitUBFMXri(MI);
1069 break;
1070 case AArch64::COPY:
1071 Changed |= visitCopy(MI);
1072 break;
1073 }
1074 }
1075 }
1076
1077 return Changed;
1078}
1079
1080bool AArch64MIPeepholeOptLegacy::runOnMachineFunction(MachineFunction &MF) {
1081 if (skipFunction(F: MF.getFunction()))
1082 return false;
1083
1084 MachineLoopInfo &MLI = getAnalysis<MachineLoopInfoWrapperPass>().getLI();
1085 return AArch64MIPeepholeOptImpl(MLI).run(MF);
1086}
1087
1088FunctionPass *llvm::createAArch64MIPeepholeOptLegacyPass() {
1089 return new AArch64MIPeepholeOptLegacy();
1090}
1091
1092PreservedAnalyses
1093AArch64MIPeepholeOptPass::run(MachineFunction &MF,
1094 MachineFunctionAnalysisManager &MFAM) {
1095 MachineLoopInfo &MLI = MFAM.getResult<MachineLoopAnalysis>(IR&: MF);
1096 const bool Changed = AArch64MIPeepholeOptImpl(MLI).run(MF);
1097 if (!Changed)
1098 return PreservedAnalyses::all();
1099 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
1100 PA.preserveSet<CFGAnalyses>();
1101 return PA;
1102}
1103