1//===- AArch64MIPeepholeOpt.cpp - AArch64 MI peephole optimization pass ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass performs below peephole optimizations on MIR level.
10//
11// 1. MOVi32imm + (ANDS?|EOR|ORR)Wrr ==> (AND|EOR|ORR)Wri + (ANDS?|EOR|ORR)Wri
12// MOVi64imm + (ANDS?|EOR|ORR)Xrr ==> (AND|EOR|ORR)Xri + (ANDS?|EOR|ORR)Xri
13//
14// 2. MOVi32imm + ADDWrr ==> ADDWRi + ADDWRi
15// MOVi64imm + ADDXrr ==> ADDXri + ADDXri
16//
17// 3. MOVi32imm + SUBWrr ==> SUBWRi + SUBWRi
18// MOVi64imm + SUBXrr ==> SUBXri + SUBXri
19//
20// The mov pseudo instruction could be expanded to multiple mov instructions
21// later. In this case, we could try to split the constant operand of mov
22// instruction into two immediates which can be directly encoded into
23// *Wri/*Xri instructions. It makes two AND/ADD/SUB instructions instead of
24// multiple `mov` + `and/add/sub` instructions.
25//
26// 4. Remove redundant ORRWrs which is generated by zero-extend.
27//
28// %3:gpr32 = ORRWrs $wzr, %2, 0
29// %4:gpr64 = SUBREG_TO_REG %3, %subreg.sub_32
30//
31// If AArch64's 32-bit form of instruction defines the source operand of
32// ORRWrs, we can remove the ORRWrs because the upper 32 bits of the source
33// operand are set to zero.
34//
35// 5. %reg = INSERT_SUBREG %reg(tied-def 0), %subreg, subidx
36// ==> %reg:subidx = SUBREG_TO_REG %subreg, subidx
37//
38// 6. %intermediate:gpr32 = COPY %src:fpr128
39// %dst:fpr128 = INSvi32gpr %dst_vec:fpr128, dst_index, %intermediate:gpr32
40// ==> %dst:fpr128 = INSvi32lane %dst_vec:fpr128, dst_index, %src:fpr128, 0
41//
42// In cases where a source FPR is copied to a GPR in order to be copied
43// to a destination FPR, we can directly copy the values between the FPRs,
44// eliminating the use of the Integer unit. When we match a pattern of
45// INSvi[X]gpr that is preceded by a chain of COPY instructions from a FPR
46// source, we use the INSvi[X]lane to replace the COPY & INSvi[X]gpr
47// instructions.
48//
49// 7. If MI sets zero for high 64-bits implicitly, remove `mov 0` for high
50// 64-bits. For example,
51//
52// %1:fpr64 = nofpexcept FCVTNv4i16 %0:fpr128, implicit $fpcr
53// %2:fpr64 = MOVID 0
54// %4:fpr128 = IMPLICIT_DEF
55// %3:fpr128 = INSERT_SUBREG %4:fpr128(tied-def 0), %2:fpr64, %subreg.dsub
56// %6:fpr128 = IMPLICIT_DEF
57// %5:fpr128 = INSERT_SUBREG %6:fpr128(tied-def 0), %1:fpr64, %subreg.dsub
58// %7:fpr128 = INSvi64lane %5:fpr128(tied-def 0), 1, %3:fpr128, 0
59// ==>
60// %1:fpr64 = nofpexcept FCVTNv4i16 %0:fpr128, implicit $fpcr
61// %6:fpr128 = IMPLICIT_DEF
62// %7:fpr128 = INSERT_SUBREG %6:fpr128(tied-def 0), %1:fpr64, %subreg.dsub
63//
64// 8. Remove redundant CSELs that select between identical registers, by
65// replacing them with unconditional moves.
66//
67// 9. Replace UBFMXri with UBFMWri if the instruction is equivalent to a 32 bit
68// LSR or LSL alias of UBFM.
69//
70//===----------------------------------------------------------------------===//
71
72#include "AArch64ExpandImm.h"
73#include "AArch64InstrInfo.h"
74#include "MCTargetDesc/AArch64AddressingModes.h"
75#include "llvm/CodeGen/MachineDominators.h"
76#include "llvm/CodeGen/MachineLoopInfo.h"
77
78using namespace llvm;
79
80#define DEBUG_TYPE "aarch64-mi-peephole-opt"
81
82namespace {
83
84class AArch64MIPeepholeOptImpl {
85public:
86 const AArch64InstrInfo *TII;
87 const AArch64RegisterInfo *TRI;
88 MachineLoopInfo *MLI;
89 MachineRegisterInfo *MRI;
90
91 explicit AArch64MIPeepholeOptImpl(MachineLoopInfo &MLI) : MLI(&MLI) {}
92
93 bool run(MachineFunction &MF);
94
95private:
96 using OpcodePair = std::pair<unsigned, unsigned>;
97 template <typename T>
98 using SplitAndOpcFunc =
99 std::function<std::optional<OpcodePair>(T, unsigned, T &, T &)>;
100 using BuildMIFunc =
101 std::function<void(MachineInstr &, OpcodePair, unsigned, unsigned,
102 Register, Register, Register)>;
103
104 /// For instructions where an immediate operand could be split into two
105 /// separate immediate instructions, use the splitTwoPartImm two handle the
106 /// optimization.
107 ///
108 /// To implement, the following function types must be passed to
109 /// splitTwoPartImm. A SplitAndOpcFunc must be implemented that determines if
110 /// splitting the immediate is valid and returns the associated new opcode. A
111 /// BuildMIFunc must be implemented to build the two immediate instructions.
112 ///
113 /// Example Pattern (where IMM would require 2+ MOV instructions):
114 /// %dst = <Instr>rr %src IMM [...]
115 /// becomes:
116 /// %tmp = <Instr>ri %src (encode half IMM) [...]
117 /// %dst = <Instr>ri %tmp (encode half IMM) [...]
118 template <typename T>
119 bool splitTwoPartImm(MachineInstr &MI,
120 SplitAndOpcFunc<T> SplitAndOpc, BuildMIFunc BuildInstr);
121
122 bool checkMovImmInstr(MachineInstr &MI, MachineInstr *&MovMI,
123 MachineInstr *&SubregToRegMI);
124
125 template <typename T>
126 bool visitADDSUB(unsigned PosOpc, unsigned NegOpc, MachineInstr &MI);
127 template <typename T>
128 bool visitADDSSUBS(OpcodePair PosOpcs, OpcodePair NegOpcs, MachineInstr &MI);
129
130 // Strategy used to split logical immediate bitmasks.
131 enum class SplitStrategy {
132 Intersect,
133 Disjoint,
134 };
135 template <typename T>
136 bool trySplitLogicalImm(unsigned Opc, MachineInstr &MI,
137 SplitStrategy Strategy, unsigned OtherOpc = 0);
138 bool visitORR(MachineInstr &MI);
139 bool visitCSEL(MachineInstr &MI);
140 bool visitINSERT(MachineInstr &MI);
141 bool visitINSviGPR(MachineInstr &MI, unsigned Opc);
142 bool visitINSvi64lane(MachineInstr &MI);
143 bool visitFMOVDr(MachineInstr &MI);
144 bool visitUBFMXri(MachineInstr &MI);
145 bool visitCopy(MachineInstr &MI);
146};
147
148struct AArch64MIPeepholeOptLegacy : public MachineFunctionPass {
149 static char ID;
150
151 AArch64MIPeepholeOptLegacy() : MachineFunctionPass(ID) {}
152
153 bool runOnMachineFunction(MachineFunction &MF) override;
154
155 StringRef getPassName() const override {
156 return "AArch64 MI Peephole Optimization pass";
157 }
158
159 void getAnalysisUsage(AnalysisUsage &AU) const override {
160 AU.setPreservesCFG();
161 AU.addRequired<MachineLoopInfoWrapperPass>();
162 MachineFunctionPass::getAnalysisUsage(AU);
163 }
164};
165
166char AArch64MIPeepholeOptLegacy::ID = 0;
167
168} // end anonymous namespace
169
170INITIALIZE_PASS(AArch64MIPeepholeOptLegacy, "aarch64-mi-peephole-opt",
171 "AArch64 MI Peephole Optimization", false, false)
172
173template <typename T>
174static bool splitBitmaskImm(T Imm, unsigned RegSize, T &Imm1Enc, T &Imm2Enc) {
175 T UImm = static_cast<T>(Imm);
176 assert(UImm && (UImm != ~static_cast<T>(0)) && "Invalid immediate!");
177
178 // The bitmask immediate consists of consecutive ones. Let's say there is
179 // constant 0b00000000001000000000010000000000 which does not consist of
180 // consecutive ones. We can split it in to two bitmask immediate like
181 // 0b00000000001111111111110000000000 and 0b11111111111000000000011111111111.
182 // If we do AND with these two bitmask immediate, we can see original one.
183 unsigned LowestBitSet = llvm::countr_zero(UImm);
184 unsigned HighestBitSet = Log2_64(UImm);
185
186 // Create a mask which is filled with one from the position of lowest bit set
187 // to the position of highest bit set.
188 T NewImm1 = (static_cast<T>(2) << HighestBitSet) -
189 (static_cast<T>(1) << LowestBitSet);
190 // Create a mask which is filled with one outside the position of lowest bit
191 // set and the position of highest bit set.
192 T NewImm2 = UImm | ~NewImm1;
193
194 // If the split value is not valid bitmask immediate, do not split this
195 // constant.
196 if (!AArch64_AM::isLogicalImmediate(imm: NewImm2, regSize: RegSize))
197 return false;
198
199 Imm1Enc = AArch64_AM::encodeLogicalImmediate(imm: NewImm1, regSize: RegSize);
200 Imm2Enc = AArch64_AM::encodeLogicalImmediate(imm: NewImm2, regSize: RegSize);
201 return true;
202}
203
204template <typename T>
205static bool splitDisjointBitmaskImm(T Imm, unsigned RegSize, T &Imm1Enc,
206 T &Imm2Enc) {
207 assert(Imm && (Imm != ~static_cast<T>(0)) && "Invalid immediate!");
208
209 // Try to split a bitmask of the form 0b00000000011000000000011110000000 into
210 // two disjoint masks such as 0b00000000011000000000000000000000 and
211 // 0b00000000000000000000011110000000 where the inclusive/exclusive OR of the
212 // new masks match the original mask.
213 unsigned LowestBitSet = llvm::countr_zero(Imm);
214 unsigned LowestGapBitUnset =
215 LowestBitSet + llvm::countr_one(Imm >> LowestBitSet);
216
217 // Create a mask for the least significant group of consecutive ones.
218 assert(LowestGapBitUnset < sizeof(T) * CHAR_BIT && "Undefined behaviour!");
219 T NewImm1 = (static_cast<T>(1) << LowestGapBitUnset) -
220 (static_cast<T>(1) << LowestBitSet);
221 // Create a disjoint mask for the remaining ones.
222 T NewImm2 = Imm & ~NewImm1;
223
224 // Do not split if NewImm2 is not a valid bitmask immediate.
225 if (!AArch64_AM::isLogicalImmediate(imm: NewImm2, regSize: RegSize))
226 return false;
227
228 Imm1Enc = AArch64_AM::encodeLogicalImmediate(imm: NewImm1, regSize: RegSize);
229 Imm2Enc = AArch64_AM::encodeLogicalImmediate(imm: NewImm2, regSize: RegSize);
230 return true;
231}
232
233template <typename T>
234bool AArch64MIPeepholeOptImpl::trySplitLogicalImm(unsigned Opc,
235 MachineInstr &MI,
236 SplitStrategy Strategy,
237 unsigned OtherOpc) {
238 // Try below transformations.
239 //
240 // MOVi32imm + (ANDS?|EOR|ORR)Wrr ==> (AND|EOR|ORR)Wri + (ANDS?|EOR|ORR)Wri
241 // MOVi64imm + (ANDS?|EOR|ORR)Xrr ==> (AND|EOR|ORR)Xri + (ANDS?|EOR|ORR)Xri
242 //
243 // The mov pseudo instruction could be expanded to multiple mov instructions
244 // later. Let's try to split the constant operand of mov instruction into two
245 // bitmask immediates based on the given split strategy. It makes only two
246 // logical instructions instead of multiple mov + logic instructions.
247
248 return splitTwoPartImm<T>(
249 MI,
250 [Opc, Strategy, OtherOpc](T Imm, unsigned RegSize, T &Imm0,
251 T &Imm1) -> std::optional<OpcodePair> {
252 // If this immediate is already a suitable bitmask, don't split it.
253 // TODO: Should we just combine the two instructions in this case?
254 if (AArch64_AM::isLogicalImmediate(imm: Imm, regSize: RegSize))
255 return std::nullopt;
256
257 // If this immediate can be handled by one instruction, don't split it.
258 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
259 AArch64_IMM::expandMOVImm(Imm, BitSize: RegSize, Insn);
260 if (Insn.size() == 1)
261 return std::nullopt;
262
263 bool SplitSucc = false;
264 switch (Strategy) {
265 case SplitStrategy::Intersect:
266 SplitSucc = splitBitmaskImm(Imm, RegSize, Imm0, Imm1);
267 break;
268 case SplitStrategy::Disjoint:
269 SplitSucc = splitDisjointBitmaskImm(Imm, RegSize, Imm0, Imm1);
270 break;
271 }
272 if (SplitSucc)
273 return std::make_pair(x: Opc, y: !OtherOpc ? Opc : OtherOpc);
274 return std::nullopt;
275 },
276 [&TII = TII](MachineInstr &MI, OpcodePair Opcode, unsigned Imm0,
277 unsigned Imm1, Register SrcReg, Register NewTmpReg,
278 Register NewDstReg) {
279 DebugLoc DL = MI.getDebugLoc();
280 MachineBasicBlock *MBB = MI.getParent();
281 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opcode.first), DestReg: NewTmpReg)
282 .addReg(RegNo: SrcReg)
283 .addImm(Val: Imm0);
284 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opcode.second), DestReg: NewDstReg)
285 .addReg(RegNo: NewTmpReg)
286 .addImm(Val: Imm1);
287 });
288}
289
290bool AArch64MIPeepholeOptImpl::visitORR(MachineInstr &MI) {
291 // Check this ORR comes from below zero-extend pattern.
292 //
293 // def : Pat<(i64 (zext GPR32:$src)),
294 // (SUBREG_TO_REG (ORRWrs WZR, GPR32:$src, 0), sub_32)>;
295 if (MI.getOperand(i: 3).getImm() != 0)
296 return false;
297
298 if (MI.getOperand(i: 1).getReg() != AArch64::WZR)
299 return false;
300
301 if (MI.getOperand(i: 2).getSubReg())
302 return false;
303
304 MachineInstr *SrcMI = MRI->getUniqueVRegDef(Reg: MI.getOperand(i: 2).getReg());
305 if (!SrcMI)
306 return false;
307
308 // From https://developer.arm.com/documentation/dui0801/b/BABBGCAC
309 //
310 // When you use the 32-bit form of an instruction, the upper 32 bits of the
311 // source registers are ignored and the upper 32 bits of the destination
312 // register are set to zero.
313 //
314 // If AArch64's 32-bit form of instruction defines the source operand of
315 // zero-extend, we do not need the zero-extend. Let's check the MI's opcode is
316 // real AArch64 instruction and if it is not, do not process the opcode
317 // conservatively.
318 if (SrcMI->getOpcode() == TargetOpcode::COPY &&
319 SrcMI->getOperand(i: 1).getReg().isVirtual()) {
320 const TargetRegisterClass *RC =
321 MRI->getRegClass(Reg: SrcMI->getOperand(i: 1).getReg());
322
323 // A COPY from an FPR will become a FMOVSWr, so do so now so that we know
324 // that the upper bits are zero.
325 if (RC != &AArch64::FPR32RegClass &&
326 ((RC != &AArch64::FPR64RegClass && RC != &AArch64::FPR128RegClass &&
327 RC != &AArch64::ZPRRegClass) ||
328 SrcMI->getOperand(i: 1).getSubReg() != AArch64::ssub))
329 return false;
330 Register CpySrc;
331 if (SrcMI->getOperand(i: 1).getSubReg() == AArch64::ssub) {
332 CpySrc = MRI->createVirtualRegister(RegClass: &AArch64::FPR32RegClass);
333 BuildMI(BB&: *SrcMI->getParent(), I: SrcMI, MIMD: SrcMI->getDebugLoc(),
334 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: CpySrc)
335 .add(MO: SrcMI->getOperand(i: 1));
336 } else {
337 CpySrc = SrcMI->getOperand(i: 1).getReg();
338 }
339 BuildMI(BB&: *SrcMI->getParent(), I: SrcMI, MIMD: SrcMI->getDebugLoc(),
340 MCID: TII->get(Opcode: AArch64::FMOVSWr), DestReg: SrcMI->getOperand(i: 0).getReg())
341 .addReg(RegNo: CpySrc);
342 SrcMI->eraseFromParent();
343 }
344 else if (SrcMI->getOpcode() <= TargetOpcode::GENERIC_OP_END)
345 return false;
346
347 Register DefReg = MI.getOperand(i: 0).getReg();
348 Register SrcReg = MI.getOperand(i: 2).getReg();
349 MRI->replaceRegWith(FromReg: DefReg, ToReg: SrcReg);
350 MRI->clearKillFlags(Reg: SrcReg);
351 LLVM_DEBUG(dbgs() << "Removed: " << MI << "\n");
352 MI.eraseFromParent();
353
354 return true;
355}
356
357bool AArch64MIPeepholeOptImpl::visitCSEL(MachineInstr &MI) {
358 // Replace CSEL with MOV when both inputs are the same register.
359 if (MI.getOperand(i: 1).getReg() != MI.getOperand(i: 2).getReg())
360 return false;
361
362 auto ZeroReg =
363 MI.getOpcode() == AArch64::CSELXr ? AArch64::XZR : AArch64::WZR;
364 auto OrOpcode =
365 MI.getOpcode() == AArch64::CSELXr ? AArch64::ORRXrs : AArch64::ORRWrs;
366
367 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: OrOpcode))
368 .addReg(RegNo: MI.getOperand(i: 0).getReg(), Flags: RegState::Define)
369 .addReg(RegNo: ZeroReg)
370 .addReg(RegNo: MI.getOperand(i: 1).getReg())
371 .addImm(Val: 0);
372
373 MI.eraseFromParent();
374 return true;
375}
376
377bool AArch64MIPeepholeOptImpl::visitINSERT(MachineInstr &MI) {
378 // Check this INSERT_SUBREG comes from below zero-extend pattern.
379 //
380 // From %reg = INSERT_SUBREG %reg(tied-def 0), %subreg, subidx
381 // To %reg:subidx = SUBREG_TO_REG %subreg, subidx
382 //
383 // We're assuming the first operand to INSERT_SUBREG is irrelevant because a
384 // COPY would destroy the upper part of the register anyway
385 if (!MI.isRegTiedToDefOperand(UseOpIdx: 1))
386 return false;
387
388 Register DstReg = MI.getOperand(i: 0).getReg();
389 const TargetRegisterClass *RC = MRI->getRegClass(Reg: DstReg);
390 MachineInstr *SrcMI = MRI->getUniqueVRegDef(Reg: MI.getOperand(i: 2).getReg());
391 if (!SrcMI)
392 return false;
393
394 // From https://developer.arm.com/documentation/dui0801/b/BABBGCAC
395 //
396 // When you use the 32-bit form of an instruction, the upper 32 bits of the
397 // source registers are ignored and the upper 32 bits of the destination
398 // register are set to zero.
399 //
400 // If AArch64's 32-bit form of instruction defines the source operand of
401 // zero-extend, we do not need the zero-extend. Let's check the MI's opcode is
402 // real AArch64 instruction and if it is not, do not process the opcode
403 // conservatively.
404 if ((SrcMI->getOpcode() <= TargetOpcode::GENERIC_OP_END) ||
405 !AArch64::GPR64allRegClass.hasSubClassEq(RC))
406 return false;
407
408 // Build a SUBREG_TO_REG instruction
409 MachineInstr *SubregMI =
410 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
411 MCID: TII->get(Opcode: TargetOpcode::SUBREG_TO_REG), DestReg: DstReg)
412 .add(MO: MI.getOperand(i: 2))
413 .add(MO: MI.getOperand(i: 3));
414 LLVM_DEBUG(dbgs() << MI << " replace by:\n: " << *SubregMI << "\n");
415 (void)SubregMI;
416 MI.eraseFromParent();
417
418 return true;
419}
420
421template <typename T>
422static bool splitAddSubImm(T Imm, unsigned RegSize, T &Imm0, T &Imm1) {
423 // The immediate must be in the form of ((imm0 << 12) + imm1), in which both
424 // imm0 and imm1 are non-zero 12-bit unsigned int.
425 if ((Imm & 0xfff000) == 0 || (Imm & 0xfff) == 0 ||
426 (Imm & ~static_cast<T>(0xffffff)) != 0)
427 return false;
428
429 // The immediate can not be composed via a single instruction.
430 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
431 AArch64_IMM::expandMOVImm(Imm, BitSize: RegSize, Insn);
432 if (Insn.size() == 1)
433 return false;
434
435 // Split Imm into (Imm0 << 12) + Imm1;
436 Imm0 = (Imm >> 12) & 0xfff;
437 Imm1 = Imm & 0xfff;
438 return true;
439}
440
441template <typename T>
442bool AArch64MIPeepholeOptImpl::visitADDSUB(unsigned PosOpc, unsigned NegOpc,
443 MachineInstr &MI) {
444 // Try below transformation.
445 //
446 // ADDWrr X, MOVi32imm ==> ADDWri + ADDWri
447 // ADDXrr X, MOVi64imm ==> ADDXri + ADDXri
448 //
449 // SUBWrr X, MOVi32imm ==> SUBWri + SUBWri
450 // SUBXrr X, MOVi64imm ==> SUBXri + SUBXri
451 //
452 // The mov pseudo instruction could be expanded to multiple mov instructions
453 // later. Let's try to split the constant operand of mov instruction into two
454 // legal add/sub immediates. It makes only two ADD/SUB instructions instead of
455 // multiple `mov` + `and/sub` instructions.
456
457 // We can sometimes have ADDWrr WZR, MULi32imm that have not been constant
458 // folded. Make sure that we don't generate invalid instructions that use XZR
459 // in those cases.
460 if (MI.getOperand(i: 1).getReg() == AArch64::XZR ||
461 MI.getOperand(i: 1).getReg() == AArch64::WZR)
462 return false;
463
464 return splitTwoPartImm<T>(
465 MI,
466 [PosOpc, NegOpc](T Imm, unsigned RegSize, T &Imm0,
467 T &Imm1) -> std::optional<OpcodePair> {
468 if (splitAddSubImm(Imm, RegSize, Imm0, Imm1))
469 return std::make_pair(x: PosOpc, y: PosOpc);
470 if (splitAddSubImm(-Imm, RegSize, Imm0, Imm1))
471 return std::make_pair(x: NegOpc, y: NegOpc);
472 return std::nullopt;
473 },
474 [&TII = TII](MachineInstr &MI, OpcodePair Opcode, unsigned Imm0,
475 unsigned Imm1, Register SrcReg, Register NewTmpReg,
476 Register NewDstReg) {
477 DebugLoc DL = MI.getDebugLoc();
478 MachineBasicBlock *MBB = MI.getParent();
479 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opcode.first), DestReg: NewTmpReg)
480 .addReg(RegNo: SrcReg)
481 .addImm(Val: Imm0)
482 .addImm(Val: 12);
483 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opcode.second), DestReg: NewDstReg)
484 .addReg(RegNo: NewTmpReg)
485 .addImm(Val: Imm1)
486 .addImm(Val: 0);
487 });
488}
489
490template <typename T>
491bool AArch64MIPeepholeOptImpl::visitADDSSUBS(OpcodePair PosOpcs,
492 OpcodePair NegOpcs,
493 MachineInstr &MI) {
494 // Try the same transformation as ADDSUB but with additional requirement
495 // that the condition code usages are only for Equal and Not Equal
496
497 if (MI.getOperand(i: 1).getReg() == AArch64::XZR ||
498 MI.getOperand(i: 1).getReg() == AArch64::WZR)
499 return false;
500
501 return splitTwoPartImm<T>(
502 MI,
503 [PosOpcs, NegOpcs, &MI, &TRI = TRI,
504 &MRI = MRI](T Imm, unsigned RegSize, T &Imm0,
505 T &Imm1) -> std::optional<OpcodePair> {
506 OpcodePair OP;
507 if (splitAddSubImm(Imm, RegSize, Imm0, Imm1))
508 OP = PosOpcs;
509 else if (splitAddSubImm(-Imm, RegSize, Imm0, Imm1))
510 OP = NegOpcs;
511 else
512 return std::nullopt;
513 // Check conditional uses last since it is expensive for scanning
514 // proceeding instructions
515 MachineInstr *SrcMI = MRI->getVRegDef(Reg: MI.getOperand(i: 1).getReg());
516 if (!SrcMI)
517 return std::nullopt;
518 std::optional<UsedNZCV> NZCVUsed = examineCFlagsUse(MI&: *SrcMI, CmpInstr&: MI, TRI: *TRI);
519 if (!NZCVUsed || NZCVUsed->C || NZCVUsed->V)
520 return std::nullopt;
521 return OP;
522 },
523 [&TII = TII](MachineInstr &MI, OpcodePair Opcode, unsigned Imm0,
524 unsigned Imm1, Register SrcReg, Register NewTmpReg,
525 Register NewDstReg) {
526 DebugLoc DL = MI.getDebugLoc();
527 MachineBasicBlock *MBB = MI.getParent();
528 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opcode.first), DestReg: NewTmpReg)
529 .addReg(RegNo: SrcReg)
530 .addImm(Val: Imm0)
531 .addImm(Val: 12);
532 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opcode.second), DestReg: NewDstReg)
533 .addReg(RegNo: NewTmpReg)
534 .addImm(Val: Imm1)
535 .addImm(Val: 0);
536 });
537}
538
539// Checks if the corresponding MOV immediate instruction is applicable for
540// this peephole optimization.
541bool AArch64MIPeepholeOptImpl::checkMovImmInstr(MachineInstr &MI,
542 MachineInstr *&MovMI,
543 MachineInstr *&SubregToRegMI) {
544 // Check whether current MBB is in loop and the AND is loop invariant.
545 MachineBasicBlock *MBB = MI.getParent();
546 MachineLoop *L = MLI->getLoopFor(BB: MBB);
547 if (L && !L->isLoopInvariant(I&: MI))
548 return false;
549
550 // Check whether current MI's operand is MOV with immediate.
551 MovMI = MRI->getUniqueVRegDef(Reg: MI.getOperand(i: 2).getReg());
552 if (!MovMI)
553 return false;
554
555 // If it is SUBREG_TO_REG, check its operand.
556 SubregToRegMI = nullptr;
557 if (MovMI->getOpcode() == TargetOpcode::SUBREG_TO_REG) {
558 SubregToRegMI = MovMI;
559 MovMI = MRI->getUniqueVRegDef(Reg: MovMI->getOperand(i: 1).getReg());
560 if (!MovMI)
561 return false;
562 }
563
564 if (MovMI->getOpcode() != AArch64::MOVi32imm &&
565 MovMI->getOpcode() != AArch64::MOVi64imm)
566 return false;
567
568 // If the MOV has multiple uses, do not split the immediate because it causes
569 // more instructions.
570 if (!MRI->hasOneNonDBGUse(RegNo: MovMI->getOperand(i: 0).getReg()))
571 return false;
572 if (SubregToRegMI &&
573 !MRI->hasOneNonDBGUse(RegNo: SubregToRegMI->getOperand(i: 0).getReg()))
574 return false;
575
576 // It is OK to perform this peephole optimization.
577 return true;
578}
579
580template <typename T>
581bool AArch64MIPeepholeOptImpl::splitTwoPartImm(MachineInstr &MI,
582 SplitAndOpcFunc<T> SplitAndOpc,
583 BuildMIFunc BuildInstr) {
584 unsigned RegSize = sizeof(T) * 8;
585 assert((RegSize == 32 || RegSize == 64) &&
586 "Invalid RegSize for legal immediate peephole optimization");
587
588 // Perform several essential checks against current MI.
589 MachineInstr *MovMI, *SubregToRegMI;
590 if (!checkMovImmInstr(MI, MovMI, SubregToRegMI))
591 return false;
592
593 // Split the immediate to Imm0 and Imm1, and calculate the Opcode.
594 T Imm = static_cast<T>(MovMI->getOperand(i: 1).getImm()), Imm0, Imm1;
595 // For the 32 bit form of instruction, the upper 32 bits of the destination
596 // register are set to zero. If there is SUBREG_TO_REG, set the upper 32 bits
597 // of Imm to zero. This is essential if the Immediate value was a negative
598 // number since it was sign extended when we assign to the 64-bit Imm.
599 if (SubregToRegMI)
600 Imm &= 0xFFFFFFFF;
601 OpcodePair Opcode;
602 if (auto R = SplitAndOpc(Imm, RegSize, Imm0, Imm1))
603 Opcode = *R;
604 else
605 return false;
606
607 // Create new MIs using the first and second opcodes. Opcodes might differ for
608 // flag setting operations that should only set flags on second instruction.
609 // NewTmpReg = Opcode.first SrcReg Imm0
610 // NewDstReg = Opcode.second NewTmpReg Imm1
611
612 // Determine register classes for destinations and register operands
613 const TargetRegisterClass *FirstInstrDstRC =
614 TII->getRegClass(MCID: TII->get(Opcode: Opcode.first), OpNum: 0);
615 const TargetRegisterClass *FirstInstrOperandRC =
616 TII->getRegClass(MCID: TII->get(Opcode: Opcode.first), OpNum: 1);
617 const TargetRegisterClass *SecondInstrDstRC =
618 (Opcode.first == Opcode.second)
619 ? FirstInstrDstRC
620 : TII->getRegClass(MCID: TII->get(Opcode: Opcode.second), OpNum: 0);
621 const TargetRegisterClass *SecondInstrOperandRC =
622 (Opcode.first == Opcode.second)
623 ? FirstInstrOperandRC
624 : TII->getRegClass(MCID: TII->get(Opcode: Opcode.second), OpNum: 1);
625
626 // Get old registers destinations and new register destinations
627 Register DstReg = MI.getOperand(i: 0).getReg();
628 Register SrcReg = MI.getOperand(i: 1).getReg();
629 Register NewTmpReg = MRI->createVirtualRegister(RegClass: FirstInstrDstRC);
630 // In the situation that DstReg is not Virtual (likely WZR or XZR), we want to
631 // reuse that same destination register.
632 Register NewDstReg = DstReg.isVirtual()
633 ? MRI->createVirtualRegister(RegClass: SecondInstrDstRC)
634 : DstReg;
635
636 // Constrain registers based on their new uses
637 MRI->constrainRegClass(Reg: SrcReg, RC: FirstInstrOperandRC);
638 MRI->constrainRegClass(Reg: NewTmpReg, RC: SecondInstrOperandRC);
639 if (DstReg != NewDstReg)
640 MRI->constrainRegClass(Reg: NewDstReg, RC: MRI->getRegClass(Reg: DstReg));
641
642 // Call the delegating operation to build the instruction
643 BuildInstr(MI, Opcode, Imm0, Imm1, SrcReg, NewTmpReg, NewDstReg);
644
645 // replaceRegWith changes MI's definition register. Keep it for SSA form until
646 // deleting MI. Only if we made a new destination register.
647 if (DstReg != NewDstReg) {
648 MRI->replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
649 MI.getOperand(i: 0).setReg(DstReg);
650 }
651
652 auto SalvageDebugUses = [&](Register Reg, int64_t Val) {
653 for (MachineOperand &MO : make_early_inc_range(Range: MRI->use_operands(Reg))) {
654 MachineInstr &DbgMI = *MO.getParent();
655 if (!DbgMI.isDebugValue())
656 continue;
657 // An immediate cannot express a value in memory at that address.
658 if (DbgMI.isNonListDebugValue() && DbgMI.isIndirectDebugValue())
659 DbgMI.setDebugValueUndef();
660 else
661 MO.ChangeToImmediate(ImmVal: Val);
662 }
663 };
664
665 // Record the MIs need to be removed.
666 MI.eraseFromParent();
667 if (SubregToRegMI) {
668 SalvageDebugUses(SubregToRegMI->getOperand(i: 0).getReg(), Imm);
669 SubregToRegMI->eraseFromParent();
670 }
671 SalvageDebugUses(MovMI->getOperand(i: 0).getReg(),
672 MovMI->getOperand(i: 1).getImm());
673 MovMI->eraseFromParent();
674
675 return true;
676}
677
678bool AArch64MIPeepholeOptImpl::visitINSviGPR(MachineInstr &MI, unsigned Opc) {
679 // Check if this INSvi[X]gpr comes from COPY of a source FPR128
680 //
681 // From
682 // %intermediate1:gpr64 = COPY %src:fpr128
683 // %intermediate2:gpr32 = COPY %intermediate1:gpr64
684 // %dst:fpr128 = INSvi[X]gpr %dst_vec:fpr128, dst_index, %intermediate2:gpr32
685 // To
686 // %dst:fpr128 = INSvi[X]lane %dst_vec:fpr128, dst_index, %src:fpr128,
687 // src_index
688 // where src_index = 0, X = [8|16|32|64]
689
690 MachineInstr *SrcMI = MRI->getUniqueVRegDef(Reg: MI.getOperand(i: 3).getReg());
691
692 // For a chain of COPY instructions, find the initial source register
693 // and check if it's an FPR128
694 while (true) {
695 if (!SrcMI || SrcMI->getOpcode() != TargetOpcode::COPY)
696 return false;
697
698 if (!SrcMI->getOperand(i: 1).getReg().isVirtual())
699 return false;
700
701 if (MRI->getRegClass(Reg: SrcMI->getOperand(i: 1).getReg()) ==
702 &AArch64::FPR128RegClass) {
703 break;
704 }
705 SrcMI = MRI->getUniqueVRegDef(Reg: SrcMI->getOperand(i: 1).getReg());
706 }
707
708 Register DstReg = MI.getOperand(i: 0).getReg();
709 Register SrcReg = SrcMI->getOperand(i: 1).getReg();
710 MachineInstr *INSvilaneMI =
711 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc), DestReg: DstReg)
712 .add(MO: MI.getOperand(i: 1))
713 .add(MO: MI.getOperand(i: 2))
714 .addUse(RegNo: SrcReg, Flags: getRegState(RegOp: SrcMI->getOperand(i: 1)))
715 .addImm(Val: 0);
716
717 LLVM_DEBUG(dbgs() << MI << " replace by:\n: " << *INSvilaneMI << "\n");
718 (void)INSvilaneMI;
719 MI.eraseFromParent();
720 return true;
721}
722
723// All instructions that set a FPR64 will implicitly zero the top bits of the
724// register. When the def is expressed as a COPY from a GPR, turn it into an
725// explicit FMOV so it cannot be elided later in further passes.
726static bool is64bitDefwithZeroHigh64bit(MachineInstr *MI,
727 MachineRegisterInfo *MRI,
728 const AArch64InstrInfo *TII) {
729 if (!MI->getOperand(i: 0).isReg() || !MI->getOperand(i: 0).isDef())
730 return false;
731 const TargetRegisterClass *RC = MRI->getRegClass(Reg: MI->getOperand(i: 0).getReg());
732 if (RC != &AArch64::FPR64RegClass)
733 return false;
734 if (MI->getOpcode() == TargetOpcode::COPY) {
735 MachineOperand &SrcOp = MI->getOperand(i: 1);
736 if (!SrcOp.isReg())
737 return false;
738 Register SrcReg = SrcOp.getReg();
739 if (SrcOp.getSubReg()) {
740 // If operand is defined by a LD1/2/3/4 that define a D subreg tuple
741 // then upper bits of the tuple's registers are implicitly zeroed
742 // and the FMOV is unneeded.
743 if (!SrcReg.isVirtual())
744 return false;
745
746 MachineInstr *SrcDef = MRI->getUniqueVRegDef(Reg: SrcReg);
747 if (!SrcDef || SrcDef->getOpcode() <= TargetOpcode::GENERIC_OP_END ||
748 SrcDef->getDesc().isPseudo() || !SrcDef->mayLoad())
749 return false;
750
751 const TargetRegisterClass *RC = MRI->getRegClass(Reg: SrcReg);
752 return RC == &AArch64::DDRegClass || RC == &AArch64::DDDRegClass ||
753 RC == &AArch64::DDDDRegClass;
754 }
755 auto IsGPR64Like = [&]() -> bool {
756 if (SrcReg.isVirtual())
757 return AArch64::GPR64allRegClass.hasSubClassEq(
758 RC: MRI->getRegClass(Reg: SrcReg));
759 return AArch64::GPR64allRegClass.contains(Reg: SrcReg);
760 };
761 if (!IsGPR64Like())
762 return false;
763 assert(TII && "Expected InstrInfo when materializing COPYs");
764 // FMOVXDr insists on strict GPR64 operands, so fix up the COPY source.
765 MachineOperand &SrcMO = MI->getOperand(i: 1);
766 bool SrcKill = SrcMO.isKill();
767 if (SrcReg.isVirtual()) {
768 if (MRI->getRegClass(Reg: SrcReg) != &AArch64::GPR64RegClass) {
769 // Pass the value through a temporary GPR64 vreg to satisfy the
770 // verifier.
771 Register NewSrc = MRI->createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
772 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
773 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewSrc)
774 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: SrcKill));
775 SrcReg = NewSrc;
776 SrcKill = true;
777 }
778 } else if (!AArch64::GPR64RegClass.contains(Reg: SrcReg)) {
779 return false;
780 }
781 SrcMO.setReg(SrcReg);
782 SrcMO.setSubReg(0);
783 SrcMO.setIsKill(SrcKill);
784 // Replace the COPY with an explicit FMOV so the zeroing behaviour stays
785 // visible.
786 MI->setDesc(TII->get(Opcode: AArch64::FMOVXDr));
787 return true;
788 }
789 return MI->getOpcode() > TargetOpcode::GENERIC_OP_END;
790}
791
792bool AArch64MIPeepholeOptImpl::visitINSvi64lane(MachineInstr &MI) {
793 // Check the MI for low 64-bits sets zero for high 64-bits implicitly.
794 // We are expecting below case.
795 //
796 // %1:fpr64 = nofpexcept FCVTNv4i16 %0:fpr128, implicit $fpcr
797 // %6:fpr128 = IMPLICIT_DEF
798 // %5:fpr128 = INSERT_SUBREG %6:fpr128(tied-def 0), killed %1:fpr64, %subreg.dsub
799 // %7:fpr128 = INSvi64lane %5:fpr128(tied-def 0), 1, killed %3:fpr128, 0
800 MachineInstr *Low64MI = MRI->getVRegDef(Reg: MI.getOperand(i: 1).getReg());
801 if (!Low64MI || Low64MI->getOpcode() != AArch64::INSERT_SUBREG)
802 return false;
803 Low64MI = MRI->getUniqueVRegDef(Reg: Low64MI->getOperand(i: 2).getReg());
804 if (!Low64MI || !is64bitDefwithZeroHigh64bit(MI: Low64MI, MRI, TII))
805 return false;
806
807 // Check there is `mov 0` MI for high 64-bits.
808 // We are expecting below cases.
809 //
810 // %2:fpr64 = MOVID 0
811 // %4:fpr128 = IMPLICIT_DEF
812 // %3:fpr128 = INSERT_SUBREG %4:fpr128(tied-def 0), killed %2:fpr64, %subreg.dsub
813 // %7:fpr128 = INSvi64lane %5:fpr128(tied-def 0), 1, killed %3:fpr128, 0
814 // or
815 // %5:fpr128 = MOVIv2d_ns 0
816 // %6:fpr64 = COPY %5.dsub:fpr128
817 // %8:fpr128 = IMPLICIT_DEF
818 // %7:fpr128 = INSERT_SUBREG %8:fpr128(tied-def 0), killed %6:fpr64, %subreg.dsub
819 // %11:fpr128 = INSvi64lane %9:fpr128(tied-def 0), 1, killed %7:fpr128, 0
820 MachineInstr *High64MI = MRI->getUniqueVRegDef(Reg: MI.getOperand(i: 3).getReg());
821 if (!High64MI || High64MI->getOpcode() != AArch64::INSERT_SUBREG)
822 return false;
823 High64MI = MRI->getUniqueVRegDef(Reg: High64MI->getOperand(i: 2).getReg());
824 if (High64MI && High64MI->getOpcode() == TargetOpcode::COPY)
825 High64MI = MRI->getUniqueVRegDef(Reg: High64MI->getOperand(i: 1).getReg());
826 if (!High64MI || (High64MI->getOpcode() != AArch64::MOVID &&
827 High64MI->getOpcode() != AArch64::MOVIv2d_ns))
828 return false;
829 if (High64MI->getOperand(i: 1).getImm() != 0)
830 return false;
831
832 // Let's remove MIs for high 64-bits.
833 Register OldDef = MI.getOperand(i: 0).getReg();
834 Register NewDef = MI.getOperand(i: 1).getReg();
835 LLVM_DEBUG(dbgs() << "Removing: " << MI << "\n");
836 MRI->constrainRegClass(Reg: NewDef, RC: MRI->getRegClass(Reg: OldDef));
837 MRI->replaceRegWith(FromReg: OldDef, ToReg: NewDef);
838 MRI->clearKillFlags(Reg: NewDef);
839 MI.eraseFromParent();
840
841 return true;
842}
843
844bool AArch64MIPeepholeOptImpl::visitFMOVDr(MachineInstr &MI) {
845 // An FMOVDr sets the high 64-bits to zero implicitly, similar to ORR for GPR.
846 MachineInstr *Low64MI = MRI->getUniqueVRegDef(Reg: MI.getOperand(i: 1).getReg());
847 if (!Low64MI || !is64bitDefwithZeroHigh64bit(MI: Low64MI, MRI, TII))
848 return false;
849
850 // Let's remove MIs for high 64-bits.
851 Register OldDef = MI.getOperand(i: 0).getReg();
852 Register NewDef = MI.getOperand(i: 1).getReg();
853 LLVM_DEBUG(dbgs() << "Removing: " << MI << "\n");
854 MRI->clearKillFlags(Reg: OldDef);
855 MRI->clearKillFlags(Reg: NewDef);
856 MRI->constrainRegClass(Reg: NewDef, RC: MRI->getRegClass(Reg: OldDef));
857 MRI->replaceRegWith(FromReg: OldDef, ToReg: NewDef);
858 MI.eraseFromParent();
859
860 return true;
861}
862
863bool AArch64MIPeepholeOptImpl::visitUBFMXri(MachineInstr &MI) {
864 // Check if the instruction is equivalent to a 32 bit LSR or LSL alias of
865 // UBFM, and replace the UBFMXri instruction with its 32 bit variant, UBFMWri.
866 int64_t Immr = MI.getOperand(i: 2).getImm();
867 int64_t Imms = MI.getOperand(i: 3).getImm();
868
869 bool IsLSR = Imms == 31 && Immr <= Imms;
870 bool IsLSL = Immr == Imms + 33;
871 if (!IsLSR && !IsLSL)
872 return false;
873
874 if (IsLSL) {
875 Immr -= 32;
876 }
877
878 const TargetRegisterClass *DstRC64 =
879 TII->getRegClass(MCID: TII->get(Opcode: MI.getOpcode()), OpNum: 0);
880 const TargetRegisterClass *DstRC32 =
881 TRI->getSubRegisterClass(DstRC64, AArch64::sub_32);
882 assert(DstRC32 && "Destination register class of UBFMXri doesn't have a "
883 "sub_32 subregister class");
884
885 const TargetRegisterClass *SrcRC64 =
886 TII->getRegClass(MCID: TII->get(Opcode: MI.getOpcode()), OpNum: 1);
887 const TargetRegisterClass *SrcRC32 =
888 TRI->getSubRegisterClass(SrcRC64, AArch64::sub_32);
889 assert(SrcRC32 && "Source register class of UBFMXri doesn't have a sub_32 "
890 "subregister class");
891
892 Register DstReg64 = MI.getOperand(i: 0).getReg();
893 Register DstReg32 = MRI->createVirtualRegister(RegClass: DstRC32);
894 Register SrcReg64 = MI.getOperand(i: 1).getReg();
895 Register SrcReg32 = MRI->createVirtualRegister(RegClass: SrcRC32);
896
897 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::COPY),
898 DestReg: SrcReg32)
899 .addReg(RegNo: SrcReg64, Flags: {}, SubReg: AArch64::sub_32);
900 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::UBFMWri),
901 DestReg: DstReg32)
902 .addReg(RegNo: SrcReg32)
903 .addImm(Val: Immr)
904 .addImm(Val: Imms);
905 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
906 MCID: TII->get(Opcode: AArch64::SUBREG_TO_REG), DestReg: DstReg64)
907 .addReg(RegNo: DstReg32)
908 .addImm(Val: AArch64::sub_32);
909 MI.eraseFromParent();
910 return true;
911}
912
913// Across a basic-block we might have in i32 extract from a value that only
914// operates on upper bits (for example a sxtw). We can replace the COPY with a
915// new version skipping the sxtw.
916bool AArch64MIPeepholeOptImpl::visitCopy(MachineInstr &MI) {
917 Register InputReg = MI.getOperand(i: 1).getReg();
918 if (MI.getOperand(i: 1).getSubReg() != AArch64::sub_32 ||
919 !MRI->hasOneNonDBGUse(RegNo: InputReg))
920 return false;
921
922 MachineInstr *SrcMI = MRI->getUniqueVRegDef(Reg: InputReg);
923 SmallPtrSet<MachineInstr *, 4> DeadInstrs;
924 DeadInstrs.insert(Ptr: SrcMI);
925 while (SrcMI && SrcMI->isFullCopy() &&
926 MRI->hasOneNonDBGUse(RegNo: SrcMI->getOperand(i: 1).getReg())) {
927 SrcMI = MRI->getUniqueVRegDef(Reg: SrcMI->getOperand(i: 1).getReg());
928 DeadInstrs.insert(Ptr: SrcMI);
929 }
930
931 if (!SrcMI)
932 return false;
933
934 // Look for SXTW(X) and return Reg.
935 auto getSXTWSrcReg = [](MachineInstr *SrcMI) -> Register {
936 if (SrcMI->getOpcode() != AArch64::SBFMXri ||
937 SrcMI->getOperand(i: 2).getImm() != 0 ||
938 SrcMI->getOperand(i: 3).getImm() != 31)
939 return Register();
940 return SrcMI->getOperand(i: 1).getReg();
941 };
942 // Look for SUBREG_TO_REG(ORRWrr(WZR, COPY(X.sub_32)))
943 auto getUXTWSrcReg = [&](MachineInstr *SrcMI) -> Register {
944 if (SrcMI->getOpcode() != AArch64::SUBREG_TO_REG ||
945 SrcMI->getOperand(i: 2).getImm() != AArch64::sub_32 ||
946 !MRI->hasOneNonDBGUse(RegNo: SrcMI->getOperand(i: 1).getReg()))
947 return Register();
948 MachineInstr *Orr = MRI->getUniqueVRegDef(Reg: SrcMI->getOperand(i: 1).getReg());
949 if (!Orr || Orr->getOpcode() != AArch64::ORRWrr ||
950 Orr->getOperand(i: 1).getReg() != AArch64::WZR ||
951 !MRI->hasOneNonDBGUse(RegNo: Orr->getOperand(i: 2).getReg()))
952 return Register();
953 MachineInstr *Cpy = MRI->getUniqueVRegDef(Reg: Orr->getOperand(i: 2).getReg());
954 if (!Cpy || Cpy->getOpcode() != AArch64::COPY ||
955 Cpy->getOperand(i: 1).getSubReg() != AArch64::sub_32)
956 return Register();
957 DeadInstrs.insert(Ptr: Orr);
958 return Cpy->getOperand(i: 1).getReg();
959 };
960
961 Register SrcReg = getSXTWSrcReg(SrcMI);
962 if (!SrcReg)
963 SrcReg = getUXTWSrcReg(SrcMI);
964 if (!SrcReg)
965 return false;
966
967 MRI->constrainRegClass(Reg: SrcReg, RC: MRI->getRegClass(Reg: InputReg));
968 LLVM_DEBUG(dbgs() << "Optimizing: " << MI);
969 MI.getOperand(i: 1).setReg(SrcReg);
970 LLVM_DEBUG(dbgs() << " to: " << MI);
971 for (auto *DeadMI : DeadInstrs) {
972 LLVM_DEBUG(dbgs() << " Removing: " << *DeadMI);
973 DeadMI->eraseFromParent();
974 }
975 return true;
976}
977
978bool AArch64MIPeepholeOptImpl::run(MachineFunction &MF) {
979 TII = static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
980 TRI = static_cast<const AArch64RegisterInfo *>(
981 MF.getSubtarget().getRegisterInfo());
982 MRI = &MF.getRegInfo();
983
984 assert(MRI->isSSA() && "Expected to be run on SSA form!");
985
986 bool Changed = false;
987
988 for (MachineBasicBlock &MBB : MF) {
989 for (MachineInstr &MI : make_early_inc_range(Range&: MBB)) {
990 switch (MI.getOpcode()) {
991 default:
992 break;
993 case AArch64::INSERT_SUBREG:
994 Changed |= visitINSERT(MI);
995 break;
996 case AArch64::ANDWrr:
997 Changed |= trySplitLogicalImm<uint32_t>(Opc: AArch64::ANDWri, MI,
998 Strategy: SplitStrategy::Intersect);
999 break;
1000 case AArch64::ANDXrr:
1001 Changed |= trySplitLogicalImm<uint64_t>(Opc: AArch64::ANDXri, MI,
1002 Strategy: SplitStrategy::Intersect);
1003 break;
1004 case AArch64::ANDSWrr:
1005 Changed |= trySplitLogicalImm<uint32_t>(
1006 Opc: AArch64::ANDWri, MI, Strategy: SplitStrategy::Intersect, OtherOpc: AArch64::ANDSWri);
1007 break;
1008 case AArch64::ANDSXrr:
1009 Changed |= trySplitLogicalImm<uint64_t>(
1010 Opc: AArch64::ANDXri, MI, Strategy: SplitStrategy::Intersect, OtherOpc: AArch64::ANDSXri);
1011 break;
1012 case AArch64::EORWrr:
1013 Changed |= trySplitLogicalImm<uint32_t>(Opc: AArch64::EORWri, MI,
1014 Strategy: SplitStrategy::Disjoint);
1015 break;
1016 case AArch64::EORXrr:
1017 Changed |= trySplitLogicalImm<uint64_t>(Opc: AArch64::EORXri, MI,
1018 Strategy: SplitStrategy::Disjoint);
1019 break;
1020 case AArch64::ORRWrr:
1021 Changed |= trySplitLogicalImm<uint32_t>(Opc: AArch64::ORRWri, MI,
1022 Strategy: SplitStrategy::Disjoint);
1023 break;
1024 case AArch64::ORRXrr:
1025 Changed |= trySplitLogicalImm<uint64_t>(Opc: AArch64::ORRXri, MI,
1026 Strategy: SplitStrategy::Disjoint);
1027 break;
1028 case AArch64::ORRWrs:
1029 Changed |= visitORR(MI);
1030 break;
1031 case AArch64::ADDWrr:
1032 Changed |= visitADDSUB<uint32_t>(PosOpc: AArch64::ADDWri, NegOpc: AArch64::SUBWri, MI);
1033 break;
1034 case AArch64::SUBWrr:
1035 Changed |= visitADDSUB<uint32_t>(PosOpc: AArch64::SUBWri, NegOpc: AArch64::ADDWri, MI);
1036 break;
1037 case AArch64::ADDXrr:
1038 Changed |= visitADDSUB<uint64_t>(PosOpc: AArch64::ADDXri, NegOpc: AArch64::SUBXri, MI);
1039 break;
1040 case AArch64::SUBXrr:
1041 Changed |= visitADDSUB<uint64_t>(PosOpc: AArch64::SUBXri, NegOpc: AArch64::ADDXri, MI);
1042 break;
1043 case AArch64::ADDSWrr:
1044 Changed |=
1045 visitADDSSUBS<uint32_t>(PosOpcs: {AArch64::ADDWri, AArch64::ADDSWri},
1046 NegOpcs: {AArch64::SUBWri, AArch64::SUBSWri}, MI);
1047 break;
1048 case AArch64::SUBSWrr:
1049 Changed |=
1050 visitADDSSUBS<uint32_t>(PosOpcs: {AArch64::SUBWri, AArch64::SUBSWri},
1051 NegOpcs: {AArch64::ADDWri, AArch64::ADDSWri}, MI);
1052 break;
1053 case AArch64::ADDSXrr:
1054 Changed |=
1055 visitADDSSUBS<uint64_t>(PosOpcs: {AArch64::ADDXri, AArch64::ADDSXri},
1056 NegOpcs: {AArch64::SUBXri, AArch64::SUBSXri}, MI);
1057 break;
1058 case AArch64::SUBSXrr:
1059 Changed |=
1060 visitADDSSUBS<uint64_t>(PosOpcs: {AArch64::SUBXri, AArch64::SUBSXri},
1061 NegOpcs: {AArch64::ADDXri, AArch64::ADDSXri}, MI);
1062 break;
1063 case AArch64::CSELWr:
1064 case AArch64::CSELXr:
1065 Changed |= visitCSEL(MI);
1066 break;
1067 case AArch64::INSvi64gpr:
1068 Changed |= visitINSviGPR(MI, Opc: AArch64::INSvi64lane);
1069 break;
1070 case AArch64::INSvi32gpr:
1071 Changed |= visitINSviGPR(MI, Opc: AArch64::INSvi32lane);
1072 break;
1073 case AArch64::INSvi16gpr:
1074 Changed |= visitINSviGPR(MI, Opc: AArch64::INSvi16lane);
1075 break;
1076 case AArch64::INSvi8gpr:
1077 Changed |= visitINSviGPR(MI, Opc: AArch64::INSvi8lane);
1078 break;
1079 case AArch64::INSvi64lane:
1080 Changed |= visitINSvi64lane(MI);
1081 break;
1082 case AArch64::FMOVDr:
1083 Changed |= visitFMOVDr(MI);
1084 break;
1085 case AArch64::UBFMXri:
1086 Changed |= visitUBFMXri(MI);
1087 break;
1088 case AArch64::COPY:
1089 Changed |= visitCopy(MI);
1090 break;
1091 }
1092 }
1093 }
1094
1095 return Changed;
1096}
1097
1098bool AArch64MIPeepholeOptLegacy::runOnMachineFunction(MachineFunction &MF) {
1099 if (skipFunction(F: MF.getFunction()))
1100 return false;
1101
1102 MachineLoopInfo &MLI = getAnalysis<MachineLoopInfoWrapperPass>().getLI();
1103 return AArch64MIPeepholeOptImpl(MLI).run(MF);
1104}
1105
1106FunctionPass *llvm::createAArch64MIPeepholeOptLegacyPass() {
1107 return new AArch64MIPeepholeOptLegacy();
1108}
1109
1110PreservedAnalyses
1111AArch64MIPeepholeOptPass::run(MachineFunction &MF,
1112 MachineFunctionAnalysisManager &MFAM) {
1113 MachineLoopInfo &MLI = MFAM.getResult<MachineLoopAnalysis>(IR&: MF);
1114 const bool Changed = AArch64MIPeepholeOptImpl(MLI).run(MF);
1115 if (!Changed)
1116 return PreservedAnalyses::all();
1117 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
1118 PA.preserveSet<CFGAnalyses>();
1119 return PA;
1120}
1121