1//===- AArch64MIPeepholeOpt.cpp - AArch64 MI peephole optimization pass ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass performs below peephole optimizations on MIR level.
10//
11// 1. MOVi32imm + (ANDS?|EOR|ORR)Wrr ==> (AND|EOR|ORR)Wri + (ANDS?|EOR|ORR)Wri
12// MOVi64imm + (ANDS?|EOR|ORR)Xrr ==> (AND|EOR|ORR)Xri + (ANDS?|EOR|ORR)Xri
13//
14// 2. MOVi32imm + ADDWrr ==> ADDWRi + ADDWRi
15// MOVi64imm + ADDXrr ==> ADDXri + ADDXri
16//
17// 3. MOVi32imm + SUBWrr ==> SUBWRi + SUBWRi
18// MOVi64imm + SUBXrr ==> SUBXri + SUBXri
19//
20// The mov pseudo instruction could be expanded to multiple mov instructions
21// later. In this case, we could try to split the constant operand of mov
22// instruction into two immediates which can be directly encoded into
23// *Wri/*Xri instructions. It makes two AND/ADD/SUB instructions instead of
24// multiple `mov` + `and/add/sub` instructions.
25//
26// 4. Remove redundant ORRWrs which is generated by zero-extend.
27//
28// %3:gpr32 = ORRWrs $wzr, %2, 0
29// %4:gpr64 = SUBREG_TO_REG %3, %subreg.sub_32
30//
31// If AArch64's 32-bit form of instruction defines the source operand of
32// ORRWrs, we can remove the ORRWrs because the upper 32 bits of the source
33// operand are set to zero.
34//
35// 5. %reg = INSERT_SUBREG %reg(tied-def 0), %subreg, subidx
36// ==> %reg:subidx = SUBREG_TO_REG %subreg, subidx
37//
38// 6. %intermediate:gpr32 = COPY %src:fpr128
39// %dst:fpr128 = INSvi32gpr %dst_vec:fpr128, dst_index, %intermediate:gpr32
40// ==> %dst:fpr128 = INSvi32lane %dst_vec:fpr128, dst_index, %src:fpr128, 0
41//
42// In cases where a source FPR is copied to a GPR in order to be copied
43// to a destination FPR, we can directly copy the values between the FPRs,
44// eliminating the use of the Integer unit. When we match a pattern of
45// INSvi[X]gpr that is preceded by a chain of COPY instructions from a FPR
46// source, we use the INSvi[X]lane to replace the COPY & INSvi[X]gpr
47// instructions.
48//
49// 7. If MI sets zero for high 64-bits implicitly, remove `mov 0` for high
50// 64-bits. For example,
51//
52// %1:fpr64 = nofpexcept FCVTNv4i16 %0:fpr128, implicit $fpcr
53// %2:fpr64 = MOVID 0
54// %4:fpr128 = IMPLICIT_DEF
55// %3:fpr128 = INSERT_SUBREG %4:fpr128(tied-def 0), %2:fpr64, %subreg.dsub
56// %6:fpr128 = IMPLICIT_DEF
57// %5:fpr128 = INSERT_SUBREG %6:fpr128(tied-def 0), %1:fpr64, %subreg.dsub
58// %7:fpr128 = INSvi64lane %5:fpr128(tied-def 0), 1, %3:fpr128, 0
59// ==>
60// %1:fpr64 = nofpexcept FCVTNv4i16 %0:fpr128, implicit $fpcr
61// %6:fpr128 = IMPLICIT_DEF
62// %7:fpr128 = INSERT_SUBREG %6:fpr128(tied-def 0), %1:fpr64, %subreg.dsub
63//
64// 8. Remove redundant CSELs that select between identical registers, by
65// replacing them with unconditional moves.
66//
67// 9. Replace UBFMXri with UBFMWri if the instruction is equivalent to a 32 bit
68// LSR or LSL alias of UBFM.
69//
70//===----------------------------------------------------------------------===//
71
72#include "AArch64ExpandImm.h"
73#include "AArch64InstrInfo.h"
74#include "MCTargetDesc/AArch64AddressingModes.h"
75#include "llvm/ADT/DenseMap.h"
76#include "llvm/ADT/STLExtras.h"
77#include "llvm/CodeGen/MachineDominators.h"
78#include "llvm/CodeGen/MachineLoopInfo.h"
79
80using namespace llvm;
81
82#define DEBUG_TYPE "aarch64-mi-peephole-opt"
83
84namespace {
85
86class AArch64MIPeepholeOptImpl {
87public:
88 const AArch64InstrInfo *TII;
89 const AArch64RegisterInfo *TRI;
90 MachineLoopInfo *MLI;
91 MachineRegisterInfo *MRI;
92
93 explicit AArch64MIPeepholeOptImpl(MachineLoopInfo &MLI) : MLI(&MLI) {}
94
95 bool run(MachineFunction &MF);
96
97private:
98 using OpcodePair = std::pair<unsigned, unsigned>;
99 template <typename T>
100 using SplitAndOpcFunc =
101 std::function<std::optional<OpcodePair>(T, unsigned, T &, T &)>;
102 using BuildMIFunc =
103 std::function<void(MachineInstr &, OpcodePair, unsigned, unsigned,
104 Register, Register, Register)>;
105
106 /// For instructions where an immediate operand could be split into two
107 /// separate immediate instructions, use the splitTwoPartImm two handle the
108 /// optimization.
109 ///
110 /// To implement, the following function types must be passed to
111 /// splitTwoPartImm. A SplitAndOpcFunc must be implemented that determines if
112 /// splitting the immediate is valid and returns the associated new opcode. A
113 /// BuildMIFunc must be implemented to build the two immediate instructions.
114 ///
115 /// Example Pattern (where IMM would require 2+ MOV instructions):
116 /// %dst = <Instr>rr %src IMM [...]
117 /// becomes:
118 /// %tmp = <Instr>ri %src (encode half IMM) [...]
119 /// %dst = <Instr>ri %tmp (encode half IMM) [...]
120 template <typename T>
121 bool splitTwoPartImm(MachineInstr &MI,
122 SplitAndOpcFunc<T> SplitAndOpc, BuildMIFunc BuildInstr);
123
124 bool checkMovImmInstr(MachineInstr &MI, MachineInstr *&MovMI,
125 MachineInstr *&SubregToRegMI);
126
127 bool getConstantDef(Register Reg, uint64_t &Value, MachineInstr *&MovMI,
128 MachineInstr *&SubregToRegMI);
129 template <typename T>
130 void recordConstantDef(MachineInstr &MI,
131 DenseMap<uint64_t, Register> &Consts);
132 bool foldSharedAddSubConstant(MachineInstr &MI, unsigned AddOpc,
133 unsigned SubOpc, bool Is64Bit,
134 DenseMap<uint64_t, Register> &SameWidth);
135
136 template <typename T>
137 bool visitADDSUB(unsigned PosOpc, unsigned NegOpc, MachineInstr &MI);
138 template <typename T>
139 bool visitADDSSUBS(OpcodePair PosOpcs, OpcodePair NegOpcs, MachineInstr &MI);
140
141 // Strategy used to split logical immediate bitmasks.
142 enum class SplitStrategy {
143 Intersect,
144 Disjoint,
145 };
146 template <typename T>
147 bool trySplitLogicalImm(unsigned Opc, MachineInstr &MI,
148 SplitStrategy Strategy, unsigned OtherOpc = 0);
149 bool visitORR(MachineInstr &MI);
150 bool visitCSEL(MachineInstr &MI);
151 bool visitINSERT(MachineInstr &MI);
152 bool visitINSviGPR(MachineInstr &MI, unsigned Opc);
153 bool visitINSvi64lane(MachineInstr &MI);
154 bool visitFMOVDr(MachineInstr &MI);
155 bool visitUBFMXri(MachineInstr &MI);
156 bool visitCopy(MachineInstr &MI);
157};
158
159struct AArch64MIPeepholeOptLegacy : public MachineFunctionPass {
160 static char ID;
161
162 AArch64MIPeepholeOptLegacy() : MachineFunctionPass(ID) {}
163
164 bool runOnMachineFunction(MachineFunction &MF) override;
165
166 StringRef getPassName() const override {
167 return "AArch64 MI Peephole Optimization pass";
168 }
169
170 void getAnalysisUsage(AnalysisUsage &AU) const override {
171 AU.setPreservesCFG();
172 AU.addRequired<MachineLoopInfoWrapperPass>();
173 MachineFunctionPass::getAnalysisUsage(AU);
174 }
175};
176
177char AArch64MIPeepholeOptLegacy::ID = 0;
178
179} // end anonymous namespace
180
181INITIALIZE_PASS(AArch64MIPeepholeOptLegacy, "aarch64-mi-peephole-opt",
182 "AArch64 MI Peephole Optimization", false, false)
183
184template <typename T>
185static bool splitBitmaskImm(T Imm, unsigned RegSize, T &Imm1Enc, T &Imm2Enc) {
186 T UImm = static_cast<T>(Imm);
187 assert(UImm && (UImm != ~static_cast<T>(0)) && "Invalid immediate!");
188
189 // The bitmask immediate consists of consecutive ones. Let's say there is
190 // constant 0b00000000001000000000010000000000 which does not consist of
191 // consecutive ones. We can split it in to two bitmask immediate like
192 // 0b00000000001111111111110000000000 and 0b11111111111000000000011111111111.
193 // If we do AND with these two bitmask immediate, we can see original one.
194 unsigned LowestBitSet = llvm::countr_zero(UImm);
195 unsigned HighestBitSet = Log2_64(UImm);
196
197 // Create a mask which is filled with one from the position of lowest bit set
198 // to the position of highest bit set.
199 T NewImm1 = (static_cast<T>(2) << HighestBitSet) -
200 (static_cast<T>(1) << LowestBitSet);
201 // Create a mask which is filled with one outside the position of lowest bit
202 // set and the position of highest bit set.
203 T NewImm2 = UImm | ~NewImm1;
204
205 // If the split value is not valid bitmask immediate, do not split this
206 // constant.
207 if (!AArch64_AM::isLogicalImmediate(imm: NewImm2, regSize: RegSize))
208 return false;
209
210 Imm1Enc = AArch64_AM::encodeLogicalImmediate(imm: NewImm1, regSize: RegSize);
211 Imm2Enc = AArch64_AM::encodeLogicalImmediate(imm: NewImm2, regSize: RegSize);
212 return true;
213}
214
215template <typename T>
216static bool splitDisjointBitmaskImm(T Imm, unsigned RegSize, T &Imm1Enc,
217 T &Imm2Enc) {
218 assert(Imm && (Imm != ~static_cast<T>(0)) && "Invalid immediate!");
219
220 // Try to split a bitmask of the form 0b00000000011000000000011110000000 into
221 // two disjoint masks such as 0b00000000011000000000000000000000 and
222 // 0b00000000000000000000011110000000 where the inclusive/exclusive OR of the
223 // new masks match the original mask.
224 unsigned LowestBitSet = llvm::countr_zero(Imm);
225 unsigned LowestGapBitUnset =
226 LowestBitSet + llvm::countr_one(Imm >> LowestBitSet);
227
228 // Create a mask for the least significant group of consecutive ones.
229 assert(LowestGapBitUnset < sizeof(T) * CHAR_BIT && "Undefined behaviour!");
230 T NewImm1 = (static_cast<T>(1) << LowestGapBitUnset) -
231 (static_cast<T>(1) << LowestBitSet);
232 // Create a disjoint mask for the remaining ones.
233 T NewImm2 = Imm & ~NewImm1;
234
235 // Do not split if NewImm2 is not a valid bitmask immediate.
236 if (!AArch64_AM::isLogicalImmediate(imm: NewImm2, regSize: RegSize))
237 return false;
238
239 Imm1Enc = AArch64_AM::encodeLogicalImmediate(imm: NewImm1, regSize: RegSize);
240 Imm2Enc = AArch64_AM::encodeLogicalImmediate(imm: NewImm2, regSize: RegSize);
241 return true;
242}
243
244template <typename T>
245bool AArch64MIPeepholeOptImpl::trySplitLogicalImm(unsigned Opc,
246 MachineInstr &MI,
247 SplitStrategy Strategy,
248 unsigned OtherOpc) {
249 // Try below transformations.
250 //
251 // MOVi32imm + (ANDS?|EOR|ORR)Wrr ==> (AND|EOR|ORR)Wri + (ANDS?|EOR|ORR)Wri
252 // MOVi64imm + (ANDS?|EOR|ORR)Xrr ==> (AND|EOR|ORR)Xri + (ANDS?|EOR|ORR)Xri
253 //
254 // The mov pseudo instruction could be expanded to multiple mov instructions
255 // later. Let's try to split the constant operand of mov instruction into two
256 // bitmask immediates based on the given split strategy. It makes only two
257 // logical instructions instead of multiple mov + logic instructions.
258
259 return splitTwoPartImm<T>(
260 MI,
261 [Opc, Strategy, OtherOpc](T Imm, unsigned RegSize, T &Imm0,
262 T &Imm1) -> std::optional<OpcodePair> {
263 // If this immediate is already a suitable bitmask, don't split it.
264 // TODO: Should we just combine the two instructions in this case?
265 if (AArch64_AM::isLogicalImmediate(imm: Imm, regSize: RegSize))
266 return std::nullopt;
267
268 // If this immediate can be handled by one instruction, don't split it.
269 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
270 AArch64_IMM::expandMOVImm(Imm, BitSize: RegSize, Insn);
271 if (Insn.size() == 1)
272 return std::nullopt;
273
274 bool SplitSucc = false;
275 switch (Strategy) {
276 case SplitStrategy::Intersect:
277 SplitSucc = splitBitmaskImm(Imm, RegSize, Imm0, Imm1);
278 break;
279 case SplitStrategy::Disjoint:
280 SplitSucc = splitDisjointBitmaskImm(Imm, RegSize, Imm0, Imm1);
281 break;
282 }
283 if (SplitSucc)
284 return std::make_pair(x: Opc, y: !OtherOpc ? Opc : OtherOpc);
285 return std::nullopt;
286 },
287 [&TII = TII](MachineInstr &MI, OpcodePair Opcode, unsigned Imm0,
288 unsigned Imm1, Register SrcReg, Register NewTmpReg,
289 Register NewDstReg) {
290 DebugLoc DL = MI.getDebugLoc();
291 MachineBasicBlock *MBB = MI.getParent();
292 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opcode.first), DestReg: NewTmpReg)
293 .addReg(RegNo: SrcReg)
294 .addImm(Val: Imm0);
295 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opcode.second), DestReg: NewDstReg)
296 .addReg(RegNo: NewTmpReg)
297 .addImm(Val: Imm1);
298 });
299}
300
301bool AArch64MIPeepholeOptImpl::visitORR(MachineInstr &MI) {
302 // Check this ORR comes from below zero-extend pattern.
303 //
304 // def : Pat<(i64 (zext GPR32:$src)),
305 // (SUBREG_TO_REG (ORRWrs WZR, GPR32:$src, 0), sub_32)>;
306 if (MI.getOperand(i: 3).getImm() != 0)
307 return false;
308
309 if (MI.getOperand(i: 1).getReg() != AArch64::WZR)
310 return false;
311
312 if (MI.getOperand(i: 2).getSubReg())
313 return false;
314
315 Register DefReg = MI.getOperand(i: 0).getReg();
316 Register SrcReg = MI.getOperand(i: 2).getReg();
317 if (!DefReg.isVirtual() || !SrcReg.isVirtual())
318 return false;
319
320 MachineInstr *SrcMI = MRI->getVRegDef(Reg: SrcReg);
321 if (!SrcMI)
322 return false;
323
324 // From https://developer.arm.com/documentation/dui0801/b/BABBGCAC
325 //
326 // When you use the 32-bit form of an instruction, the upper 32 bits of the
327 // source registers are ignored and the upper 32 bits of the destination
328 // register are set to zero.
329 //
330 // If AArch64's 32-bit form of instruction defines the source operand of
331 // zero-extend, we do not need the zero-extend. Let's check the MI's opcode is
332 // real AArch64 instruction and if it is not, do not process the opcode
333 // conservatively.
334 if (SrcMI->getOpcode() == TargetOpcode::COPY &&
335 SrcMI->getOperand(i: 1).getReg().isVirtual()) {
336 const TargetRegisterClass *RC =
337 MRI->getRegClass(Reg: SrcMI->getOperand(i: 1).getReg());
338
339 // A COPY from an FPR will become a FMOVSWr, so do so now so that we know
340 // that the upper bits are zero.
341 if (RC != &AArch64::FPR32RegClass &&
342 ((RC != &AArch64::FPR64RegClass && RC != &AArch64::FPR128RegClass &&
343 RC != &AArch64::ZPRRegClass) ||
344 SrcMI->getOperand(i: 1).getSubReg() != AArch64::ssub))
345 return false;
346 Register CpySrc;
347 if (SrcMI->getOperand(i: 1).getSubReg() == AArch64::ssub) {
348 CpySrc = MRI->createVirtualRegister(RegClass: &AArch64::FPR32RegClass);
349 BuildMI(BB&: *SrcMI->getParent(), I: SrcMI, MIMD: SrcMI->getDebugLoc(),
350 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: CpySrc)
351 .add(MO: SrcMI->getOperand(i: 1));
352 } else {
353 CpySrc = SrcMI->getOperand(i: 1).getReg();
354 }
355 BuildMI(BB&: *SrcMI->getParent(), I: SrcMI, MIMD: SrcMI->getDebugLoc(),
356 MCID: TII->get(Opcode: AArch64::FMOVSWr), DestReg: SrcMI->getOperand(i: 0).getReg())
357 .addReg(RegNo: CpySrc);
358 SrcMI->eraseFromParent();
359 }
360 else if (SrcMI->getOpcode() <= TargetOpcode::GENERIC_OP_END)
361 return false;
362
363 MRI->replaceRegWith(FromReg: DefReg, ToReg: SrcReg);
364 LLVM_DEBUG(dbgs() << "Removed: " << MI << "\n");
365 MI.eraseFromParent();
366
367 return true;
368}
369
370bool AArch64MIPeepholeOptImpl::visitCSEL(MachineInstr &MI) {
371 // Replace CSEL with MOV when both inputs are the same register.
372 if (MI.getOperand(i: 1).getReg() != MI.getOperand(i: 2).getReg())
373 return false;
374
375 auto ZeroReg =
376 MI.getOpcode() == AArch64::CSELXr ? AArch64::XZR : AArch64::WZR;
377 auto OrOpcode =
378 MI.getOpcode() == AArch64::CSELXr ? AArch64::ORRXrs : AArch64::ORRWrs;
379
380 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: OrOpcode))
381 .addReg(RegNo: MI.getOperand(i: 0).getReg(), Flags: RegState::Define)
382 .addReg(RegNo: ZeroReg)
383 .addReg(RegNo: MI.getOperand(i: 1).getReg())
384 .addImm(Val: 0);
385
386 MI.eraseFromParent();
387 return true;
388}
389
390bool AArch64MIPeepholeOptImpl::visitINSERT(MachineInstr &MI) {
391 // Check this INSERT_SUBREG comes from below zero-extend pattern.
392 //
393 // From %reg = INSERT_SUBREG %reg(tied-def 0), %subreg, subidx
394 // To %reg:subidx = SUBREG_TO_REG %subreg, subidx
395 //
396 // We're assuming the first operand to INSERT_SUBREG is irrelevant because a
397 // COPY would destroy the upper part of the register anyway
398 if (!MI.isRegTiedToDefOperand(UseOpIdx: 1))
399 return false;
400
401 Register DstReg = MI.getOperand(i: 0).getReg();
402 const TargetRegisterClass *RC = MRI->getRegClass(Reg: DstReg);
403 MachineInstr *SrcMI = MRI->getUniqueVRegDef(Reg: MI.getOperand(i: 2).getReg());
404 if (!SrcMI)
405 return false;
406
407 // From https://developer.arm.com/documentation/dui0801/b/BABBGCAC
408 //
409 // When you use the 32-bit form of an instruction, the upper 32 bits of the
410 // source registers are ignored and the upper 32 bits of the destination
411 // register are set to zero.
412 //
413 // If AArch64's 32-bit form of instruction defines the source operand of
414 // zero-extend, we do not need the zero-extend. Let's check the MI's opcode is
415 // real AArch64 instruction and if it is not, do not process the opcode
416 // conservatively.
417 if ((SrcMI->getOpcode() <= TargetOpcode::GENERIC_OP_END) ||
418 !AArch64::GPR64allRegClass.hasSubClassEq(RC))
419 return false;
420
421 // Build a SUBREG_TO_REG instruction
422 MachineInstr *SubregMI =
423 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
424 MCID: TII->get(Opcode: TargetOpcode::SUBREG_TO_REG), DestReg: DstReg)
425 .add(MO: MI.getOperand(i: 2))
426 .add(MO: MI.getOperand(i: 3));
427 LLVM_DEBUG(dbgs() << MI << " replace by:\n: " << *SubregMI << "\n");
428 (void)SubregMI;
429 MI.eraseFromParent();
430
431 return true;
432}
433
434template <typename T>
435static bool splitAddSubImm(T Imm, unsigned RegSize, T &Imm0, T &Imm1) {
436 // The immediate must be in the form of ((imm0 << 12) + imm1), in which both
437 // imm0 and imm1 are non-zero 12-bit unsigned int.
438 if ((Imm & 0xfff000) == 0 || (Imm & 0xfff) == 0 ||
439 (Imm & ~static_cast<T>(0xffffff)) != 0)
440 return false;
441
442 // The immediate can not be composed via a single instruction.
443 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
444 AArch64_IMM::expandMOVImm(Imm, BitSize: RegSize, Insn);
445 if (Insn.size() == 1)
446 return false;
447
448 // Split Imm into (Imm0 << 12) + Imm1;
449 Imm0 = (Imm >> 12) & 0xfff;
450 Imm1 = Imm & 0xfff;
451 return true;
452}
453
454template <typename T>
455bool AArch64MIPeepholeOptImpl::visitADDSUB(unsigned PosOpc, unsigned NegOpc,
456 MachineInstr &MI) {
457 // Try below transformation.
458 //
459 // ADDWrr X, MOVi32imm ==> ADDWri + ADDWri
460 // ADDXrr X, MOVi64imm ==> ADDXri + ADDXri
461 //
462 // SUBWrr X, MOVi32imm ==> SUBWri + SUBWri
463 // SUBXrr X, MOVi64imm ==> SUBXri + SUBXri
464 //
465 // The mov pseudo instruction could be expanded to multiple mov instructions
466 // later. Let's try to split the constant operand of mov instruction into two
467 // legal add/sub immediates. It makes only two ADD/SUB instructions instead of
468 // multiple `mov` + `and/sub` instructions.
469
470 // We can sometimes have ADDWrr WZR, MULi32imm that have not been constant
471 // folded. Make sure that we don't generate invalid instructions that use XZR
472 // in those cases.
473 if (MI.getOperand(i: 1).getReg() == AArch64::XZR ||
474 MI.getOperand(i: 1).getReg() == AArch64::WZR)
475 return false;
476
477 return splitTwoPartImm<T>(
478 MI,
479 [PosOpc, NegOpc](T Imm, unsigned RegSize, T &Imm0,
480 T &Imm1) -> std::optional<OpcodePair> {
481 if (splitAddSubImm(Imm, RegSize, Imm0, Imm1))
482 return std::make_pair(x: PosOpc, y: PosOpc);
483 if (splitAddSubImm(-Imm, RegSize, Imm0, Imm1))
484 return std::make_pair(x: NegOpc, y: NegOpc);
485 return std::nullopt;
486 },
487 [&TII = TII](MachineInstr &MI, OpcodePair Opcode, unsigned Imm0,
488 unsigned Imm1, Register SrcReg, Register NewTmpReg,
489 Register NewDstReg) {
490 DebugLoc DL = MI.getDebugLoc();
491 MachineBasicBlock *MBB = MI.getParent();
492 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opcode.first), DestReg: NewTmpReg)
493 .addReg(RegNo: SrcReg)
494 .addImm(Val: Imm0)
495 .addImm(Val: 12);
496 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opcode.second), DestReg: NewDstReg)
497 .addReg(RegNo: NewTmpReg)
498 .addImm(Val: Imm1)
499 .addImm(Val: 0);
500 });
501}
502
503template <typename T>
504bool AArch64MIPeepholeOptImpl::visitADDSSUBS(OpcodePair PosOpcs,
505 OpcodePair NegOpcs,
506 MachineInstr &MI) {
507 // Try the same transformation as ADDSUB but with additional requirement
508 // that the condition code usages are only for Equal and Not Equal
509
510 if (MI.getOperand(i: 1).getReg() == AArch64::XZR ||
511 MI.getOperand(i: 1).getReg() == AArch64::WZR)
512 return false;
513
514 return splitTwoPartImm<T>(
515 MI,
516 [PosOpcs, NegOpcs, &MI, &TRI = TRI,
517 &MRI = MRI](T Imm, unsigned RegSize, T &Imm0,
518 T &Imm1) -> std::optional<OpcodePair> {
519 OpcodePair OP;
520 if (splitAddSubImm(Imm, RegSize, Imm0, Imm1))
521 OP = PosOpcs;
522 else if (splitAddSubImm(-Imm, RegSize, Imm0, Imm1))
523 OP = NegOpcs;
524 else
525 return std::nullopt;
526 // Check conditional uses last since it is expensive for scanning
527 // proceeding instructions
528 MachineInstr *SrcMI = MRI->getVRegDef(Reg: MI.getOperand(i: 1).getReg());
529 if (!SrcMI)
530 return std::nullopt;
531 std::optional<UsedNZCV> NZCVUsed = examineCFlagsUse(MI&: *SrcMI, CmpInstr&: MI, TRI: *TRI);
532 if (!NZCVUsed || NZCVUsed->C || NZCVUsed->V)
533 return std::nullopt;
534 return OP;
535 },
536 [&TII = TII](MachineInstr &MI, OpcodePair Opcode, unsigned Imm0,
537 unsigned Imm1, Register SrcReg, Register NewTmpReg,
538 Register NewDstReg) {
539 DebugLoc DL = MI.getDebugLoc();
540 MachineBasicBlock *MBB = MI.getParent();
541 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opcode.first), DestReg: NewTmpReg)
542 .addReg(RegNo: SrcReg)
543 .addImm(Val: Imm0)
544 .addImm(Val: 12);
545 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: Opcode.second), DestReg: NewDstReg)
546 .addReg(RegNo: NewTmpReg)
547 .addImm(Val: Imm1)
548 .addImm(Val: 0);
549 });
550}
551
552// Checks if the corresponding MOV immediate instruction is applicable for
553// this peephole optimization.
554bool AArch64MIPeepholeOptImpl::checkMovImmInstr(MachineInstr &MI,
555 MachineInstr *&MovMI,
556 MachineInstr *&SubregToRegMI) {
557 // Check whether current MBB is in loop and the AND is loop invariant.
558 MachineBasicBlock *MBB = MI.getParent();
559 MachineLoop *L = MLI->getLoopFor(BB: MBB);
560 if (L && !L->isLoopInvariant(I&: MI))
561 return false;
562
563 // Check whether current MI's operand is MOV with immediate.
564 MovMI = MRI->getUniqueVRegDef(Reg: MI.getOperand(i: 2).getReg());
565 if (!MovMI)
566 return false;
567
568 // If it is SUBREG_TO_REG, check its operand.
569 SubregToRegMI = nullptr;
570 if (MovMI->getOpcode() == TargetOpcode::SUBREG_TO_REG) {
571 SubregToRegMI = MovMI;
572 MovMI = MRI->getUniqueVRegDef(Reg: MovMI->getOperand(i: 1).getReg());
573 if (!MovMI)
574 return false;
575 }
576
577 if (MovMI->getOpcode() != AArch64::MOVi32imm &&
578 MovMI->getOpcode() != AArch64::MOVi64imm)
579 return false;
580
581 // If the MOV has multiple uses, do not split the immediate because it causes
582 // more instructions.
583 if (!MRI->hasOneNonDBGUse(RegNo: MovMI->getOperand(i: 0).getReg()))
584 return false;
585 if (SubregToRegMI &&
586 !MRI->hasOneNonDBGUse(RegNo: SubregToRegMI->getOperand(i: 0).getReg()))
587 return false;
588
589 // It is OK to perform this peephole optimization.
590 return true;
591}
592
593bool AArch64MIPeepholeOptImpl::getConstantDef(Register Reg, uint64_t &Value,
594 MachineInstr *&MovMI,
595 MachineInstr *&SubregToRegMI) {
596 SubregToRegMI = nullptr;
597 MovMI = MRI->getUniqueVRegDef(Reg);
598 if (!MovMI)
599 return false;
600 if (MovMI->getOpcode() == TargetOpcode::SUBREG_TO_REG) {
601 SubregToRegMI = MovMI;
602 MovMI = MRI->getUniqueVRegDef(Reg: MovMI->getOperand(i: 1).getReg());
603 if (!MovMI)
604 return false;
605 }
606 if (MovMI->getOpcode() != AArch64::MOVi32imm &&
607 MovMI->getOpcode() != AArch64::MOVi64imm)
608 return false;
609 Value = static_cast<uint64_t>(MovMI->getOperand(i: 1).getImm());
610 if (SubregToRegMI)
611 Value = static_cast<uint32_t>(Value);
612 return true;
613}
614
615static bool hasRealUse(const MachineRegisterInfo &MRI, Register Reg) {
616 return any_of(Range: MRI.use_nodbg_instructions(Reg),
617 P: [](const MachineInstr &UseMI) { return !UseMI.isFakeUse(); });
618}
619
620template <typename T>
621void AArch64MIPeepholeOptImpl::recordConstantDef(
622 MachineInstr &MI, DenseMap<uint64_t, Register> &Consts) {
623 if (hasRealUse(MRI: *MRI, Reg: MI.getOperand(i: 0).getReg()))
624 Consts[static_cast<T>(MI.getOperand(i: 1).getImm())] =
625 MI.getOperand(i: 0).getReg();
626}
627
628bool AArch64MIPeepholeOptImpl::foldSharedAddSubConstant(
629 MachineInstr &MI, unsigned AddOpc, unsigned SubOpc, bool Is64Bit,
630 DenseMap<uint64_t, Register> &SameWidth) {
631 unsigned Opc = MI.getOpcode();
632
633 Register SrcReg = MI.getOperand(i: 2).getReg();
634 if (!SrcReg.isVirtual())
635 return false;
636
637 uint64_t RawValue;
638 MachineInstr *MovMI, *SubregToRegMI;
639 if (!getConstantDef(Reg: SrcReg, Value&: RawValue, MovMI, SubregToRegMI))
640 return false;
641
642 uint64_t NegValue =
643 Is64Bit ? (0ULL - RawValue)
644 : static_cast<uint32_t>(0U - static_cast<uint32_t>(RawValue));
645
646 auto SameIt = SameWidth.find(Val: NegValue);
647 if (SameIt == SameWidth.end() || SameIt->second == SrcReg ||
648 MRI->def_empty(RegNo: SameIt->second))
649 return false;
650 Register CanonReg = SameIt->second;
651
652 unsigned NewOpc = (Opc == AddOpc) ? SubOpc : AddOpc;
653 MI.getOperand(i: 2).setReg(CanonReg);
654 MI.setDesc(TII->get(Opcode: NewOpc));
655 MRI->clearKillFlags(Reg: CanonReg);
656
657 if (MRI->use_nodbg_empty(RegNo: SrcReg)) {
658 if (SubregToRegMI) {
659 SubregToRegMI->eraseFromParent();
660 if (MRI->use_nodbg_empty(RegNo: MovMI->getOperand(i: 0).getReg()))
661 MovMI->eraseFromParent();
662 } else {
663 MovMI->eraseFromParent();
664 }
665 }
666 return true;
667}
668
669template <typename T>
670bool AArch64MIPeepholeOptImpl::splitTwoPartImm(MachineInstr &MI,
671 SplitAndOpcFunc<T> SplitAndOpc,
672 BuildMIFunc BuildInstr) {
673 unsigned RegSize = sizeof(T) * 8;
674 assert((RegSize == 32 || RegSize == 64) &&
675 "Invalid RegSize for legal immediate peephole optimization");
676
677 // Perform several essential checks against current MI.
678 MachineInstr *MovMI, *SubregToRegMI;
679 if (!checkMovImmInstr(MI, MovMI, SubregToRegMI))
680 return false;
681
682 // Split the immediate to Imm0 and Imm1, and calculate the Opcode.
683 T Imm = static_cast<T>(MovMI->getOperand(i: 1).getImm()), Imm0, Imm1;
684 // For the 32 bit form of instruction, the upper 32 bits of the destination
685 // register are set to zero. If there is SUBREG_TO_REG, set the upper 32 bits
686 // of Imm to zero. This is essential if the Immediate value was a negative
687 // number since it was sign extended when we assign to the 64-bit Imm.
688 if (SubregToRegMI)
689 Imm &= 0xFFFFFFFF;
690 OpcodePair Opcode;
691 if (auto R = SplitAndOpc(Imm, RegSize, Imm0, Imm1))
692 Opcode = *R;
693 else
694 return false;
695
696 // Create new MIs using the first and second opcodes. Opcodes might differ for
697 // flag setting operations that should only set flags on second instruction.
698 // NewTmpReg = Opcode.first SrcReg Imm0
699 // NewDstReg = Opcode.second NewTmpReg Imm1
700
701 // Determine register classes for destinations and register operands
702 const TargetRegisterClass *FirstInstrDstRC =
703 TII->getRegClass(MCID: TII->get(Opcode: Opcode.first), OpNum: 0);
704 const TargetRegisterClass *FirstInstrOperandRC =
705 TII->getRegClass(MCID: TII->get(Opcode: Opcode.first), OpNum: 1);
706 const TargetRegisterClass *SecondInstrDstRC =
707 (Opcode.first == Opcode.second)
708 ? FirstInstrDstRC
709 : TII->getRegClass(MCID: TII->get(Opcode: Opcode.second), OpNum: 0);
710 const TargetRegisterClass *SecondInstrOperandRC =
711 (Opcode.first == Opcode.second)
712 ? FirstInstrOperandRC
713 : TII->getRegClass(MCID: TII->get(Opcode: Opcode.second), OpNum: 1);
714
715 // Get old registers destinations and new register destinations
716 Register DstReg = MI.getOperand(i: 0).getReg();
717 Register SrcReg = MI.getOperand(i: 1).getReg();
718 Register NewTmpReg = MRI->createVirtualRegister(RegClass: FirstInstrDstRC);
719 // In the situation that DstReg is not Virtual (likely WZR or XZR), we want to
720 // reuse that same destination register.
721 Register NewDstReg = DstReg.isVirtual()
722 ? MRI->createVirtualRegister(RegClass: SecondInstrDstRC)
723 : DstReg;
724
725 // Constrain registers based on their new uses
726 MRI->constrainRegClass(Reg: SrcReg, RC: FirstInstrOperandRC);
727 MRI->constrainRegClass(Reg: NewTmpReg, RC: SecondInstrOperandRC);
728 if (DstReg != NewDstReg)
729 MRI->constrainRegClass(Reg: NewDstReg, RC: MRI->getRegClass(Reg: DstReg));
730
731 // Call the delegating operation to build the instruction
732 BuildInstr(MI, Opcode, Imm0, Imm1, SrcReg, NewTmpReg, NewDstReg);
733
734 // replaceRegWith changes MI's definition register. Keep it for SSA form until
735 // deleting MI. Only if we made a new destination register.
736 if (DstReg != NewDstReg) {
737 MRI->replaceRegWith(FromReg: DstReg, ToReg: NewDstReg);
738 MI.getOperand(i: 0).setReg(DstReg);
739 }
740
741 auto SalvageDebugUses = [&](Register Reg, int64_t Val) {
742 for (MachineOperand &MO : make_early_inc_range(Range: MRI->use_operands(Reg))) {
743 MachineInstr &DbgMI = *MO.getParent();
744 if (!DbgMI.isDebugValue())
745 continue;
746 // An immediate cannot express a value in memory at that address.
747 if (DbgMI.isNonListDebugValue() && DbgMI.isIndirectDebugValue())
748 DbgMI.setDebugValueUndef();
749 else
750 MO.ChangeToImmediate(ImmVal: Val);
751 }
752 };
753
754 // Record the MIs need to be removed.
755 MI.eraseFromParent();
756 if (SubregToRegMI) {
757 SalvageDebugUses(SubregToRegMI->getOperand(i: 0).getReg(), Imm);
758 SubregToRegMI->eraseFromParent();
759 }
760 SalvageDebugUses(MovMI->getOperand(i: 0).getReg(),
761 MovMI->getOperand(i: 1).getImm());
762 MovMI->eraseFromParent();
763
764 return true;
765}
766
767bool AArch64MIPeepholeOptImpl::visitINSviGPR(MachineInstr &MI, unsigned Opc) {
768 // Check if this INSvi[X]gpr comes from COPY of a source FPR128
769 //
770 // From
771 // %intermediate1:gpr64 = COPY %src:fpr128
772 // %intermediate2:gpr32 = COPY %intermediate1:gpr64
773 // %dst:fpr128 = INSvi[X]gpr %dst_vec:fpr128, dst_index, %intermediate2:gpr32
774 // To
775 // %dst:fpr128 = INSvi[X]lane %dst_vec:fpr128, dst_index, %src:fpr128,
776 // src_index
777 // where src_index = 0, X = [8|16|32|64]
778
779 MachineInstr *SrcMI = MRI->getUniqueVRegDef(Reg: MI.getOperand(i: 3).getReg());
780
781 // For a chain of COPY instructions, find the initial source register
782 // and check if it's an FPR128
783 while (true) {
784 if (!SrcMI || SrcMI->getOpcode() != TargetOpcode::COPY)
785 return false;
786
787 if (!SrcMI->getOperand(i: 1).getReg().isVirtual())
788 return false;
789
790 if (MRI->getRegClass(Reg: SrcMI->getOperand(i: 1).getReg()) ==
791 &AArch64::FPR128RegClass) {
792 break;
793 }
794 SrcMI = MRI->getUniqueVRegDef(Reg: SrcMI->getOperand(i: 1).getReg());
795 }
796
797 Register DstReg = MI.getOperand(i: 0).getReg();
798 Register SrcReg = SrcMI->getOperand(i: 1).getReg();
799 MachineInstr *INSvilaneMI =
800 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc), DestReg: DstReg)
801 .add(MO: MI.getOperand(i: 1))
802 .add(MO: MI.getOperand(i: 2))
803 .addUse(RegNo: SrcReg, Flags: getRegState(RegOp: SrcMI->getOperand(i: 1)))
804 .addImm(Val: 0);
805
806 LLVM_DEBUG(dbgs() << MI << " replace by:\n: " << *INSvilaneMI << "\n");
807 (void)INSvilaneMI;
808 MI.eraseFromParent();
809 return true;
810}
811
812// All instructions that set a FPR64 will implicitly zero the top bits of the
813// register. When the def is expressed as a COPY from a GPR, turn it into an
814// explicit FMOV so it cannot be elided later in further passes.
815static bool is64bitDefwithZeroHigh64bit(MachineInstr *MI,
816 MachineRegisterInfo *MRI,
817 const AArch64InstrInfo *TII) {
818 if (!MI->getOperand(i: 0).isReg() || !MI->getOperand(i: 0).isDef())
819 return false;
820 const TargetRegisterClass *RC = MRI->getRegClass(Reg: MI->getOperand(i: 0).getReg());
821 if (RC != &AArch64::FPR64RegClass)
822 return false;
823 if (MI->getOpcode() == TargetOpcode::COPY) {
824 MachineOperand &SrcOp = MI->getOperand(i: 1);
825 if (!SrcOp.isReg())
826 return false;
827 Register SrcReg = SrcOp.getReg();
828 if (SrcOp.getSubReg()) {
829 // If operand is defined by a LD1/2/3/4 that define a D subreg tuple
830 // then upper bits of the tuple's registers are implicitly zeroed
831 // and the FMOV is unneeded.
832 if (!SrcReg.isVirtual())
833 return false;
834
835 MachineInstr *SrcDef = MRI->getUniqueVRegDef(Reg: SrcReg);
836 if (!SrcDef || SrcDef->getOpcode() <= TargetOpcode::GENERIC_OP_END ||
837 SrcDef->getDesc().isPseudo() || !SrcDef->mayLoad())
838 return false;
839
840 const TargetRegisterClass *RC = MRI->getRegClass(Reg: SrcReg);
841 return RC == &AArch64::DDRegClass || RC == &AArch64::DDDRegClass ||
842 RC == &AArch64::DDDDRegClass;
843 }
844 auto IsGPR64Like = [&]() -> bool {
845 if (SrcReg.isVirtual())
846 return AArch64::GPR64allRegClass.hasSubClassEq(
847 RC: MRI->getRegClass(Reg: SrcReg));
848 return AArch64::GPR64allRegClass.contains(Reg: SrcReg);
849 };
850 if (!IsGPR64Like())
851 return false;
852 assert(TII && "Expected InstrInfo when materializing COPYs");
853 // FMOVXDr insists on strict GPR64 operands, so fix up the COPY source.
854 MachineOperand &SrcMO = MI->getOperand(i: 1);
855 bool SrcKill = SrcMO.isKill();
856 if (SrcReg.isVirtual()) {
857 if (MRI->getRegClass(Reg: SrcReg) != &AArch64::GPR64RegClass) {
858 // Pass the value through a temporary GPR64 vreg to satisfy the
859 // verifier.
860 Register NewSrc = MRI->createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
861 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: MI->getDebugLoc(),
862 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewSrc)
863 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: SrcKill));
864 SrcReg = NewSrc;
865 }
866 } else if (!AArch64::GPR64RegClass.contains(Reg: SrcReg)) {
867 return false;
868 }
869 SrcMO.setReg(SrcReg);
870 SrcMO.setSubReg(0);
871 SrcMO.setIsKill(SrcKill);
872 // Replace the COPY with an explicit FMOV so the zeroing behaviour stays
873 // visible.
874 MI->setDesc(TII->get(Opcode: AArch64::FMOVXDr));
875 return true;
876 }
877 return MI->getOpcode() > TargetOpcode::GENERIC_OP_END;
878}
879
880bool AArch64MIPeepholeOptImpl::visitINSvi64lane(MachineInstr &MI) {
881 // Check the MI for low 64-bits sets zero for high 64-bits implicitly.
882 // We are expecting below case.
883 //
884 // %1:fpr64 = nofpexcept FCVTNv4i16 %0:fpr128, implicit $fpcr
885 // %6:fpr128 = IMPLICIT_DEF
886 // %5:fpr128 = INSERT_SUBREG %6:fpr128(tied-def 0), %1:fpr64, %subreg.dsub
887 // %7:fpr128 = INSvi64lane %5:fpr128(tied-def 0), 1, %3:fpr128, 0
888 if (MI.getOperand(i: 2).getImm() != 1)
889 return false;
890 MachineInstr *Low64MI = MRI->getVRegDef(Reg: MI.getOperand(i: 1).getReg());
891 if (!Low64MI || Low64MI->getOpcode() != AArch64::INSERT_SUBREG)
892 return false;
893 Low64MI = MRI->getUniqueVRegDef(Reg: Low64MI->getOperand(i: 2).getReg());
894 if (!Low64MI || !is64bitDefwithZeroHigh64bit(MI: Low64MI, MRI, TII))
895 return false;
896
897 // Check there is `mov 0` MI for high 64-bits.
898 // We are expecting below cases.
899 //
900 // %2:fpr64 = MOVID 0
901 // %4:fpr128 = IMPLICIT_DEF
902 // %3:fpr128 = INSERT_SUBREG %4:fpr128(tied-def 0), %2:fpr64, %subreg.dsub
903 // %7:fpr128 = INSvi64lane %5:fpr128(tied-def 0), 1, %3:fpr128, 0
904 // or
905 // %5:fpr128 = MOVIv2d_ns 0
906 // %6:fpr64 = COPY %5.dsub:fpr128
907 // %8:fpr128 = IMPLICIT_DEF
908 // %7:fpr128 = INSERT_SUBREG %8:fpr128(tied-def 0), %6:fpr64, %subreg.dsub
909 // %11:fpr128 = INSvi64lane %9:fpr128(tied-def 0), 1, %7:fpr128, 0
910 MachineInstr *High64MI = MRI->getUniqueVRegDef(Reg: MI.getOperand(i: 3).getReg());
911 if (!High64MI || High64MI->getOpcode() != AArch64::INSERT_SUBREG)
912 return false;
913 High64MI = MRI->getUniqueVRegDef(Reg: High64MI->getOperand(i: 2).getReg());
914 if (High64MI && High64MI->getOpcode() == TargetOpcode::COPY)
915 High64MI = MRI->getUniqueVRegDef(Reg: High64MI->getOperand(i: 1).getReg());
916 if (!High64MI || (High64MI->getOpcode() != AArch64::MOVID &&
917 High64MI->getOpcode() != AArch64::MOVIv2d_ns))
918 return false;
919 if (High64MI->getOperand(i: 1).getImm() != 0)
920 return false;
921
922 // Let's remove MIs for high 64-bits.
923 Register OldDef = MI.getOperand(i: 0).getReg();
924 Register NewDef = MI.getOperand(i: 1).getReg();
925 LLVM_DEBUG(dbgs() << "Removing: " << MI << "\n");
926 MRI->constrainRegClass(Reg: NewDef, RC: MRI->getRegClass(Reg: OldDef));
927 MRI->replaceRegWith(FromReg: OldDef, ToReg: NewDef);
928 MI.eraseFromParent();
929
930 return true;
931}
932
933bool AArch64MIPeepholeOptImpl::visitFMOVDr(MachineInstr &MI) {
934 // An FMOVDr sets the high 64-bits to zero implicitly, similar to ORR for GPR.
935 MachineInstr *Low64MI = MRI->getUniqueVRegDef(Reg: MI.getOperand(i: 1).getReg());
936 if (!Low64MI || !is64bitDefwithZeroHigh64bit(MI: Low64MI, MRI, TII))
937 return false;
938
939 // Let's remove MIs for high 64-bits.
940 Register OldDef = MI.getOperand(i: 0).getReg();
941 Register NewDef = MI.getOperand(i: 1).getReg();
942 LLVM_DEBUG(dbgs() << "Removing: " << MI << "\n");
943 MRI->constrainRegClass(Reg: NewDef, RC: MRI->getRegClass(Reg: OldDef));
944 MRI->replaceRegWith(FromReg: OldDef, ToReg: NewDef);
945 MI.eraseFromParent();
946
947 return true;
948}
949
950bool AArch64MIPeepholeOptImpl::visitUBFMXri(MachineInstr &MI) {
951 // Check if the instruction is equivalent to a 32 bit LSR or LSL alias of
952 // UBFM, and replace the UBFMXri instruction with its 32 bit variant, UBFMWri.
953 int64_t Immr = MI.getOperand(i: 2).getImm();
954 int64_t Imms = MI.getOperand(i: 3).getImm();
955
956 bool IsLSR = Imms == 31 && Immr <= Imms;
957 bool IsLSL = Immr == Imms + 33;
958 if (!IsLSR && !IsLSL)
959 return false;
960
961 if (IsLSL) {
962 Immr -= 32;
963 }
964
965 const TargetRegisterClass *DstRC64 =
966 TII->getRegClass(MCID: TII->get(Opcode: MI.getOpcode()), OpNum: 0);
967 const TargetRegisterClass *DstRC32 =
968 TRI->getSubRegisterClass(DstRC64, AArch64::sub_32);
969 assert(DstRC32 && "Destination register class of UBFMXri doesn't have a "
970 "sub_32 subregister class");
971
972 const TargetRegisterClass *SrcRC64 =
973 TII->getRegClass(MCID: TII->get(Opcode: MI.getOpcode()), OpNum: 1);
974 const TargetRegisterClass *SrcRC32 =
975 TRI->getSubRegisterClass(SrcRC64, AArch64::sub_32);
976 assert(SrcRC32 && "Source register class of UBFMXri doesn't have a sub_32 "
977 "subregister class");
978
979 Register DstReg64 = MI.getOperand(i: 0).getReg();
980 Register DstReg32 = MRI->createVirtualRegister(RegClass: DstRC32);
981 Register SrcReg64 = MI.getOperand(i: 1).getReg();
982 Register SrcReg32 = MRI->createVirtualRegister(RegClass: SrcRC32);
983
984 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::COPY),
985 DestReg: SrcReg32)
986 .addReg(RegNo: SrcReg64, Flags: {}, SubReg: AArch64::sub_32);
987 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::UBFMWri),
988 DestReg: DstReg32)
989 .addReg(RegNo: SrcReg32)
990 .addImm(Val: Immr)
991 .addImm(Val: Imms);
992 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
993 MCID: TII->get(Opcode: AArch64::SUBREG_TO_REG), DestReg: DstReg64)
994 .addReg(RegNo: DstReg32)
995 .addImm(Val: AArch64::sub_32);
996 MI.eraseFromParent();
997 return true;
998}
999
1000// Across a basic-block we might have in i32 extract from a value that only
1001// operates on upper bits (for example a sxtw). We can replace the COPY with a
1002// new version skipping the sxtw.
1003bool AArch64MIPeepholeOptImpl::visitCopy(MachineInstr &MI) {
1004 Register InputReg = MI.getOperand(i: 1).getReg();
1005 if (MI.getOperand(i: 1).getSubReg() != AArch64::sub_32 ||
1006 !MRI->hasOneNonDBGUse(RegNo: InputReg))
1007 return false;
1008
1009 MachineInstr *SrcMI = MRI->getUniqueVRegDef(Reg: InputReg);
1010 SmallPtrSet<MachineInstr *, 4> DeadInstrs;
1011 DeadInstrs.insert(Ptr: SrcMI);
1012 while (SrcMI && SrcMI->isFullCopy() &&
1013 MRI->hasOneNonDBGUse(RegNo: SrcMI->getOperand(i: 1).getReg())) {
1014 SrcMI = MRI->getUniqueVRegDef(Reg: SrcMI->getOperand(i: 1).getReg());
1015 DeadInstrs.insert(Ptr: SrcMI);
1016 }
1017
1018 if (!SrcMI)
1019 return false;
1020
1021 // Look for SXTW(X) and return Reg.
1022 auto getSXTWSrcReg = [](MachineInstr *SrcMI) -> Register {
1023 if (SrcMI->getOpcode() != AArch64::SBFMXri ||
1024 SrcMI->getOperand(i: 2).getImm() != 0 ||
1025 SrcMI->getOperand(i: 3).getImm() != 31)
1026 return Register();
1027 return SrcMI->getOperand(i: 1).getReg();
1028 };
1029 // Look for SUBREG_TO_REG(ORRWrr(WZR, COPY(X.sub_32)))
1030 auto getUXTWSrcReg = [&](MachineInstr *SrcMI) -> Register {
1031 if (SrcMI->getOpcode() != AArch64::SUBREG_TO_REG ||
1032 SrcMI->getOperand(i: 2).getImm() != AArch64::sub_32 ||
1033 !MRI->hasOneNonDBGUse(RegNo: SrcMI->getOperand(i: 1).getReg()))
1034 return Register();
1035 MachineInstr *Orr = MRI->getUniqueVRegDef(Reg: SrcMI->getOperand(i: 1).getReg());
1036 if (!Orr || Orr->getOpcode() != AArch64::ORRWrr ||
1037 Orr->getOperand(i: 1).getReg() != AArch64::WZR ||
1038 !MRI->hasOneNonDBGUse(RegNo: Orr->getOperand(i: 2).getReg()))
1039 return Register();
1040 MachineInstr *Cpy = MRI->getUniqueVRegDef(Reg: Orr->getOperand(i: 2).getReg());
1041 if (!Cpy || Cpy->getOpcode() != AArch64::COPY ||
1042 Cpy->getOperand(i: 1).getSubReg() != AArch64::sub_32)
1043 return Register();
1044 DeadInstrs.insert(Ptr: Orr);
1045 return Cpy->getOperand(i: 1).getReg();
1046 };
1047
1048 Register SrcReg = getSXTWSrcReg(SrcMI);
1049 if (!SrcReg)
1050 SrcReg = getUXTWSrcReg(SrcMI);
1051 if (!SrcReg)
1052 return false;
1053
1054 MRI->constrainRegClass(Reg: SrcReg, RC: MRI->getRegClass(Reg: InputReg));
1055 LLVM_DEBUG(dbgs() << "Optimizing: " << MI);
1056 MI.getOperand(i: 1).setReg(SrcReg);
1057 LLVM_DEBUG(dbgs() << " to: " << MI);
1058 for (auto *DeadMI : DeadInstrs) {
1059 LLVM_DEBUG(dbgs() << " Removing: " << *DeadMI);
1060 DeadMI->eraseFromParent();
1061 }
1062 return true;
1063}
1064
1065bool AArch64MIPeepholeOptImpl::run(MachineFunction &MF) {
1066 TII = static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
1067 TRI = static_cast<const AArch64RegisterInfo *>(
1068 MF.getSubtarget().getRegisterInfo());
1069 MRI = &MF.getRegInfo();
1070
1071 assert(MRI->isSSA() && "Expected to be run on SSA form!");
1072
1073 bool Changed = false;
1074
1075 for (MachineBasicBlock &MBB : MF) {
1076 DenseMap<uint64_t, Register> SharedConsts32, SharedConsts64;
1077
1078 for (MachineInstr &MI : make_early_inc_range(Range&: MBB)) {
1079 switch (MI.getOpcode()) {
1080 default:
1081 break;
1082 case AArch64::MOVi32imm:
1083 recordConstantDef<uint32_t>(MI, Consts&: SharedConsts32);
1084 break;
1085 case AArch64::MOVi64imm:
1086 recordConstantDef<uint64_t>(MI, Consts&: SharedConsts64);
1087 break;
1088 case AArch64::INSERT_SUBREG:
1089 Changed |= visitINSERT(MI);
1090 break;
1091 case AArch64::ANDWrr:
1092 Changed |= trySplitLogicalImm<uint32_t>(Opc: AArch64::ANDWri, MI,
1093 Strategy: SplitStrategy::Intersect);
1094 break;
1095 case AArch64::ANDXrr:
1096 Changed |= trySplitLogicalImm<uint64_t>(Opc: AArch64::ANDXri, MI,
1097 Strategy: SplitStrategy::Intersect);
1098 break;
1099 case AArch64::ANDSWrr:
1100 Changed |= trySplitLogicalImm<uint32_t>(
1101 Opc: AArch64::ANDWri, MI, Strategy: SplitStrategy::Intersect, OtherOpc: AArch64::ANDSWri);
1102 break;
1103 case AArch64::ANDSXrr:
1104 Changed |= trySplitLogicalImm<uint64_t>(
1105 Opc: AArch64::ANDXri, MI, Strategy: SplitStrategy::Intersect, OtherOpc: AArch64::ANDSXri);
1106 break;
1107 case AArch64::EORWrr:
1108 Changed |= trySplitLogicalImm<uint32_t>(Opc: AArch64::EORWri, MI,
1109 Strategy: SplitStrategy::Disjoint);
1110 break;
1111 case AArch64::EORXrr:
1112 Changed |= trySplitLogicalImm<uint64_t>(Opc: AArch64::EORXri, MI,
1113 Strategy: SplitStrategy::Disjoint);
1114 break;
1115 case AArch64::ORRWrr:
1116 Changed |= trySplitLogicalImm<uint32_t>(Opc: AArch64::ORRWri, MI,
1117 Strategy: SplitStrategy::Disjoint);
1118 break;
1119 case AArch64::ORRXrr:
1120 Changed |= trySplitLogicalImm<uint64_t>(Opc: AArch64::ORRXri, MI,
1121 Strategy: SplitStrategy::Disjoint);
1122 break;
1123 case AArch64::ORRWrs:
1124 Changed |= visitORR(MI);
1125 break;
1126 case AArch64::ADDWrr:
1127 Changed |=
1128 foldSharedAddSubConstant(MI, AddOpc: AArch64::ADDWrr, SubOpc: AArch64::SUBWrr,
1129 Is64Bit: false, SameWidth&: SharedConsts32) ||
1130 visitADDSUB<uint32_t>(PosOpc: AArch64::ADDWri, NegOpc: AArch64::SUBWri, MI);
1131 break;
1132 case AArch64::SUBWrr:
1133 Changed |=
1134 foldSharedAddSubConstant(MI, AddOpc: AArch64::ADDWrr, SubOpc: AArch64::SUBWrr,
1135 Is64Bit: false, SameWidth&: SharedConsts32) ||
1136 visitADDSUB<uint32_t>(PosOpc: AArch64::SUBWri, NegOpc: AArch64::ADDWri, MI);
1137 break;
1138 case AArch64::ADDXrr:
1139 Changed |=
1140 foldSharedAddSubConstant(MI, AddOpc: AArch64::ADDXrr, SubOpc: AArch64::SUBXrr, Is64Bit: true,
1141 SameWidth&: SharedConsts64) ||
1142 visitADDSUB<uint64_t>(PosOpc: AArch64::ADDXri, NegOpc: AArch64::SUBXri, MI);
1143 break;
1144 case AArch64::SUBXrr:
1145 Changed |=
1146 foldSharedAddSubConstant(MI, AddOpc: AArch64::ADDXrr, SubOpc: AArch64::SUBXrr, Is64Bit: true,
1147 SameWidth&: SharedConsts64) ||
1148 visitADDSUB<uint64_t>(PosOpc: AArch64::SUBXri, NegOpc: AArch64::ADDXri, MI);
1149 break;
1150 case AArch64::ADDSWrr:
1151 Changed |=
1152 visitADDSSUBS<uint32_t>(PosOpcs: {AArch64::ADDWri, AArch64::ADDSWri},
1153 NegOpcs: {AArch64::SUBWri, AArch64::SUBSWri}, MI);
1154 break;
1155 case AArch64::SUBSWrr:
1156 Changed |=
1157 visitADDSSUBS<uint32_t>(PosOpcs: {AArch64::SUBWri, AArch64::SUBSWri},
1158 NegOpcs: {AArch64::ADDWri, AArch64::ADDSWri}, MI);
1159 break;
1160 case AArch64::ADDSXrr:
1161 Changed |=
1162 visitADDSSUBS<uint64_t>(PosOpcs: {AArch64::ADDXri, AArch64::ADDSXri},
1163 NegOpcs: {AArch64::SUBXri, AArch64::SUBSXri}, MI);
1164 break;
1165 case AArch64::SUBSXrr:
1166 Changed |=
1167 visitADDSSUBS<uint64_t>(PosOpcs: {AArch64::SUBXri, AArch64::SUBSXri},
1168 NegOpcs: {AArch64::ADDXri, AArch64::ADDSXri}, MI);
1169 break;
1170 case AArch64::CSELWr:
1171 case AArch64::CSELXr:
1172 Changed |= visitCSEL(MI);
1173 break;
1174 case AArch64::INSvi64gpr:
1175 Changed |= visitINSviGPR(MI, Opc: AArch64::INSvi64lane);
1176 break;
1177 case AArch64::INSvi32gpr:
1178 Changed |= visitINSviGPR(MI, Opc: AArch64::INSvi32lane);
1179 break;
1180 case AArch64::INSvi16gpr:
1181 Changed |= visitINSviGPR(MI, Opc: AArch64::INSvi16lane);
1182 break;
1183 case AArch64::INSvi8gpr:
1184 Changed |= visitINSviGPR(MI, Opc: AArch64::INSvi8lane);
1185 break;
1186 case AArch64::INSvi64lane:
1187 Changed |= visitINSvi64lane(MI);
1188 break;
1189 case AArch64::FMOVDr:
1190 Changed |= visitFMOVDr(MI);
1191 break;
1192 case AArch64::UBFMXri:
1193 Changed |= visitUBFMXri(MI);
1194 break;
1195 case AArch64::COPY:
1196 Changed |= visitCopy(MI);
1197 break;
1198 }
1199 }
1200 }
1201
1202 return Changed;
1203}
1204
1205bool AArch64MIPeepholeOptLegacy::runOnMachineFunction(MachineFunction &MF) {
1206 if (skipFunction(F: MF.getFunction()))
1207 return false;
1208
1209 MachineLoopInfo &MLI = getAnalysis<MachineLoopInfoWrapperPass>().getLI();
1210 return AArch64MIPeepholeOptImpl(MLI).run(MF);
1211}
1212
1213FunctionPass *llvm::createAArch64MIPeepholeOptLegacyPass() {
1214 return new AArch64MIPeepholeOptLegacy();
1215}
1216
1217PreservedAnalyses
1218AArch64MIPeepholeOptPass::run(MachineFunction &MF,
1219 MachineFunctionAnalysisManager &MFAM) {
1220 MachineLoopInfo &MLI = MFAM.getResult<MachineLoopAnalysis>(IR&: MF);
1221 const bool Changed = AArch64MIPeepholeOptImpl(MLI).run(MF);
1222 if (!Changed)
1223 return PreservedAnalyses::all();
1224 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
1225 PA.preserveSet<CFGAnalyses>();
1226 return PA;
1227}
1228