1//===-- RISCVMoveMerger.cpp - RISC-V move merge pass ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that performs move related peephole optimizations
10// as Zcmp has specified. This pass should be run after register allocation.
11//
12// This pass also supports Xqccmp, which has identical instructions.
13//
14//===----------------------------------------------------------------------===//
15
16#include "RISCVInstrInfo.h"
17#include "RISCVSubtarget.h"
18
19using namespace llvm;
20
21#define RISCV_MOVE_MERGE_NAME "RISC-V Zcmp move merging pass"
22
23namespace {
24struct RISCVMoveMerge : public MachineFunctionPass {
25 static char ID;
26
27 RISCVMoveMerge() : MachineFunctionPass(ID) {}
28
29 const RISCVSubtarget *ST;
30 const RISCVInstrInfo *TII;
31 const TargetRegisterInfo *TRI;
32
33 // Track which register units have been modified and used.
34 LiveRegUnits ModifiedRegUnits, UsedRegUnits;
35
36 bool isGPRPairCopyCandidate(const DestSourcePair &RegPair, bool EvenRegPair);
37
38 bool isCandidateToMergeMVA01S(const DestSourcePair &RegPair);
39 bool isCandidateToMergeMVSA01(const DestSourcePair &RegPair);
40
41 bool isPLIPairCandidate(const MachineInstr &MI, bool EvenRegPair);
42
43 // Merge the two instructions indicated into a single pair instruction.
44 MachineBasicBlock::iterator
45 mergeGPRPairInsns(MachineBasicBlock::iterator I,
46 MachineBasicBlock::iterator Paired, bool RegPairIsEven);
47 MachineBasicBlock::iterator
48 mergePairedInsns(MachineBasicBlock::iterator I,
49 MachineBasicBlock::iterator Paired, bool MoveFromSToA);
50 MachineBasicBlock::iterator mergePLIPair(MachineBasicBlock::iterator I,
51 MachineBasicBlock::iterator Paired,
52 bool RegPairIsEven);
53
54 MachineBasicBlock::iterator
55 findMatchingGPRPairCopy(MachineBasicBlock::iterator &MBBI, bool EvenRegPair,
56 const DestSourcePair &RegPair);
57 // Look for C.MV instruction that can be combined with
58 // the given instruction into CM.MVA01S or CM.MVSA01. Return the matching
59 // instruction if one exists.
60 MachineBasicBlock::iterator
61 findMatchingSACopy(MachineBasicBlock::iterator &MBBI, bool MoveFromSToA,
62 const DestSourcePair &RegPair);
63 MachineBasicBlock::iterator findMatchingPLI(MachineBasicBlock::iterator &MBBI,
64 bool EvenRegPair);
65 bool mergeMovePairs(MachineBasicBlock &MBB);
66 bool runOnMachineFunction(MachineFunction &Fn) override;
67
68 StringRef getPassName() const override { return RISCV_MOVE_MERGE_NAME; }
69};
70
71char RISCVMoveMerge::ID = 0;
72
73} // end of anonymous namespace
74
75INITIALIZE_PASS(RISCVMoveMerge, "riscv-move-merge", RISCV_MOVE_MERGE_NAME,
76 false, false)
77
78static unsigned getCM_MVOpcode(const RISCVSubtarget &ST, bool MoveFromSToA) {
79 if (ST.hasStdExtZcmp())
80 return MoveFromSToA ? RISCV::CM_MVA01S : RISCV::CM_MVSA01;
81
82 if (ST.hasVendorXqccmp())
83 return MoveFromSToA ? RISCV::QC_CM_MVA01S : RISCV::QC_CM_MVSA01;
84
85 llvm_unreachable("Unhandled subtarget with paired move.");
86}
87
88// Returns 0 if Opc has no paired form.
89static unsigned getPairedPLIOpcode(unsigned Opc) {
90 switch (Opc) {
91 case RISCV::PLI_B:
92 return RISCV::PLI_DB;
93 case RISCV::PLI_H:
94 return RISCV::PLI_DH;
95 case RISCV::PLUI_H:
96 return RISCV::PLUI_DH;
97 default:
98 return 0;
99 }
100}
101
102bool RISCVMoveMerge::isGPRPairCopyCandidate(const DestSourcePair &RegPair,
103 bool EvenRegPair) {
104 Register Destination = RegPair.Destination->getReg();
105 Register Source = RegPair.Source->getReg();
106
107 if (Source == Destination)
108 return false;
109
110 if ((!ST->hasStdExtZdinx() && !ST->hasStdExtP()) || ST->is64Bit())
111 return false;
112
113 unsigned SubIdx = EvenRegPair ? RISCV::sub_gpr_even : RISCV::sub_gpr_odd;
114
115 Register SrcPair =
116 TRI->getMatchingSuperReg(Reg: Source, SubIdx, RC: &RISCV::GPRPairRegClass);
117 Register DestPair =
118 TRI->getMatchingSuperReg(Reg: Destination, SubIdx, RC: &RISCV::GPRPairRegClass);
119
120 return SrcPair.isValid() && DestPair.isValid();
121}
122
123// Check if registers meet CM.MVA01S constraints.
124bool RISCVMoveMerge::isCandidateToMergeMVA01S(const DestSourcePair &RegPair) {
125 Register Destination = RegPair.Destination->getReg();
126 Register Source = RegPair.Source->getReg();
127 // If destination is not a0 or a1.
128 if ((ST->hasStdExtZcmp() || ST->hasVendorXqccmp()) &&
129 (Destination == RISCV::X10 || Destination == RISCV::X11) &&
130 RISCV::SR07RegClass.contains(Reg: Source))
131 return true;
132 return false;
133}
134
135// Check if registers meet CM.MVSA01 constraints.
136bool RISCVMoveMerge::isCandidateToMergeMVSA01(const DestSourcePair &RegPair) {
137 Register Destination = RegPair.Destination->getReg();
138 Register Source = RegPair.Source->getReg();
139 // If Source is s0 - s7.
140 if ((ST->hasStdExtZcmp() || ST->hasVendorXqccmp()) &&
141 (Source == RISCV::X10 || Source == RISCV::X11) &&
142 RISCV::SR07RegClass.contains(Reg: Destination))
143 return true;
144 return false;
145}
146
147// Check if MI is a single-reg pli/plui whose destination is a half of a
148// GPRPair.
149bool RISCVMoveMerge::isPLIPairCandidate(const MachineInstr &MI,
150 bool EvenRegPair) {
151 if (!ST->hasStdExtP() || ST->is64Bit())
152 return false;
153 if (!getPairedPLIOpcode(Opc: MI.getOpcode()))
154 return false;
155 unsigned SubIdx = EvenRegPair ? RISCV::sub_gpr_even : RISCV::sub_gpr_odd;
156 return TRI
157 ->getMatchingSuperReg(Reg: MI.getOperand(i: 0).getReg(), SubIdx,
158 RC: &RISCV::GPRPairRegClass)
159 .isValid();
160}
161
162MachineBasicBlock::iterator
163RISCVMoveMerge::mergeGPRPairInsns(MachineBasicBlock::iterator I,
164 MachineBasicBlock::iterator Paired,
165 bool RegPairIsEven) {
166 MachineBasicBlock::iterator E = I->getParent()->end();
167 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
168 DestSourcePair FirstPair = *TII->isCopyInstrImpl(MI: *I);
169 DestSourcePair SecondPair = *TII->isCopyInstrImpl(MI: *Paired);
170
171 if (NextI == Paired)
172 NextI = next_nodbg(It: NextI, End: E);
173 DebugLoc DL = I->getDebugLoc();
174
175 // Make a copy of the second instruction to update the kill
176 // flag.
177 MachineOperand PairedSource = *SecondPair.Source;
178
179 for (auto It = std::next(x: I); It != Paired && PairedSource.isKill(); ++It)
180 if (It->readsRegister(Reg: PairedSource.getReg(), TRI))
181 PairedSource.setIsKill(false);
182
183 unsigned GPRPairIdx =
184 RegPairIsEven ? RISCV::sub_gpr_even : RISCV::sub_gpr_odd;
185 MCRegister SrcReg = TRI->getMatchingSuperReg(
186 Reg: FirstPair.Source->getReg(), SubIdx: GPRPairIdx, RC: &RISCV::GPRPairRegClass);
187 MCRegister DestReg = TRI->getMatchingSuperReg(
188 Reg: FirstPair.Destination->getReg(), SubIdx: GPRPairIdx, RC: &RISCV::GPRPairRegClass);
189 bool KillSrc = PairedSource.isKill() && FirstPair.Source->isKill();
190
191 TII->copyPhysReg(MBB&: *I->getParent(), MBBI: I, DL, DstReg: DestReg, SrcReg, KillSrc);
192
193 I->eraseFromParent();
194 Paired->eraseFromParent();
195 return NextI;
196}
197
198MachineBasicBlock::iterator
199RISCVMoveMerge::mergePairedInsns(MachineBasicBlock::iterator I,
200 MachineBasicBlock::iterator Paired,
201 bool MoveFromSToA) {
202 const MachineOperand *Sreg1, *Sreg2;
203 MachineBasicBlock::iterator E = I->getParent()->end();
204 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
205 DestSourcePair FirstPair = *TII->isCopyInstrImpl(MI: *I);
206 DestSourcePair PairedRegs = *TII->isCopyInstrImpl(MI: *Paired);
207
208 if (NextI == Paired)
209 NextI = next_nodbg(It: NextI, End: E);
210 DebugLoc DL = I->getDebugLoc();
211
212 // Make a copy so we can update the kill flag in the MoveFromSToA case. The
213 // copied operand needs to be scoped outside the if since we make a pointer
214 // to it.
215 MachineOperand PairedSource = *PairedRegs.Source;
216
217 // The order of S-reg depends on which instruction holds A0, instead of
218 // the order of register pair.
219 // e,g.
220 // mv a1, s1
221 // mv a0, s2 => cm.mva01s s2,s1
222 //
223 // mv a0, s2
224 // mv a1, s1 => cm.mva01s s2,s1
225 unsigned Opcode = getCM_MVOpcode(ST: *ST, MoveFromSToA);
226 if (MoveFromSToA) {
227 // We are moving one of the copies earlier so its kill flag may become
228 // invalid. Clear the copied kill flag if there are any reads of the
229 // register between the new location and the old location.
230 for (auto It = std::next(x: I); It != Paired && PairedSource.isKill(); ++It)
231 if (It->readsRegister(Reg: PairedSource.getReg(), TRI))
232 PairedSource.setIsKill(false);
233
234 Sreg1 = FirstPair.Source;
235 Sreg2 = &PairedSource;
236 if (FirstPair.Destination->getReg() != RISCV::X10)
237 std::swap(a&: Sreg1, b&: Sreg2);
238 } else {
239 Sreg1 = FirstPair.Destination;
240 Sreg2 = PairedRegs.Destination;
241 if (FirstPair.Source->getReg() != RISCV::X10)
242 std::swap(a&: Sreg1, b&: Sreg2);
243 }
244
245 BuildMI(BB&: *I->getParent(), I, MIMD: DL, MCID: TII->get(Opcode)).add(MO: *Sreg1).add(MO: *Sreg2);
246
247 I->eraseFromParent();
248 Paired->eraseFromParent();
249 return NextI;
250}
251
252MachineBasicBlock::iterator
253RISCVMoveMerge::mergePLIPair(MachineBasicBlock::iterator I,
254 MachineBasicBlock::iterator Paired,
255 bool RegPairIsEven) {
256 MachineBasicBlock::iterator E = I->getParent()->end();
257 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
258
259 if (NextI == Paired)
260 NextI = next_nodbg(It: NextI, End: E);
261 DebugLoc DL = I->getDebugLoc();
262
263 unsigned Opcode = getPairedPLIOpcode(Opc: I->getOpcode());
264 unsigned GPRPairIdx =
265 RegPairIsEven ? RISCV::sub_gpr_even : RISCV::sub_gpr_odd;
266 Register DestReg = TRI->getMatchingSuperReg(
267 Reg: I->getOperand(i: 0).getReg(), SubIdx: GPRPairIdx, RC: &RISCV::GPRPairRegClass);
268
269 BuildMI(BB&: *I->getParent(), I, MIMD: DL, MCID: TII->get(Opcode), DestReg)
270 .addImm(Val: I->getOperand(i: 1).getImm());
271
272 I->eraseFromParent();
273 Paired->eraseFromParent();
274 return NextI;
275}
276
277MachineBasicBlock::iterator
278RISCVMoveMerge::findMatchingGPRPairCopy(MachineBasicBlock::iterator &MBBI,
279 bool EvenRegPair,
280 const DestSourcePair &RegPair) {
281 MachineBasicBlock::iterator E = MBBI->getParent()->end();
282 ModifiedRegUnits.clear();
283 UsedRegUnits.clear();
284 unsigned RegPairIdx = EvenRegPair ? RISCV::sub_gpr_even : RISCV::sub_gpr_odd;
285 unsigned SecondPairIdx =
286 !EvenRegPair ? RISCV::sub_gpr_even : RISCV::sub_gpr_odd;
287
288 // Get the expected source/destination registers of the matching lane.
289 Register SrcGPRPair = TRI->getMatchingSuperReg(
290 Reg: RegPair.Source->getReg(), SubIdx: RegPairIdx, RC: &RISCV::GPRPairRegClass);
291 Register DestGPRPair = TRI->getMatchingSuperReg(
292 Reg: RegPair.Destination->getReg(), SubIdx: RegPairIdx, RC: &RISCV::GPRPairRegClass);
293 Register ExpectedSourceReg = TRI->getSubReg(Reg: SrcGPRPair, Idx: SecondPairIdx);
294 Register ExpectedDestReg = TRI->getSubReg(Reg: DestGPRPair, Idx: SecondPairIdx);
295
296 for (MachineBasicBlock::iterator I = next_nodbg(It: MBBI, End: E); I != E;
297 I = next_nodbg(It: I, End: E)) {
298
299 MachineInstr &MI = *I;
300
301 if (auto SecondPair = TII->isCopyInstrImpl(MI)) {
302 Register SourceReg = SecondPair->Source->getReg();
303 Register DestReg = SecondPair->Destination->getReg();
304
305 if (RegPair.Destination->getReg() == DestReg ||
306 RegPair.Source->getReg() == SourceReg)
307 return E;
308
309 // Check if the second pair's registers match the other lane of the
310 // GPRPairs.
311 if (SourceReg == ExpectedSourceReg && DestReg == ExpectedDestReg)
312 return I;
313 }
314 // Update modified / used register units.
315 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
316 // Once expected lane registers are clobbered/read in-between, we can stop
317 // scanning since the pair cannot be legally merged anymore.
318 if (!ModifiedRegUnits.available(Reg: ExpectedDestReg) ||
319 !UsedRegUnits.available(Reg: ExpectedDestReg) ||
320 !ModifiedRegUnits.available(Reg: ExpectedSourceReg))
321 return E;
322 }
323 return E;
324}
325
326MachineBasicBlock::iterator
327RISCVMoveMerge::findMatchingSACopy(MachineBasicBlock::iterator &MBBI,
328 bool MoveFromSToA,
329 const DestSourcePair &RegPair) {
330 MachineBasicBlock::iterator E = MBBI->getParent()->end();
331
332 // Track which register units have been modified and used between the first
333 // insn and the second insn.
334 ModifiedRegUnits.clear();
335 UsedRegUnits.clear();
336
337 for (MachineBasicBlock::iterator I = next_nodbg(It: MBBI, End: E); I != E;
338 I = next_nodbg(It: I, End: E)) {
339
340 MachineInstr &MI = *I;
341
342 if (auto SecondPair = TII->isCopyInstrImpl(MI)) {
343 Register SourceReg = SecondPair->Source->getReg();
344 Register DestReg = SecondPair->Destination->getReg();
345
346 bool IsCandidate = MoveFromSToA ? isCandidateToMergeMVA01S(RegPair: *SecondPair)
347 : isCandidateToMergeMVSA01(RegPair: *SecondPair);
348 if (IsCandidate) {
349 // Second destination must be different.
350 if (RegPair.Destination->getReg() == DestReg)
351 return E;
352
353 // For AtoS the source must also be different.
354 if (!MoveFromSToA && RegPair.Source->getReg() == SourceReg)
355 return E;
356
357 // If paired destination register was modified or used, the source reg
358 // was modified, there is no possibility of finding matching
359 // instruction so exit early.
360 if (!ModifiedRegUnits.available(Reg: DestReg) ||
361 !UsedRegUnits.available(Reg: DestReg) ||
362 !ModifiedRegUnits.available(Reg: SourceReg))
363 return E;
364
365 return I;
366 }
367 }
368 // Update modified / used register units.
369 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
370 }
371 return E;
372}
373
374// Look for a same-opcode pli/plui writing the other lane of the same GPRPair
375// with the same immediate. Return the matching instruction if one exists.
376MachineBasicBlock::iterator
377RISCVMoveMerge::findMatchingPLI(MachineBasicBlock::iterator &MBBI,
378 bool EvenRegPair) {
379 MachineBasicBlock::iterator E = MBBI->getParent()->end();
380 ModifiedRegUnits.clear();
381 UsedRegUnits.clear();
382 unsigned Opc = MBBI->getOpcode();
383 Register FirstDestReg = MBBI->getOperand(i: 0).getReg();
384 int64_t FirstImm = MBBI->getOperand(i: 1).getImm();
385 unsigned RegPairIdx = EvenRegPair ? RISCV::sub_gpr_even : RISCV::sub_gpr_odd;
386 unsigned SecondPairIdx =
387 !EvenRegPair ? RISCV::sub_gpr_even : RISCV::sub_gpr_odd;
388
389 // Get the expected destination register of the matching lane.
390 Register DestGPRPair = TRI->getMatchingSuperReg(Reg: FirstDestReg, SubIdx: RegPairIdx,
391 RC: &RISCV::GPRPairRegClass);
392 Register ExpectedDestReg = TRI->getSubReg(Reg: DestGPRPair, Idx: SecondPairIdx);
393
394 for (MachineBasicBlock::iterator I = next_nodbg(It: MBBI, End: E); I != E;
395 I = next_nodbg(It: I, End: E)) {
396
397 MachineInstr &MI = *I;
398
399 if (MI.getOpcode() == Opc) {
400 Register DestReg = MI.getOperand(i: 0).getReg();
401 int64_t Imm = MI.getOperand(i: 1).getImm();
402
403 if (FirstDestReg == DestReg)
404 return E;
405
406 // Check if the second PLI matches the other lane and immediate.
407 if (DestReg == ExpectedDestReg && Imm == FirstImm)
408 return I;
409 }
410 // Update modified / used register units.
411 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
412 // Once the expected lane register is clobbered/read in-between, we can
413 // stop scanning since the pair cannot be legally merged anymore.
414 if (!ModifiedRegUnits.available(Reg: ExpectedDestReg) ||
415 !UsedRegUnits.available(Reg: ExpectedDestReg))
416 return E;
417 }
418 return E;
419}
420
421// Finds instructions, which could be represented as C.MV instructions and
422// merged into CM.MVA01S or CM.MVSA01.
423bool RISCVMoveMerge::mergeMovePairs(MachineBasicBlock &MBB) {
424 bool Modified = false;
425
426 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
427 MBBI != E;) {
428 // Try merging a pair of single-reg PLI/PLUI into a paired form.
429 bool IsPLIEven = isPLIPairCandidate(MI: *MBBI, /*EvenRegPair=*/true);
430 bool IsPLIOdd = isPLIPairCandidate(MI: *MBBI, /*EvenRegPair=*/false);
431 if (IsPLIEven != IsPLIOdd) {
432 MachineBasicBlock::iterator Paired = findMatchingPLI(MBBI, EvenRegPair: IsPLIEven);
433 if (Paired != E) {
434 MBBI = mergePLIPair(I: MBBI, Paired, RegPairIsEven: IsPLIEven);
435 Modified = true;
436 continue;
437 }
438 }
439
440 // Check if the instruction can be compressed to C.MV instruction. If it
441 // can, return Dest/Src register pair.
442 auto RegPair = TII->isCopyInstrImpl(MI: *MBBI);
443 if (RegPair.has_value()) {
444 bool MoveFromSToA = isCandidateToMergeMVA01S(RegPair: *RegPair);
445 bool MoveFromAToS = isCandidateToMergeMVSA01(RegPair: *RegPair);
446 bool IsEven = isGPRPairCopyCandidate(RegPair: *RegPair, /*EvenRegPair=*/true);
447 bool IsOdd = isGPRPairCopyCandidate(RegPair: *RegPair, /*EvenRegPair=*/false);
448 if (!MoveFromSToA && !MoveFromAToS && !IsEven && !IsOdd) {
449 ++MBBI;
450 continue;
451 }
452
453 MachineBasicBlock::iterator Paired = E;
454 if (MoveFromSToA || MoveFromAToS) {
455 Paired = findMatchingSACopy(MBBI, MoveFromSToA, RegPair: *RegPair);
456 if (Paired != E) {
457 MBBI = mergePairedInsns(I: MBBI, Paired, MoveFromSToA);
458 Modified = true;
459 continue;
460 }
461 }
462 if (IsEven != IsOdd) {
463 Paired = findMatchingGPRPairCopy(MBBI, EvenRegPair: IsEven, RegPair: *RegPair);
464 if (Paired != E) {
465 MBBI = mergeGPRPairInsns(I: MBBI, Paired, RegPairIsEven: IsEven);
466 Modified = true;
467 continue;
468 }
469 }
470 }
471 ++MBBI;
472 }
473 return Modified;
474}
475
476bool RISCVMoveMerge::runOnMachineFunction(MachineFunction &Fn) {
477 if (skipFunction(F: Fn.getFunction()))
478 return false;
479
480 ST = &Fn.getSubtarget<RISCVSubtarget>();
481 bool HasGPRPairCopy =
482 !ST->is64Bit() && (ST->hasStdExtZdinx() || ST->hasStdExtP());
483 if (!ST->hasStdExtZcmp() && !ST->hasVendorXqccmp() && !HasGPRPairCopy)
484 return false;
485
486 TII = ST->getInstrInfo();
487 TRI = ST->getRegisterInfo();
488 // Resize the modified and used register unit trackers. We do this once
489 // per function and then clear the register units each time we optimize a
490 // move.
491 ModifiedRegUnits.init(TRI: *TRI);
492 UsedRegUnits.init(TRI: *TRI);
493 bool Modified = false;
494 for (auto &MBB : Fn)
495 Modified |= mergeMovePairs(MBB);
496 return Modified;
497}
498
499/// createRISCVMoveMergePass - returns an instance of the
500/// move merge pass.
501FunctionPass *llvm::createRISCVMoveMergePass() { return new RISCVMoveMerge(); }
502