1//===-- RISCVMoveMerger.cpp - RISC-V move merge pass ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that performs move related peephole optimizations
10// as Zcmp has specified. This pass should be run after register allocation.
11//
12// This pass also supports Xqccmp, which has identical instructions.
13//
14//===----------------------------------------------------------------------===//
15
16#include "RISCVInstrInfo.h"
17#include "RISCVSubtarget.h"
18
19using namespace llvm;
20
21#define RISCV_MOVE_MERGE_NAME "RISC-V Zcmp move merging pass"
22
23namespace {
24struct RISCVMoveMerge : public MachineFunctionPass {
25 static char ID;
26
27 RISCVMoveMerge() : MachineFunctionPass(ID) {}
28
29 const RISCVSubtarget *ST;
30 const RISCVInstrInfo *TII;
31 const TargetRegisterInfo *TRI;
32
33 // Track which register units have been modified and used.
34 LiveRegUnits ModifiedRegUnits, UsedRegUnits;
35
36 bool isGPRPairCopyCandidate(const DestSourcePair &RegPair, bool EvenRegPair);
37
38 bool isCandidateToMergeMVA01S(const DestSourcePair &RegPair);
39 bool isCandidateToMergeMVSA01(const DestSourcePair &RegPair);
40
41 bool isPLIPairCandidate(const MachineInstr &MI, bool EvenRegPair);
42
43 // Merge the two instructions indicated into a single pair instruction.
44 MachineBasicBlock::iterator
45 mergeGPRPairInsns(MachineBasicBlock::iterator I,
46 MachineBasicBlock::iterator Paired, bool RegPairIsEven);
47 MachineBasicBlock::iterator
48 mergePairedInsns(MachineBasicBlock::iterator I,
49 MachineBasicBlock::iterator Paired, bool MoveFromSToA);
50 MachineBasicBlock::iterator mergePLIPair(MachineBasicBlock::iterator I,
51 MachineBasicBlock::iterator Paired,
52 bool RegPairIsEven);
53
54 MachineBasicBlock::iterator
55 findMatchingGPRPairCopy(MachineBasicBlock::iterator &MBBI, bool EvenRegPair,
56 const DestSourcePair &RegPair);
57 // Look for C.MV instruction that can be combined with
58 // the given instruction into CM.MVA01S or CM.MVSA01. Return the matching
59 // instruction if one exists.
60 MachineBasicBlock::iterator
61 findMatchingSACopy(MachineBasicBlock::iterator &MBBI, bool MoveFromSToA,
62 const DestSourcePair &RegPair);
63 MachineBasicBlock::iterator findMatchingPLI(MachineBasicBlock::iterator &MBBI,
64 bool EvenRegPair);
65 bool mergeMovePairs(MachineBasicBlock &MBB);
66 bool runOnMachineFunction(MachineFunction &Fn) override;
67
68 StringRef getPassName() const override { return RISCV_MOVE_MERGE_NAME; }
69};
70
71char RISCVMoveMerge::ID = 0;
72
73} // end of anonymous namespace
74
75INITIALIZE_PASS(RISCVMoveMerge, "riscv-move-merge", RISCV_MOVE_MERGE_NAME,
76 false, false)
77
78static unsigned getCM_MVOpcode(const RISCVSubtarget &ST, bool MoveFromSToA) {
79 if (ST.hasStdExtZcmp())
80 return MoveFromSToA ? RISCV::CM_MVA01S : RISCV::CM_MVSA01;
81
82 if (ST.hasVendorXqccmp())
83 return MoveFromSToA ? RISCV::QC_CM_MVA01S : RISCV::QC_CM_MVSA01;
84
85 llvm_unreachable("Unhandled subtarget with paired move.");
86}
87
88// Returns 0 if Opc has no paired form.
89static unsigned getPairedPLIOpcode(unsigned Opc) {
90 switch (Opc) {
91 case RISCV::PLI_B:
92 return RISCV::PLI_DB;
93 case RISCV::PLI_H:
94 return RISCV::PLI_DH;
95 case RISCV::PLUI_H:
96 return RISCV::PLUI_DH;
97 default:
98 return 0;
99 }
100}
101
102bool RISCVMoveMerge::isGPRPairCopyCandidate(const DestSourcePair &RegPair,
103 bool EvenRegPair) {
104 Register Destination = RegPair.Destination->getReg();
105 Register Source = RegPair.Source->getReg();
106
107 if (Source == Destination)
108 return false;
109
110 if ((!ST->hasStdExtZdinx() && !ST->hasStdExtP()) || ST->is64Bit())
111 return false;
112
113 unsigned SubIdx = EvenRegPair ? RISCV::sub_gpr_even : RISCV::sub_gpr_odd;
114
115 Register SrcPair =
116 TRI->getMatchingSuperReg(Reg: Source, SubIdx, RC: &RISCV::GPRPairRegClass);
117 Register DestPair =
118 TRI->getMatchingSuperReg(Reg: Destination, SubIdx, RC: &RISCV::GPRPairRegClass);
119
120 return SrcPair.isValid() && DestPair.isValid();
121}
122
123// Check if registers meet CM.MVA01S constraints.
124bool RISCVMoveMerge::isCandidateToMergeMVA01S(const DestSourcePair &RegPair) {
125 Register Destination = RegPair.Destination->getReg();
126 Register Source = RegPair.Source->getReg();
127 // If destination is not a0 or a1.
128 if ((ST->hasStdExtZcmp() || ST->hasVendorXqccmp()) &&
129 (Destination == RISCV::X10 || Destination == RISCV::X11) &&
130 RISCV::GPRS07RegClass.contains(Reg: Source))
131 return true;
132 return false;
133}
134
135// Check if registers meet CM.MVSA01 constraints.
136bool RISCVMoveMerge::isCandidateToMergeMVSA01(const DestSourcePair &RegPair) {
137 Register Destination = RegPair.Destination->getReg();
138 Register Source = RegPair.Source->getReg();
139 // If Source is s0 - s7.
140 if ((ST->hasStdExtZcmp() || ST->hasVendorXqccmp()) &&
141 (Source == RISCV::X10 || Source == RISCV::X11) &&
142 RISCV::GPRS07RegClass.contains(Reg: Destination))
143 return true;
144 return false;
145}
146
147// Check if MI is a single-reg pli/plui whose destination is a half of a
148// GPRPair.
149bool RISCVMoveMerge::isPLIPairCandidate(const MachineInstr &MI,
150 bool EvenRegPair) {
151 if (!ST->hasStdExtP() || ST->is64Bit())
152 return false;
153 if (!getPairedPLIOpcode(Opc: MI.getOpcode()))
154 return false;
155 unsigned SubIdx = EvenRegPair ? RISCV::sub_gpr_even : RISCV::sub_gpr_odd;
156 return TRI
157 ->getMatchingSuperReg(Reg: MI.getOperand(i: 0).getReg(), SubIdx,
158 RC: &RISCV::GPRPairRegClass)
159 .isValid();
160}
161
162MachineBasicBlock::iterator
163RISCVMoveMerge::mergeGPRPairInsns(MachineBasicBlock::iterator I,
164 MachineBasicBlock::iterator Paired,
165 bool RegPairIsEven) {
166 MachineBasicBlock::iterator E = I->getParent()->end();
167 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
168 DestSourcePair FirstPair = *TII->isCopyInstrImpl(MI: *I);
169 DestSourcePair SecondPair = *TII->isCopyInstrImpl(MI: *Paired);
170
171 if (NextI == Paired)
172 NextI = next_nodbg(It: NextI, End: E);
173 DebugLoc DL = I->getDebugLoc();
174
175 // Make a copy of the second instruction to update the kill
176 // flag.
177 MachineOperand PairedSource = *SecondPair.Source;
178
179 for (auto It = std::next(x: I); It != Paired && PairedSource.isKill(); ++It)
180 if (It->readsRegister(Reg: PairedSource.getReg(), TRI))
181 PairedSource.setIsKill(false);
182
183 unsigned GPRPairIdx =
184 RegPairIsEven ? RISCV::sub_gpr_even : RISCV::sub_gpr_odd;
185 MCRegister SrcReg = TRI->getMatchingSuperReg(
186 Reg: FirstPair.Source->getReg(), SubIdx: GPRPairIdx, RC: &RISCV::GPRPairRegClass);
187 MCRegister DestReg = TRI->getMatchingSuperReg(
188 Reg: FirstPair.Destination->getReg(), SubIdx: GPRPairIdx, RC: &RISCV::GPRPairRegClass);
189 bool KillSrc = PairedSource.isKill() && FirstPair.Source->isKill();
190
191 TII->copyPhysReg(MBB&: *I->getParent(), MBBI: I, DL, DstReg: DestReg, SrcReg, KillSrc);
192
193 I->eraseFromParent();
194 Paired->eraseFromParent();
195 return NextI;
196}
197
198MachineBasicBlock::iterator
199RISCVMoveMerge::mergePairedInsns(MachineBasicBlock::iterator I,
200 MachineBasicBlock::iterator Paired,
201 bool MoveFromSToA) {
202 const MachineOperand *Sreg1, *Sreg2;
203 MachineBasicBlock::iterator E = I->getParent()->end();
204 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
205 DestSourcePair FirstPair = *TII->isCopyInstrImpl(MI: *I);
206 DestSourcePair PairedRegs = *TII->isCopyInstrImpl(MI: *Paired);
207
208 if (NextI == Paired)
209 NextI = next_nodbg(It: NextI, End: E);
210 DebugLoc DL = I->getDebugLoc();
211
212 // Make a copy so we can update the kill flag in the MoveFromSToA case. The
213 // copied operand needs to be scoped outside the if since we make a pointer
214 // to it.
215 MachineOperand PairedSource = *PairedRegs.Source;
216
217 // The order of S-reg depends on which instruction holds A0, instead of
218 // the order of register pair.
219 // e,g.
220 // mv a1, s1
221 // mv a0, s2 => cm.mva01s s2,s1
222 //
223 // mv a0, s2
224 // mv a1, s1 => cm.mva01s s2,s1
225 unsigned Opcode = getCM_MVOpcode(ST: *ST, MoveFromSToA);
226 if (MoveFromSToA) {
227 // We are moving one of the copies earlier so its kill flag may become
228 // invalid. Clear the copied kill flag if there are any reads of the
229 // register between the new location and the old location.
230 for (auto It = std::next(x: I); It != Paired && PairedSource.isKill(); ++It)
231 if (It->readsRegister(Reg: PairedSource.getReg(), TRI))
232 PairedSource.setIsKill(false);
233
234 Sreg1 = FirstPair.Source;
235 Sreg2 = &PairedSource;
236 if (FirstPair.Destination->getReg() != RISCV::X10)
237 std::swap(a&: Sreg1, b&: Sreg2);
238 } else {
239 Sreg1 = FirstPair.Destination;
240 Sreg2 = PairedRegs.Destination;
241 if (FirstPair.Source->getReg() != RISCV::X10)
242 std::swap(a&: Sreg1, b&: Sreg2);
243 }
244
245 BuildMI(BB&: *I->getParent(), I, MIMD: DL, MCID: TII->get(Opcode)).add(MO: *Sreg1).add(MO: *Sreg2);
246
247 I->eraseFromParent();
248 Paired->eraseFromParent();
249 return NextI;
250}
251
252MachineBasicBlock::iterator
253RISCVMoveMerge::mergePLIPair(MachineBasicBlock::iterator I,
254 MachineBasicBlock::iterator Paired,
255 bool RegPairIsEven) {
256 MachineBasicBlock::iterator E = I->getParent()->end();
257 MachineBasicBlock::iterator NextI = next_nodbg(It: I, End: E);
258
259 if (NextI == Paired)
260 NextI = next_nodbg(It: NextI, End: E);
261 DebugLoc DL = I->getDebugLoc();
262
263 unsigned Opcode = getPairedPLIOpcode(Opc: I->getOpcode());
264 unsigned GPRPairIdx =
265 RegPairIsEven ? RISCV::sub_gpr_even : RISCV::sub_gpr_odd;
266 Register DestReg = TRI->getMatchingSuperReg(
267 Reg: I->getOperand(i: 0).getReg(), SubIdx: GPRPairIdx, RC: &RISCV::GPRPairRegClass);
268
269 BuildMI(BB&: *I->getParent(), I, MIMD: DL, MCID: TII->get(Opcode), DestReg)
270 .addImm(Val: I->getOperand(i: 1).getImm());
271
272 I->eraseFromParent();
273 Paired->eraseFromParent();
274 return NextI;
275}
276
277MachineBasicBlock::iterator
278RISCVMoveMerge::findMatchingGPRPairCopy(MachineBasicBlock::iterator &MBBI,
279 bool EvenRegPair,
280 const DestSourcePair &RegPair) {
281 MachineBasicBlock::iterator E = MBBI->getParent()->end();
282 ModifiedRegUnits.clear();
283 UsedRegUnits.clear();
284 unsigned RegPairIdx = EvenRegPair ? RISCV::sub_gpr_even : RISCV::sub_gpr_odd;
285 unsigned SecondPairIdx =
286 !EvenRegPair ? RISCV::sub_gpr_even : RISCV::sub_gpr_odd;
287
288 // Get the expected source/destination registers of the matching lane.
289 Register SrcGPRPair = TRI->getMatchingSuperReg(
290 Reg: RegPair.Source->getReg(), SubIdx: RegPairIdx, RC: &RISCV::GPRPairRegClass);
291 Register DestGPRPair = TRI->getMatchingSuperReg(
292 Reg: RegPair.Destination->getReg(), SubIdx: RegPairIdx, RC: &RISCV::GPRPairRegClass);
293
294 if (!SrcGPRPair.isValid() || !DestGPRPair.isValid())
295 return E;
296
297 Register ExpectedSourceReg = TRI->getSubReg(Reg: SrcGPRPair, Idx: SecondPairIdx);
298 Register ExpectedDestReg = TRI->getSubReg(Reg: DestGPRPair, Idx: SecondPairIdx);
299
300 if (!ExpectedSourceReg.isValid() || !ExpectedDestReg.isValid())
301 return E;
302
303 for (MachineBasicBlock::iterator I = next_nodbg(It: MBBI, End: E); I != E;
304 I = next_nodbg(It: I, End: E)) {
305
306 MachineInstr &MI = *I;
307
308 if (auto SecondPair = TII->isCopyInstrImpl(MI)) {
309 Register SourceReg = SecondPair->Source->getReg();
310 Register DestReg = SecondPair->Destination->getReg();
311
312 if (RegPair.Destination->getReg() == DestReg ||
313 RegPair.Source->getReg() == SourceReg)
314 return E;
315
316 // Check if the second pair's registers match the other lane of the
317 // GPRPairs.
318 if (SourceReg == ExpectedSourceReg && DestReg == ExpectedDestReg)
319 return I;
320 }
321 // Update modified / used register units.
322 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
323 // Once expected lane registers are clobbered/read in-between, we can stop
324 // scanning since the pair cannot be legally merged anymore.
325 if (!ModifiedRegUnits.available(Reg: ExpectedDestReg) ||
326 !UsedRegUnits.available(Reg: ExpectedDestReg) ||
327 !ModifiedRegUnits.available(Reg: ExpectedSourceReg))
328 return E;
329 }
330 return E;
331}
332
333MachineBasicBlock::iterator
334RISCVMoveMerge::findMatchingSACopy(MachineBasicBlock::iterator &MBBI,
335 bool MoveFromSToA,
336 const DestSourcePair &RegPair) {
337 MachineBasicBlock::iterator E = MBBI->getParent()->end();
338
339 // Track which register units have been modified and used between the first
340 // insn and the second insn.
341 ModifiedRegUnits.clear();
342 UsedRegUnits.clear();
343
344 for (MachineBasicBlock::iterator I = next_nodbg(It: MBBI, End: E); I != E;
345 I = next_nodbg(It: I, End: E)) {
346
347 MachineInstr &MI = *I;
348
349 if (auto SecondPair = TII->isCopyInstrImpl(MI)) {
350 Register SourceReg = SecondPair->Source->getReg();
351 Register DestReg = SecondPair->Destination->getReg();
352
353 bool IsCandidate = MoveFromSToA ? isCandidateToMergeMVA01S(RegPair: *SecondPair)
354 : isCandidateToMergeMVSA01(RegPair: *SecondPair);
355 if (IsCandidate) {
356 // Second destination must be different.
357 if (RegPair.Destination->getReg() == DestReg)
358 return E;
359
360 // For AtoS the source must also be different.
361 if (!MoveFromSToA && RegPair.Source->getReg() == SourceReg)
362 return E;
363
364 // If paired destination register was modified or used, the source reg
365 // was modified, there is no possibility of finding matching
366 // instruction so exit early.
367 if (!ModifiedRegUnits.available(Reg: DestReg) ||
368 !UsedRegUnits.available(Reg: DestReg) ||
369 !ModifiedRegUnits.available(Reg: SourceReg))
370 return E;
371
372 return I;
373 }
374 }
375 // Update modified / used register units.
376 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
377 }
378 return E;
379}
380
381// Look for a same-opcode pli/plui writing the other lane of the same GPRPair
382// with the same immediate. Return the matching instruction if one exists.
383MachineBasicBlock::iterator
384RISCVMoveMerge::findMatchingPLI(MachineBasicBlock::iterator &MBBI,
385 bool EvenRegPair) {
386 MachineBasicBlock::iterator E = MBBI->getParent()->end();
387 ModifiedRegUnits.clear();
388 UsedRegUnits.clear();
389 unsigned Opc = MBBI->getOpcode();
390 Register FirstDestReg = MBBI->getOperand(i: 0).getReg();
391 int64_t FirstImm = MBBI->getOperand(i: 1).getImm();
392 unsigned RegPairIdx = EvenRegPair ? RISCV::sub_gpr_even : RISCV::sub_gpr_odd;
393 unsigned SecondPairIdx =
394 !EvenRegPair ? RISCV::sub_gpr_even : RISCV::sub_gpr_odd;
395
396 // Get the expected destination register of the matching lane.
397 Register DestGPRPair = TRI->getMatchingSuperReg(Reg: FirstDestReg, SubIdx: RegPairIdx,
398 RC: &RISCV::GPRPairRegClass);
399 assert(DestGPRPair.isValid());
400 Register ExpectedDestReg = TRI->getSubReg(Reg: DestGPRPair, Idx: SecondPairIdx);
401 assert(ExpectedDestReg.isValid());
402
403 for (MachineBasicBlock::iterator I = next_nodbg(It: MBBI, End: E); I != E;
404 I = next_nodbg(It: I, End: E)) {
405
406 MachineInstr &MI = *I;
407
408 if (MI.getOpcode() == Opc) {
409 Register DestReg = MI.getOperand(i: 0).getReg();
410 int64_t Imm = MI.getOperand(i: 1).getImm();
411
412 if (FirstDestReg == DestReg)
413 return E;
414
415 // Check if the second PLI matches the other lane and immediate.
416 if (DestReg == ExpectedDestReg && Imm == FirstImm)
417 return I;
418 }
419 // Update modified / used register units.
420 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
421 // Once the expected lane register is clobbered/read in-between, we can
422 // stop scanning since the pair cannot be legally merged anymore.
423 if (!ModifiedRegUnits.available(Reg: ExpectedDestReg) ||
424 !UsedRegUnits.available(Reg: ExpectedDestReg))
425 return E;
426 }
427 return E;
428}
429
430// Finds instructions, which could be represented as C.MV instructions and
431// merged into CM.MVA01S or CM.MVSA01.
432bool RISCVMoveMerge::mergeMovePairs(MachineBasicBlock &MBB) {
433 bool Modified = false;
434
435 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
436 MBBI != E;) {
437 // Try merging a pair of single-reg PLI/PLUI into a paired form.
438 bool IsPLIEven = isPLIPairCandidate(MI: *MBBI, /*EvenRegPair=*/true);
439 bool IsPLIOdd = isPLIPairCandidate(MI: *MBBI, /*EvenRegPair=*/false);
440 if (IsPLIEven != IsPLIOdd) {
441 MachineBasicBlock::iterator Paired = findMatchingPLI(MBBI, EvenRegPair: IsPLIEven);
442 if (Paired != E) {
443 MBBI = mergePLIPair(I: MBBI, Paired, RegPairIsEven: IsPLIEven);
444 Modified = true;
445 continue;
446 }
447 }
448
449 // Check if the instruction can be compressed to C.MV instruction. If it
450 // can, return Dest/Src register pair.
451 auto RegPair = TII->isCopyInstrImpl(MI: *MBBI);
452 if (RegPair.has_value()) {
453 bool MoveFromSToA = isCandidateToMergeMVA01S(RegPair: *RegPair);
454 bool MoveFromAToS = isCandidateToMergeMVSA01(RegPair: *RegPair);
455 bool IsEven = isGPRPairCopyCandidate(RegPair: *RegPair, /*EvenRegPair=*/true);
456 bool IsOdd = isGPRPairCopyCandidate(RegPair: *RegPair, /*EvenRegPair=*/false);
457 if (!MoveFromSToA && !MoveFromAToS && !IsEven && !IsOdd) {
458 ++MBBI;
459 continue;
460 }
461
462 MachineBasicBlock::iterator Paired = E;
463 if (MoveFromSToA || MoveFromAToS) {
464 Paired = findMatchingSACopy(MBBI, MoveFromSToA, RegPair: *RegPair);
465 if (Paired != E) {
466 MBBI = mergePairedInsns(I: MBBI, Paired, MoveFromSToA);
467 Modified = true;
468 continue;
469 }
470 }
471 if (IsEven != IsOdd) {
472 Paired = findMatchingGPRPairCopy(MBBI, EvenRegPair: IsEven, RegPair: *RegPair);
473 if (Paired != E) {
474 MBBI = mergeGPRPairInsns(I: MBBI, Paired, RegPairIsEven: IsEven);
475 Modified = true;
476 continue;
477 }
478 }
479 }
480 ++MBBI;
481 }
482 return Modified;
483}
484
485bool RISCVMoveMerge::runOnMachineFunction(MachineFunction &Fn) {
486 if (skipFunction(F: Fn.getFunction()))
487 return false;
488
489 ST = &Fn.getSubtarget<RISCVSubtarget>();
490 bool HasGPRPairCopy =
491 !ST->is64Bit() && (ST->hasStdExtZdinx() || ST->hasStdExtP());
492 if (!ST->hasStdExtZcmp() && !ST->hasVendorXqccmp() && !HasGPRPairCopy)
493 return false;
494
495 TII = ST->getInstrInfo();
496 TRI = ST->getRegisterInfo();
497 // Resize the modified and used register unit trackers. We do this once
498 // per function and then clear the register units each time we optimize a
499 // move.
500 ModifiedRegUnits.init(TRI: *TRI);
501 UsedRegUnits.init(TRI: *TRI);
502 bool Modified = false;
503 for (auto &MBB : Fn)
504 Modified |= mergeMovePairs(MBB);
505 return Modified;
506}
507
508/// createRISCVMoveMergePass - returns an instance of the
509/// move merge pass.
510FunctionPass *llvm::createRISCVMoveMergePass() { return new RISCVMoveMerge(); }
511