1//===- SIFixSGPRCopies.cpp - Remove potential VGPR => SGPR copies ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Copies from VGPR to SGPR registers are illegal and the register coalescer
11/// will sometimes generate these illegal copies in situations like this:
12///
13/// Register Class <vsrc> is the union of <vgpr> and <sgpr>
14///
15/// BB0:
16/// %0 <sgpr> = SCALAR_INST
17/// %1 <vsrc> = COPY %0 <sgpr>
18/// ...
19/// BRANCH %cond BB1, BB2
20/// BB1:
21/// %2 <vgpr> = VECTOR_INST
22/// %3 <vsrc> = COPY %2 <vgpr>
23/// BB2:
24/// %4 <vsrc> = PHI %1 <vsrc>, <%bb.0>, %3 <vrsc>, <%bb.1>
25/// %5 <vgpr> = VECTOR_INST %4 <vsrc>
26///
27///
28/// The coalescer will begin at BB0 and eliminate its copy, then the resulting
29/// code will look like this:
30///
31/// BB0:
32/// %0 <sgpr> = SCALAR_INST
33/// ...
34/// BRANCH %cond BB1, BB2
35/// BB1:
36/// %2 <vgpr> = VECTOR_INST
37/// %3 <vsrc> = COPY %2 <vgpr>
38/// BB2:
39/// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <vsrc>, <%bb.1>
40/// %5 <vgpr> = VECTOR_INST %4 <sgpr>
41///
42/// Now that the result of the PHI instruction is an SGPR, the register
43/// allocator is now forced to constrain the register class of %3 to
44/// <sgpr> so we end up with final code like this:
45///
46/// BB0:
47/// %0 <sgpr> = SCALAR_INST
48/// ...
49/// BRANCH %cond BB1, BB2
50/// BB1:
51/// %2 <vgpr> = VECTOR_INST
52/// %3 <sgpr> = COPY %2 <vgpr>
53/// BB2:
54/// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <sgpr>, <%bb.1>
55/// %5 <vgpr> = VECTOR_INST %4 <sgpr>
56///
57/// Now this code contains an illegal copy from a VGPR to an SGPR.
58///
59/// In order to avoid this problem, this pass searches for PHI instructions
60/// which define a <vsrc> register and constrains its definition class to
61/// <vgpr> if the user of the PHI's definition register is a vector instruction.
62/// If the PHI's definition class is constrained to <vgpr> then the coalescer
63/// will be unable to perform the COPY removal from the above example which
64/// ultimately led to the creation of an illegal COPY.
65//===----------------------------------------------------------------------===//
66
67#include "SIFixSGPRCopies.h"
68#include "AMDGPU.h"
69#include "AMDGPULaneMaskUtils.h"
70#include "GCNSubtarget.h"
71#include "llvm/CodeGen/MachineDominators.h"
72#include "llvm/InitializePasses.h"
73#include "llvm/Target/TargetMachine.h"
74
75using namespace llvm;
76
77#define DEBUG_TYPE "si-fix-sgpr-copies"
78
79static cl::opt<bool> EnableM0Merge(
80 "amdgpu-enable-merge-m0",
81 cl::desc("Merge and hoist M0 initializations"),
82 cl::init(Val: true));
83
84namespace {
85
86class V2SCopyInfo {
87public:
88 // VGPR to SGPR copy being processed
89 MachineInstr *Copy;
90 // All SALU instructions reachable from this copy in SSA graph
91 SetVector<MachineInstr *> SChain;
92 // Number of SGPR to VGPR copies that are used to put the SALU computation
93 // results back to VALU.
94 unsigned NumSVCopies = 0;
95
96 unsigned Score = 0;
97 // Actual count of v_readfirstlane_b32
98 // which need to be inserted to keep SChain SALU
99 unsigned NumReadfirstlanes = 0;
100 // Current score state. To speedup selection V2SCopyInfos for processing
101 bool NeedToBeConvertedToVALU = false;
102 // Marks entries lowered to VALU for bulk removal from V2SCopies.
103 bool Erased = false;
104 // Unique ID. Used as a key for mapping to keep permanent order.
105 unsigned ID;
106
107 // Count of another VGPR to SGPR copies that contribute to the
108 // current copy SChain
109 unsigned SiblingPenalty = 0;
110 SetVector<unsigned> Siblings;
111 V2SCopyInfo() : Copy(nullptr), ID(0){};
112 V2SCopyInfo(unsigned Id, MachineInstr *C, unsigned Width)
113 : Copy(C), NumReadfirstlanes(Width / 32), ID(Id){};
114#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
115 void dump() const {
116 dbgs() << ID << " : " << *Copy << "\n\tS:" << SChain.size()
117 << "\n\tSV:" << NumSVCopies << "\n\tSP: " << SiblingPenalty
118 << "\nScore: " << Score << "\n";
119 }
120#endif
121};
122
123class SIFixSGPRCopies {
124 MachineDominatorTree *MDT;
125 SmallVector<MachineInstr*, 4> SCCCopies;
126 SmallVector<MachineInstr*, 4> RegSequences;
127 SmallVector<MachineInstr*, 4> PHINodes;
128 SmallVector<MachineInstr*, 4> S2VCopies;
129 unsigned NextVGPRToSGPRCopyID = 0;
130 MapVector<unsigned, V2SCopyInfo> V2SCopies;
131 DenseMap<MachineInstr *, SetVector<unsigned>> SiblingPenalty;
132 DenseSet<MachineInstr *> PHISources;
133
134public:
135 MachineRegisterInfo *MRI;
136 const SIRegisterInfo *TRI;
137 const SIInstrInfo *TII;
138
139 SIFixSGPRCopies(MachineDominatorTree *MDT) : MDT(MDT) {}
140
141 bool run(MachineFunction &MF);
142 void fixSCCCopies(MachineFunction &MF);
143 unsigned getNextVGPRToSGPRCopyId() { return ++NextVGPRToSGPRCopyID; }
144 bool needToBeConvertedToVALU(V2SCopyInfo *I);
145 void analyzeVGPRToSGPRCopy(MachineInstr *MI);
146 void lowerVGPR2SGPRCopies(MachineFunction &MF);
147 // Handles copies which source register is:
148 // 1. Physical register
149 // 2. AGPR
150 // 3. Defined by the instruction the merely moves the immediate
151 bool lowerSpecialCase(MachineInstr &MI, MachineBasicBlock::iterator &I);
152
153 void processPHINode(MachineInstr &MI);
154
155 // Check if MO is an immediate materialized into a VGPR, and if so replace it
156 // with an SGPR immediate. The VGPR immediate is also deleted if it does not
157 // have any other uses.
158 bool tryMoveVGPRConstToSGPR(MachineOperand &MO, Register NewDst,
159 MachineBasicBlock *BlockToInsertTo,
160 MachineBasicBlock::iterator PointToInsertTo,
161 const DebugLoc &DL);
162};
163
164class SIFixSGPRCopiesLegacy : public MachineFunctionPass {
165public:
166 static char ID;
167
168 SIFixSGPRCopiesLegacy() : MachineFunctionPass(ID) {}
169
170 bool runOnMachineFunction(MachineFunction &MF) override {
171 MachineDominatorTree *MDT =
172 &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
173 SIFixSGPRCopies Impl(MDT);
174 return Impl.run(MF);
175 }
176
177 StringRef getPassName() const override { return "SI Fix SGPR copies"; }
178
179 void getAnalysisUsage(AnalysisUsage &AU) const override {
180 AU.addRequired<MachineDominatorTreeWrapperPass>();
181 AU.setPreservesCFG();
182 MachineFunctionPass::getAnalysisUsage(AU);
183 }
184
185 // Waterfall expansion may introduce Phi nodes and -verify-machineinstrs will
186 // fail.
187 MachineFunctionProperties getClearedProperties() const override {
188 return MachineFunctionProperties().setNoPHIs();
189 }
190};
191
192} // end anonymous namespace
193
194INITIALIZE_PASS_BEGIN(SIFixSGPRCopiesLegacy, DEBUG_TYPE, "SI Fix SGPR copies",
195 false, false)
196INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
197INITIALIZE_PASS_END(SIFixSGPRCopiesLegacy, DEBUG_TYPE, "SI Fix SGPR copies",
198 false, false)
199
200char SIFixSGPRCopiesLegacy::ID = 0;
201
202char &llvm::SIFixSGPRCopiesLegacyID = SIFixSGPRCopiesLegacy::ID;
203
204static std::pair<const TargetRegisterClass *, const TargetRegisterClass *>
205getCopyRegClasses(const MachineInstr &Copy,
206 const SIRegisterInfo &TRI,
207 const MachineRegisterInfo &MRI) {
208 Register DstReg = Copy.getOperand(i: 0).getReg();
209 Register SrcReg = Copy.getOperand(i: 1).getReg();
210
211 const TargetRegisterClass *SrcRC = SrcReg.isVirtual()
212 ? MRI.getRegClass(Reg: SrcReg)
213 : TRI.getPhysRegBaseClass(Reg: SrcReg);
214
215 // We don't really care about the subregister here.
216 // SrcRC = TRI.getSubRegClass(SrcRC, Copy.getOperand(1).getSubReg());
217
218 const TargetRegisterClass *DstRC = DstReg.isVirtual()
219 ? MRI.getRegClass(Reg: DstReg)
220 : TRI.getPhysRegBaseClass(Reg: DstReg);
221
222 return std::pair(SrcRC, DstRC);
223}
224
225static bool isVGPRToSGPRCopy(const TargetRegisterClass *SrcRC,
226 const TargetRegisterClass *DstRC,
227 const SIRegisterInfo &TRI) {
228 return SrcRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(RC: DstRC) &&
229 TRI.hasVectorRegisters(RC: SrcRC);
230}
231
232static bool isSGPRToVGPRCopy(const TargetRegisterClass *SrcRC,
233 const TargetRegisterClass *DstRC,
234 const SIRegisterInfo &TRI) {
235 return DstRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(RC: SrcRC) &&
236 TRI.hasVectorRegisters(RC: DstRC);
237}
238
239static bool tryChangeVGPRtoSGPRinCopy(MachineInstr &MI,
240 const SIRegisterInfo *TRI,
241 const SIInstrInfo *TII) {
242 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
243 auto &Src = MI.getOperand(i: 1);
244 Register DstReg = MI.getOperand(i: 0).getReg();
245 Register SrcReg = Src.getReg();
246 if (!SrcReg.isVirtual() || !DstReg.isVirtual())
247 return false;
248
249 for (const auto &MO : MRI.reg_nodbg_operands(Reg: DstReg)) {
250 const auto *UseMI = MO.getParent();
251 if (UseMI == &MI)
252 continue;
253 if (MO.isDef() || UseMI->getParent() != MI.getParent() ||
254 UseMI->getOpcode() <= TargetOpcode::GENERIC_OP_END)
255 return false;
256
257 unsigned OpIdx = MO.getOperandNo();
258 if (OpIdx >= UseMI->getDesc().getNumOperands() ||
259 !TII->isOperandLegal(MI: *UseMI, OpIdx, MO: &Src))
260 return false;
261 }
262 // Change VGPR to SGPR destination.
263 MRI.setRegClass(Reg: DstReg, RC: TRI->getEquivalentSGPRClass(VRC: MRI.getRegClass(Reg: DstReg)));
264 return true;
265}
266
267// Distribute an SGPR->VGPR copy of a REG_SEQUENCE into a VGPR REG_SEQUENCE.
268//
269// SGPRx = ...
270// SGPRy = REG_SEQUENCE SGPRx, sub0 ...
271// VGPRz = COPY SGPRy
272//
273// ==>
274//
275// VGPRx = COPY SGPRx
276// VGPRz = REG_SEQUENCE VGPRx, sub0
277//
278// This exposes immediate folding opportunities when materializing 64-bit
279// immediates.
280static bool foldVGPRCopyIntoRegSequence(MachineInstr &MI,
281 const SIRegisterInfo *TRI,
282 const SIInstrInfo *TII,
283 MachineRegisterInfo &MRI) {
284 assert(MI.isRegSequence());
285
286 Register DstReg = MI.getOperand(i: 0).getReg();
287 if (!TRI->isSGPRClass(RC: MRI.getRegClass(Reg: DstReg)))
288 return false;
289
290 if (!MRI.hasOneUse(RegNo: DstReg))
291 return false;
292
293 MachineInstr &CopyUse = *MRI.use_instr_begin(RegNo: DstReg);
294 if (!CopyUse.isCopy())
295 return false;
296
297 // It is illegal to have vreg inputs to a physreg defining reg_sequence.
298 if (CopyUse.getOperand(i: 0).getReg().isPhysical())
299 return false;
300
301 const TargetRegisterClass *SrcRC, *DstRC;
302 std::tie(args&: SrcRC, args&: DstRC) = getCopyRegClasses(Copy: CopyUse, TRI: *TRI, MRI);
303
304 if (!isSGPRToVGPRCopy(SrcRC, DstRC, TRI: *TRI))
305 return false;
306
307 if (tryChangeVGPRtoSGPRinCopy(MI&: CopyUse, TRI, TII))
308 return true;
309
310 // TODO: Could have multiple extracts?
311 unsigned SubReg = CopyUse.getOperand(i: 1).getSubReg();
312 if (SubReg != AMDGPU::NoSubRegister)
313 return false;
314
315 MRI.setRegClass(Reg: DstReg, RC: DstRC);
316
317 // SGPRx = ...
318 // SGPRy = REG_SEQUENCE SGPRx, sub0 ...
319 // VGPRz = COPY SGPRy
320
321 // =>
322 // VGPRx = COPY SGPRx
323 // VGPRz = REG_SEQUENCE VGPRx, sub0
324
325 MI.getOperand(i: 0).setReg(CopyUse.getOperand(i: 0).getReg());
326 bool IsAGPR = TRI->isAGPRClass(RC: DstRC);
327
328 for (unsigned I = 1, N = MI.getNumOperands(); I != N; I += 2) {
329 const TargetRegisterClass *SrcRC =
330 TRI->getRegClassForOperandReg(MRI, MO: MI.getOperand(i: I));
331 assert(TRI->isSGPRClass(SrcRC) &&
332 "Expected SGPR REG_SEQUENCE to only have SGPR inputs");
333 const TargetRegisterClass *NewSrcRC = TRI->getEquivalentVGPRClass(SRC: SrcRC);
334
335 Register TmpReg = MRI.createVirtualRegister(RegClass: NewSrcRC);
336
337 BuildMI(BB&: *MI.getParent(), I: &MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY),
338 DestReg: TmpReg)
339 .add(MO: MI.getOperand(i: I));
340
341 if (IsAGPR) {
342 const TargetRegisterClass *NewSrcRC = TRI->getEquivalentAGPRClass(SRC: SrcRC);
343 Register TmpAReg = MRI.createVirtualRegister(RegClass: NewSrcRC);
344 unsigned Opc = NewSrcRC == &AMDGPU::AGPR_32RegClass ?
345 AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::COPY;
346 BuildMI(BB&: *MI.getParent(), I: &MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc),
347 DestReg: TmpAReg)
348 .addReg(RegNo: TmpReg, Flags: RegState::Kill);
349 TmpReg = TmpAReg;
350 }
351
352 MI.getOperand(i: I).setReg(TmpReg);
353 }
354
355 CopyUse.eraseFromParent();
356 return true;
357}
358
359static bool isSafeToFoldImmIntoCopy(const MachineInstr *Copy,
360 const MachineInstr *MoveImm,
361 const SIInstrInfo *TII,
362 unsigned &SMovOp,
363 int64_t &Imm) {
364 if (Copy->getOpcode() != AMDGPU::COPY)
365 return false;
366
367 if (!MoveImm || !MoveImm->isMoveImmediate())
368 return false;
369
370 const MachineOperand *ImmOp =
371 TII->getNamedOperand(MI: *MoveImm, OperandName: AMDGPU::OpName::src0);
372 if (!ImmOp->isImm())
373 return false;
374
375 // FIXME: Handle copies with sub-regs.
376 if (Copy->getOperand(i: 1).getSubReg())
377 return false;
378
379 switch (MoveImm->getOpcode()) {
380 default:
381 return false;
382 case AMDGPU::V_MOV_B32_e32:
383 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
384 SMovOp = AMDGPU::S_MOV_B32;
385 break;
386 case AMDGPU::V_MOV_B64_e32:
387 case AMDGPU::V_MOV_B64_PSEUDO:
388 SMovOp = AMDGPU::S_MOV_B64_IMM_PSEUDO;
389 break;
390 }
391 Imm = ImmOp->getImm();
392 return true;
393}
394
395template <class UnaryPredicate>
396bool searchPredecessors(const MachineBasicBlock *MBB,
397 const MachineBasicBlock *CutOff,
398 UnaryPredicate Predicate) {
399 if (MBB == CutOff)
400 return false;
401
402 DenseSet<const MachineBasicBlock *> Visited;
403 SmallVector<MachineBasicBlock *, 4> Worklist(MBB->predecessors());
404
405 while (!Worklist.empty()) {
406 MachineBasicBlock *MBB = Worklist.pop_back_val();
407
408 if (!Visited.insert(V: MBB).second)
409 continue;
410 if (MBB == CutOff)
411 continue;
412 if (Predicate(MBB))
413 return true;
414
415 Worklist.append(in_start: MBB->pred_begin(), in_end: MBB->pred_end());
416 }
417
418 return false;
419}
420
421// Checks if there is potential path From instruction To instruction.
422// If CutOff is specified and it sits in between of that path we ignore
423// a higher portion of the path and report it is not reachable.
424static bool isReachable(const MachineInstr *From,
425 const MachineInstr *To,
426 const MachineBasicBlock *CutOff,
427 MachineDominatorTree &MDT) {
428 if (MDT.dominates(A: From, B: To))
429 return true;
430
431 const MachineBasicBlock *MBBFrom = From->getParent();
432 const MachineBasicBlock *MBBTo = To->getParent();
433
434 // Do predecessor search.
435 // We should almost never get here since we do not usually produce M0 stores
436 // other than -1.
437 return searchPredecessors(MBB: MBBTo, CutOff, Predicate: [MBBFrom]
438 (const MachineBasicBlock *MBB) { return MBB == MBBFrom; });
439}
440
441// Return the first non-prologue instruction in the block.
442static MachineBasicBlock::iterator
443getFirstNonPrologue(MachineBasicBlock *MBB, const TargetInstrInfo *TII) {
444 MachineBasicBlock::iterator I = MBB->getFirstNonPHI();
445 while (I != MBB->end() && TII->isBasicBlockPrologue(MI: *I))
446 ++I;
447
448 return I;
449}
450
451// Hoist and merge identical SGPR initializations into a common predecessor.
452// This is intended to combine M0 initializations, but can work with any
453// SGPR. A VGPR cannot be processed since we cannot guarantee vector
454// executioon.
455static bool hoistAndMergeSGPRInits(unsigned Reg,
456 ArrayRef<MachineInstr *> RegMaskInstrs,
457 const MachineRegisterInfo &MRI,
458 const TargetRegisterInfo *TRI,
459 MachineDominatorTree &MDT,
460 const TargetInstrInfo *TII) {
461 // List of inits by immediate value.
462 using InitListMap = std::map<unsigned, std::list<MachineInstr *>>;
463 InitListMap Inits;
464 // List of clobbering instructions.
465 SmallVector<MachineInstr*, 8> Clobbers;
466 // List of instructions marked for deletion.
467 SmallPtrSet<MachineInstr *, 8> MergedInstrs;
468
469 bool Changed = false;
470
471 for (auto &MI : MRI.def_instructions(Reg)) {
472 MachineOperand *Imm = nullptr;
473 for (auto &MO : MI.operands()) {
474 if ((MO.isReg() && ((MO.isDef() && MO.getReg() != Reg) || !MO.isDef())) ||
475 (!MO.isImm() && !MO.isReg()) || (MO.isImm() && Imm)) {
476 Imm = nullptr;
477 break;
478 }
479 if (MO.isImm())
480 Imm = &MO;
481 }
482 if (Imm)
483 Inits[Imm->getImm()].push_front(x: &MI);
484 else
485 Clobbers.push_back(Elt: &MI);
486 }
487
488 // A regmask clobbers Reg instead of explicitly defining it, so these are not
489 // on the def list of Reg.
490 for (MachineInstr *MI : RegMaskInstrs)
491 if (MI->modifiesRegister(Reg, TRI))
492 Clobbers.push_back(Elt: MI);
493
494 for (auto &Init : Inits) {
495 auto &Defs = Init.second;
496
497 for (auto I1 = Defs.begin(), E = Defs.end(); I1 != E; ) {
498 MachineInstr *MI1 = *I1;
499
500 for (auto I2 = std::next(x: I1); I2 != E; ) {
501 MachineInstr *MI2 = *I2;
502
503 // Check any possible interference
504 auto interferes = [&](MachineBasicBlock::iterator From,
505 MachineBasicBlock::iterator To) -> bool {
506
507 assert(MDT.dominates(&*To, &*From));
508
509 auto interferes = [&MDT, From, To](MachineInstr* &Clobber) -> bool {
510 const MachineBasicBlock *MBBFrom = From->getParent();
511 const MachineBasicBlock *MBBTo = To->getParent();
512 bool MayClobberFrom = isReachable(From: Clobber, To: &*From, CutOff: MBBTo, MDT);
513 bool MayClobberTo = isReachable(From: Clobber, To: &*To, CutOff: MBBTo, MDT);
514 if (!MayClobberFrom && !MayClobberTo)
515 return false;
516 if ((MayClobberFrom && !MayClobberTo) ||
517 (!MayClobberFrom && MayClobberTo))
518 return true;
519 // Both can clobber, this is not an interference only if both are
520 // dominated by Clobber and belong to the same block or if Clobber
521 // properly dominates To, given that To >> From, so it dominates
522 // both and located in a common dominator.
523 return !((MBBFrom == MBBTo &&
524 MDT.dominates(A: Clobber, B: &*From) &&
525 MDT.dominates(A: Clobber, B: &*To)) ||
526 MDT.properlyDominates(A: Clobber->getParent(), B: MBBTo));
527 };
528
529 return (llvm::any_of(Range&: Clobbers, P: interferes)) ||
530 (llvm::any_of(Range&: Inits, P: [&](InitListMap::value_type &C) {
531 return C.first != Init.first &&
532 llvm::any_of(Range&: C.second, P: interferes);
533 }));
534 };
535
536 if (MDT.dominates(A: MI1, B: MI2)) {
537 if (!interferes(MI2, MI1)) {
538 LLVM_DEBUG(dbgs()
539 << "Erasing from "
540 << printMBBReference(*MI2->getParent()) << " " << *MI2);
541 MergedInstrs.insert(Ptr: MI2);
542 Changed = true;
543 ++I2;
544 continue;
545 }
546 } else if (MDT.dominates(A: MI2, B: MI1)) {
547 if (!interferes(MI1, MI2)) {
548 LLVM_DEBUG(dbgs()
549 << "Erasing from "
550 << printMBBReference(*MI1->getParent()) << " " << *MI1);
551 MergedInstrs.insert(Ptr: MI1);
552 Changed = true;
553 ++I1;
554 break;
555 }
556 } else {
557 auto *MBB = MDT.findNearestCommonDominator(A: MI1->getParent(),
558 B: MI2->getParent());
559 if (!MBB) {
560 ++I2;
561 continue;
562 }
563
564 MachineBasicBlock::iterator I = getFirstNonPrologue(MBB, TII);
565 if (!interferes(MI1, I) && !interferes(MI2, I)) {
566 LLVM_DEBUG(dbgs()
567 << "Erasing from "
568 << printMBBReference(*MI1->getParent()) << " " << *MI1
569 << "and moving from "
570 << printMBBReference(*MI2->getParent()) << " to "
571 << printMBBReference(*I->getParent()) << " " << *MI2);
572 I->getParent()->splice(Where: I, Other: MI2->getParent(), From: MI2);
573 MergedInstrs.insert(Ptr: MI1);
574 Changed = true;
575 ++I1;
576 break;
577 }
578 }
579 ++I2;
580 }
581 ++I1;
582 }
583 }
584
585 // Remove initializations that were merged into another.
586 for (auto &Init : Inits) {
587 auto &Defs = Init.second;
588 auto I = Defs.begin();
589 while (I != Defs.end()) {
590 if (MergedInstrs.count(Ptr: *I)) {
591 (*I)->eraseFromParent();
592 I = Defs.erase(position: I);
593 } else
594 ++I;
595 }
596 }
597
598 // Try to schedule SGPR initializations as early as possible in the MBB.
599 for (auto &Init : Inits) {
600 auto &Defs = Init.second;
601 for (auto *MI : Defs) {
602 auto *MBB = MI->getParent();
603 MachineInstr &BoundaryMI = *getFirstNonPrologue(MBB, TII);
604 MachineBasicBlock::reverse_iterator B(BoundaryMI);
605 // Check if B should actually be a boundary. If not set the previous
606 // instruction as the boundary instead.
607 if (!TII->isBasicBlockPrologue(MI: *B))
608 B++;
609
610 auto R = std::next(x: MI->getReverseIterator());
611 const unsigned Threshold = 50;
612 // Search until B or Threshold for a place to insert the initialization.
613 for (unsigned I = 0; R != B && I < Threshold; ++R, ++I)
614 if (R->readsRegister(Reg, TRI) || R->modifiesRegister(Reg, TRI) ||
615 TII->isSchedulingBoundary(MI: *R, MBB, MF: *MBB->getParent()))
616 break;
617
618 // Move to directly after R.
619 if (&*--R != MI)
620 MBB->splice(Where: *R, Other: MBB, From: MI);
621 }
622 }
623
624 if (Changed)
625 MRI.clearKillFlags(Reg);
626
627 return Changed;
628}
629
630bool SIFixSGPRCopies::run(MachineFunction &MF) {
631 // Only need to run this in SelectionDAG path.
632 if (MF.getProperties().hasSelected())
633 return false;
634
635 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
636 MRI = &MF.getRegInfo();
637 TRI = ST.getRegisterInfo();
638 TII = ST.getInstrInfo();
639
640 // Instructions to re-legalize after changing register classes
641 SmallVector<MachineInstr *, 8> Relegalize;
642 SmallVector<MachineInstr *, 4> RegMaskInstrs;
643
644 for (MachineBasicBlock &MBB : MF) {
645 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E;
646 ++I) {
647 MachineInstr &MI = *I;
648
649 // Regmask operands clobber registers without an explicit def, so record
650 // their instructions for hoistAndMergeSGPRInits.
651 if (llvm::any_of(Range: MI.operands(),
652 P: [](const MachineOperand &MO) { return MO.isRegMask(); }))
653 RegMaskInstrs.push_back(Elt: &MI);
654
655 switch (MI.getOpcode()) {
656 default:
657 // scale_src has a register class restricted to low 256 VGPRs, changing
658 // registers to VGPR may not take it into acount.
659 if (TII->isWMMA(MI) &&
660 AMDGPU::hasNamedOperand(Opcode: MI.getOpcode(), NamedIdx: AMDGPU::OpName::scale_src0))
661 Relegalize.push_back(Elt: &MI);
662 continue;
663 case AMDGPU::COPY: {
664 const TargetRegisterClass *SrcRC, *DstRC;
665 std::tie(args&: SrcRC, args&: DstRC) = getCopyRegClasses(Copy: MI, TRI: *TRI, MRI: *MRI);
666
667 if (isSGPRToVGPRCopy(SrcRC, DstRC, TRI: *TRI)) {
668 // Since VGPR to SGPR copies affect VGPR to SGPR copy
669 // score and, hence the lowering decision, let's try to get rid of
670 // them as early as possible
671 if (tryChangeVGPRtoSGPRinCopy(MI, TRI, TII))
672 continue;
673
674 // Collect those not changed to try them after VGPR to SGPR copies
675 // lowering as there will be more opportunities.
676 S2VCopies.push_back(Elt: &MI);
677 }
678 if (!isVGPRToSGPRCopy(SrcRC, DstRC, TRI: *TRI))
679 continue;
680 if (lowerSpecialCase(MI, I))
681 continue;
682
683 analyzeVGPRToSGPRCopy(MI: &MI);
684
685 break;
686 }
687 case AMDGPU::WQM:
688 case AMDGPU::STRICT_WQM:
689 case AMDGPU::SOFT_WQM:
690 case AMDGPU::STRICT_WWM:
691 case AMDGPU::INSERT_SUBREG:
692 case AMDGPU::PHI:
693 case AMDGPU::REG_SEQUENCE: {
694 if (TRI->isSGPRClass(RC: TII->getOpRegClass(MI, OpNo: 0))) {
695 for (MachineOperand &MO : MI.operands()) {
696 if (!MO.isReg() || !MO.getReg().isVirtual())
697 continue;
698 const TargetRegisterClass *SrcRC = MRI->getRegClass(Reg: MO.getReg());
699 if (SrcRC == &AMDGPU::VReg_1RegClass)
700 continue;
701
702 if (TRI->hasVectorRegisters(RC: SrcRC)) {
703 const TargetRegisterClass *DestRC =
704 TRI->getEquivalentSGPRClass(VRC: SrcRC);
705 Register NewDst = MRI->createVirtualRegister(RegClass: DestRC);
706 MachineBasicBlock *BlockToInsertCopy =
707 MI.isPHI() ? MI.getOperand(i: MO.getOperandNo() + 1).getMBB()
708 : &MBB;
709 MachineBasicBlock::iterator PointToInsertCopy =
710 MI.isPHI() ? BlockToInsertCopy->getFirstInstrTerminator() : I;
711
712 const DebugLoc &DL = MI.getDebugLoc();
713 if (!tryMoveVGPRConstToSGPR(MO, NewDst, BlockToInsertTo: BlockToInsertCopy,
714 PointToInsertTo: PointToInsertCopy, DL)) {
715 MachineInstr *NewCopy =
716 BuildMI(BB&: *BlockToInsertCopy, I: PointToInsertCopy, MIMD: DL,
717 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: NewDst)
718 .addReg(RegNo: MO.getReg());
719 MO.setReg(NewDst);
720 analyzeVGPRToSGPRCopy(MI: NewCopy);
721 PHISources.insert(V: NewCopy);
722 }
723 }
724 }
725 }
726
727 if (MI.isPHI())
728 PHINodes.push_back(Elt: &MI);
729 else if (MI.isRegSequence())
730 RegSequences.push_back(Elt: &MI);
731
732 break;
733 }
734 case AMDGPU::V_WRITELANE_B32: {
735 // Some architectures allow more than one constant bus access without
736 // SGPR restriction
737 if (ST.getConstantBusLimit(Opcode: MI.getOpcode()) != 1)
738 break;
739
740 // Writelane is special in that it can use SGPR and M0 (which would
741 // normally count as using the constant bus twice - but in this case it
742 // is allowed since the lane selector doesn't count as a use of the
743 // constant bus). However, it is still required to abide by the 1 SGPR
744 // rule. Apply a fix here as we might have multiple SGPRs after
745 // legalizing VGPRs to SGPRs
746 int Src0Idx =
747 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src0);
748 int Src1Idx =
749 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src1);
750 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
751 MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
752
753 // Check to see if the instruction violates the 1 SGPR rule
754 if ((Src0.isReg() && TRI->isSGPRReg(MRI: *MRI, Reg: Src0.getReg()) &&
755 Src0.getReg() != AMDGPU::M0) &&
756 (Src1.isReg() && TRI->isSGPRReg(MRI: *MRI, Reg: Src1.getReg()) &&
757 Src1.getReg() != AMDGPU::M0)) {
758
759 // Check for trivially easy constant prop into one of the operands
760 // If this is the case then perform the operation now to resolve SGPR
761 // issue. If we don't do that here we will always insert a mov to m0
762 // that can't be resolved in later operand folding pass
763 bool Resolved = false;
764 for (MachineOperand *MO : {&Src0, &Src1}) {
765 if (MO->getReg().isVirtual()) {
766 MachineInstr *DefMI = MRI->getVRegDef(Reg: MO->getReg());
767 if (DefMI && TII->isFoldableCopy(MI: *DefMI)) {
768 const MachineOperand &Def = DefMI->getOperand(i: 0);
769 if (Def.isReg() &&
770 MO->getReg() == Def.getReg() &&
771 MO->getSubReg() == Def.getSubReg()) {
772 const MachineOperand &Copied = DefMI->getOperand(i: 1);
773 if (Copied.isImm() &&
774 TII->isInlineConstant(Imm: APInt(64, Copied.getImm(), true))) {
775 MO->ChangeToImmediate(ImmVal: Copied.getImm());
776 Resolved = true;
777 break;
778 }
779 }
780 }
781 }
782 }
783
784 if (!Resolved) {
785 // Haven't managed to resolve by replacing an SGPR with an immediate
786 // Move src1 to be in M0
787 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
788 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
789 .add(MO: Src1);
790 Src1.ChangeToRegister(Reg: AMDGPU::M0, isDef: false);
791 }
792 }
793 break;
794 }
795 }
796 }
797 }
798
799 lowerVGPR2SGPRCopies(MF);
800 // Postprocessing
801 fixSCCCopies(MF);
802 for (auto *MI : S2VCopies) {
803 // Check if it is still valid
804 if (MI->isCopy()) {
805 const TargetRegisterClass *SrcRC, *DstRC;
806 std::tie(args&: SrcRC, args&: DstRC) = getCopyRegClasses(Copy: *MI, TRI: *TRI, MRI: *MRI);
807 if (isSGPRToVGPRCopy(SrcRC, DstRC, TRI: *TRI))
808 tryChangeVGPRtoSGPRinCopy(MI&: *MI, TRI, TII);
809 }
810 }
811 for (auto *MI : RegSequences) {
812 // Check if it is still valid
813 if (MI->isRegSequence())
814 foldVGPRCopyIntoRegSequence(MI&: *MI, TRI, TII, MRI&: *MRI);
815 }
816 for (auto *MI : PHINodes) {
817 processPHINode(MI&: *MI);
818 }
819 while (!Relegalize.empty())
820 TII->legalizeOperands(MI&: *Relegalize.pop_back_val(), MDT);
821
822 if (MF.getTarget().getOptLevel() > CodeGenOptLevel::None && EnableM0Merge)
823 hoistAndMergeSGPRInits(Reg: AMDGPU::M0, RegMaskInstrs, MRI: *MRI, TRI, MDT&: *MDT, TII);
824
825 SiblingPenalty.clear();
826 V2SCopies.clear();
827 SCCCopies.clear();
828 RegSequences.clear();
829 PHINodes.clear();
830 S2VCopies.clear();
831 PHISources.clear();
832
833 return true;
834}
835
836void SIFixSGPRCopies::processPHINode(MachineInstr &MI) {
837 bool AllAGPRUses = true;
838 SetVector<const MachineInstr *> worklist;
839 SmallPtrSet<const MachineInstr *, 4> Visited;
840 SetVector<MachineInstr *> PHIOperands;
841 worklist.insert(X: &MI);
842 Visited.insert(Ptr: &MI);
843 // HACK to make MIR tests with no uses happy
844 bool HasUses = false;
845 while (!worklist.empty()) {
846 const MachineInstr *Instr = worklist.pop_back_val();
847 Register Reg = Instr->getOperand(i: 0).getReg();
848 for (const auto &Use : MRI->use_operands(Reg)) {
849 HasUses = true;
850 const MachineInstr *UseMI = Use.getParent();
851 AllAGPRUses &= (UseMI->isCopy() &&
852 TRI->isAGPR(MRI: *MRI, Reg: UseMI->getOperand(i: 0).getReg())) ||
853 TRI->isAGPR(MRI: *MRI, Reg: Use.getReg());
854 if (UseMI->isCopy() || UseMI->isRegSequence()) {
855 if (Visited.insert(Ptr: UseMI).second)
856 worklist.insert(X: UseMI);
857
858 continue;
859 }
860 }
861 }
862
863 Register PHIRes = MI.getOperand(i: 0).getReg();
864 const TargetRegisterClass *RC0 = MRI->getRegClass(Reg: PHIRes);
865 if (HasUses && AllAGPRUses && !TRI->isAGPRClass(RC: RC0)) {
866 LLVM_DEBUG(dbgs() << "Moving PHI to AGPR: " << MI);
867 MRI->setRegClass(Reg: PHIRes, RC: TRI->getEquivalentAGPRClass(SRC: RC0));
868 for (unsigned I = 1, N = MI.getNumOperands(); I != N; I += 2) {
869 MachineInstr *DefMI = MRI->getVRegDef(Reg: MI.getOperand(i: I).getReg());
870 if (DefMI && DefMI->isPHI())
871 PHIOperands.insert(X: DefMI);
872 }
873 }
874
875 if (TRI->hasVectorRegisters(RC: MRI->getRegClass(Reg: PHIRes)) ||
876 RC0 == &AMDGPU::VReg_1RegClass) {
877 LLVM_DEBUG(dbgs() << "Legalizing PHI: " << MI);
878 TII->legalizeOperands(MI, MDT);
879 }
880
881 // Propagate register class back to PHI operands which are PHI themselves.
882 while (!PHIOperands.empty()) {
883 processPHINode(MI&: *PHIOperands.pop_back_val());
884 }
885}
886
887bool SIFixSGPRCopies::tryMoveVGPRConstToSGPR(
888 MachineOperand &MaybeVGPRConstMO, Register DstReg,
889 MachineBasicBlock *BlockToInsertTo,
890 MachineBasicBlock::iterator PointToInsertTo, const DebugLoc &DL) {
891
892 MachineInstr *DefMI = MRI->getVRegDef(Reg: MaybeVGPRConstMO.getReg());
893 if (!DefMI || !DefMI->isMoveImmediate())
894 return false;
895
896 MachineOperand *SrcConst = TII->getNamedOperand(MI&: *DefMI, OperandName: AMDGPU::OpName::src0);
897 if (SrcConst->isReg())
898 return false;
899
900 const TargetRegisterClass *SrcRC =
901 MRI->getRegClass(Reg: MaybeVGPRConstMO.getReg());
902 unsigned MoveSize = TRI->getRegSizeInBits(RC: *SrcRC);
903 unsigned MoveOp =
904 MoveSize == 64 ? AMDGPU::S_MOV_B64_IMM_PSEUDO : AMDGPU::S_MOV_B32;
905 BuildMI(BB&: *BlockToInsertTo, I: PointToInsertTo, MIMD: DL, MCID: TII->get(Opcode: MoveOp), DestReg: DstReg)
906 .add(MO: *SrcConst);
907 if (MRI->hasOneUse(RegNo: MaybeVGPRConstMO.getReg()))
908 DefMI->eraseFromParent();
909 MaybeVGPRConstMO.setReg(DstReg);
910 return true;
911}
912
913bool SIFixSGPRCopies::lowerSpecialCase(MachineInstr &MI,
914 MachineBasicBlock::iterator &I) {
915 Register DstReg = MI.getOperand(i: 0).getReg();
916 Register SrcReg = MI.getOperand(i: 1).getReg();
917 if (!DstReg.isVirtual()) {
918 // If the destination register is a physical register there isn't
919 // really much we can do to fix this.
920 // Some special instructions use M0 as an input. Some even only use
921 // the first lane. Insert a readfirstlane and hope for the best.
922 const TargetRegisterClass *SrcRC = MRI->getRegClass(Reg: SrcReg);
923 if (DstReg == AMDGPU::M0 && TRI->hasVectorRegisters(RC: SrcRC)) {
924 Register TmpReg =
925 MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
926
927 const MCInstrDesc &ReadFirstLaneDesc =
928 TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32);
929 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: ReadFirstLaneDesc, DestReg: TmpReg)
930 .add(MO: MI.getOperand(i: 1));
931
932 unsigned SubReg = MI.getOperand(i: 1).getSubReg();
933 MI.getOperand(i: 1).setReg(TmpReg);
934 MI.getOperand(i: 1).setSubReg(AMDGPU::NoSubRegister);
935
936 const TargetRegisterClass *OpRC = TII->getRegClass(MCID: ReadFirstLaneDesc, OpNum: 1);
937 const TargetRegisterClass *ConstrainRC =
938 SubReg == AMDGPU::NoSubRegister
939 ? OpRC
940 : TRI->getMatchingSuperRegClass(A: SrcRC, B: OpRC, Idx: SubReg);
941
942 if (!MRI->constrainRegClass(Reg: SrcReg, RC: ConstrainRC))
943 llvm_unreachable("failed to constrain register");
944 return true;
945 }
946
947 if (tryMoveVGPRConstToSGPR(MaybeVGPRConstMO&: MI.getOperand(i: 1), DstReg, BlockToInsertTo: MI.getParent(), PointToInsertTo: MI,
948 DL: MI.getDebugLoc())) {
949 I = MI.eraseFromParent();
950 return true;
951 }
952
953 if (!SrcReg.isVirtual())
954 return true;
955 }
956 if (!SrcReg.isVirtual() || TRI->isAGPR(MRI: *MRI, Reg: SrcReg)) {
957 SIInstrWorklist worklist;
958 worklist.insert(MI: &MI);
959 TII->moveToVALU(Worklist&: worklist, MDT);
960 return true;
961 }
962
963 unsigned SMovOp;
964 int64_t Imm;
965 // If we are just copying an immediate, we can replace the copy with
966 // s_mov_b32.
967 if (isSafeToFoldImmIntoCopy(Copy: &MI, MoveImm: MRI->getVRegDef(Reg: SrcReg), TII, SMovOp, Imm)) {
968 MI.getOperand(i: 1).ChangeToImmediate(ImmVal: Imm);
969 MI.addImplicitDefUseOperands(MF&: *MI.getMF());
970 MI.setDesc(TII->get(Opcode: SMovOp));
971 return true;
972 }
973 return false;
974}
975
976void SIFixSGPRCopies::analyzeVGPRToSGPRCopy(MachineInstr* MI) {
977 if (PHISources.contains(V: MI))
978 return;
979 Register DstReg = MI->getOperand(i: 0).getReg();
980 const TargetRegisterClass *DstRC = TRI->getRegClassForReg(MRI: *MRI, Reg: DstReg);
981
982 V2SCopyInfo Info(getNextVGPRToSGPRCopyId(), MI,
983 TRI->getRegSizeInBits(RC: *DstRC));
984 SmallVector<MachineInstr *, 8> AnalysisWorklist;
985 // Needed because the SSA is not a tree but a graph and may have
986 // forks and joins. We should not then go same way twice.
987 DenseSet<MachineInstr *> Visited;
988 AnalysisWorklist.push_back(Elt: Info.Copy);
989 while (!AnalysisWorklist.empty()) {
990
991 MachineInstr *Inst = AnalysisWorklist.pop_back_val();
992
993 if (!Visited.insert(V: Inst).second)
994 continue;
995
996 // Copies and REG_SEQUENCE do not contribute to the final assembly
997 // So, skip them but take care of the SGPR to VGPR copies bookkeeping.
998 if (Inst->isRegSequence() &&
999 TRI->isVGPR(MRI: *MRI, Reg: Inst->getOperand(i: 0).getReg())) {
1000 Info.NumSVCopies++;
1001 continue;
1002 }
1003 if (Inst->isCopy()) {
1004 const TargetRegisterClass *SrcRC, *DstRC;
1005 std::tie(args&: SrcRC, args&: DstRC) = getCopyRegClasses(Copy: *Inst, TRI: *TRI, MRI: *MRI);
1006 if (isSGPRToVGPRCopy(SrcRC, DstRC, TRI: *TRI) &&
1007 !tryChangeVGPRtoSGPRinCopy(MI&: *Inst, TRI, TII)) {
1008 Info.NumSVCopies++;
1009 continue;
1010 }
1011 }
1012
1013 SiblingPenalty[Inst].insert(X: Info.ID);
1014
1015 SmallVector<MachineInstr *, 4> Users;
1016 if ((TII->isSALU(MI: *Inst) && Inst->isCompare()) ||
1017 (Inst->isCopy() && Inst->getOperand(i: 0).getReg() == AMDGPU::SCC)) {
1018 auto I = Inst->getIterator();
1019 auto E = Inst->getParent()->end();
1020 while (++I != E &&
1021 !I->findRegisterDefOperand(Reg: AMDGPU::SCC, /*TRI=*/nullptr)) {
1022 if (I->readsRegister(Reg: AMDGPU::SCC, /*TRI=*/nullptr))
1023 Users.push_back(Elt: &*I);
1024 }
1025 } else if (Inst->getNumExplicitDefs() != 0) {
1026 Register Reg = Inst->getOperand(i: 0).getReg();
1027 if (Reg.isVirtual() && TRI->isSGPRReg(MRI: *MRI, Reg) &&
1028 !TII->isVALU(MI: *Inst, /*AllowLDSDMA=*/true)) {
1029 for (auto &U : MRI->use_instructions(Reg))
1030 Users.push_back(Elt: &U);
1031 }
1032 }
1033 for (auto *U : Users) {
1034 if (TII->isSALU(MI: *U))
1035 Info.SChain.insert(X: U);
1036 AnalysisWorklist.push_back(Elt: U);
1037 }
1038 }
1039 V2SCopies[Info.ID] = std::move(Info);
1040}
1041
1042// The main function that computes the VGPR to SGPR copy score
1043// and determines copy further lowering way: v_readfirstlane_b32 or moveToVALU
1044bool SIFixSGPRCopies::needToBeConvertedToVALU(V2SCopyInfo *Info) {
1045 if (Info->SChain.empty()) {
1046 Info->Score = 0;
1047 return true;
1048 }
1049 Info->Siblings = SiblingPenalty[*llvm::max_element(
1050 Range&: Info->SChain, C: [&](MachineInstr *A, MachineInstr *B) -> bool {
1051 return SiblingPenalty[A].size() < SiblingPenalty[B].size();
1052 })];
1053 Info->Siblings.remove_if(P: [&](unsigned ID) { return ID == Info->ID; });
1054 // The loop below computes the number of another VGPR to SGPR V2SCopies
1055 // which contribute to the current copy SALU chain. We assume that all the
1056 // V2SCopies with the same source virtual register will be squashed to one
1057 // by regalloc. Also we take care of the V2SCopies of the differnt subregs
1058 // of the same register.
1059 SmallSet<std::pair<Register, unsigned>, 4> SrcRegs;
1060 for (auto J : Info->Siblings) {
1061 auto *InfoIt = V2SCopies.find(Key: J);
1062 if (InfoIt != V2SCopies.end()) {
1063 MachineInstr *SiblingCopy = InfoIt->second.Copy;
1064 if (SiblingCopy->isImplicitDef())
1065 // the COPY has already been MoveToVALUed
1066 continue;
1067
1068 SrcRegs.insert(V: std::pair(SiblingCopy->getOperand(i: 1).getReg(),
1069 SiblingCopy->getOperand(i: 1).getSubReg()));
1070 }
1071 }
1072 Info->SiblingPenalty = SrcRegs.size();
1073
1074 unsigned Penalty =
1075 Info->NumSVCopies + Info->SiblingPenalty + Info->NumReadfirstlanes;
1076 unsigned Profit = Info->SChain.size();
1077 Info->Score = Penalty > Profit ? 0 : Profit - Penalty;
1078 Info->NeedToBeConvertedToVALU = Info->Score < 3;
1079 return Info->NeedToBeConvertedToVALU;
1080}
1081
1082void SIFixSGPRCopies::lowerVGPR2SGPRCopies(MachineFunction &MF) {
1083
1084 SmallVector<unsigned, 8> LoweringWorklist;
1085 for (auto &C : V2SCopies) {
1086 if (needToBeConvertedToVALU(Info: &C.second))
1087 LoweringWorklist.push_back(Elt: C.second.ID);
1088 }
1089
1090 // Store all the V2S copy instructions that need to be moved to VALU
1091 // in the Copies worklist.
1092 SIInstrWorklist Copies;
1093
1094 while (!LoweringWorklist.empty()) {
1095 unsigned CurID = LoweringWorklist.pop_back_val();
1096 auto *CurInfoIt = V2SCopies.find(Key: CurID);
1097 if (CurInfoIt != V2SCopies.end() && !CurInfoIt->second.Erased) {
1098 V2SCopyInfo &C = CurInfoIt->second;
1099 LLVM_DEBUG(dbgs() << "Processing ...\n"; C.dump());
1100 for (auto S : C.Siblings) {
1101 auto *SibInfoIt = V2SCopies.find(Key: S);
1102 if (SibInfoIt != V2SCopies.end() && !SibInfoIt->second.Erased) {
1103 V2SCopyInfo &SI = SibInfoIt->second;
1104 LLVM_DEBUG(dbgs() << "Sibling:\n"; SI.dump());
1105 if (!SI.NeedToBeConvertedToVALU) {
1106 SI.SChain.set_subtract(C.SChain);
1107 if (needToBeConvertedToVALU(Info: &SI))
1108 LoweringWorklist.push_back(Elt: SI.ID);
1109 }
1110 SI.Siblings.remove_if(P: [&](unsigned ID) { return ID == C.ID; });
1111 }
1112 }
1113 LLVM_DEBUG(dbgs() << "V2S copy " << *C.Copy
1114 << " is being turned to VALU\n");
1115 Copies.insert(MI: C.Copy);
1116 C.Erased = true;
1117 }
1118 }
1119 V2SCopies.remove_if(Pred: [](const auto &P) { return P.second.Erased; });
1120
1121 TII->moveToVALU(Worklist&: Copies, MDT);
1122 Copies.clear();
1123
1124 // Now do actual lowering
1125 for (auto C : V2SCopies) {
1126 MachineInstr *MI = C.second.Copy;
1127 MachineBasicBlock *MBB = MI->getParent();
1128 // We decide to turn V2S copy to v_readfirstlane_b32
1129 // remove it from the V2SCopies and remove it from all its siblings
1130 LLVM_DEBUG(dbgs() << "V2S copy " << *MI
1131 << " is being turned to v_readfirstlane_b32"
1132 << " Score: " << C.second.Score << "\n");
1133 Register DstReg = MI->getOperand(i: 0).getReg();
1134 MRI->constrainRegClass(Reg: DstReg, RC: &AMDGPU::SReg_32_XM0RegClass);
1135
1136 Register SrcReg = MI->getOperand(i: 1).getReg();
1137 unsigned SubReg = MI->getOperand(i: 1).getSubReg();
1138 const TargetRegisterClass *SrcRC =
1139 TRI->getRegClassForOperandReg(MRI: *MRI, MO: MI->getOperand(i: 1));
1140 size_t SrcSize = TRI->getRegSizeInBits(RC: *SrcRC);
1141 if (SrcSize == 16) {
1142 assert(MF.getSubtarget<GCNSubtarget>().useRealTrue16Insts() &&
1143 "We do not expect to see 16-bit copies from VGPR to SGPR unless "
1144 "we have 16-bit VGPRs");
1145 assert(MRI->getRegClass(DstReg) == &AMDGPU::SReg_32RegClass ||
1146 MRI->getRegClass(DstReg) == &AMDGPU::SReg_32_XM0RegClass);
1147 // There is no V_READFIRSTLANE_B16, so legalize the dst/src reg to 32 bits
1148 MRI->setRegClass(Reg: DstReg, RC: &AMDGPU::SReg_32_XM0RegClass);
1149 Register VReg32 = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1150 const DebugLoc &DL = MI->getDebugLoc();
1151 Register Undef = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_16RegClass);
1152 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Undef);
1153 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: VReg32)
1154 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
1155 .addImm(Val: AMDGPU::lo16)
1156 .addReg(RegNo: Undef)
1157 .addImm(Val: AMDGPU::hi16);
1158 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
1159 .addReg(RegNo: VReg32);
1160 } else if (SrcSize == 32) {
1161 const MCInstrDesc &ReadFirstLaneDesc =
1162 TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32);
1163 const TargetRegisterClass *OpRC = TII->getRegClass(MCID: ReadFirstLaneDesc, OpNum: 1);
1164 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: ReadFirstLaneDesc, DestReg: DstReg)
1165 .addReg(RegNo: SrcReg, Flags: {}, SubReg);
1166
1167 const TargetRegisterClass *ConstrainRC =
1168 SubReg == AMDGPU::NoSubRegister
1169 ? OpRC
1170 : TRI->getMatchingSuperRegClass(A: MRI->getRegClass(Reg: SrcReg), B: OpRC,
1171 Idx: SubReg);
1172
1173 if (!MRI->constrainRegClass(Reg: SrcReg, RC: ConstrainRC))
1174 llvm_unreachable("failed to constrain register");
1175 } else {
1176 auto Result = BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(),
1177 MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
1178 int N = TRI->getRegSizeInBits(RC: *SrcRC) / 32;
1179 for (int i = 0; i < N; i++) {
1180 Register PartialSrc = TII->buildExtractSubReg(
1181 MI: Result, MRI&: *MRI, SuperReg: MI->getOperand(i: 1), SuperRC: SrcRC,
1182 SubIdx: TRI->getSubRegFromChannel(Channel: i), SubRC: &AMDGPU::VGPR_32RegClass);
1183 Register PartialDst =
1184 MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
1185 BuildMI(BB&: *MBB, I&: *Result, MIMD: Result->getDebugLoc(),
1186 MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: PartialDst)
1187 .addReg(RegNo: PartialSrc);
1188 Result.addReg(RegNo: PartialDst).addImm(Val: TRI->getSubRegFromChannel(Channel: i));
1189 }
1190 }
1191 MI->eraseFromParent();
1192 }
1193}
1194
1195void SIFixSGPRCopies::fixSCCCopies(MachineFunction &MF) {
1196 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1197 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
1198 // S_CMP_LG_U64 is only available on GFX8+. S_CMP_LG_U32 always is, and
1199 // wave32 implies GFX10+ anyway.
1200 bool HasCmp = ST.isWave32() || ST.hasScalarCompareEq64();
1201 for (MachineBasicBlock &MBB : MF) {
1202 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E;
1203 ++I) {
1204 MachineInstr &MI = *I;
1205 // May already have been lowered.
1206 if (!MI.isCopy())
1207 continue;
1208 const MachineOperand &Src = MI.getOperand(i: 1);
1209 Register SrcReg = Src.getReg();
1210 Register DstReg = MI.getOperand(i: 0).getReg();
1211 if (SrcReg == AMDGPU::SCC) {
1212 Register SCCCopy =
1213 MRI->createVirtualRegister(RegClass: TRI->getWaveMaskRegClass());
1214 I = BuildMI(BB&: *MI.getParent(), I: std::next(x: MachineBasicBlock::iterator(MI)),
1215 MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: LMC.CSelectOpc), DestReg: SCCCopy)
1216 .addImm(Val: -1)
1217 .addImm(Val: 0);
1218 I = BuildMI(BB&: *MI.getParent(), I: std::next(x: I), MIMD: I->getDebugLoc(),
1219 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: DstReg)
1220 .addReg(RegNo: SCCCopy);
1221 MI.eraseFromParent();
1222 continue;
1223 }
1224 if (DstReg == AMDGPU::SCC) {
1225 MachineBasicBlock::iterator InsPt =
1226 std::next(x: MachineBasicBlock::iterator(MI));
1227 if (HasCmp && !Src.getSubReg() &&
1228 TII->isMaskedByExec(Reg: SrcReg, Use: MI, MRI: *MRI)) {
1229 // The source already has 0 in the bits of all inactive lanes, so
1230 // SCC is just "source is non-zero". S_CMP computes that without
1231 // needing a destination register.
1232 I = BuildMI(BB&: *MI.getParent(), I: InsPt, MIMD: MI.getDebugLoc(),
1233 MCID: TII->get(Opcode: LMC.CmpLgOpc))
1234 .add(MO: Src)
1235 .addImm(Val: 0);
1236 } else {
1237 Register Tmp = MRI->createVirtualRegister(RegClass: TRI->getBoolRC());
1238 I = BuildMI(BB&: *MI.getParent(), I: InsPt, MIMD: MI.getDebugLoc(),
1239 MCID: TII->get(Opcode: LMC.AndOpc))
1240 .addReg(RegNo: Tmp, Flags: getDefRegState(B: true))
1241 .add(MO: Src)
1242 .addReg(RegNo: LMC.ExecReg);
1243 }
1244 MI.eraseFromParent();
1245 }
1246 }
1247 }
1248}
1249
1250PreservedAnalyses
1251SIFixSGPRCopiesPass::run(MachineFunction &MF,
1252 MachineFunctionAnalysisManager &MFAM) {
1253 MachineDominatorTree &MDT = MFAM.getResult<MachineDominatorTreeAnalysis>(IR&: MF);
1254 SIFixSGPRCopies Impl(&MDT);
1255 bool Changed = Impl.run(MF);
1256 if (!Changed)
1257 return PreservedAnalyses::all();
1258
1259 // TODO: We could detect CFG changed.
1260 auto PA = getMachineFunctionPassPreservedAnalyses();
1261 return PA;
1262}
1263