1//===- SIFixSGPRCopies.cpp - Remove potential VGPR => SGPR copies ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Copies from VGPR to SGPR registers are illegal and the register coalescer
11/// will sometimes generate these illegal copies in situations like this:
12///
13/// Register Class <vsrc> is the union of <vgpr> and <sgpr>
14///
15/// BB0:
16/// %0 <sgpr> = SCALAR_INST
17/// %1 <vsrc> = COPY %0 <sgpr>
18/// ...
19/// BRANCH %cond BB1, BB2
20/// BB1:
21/// %2 <vgpr> = VECTOR_INST
22/// %3 <vsrc> = COPY %2 <vgpr>
23/// BB2:
24/// %4 <vsrc> = PHI %1 <vsrc>, <%bb.0>, %3 <vrsc>, <%bb.1>
25/// %5 <vgpr> = VECTOR_INST %4 <vsrc>
26///
27///
28/// The coalescer will begin at BB0 and eliminate its copy, then the resulting
29/// code will look like this:
30///
31/// BB0:
32/// %0 <sgpr> = SCALAR_INST
33/// ...
34/// BRANCH %cond BB1, BB2
35/// BB1:
36/// %2 <vgpr> = VECTOR_INST
37/// %3 <vsrc> = COPY %2 <vgpr>
38/// BB2:
39/// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <vsrc>, <%bb.1>
40/// %5 <vgpr> = VECTOR_INST %4 <sgpr>
41///
42/// Now that the result of the PHI instruction is an SGPR, the register
43/// allocator is now forced to constrain the register class of %3 to
44/// <sgpr> so we end up with final code like this:
45///
46/// BB0:
47/// %0 <sgpr> = SCALAR_INST
48/// ...
49/// BRANCH %cond BB1, BB2
50/// BB1:
51/// %2 <vgpr> = VECTOR_INST
52/// %3 <sgpr> = COPY %2 <vgpr>
53/// BB2:
54/// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <sgpr>, <%bb.1>
55/// %5 <vgpr> = VECTOR_INST %4 <sgpr>
56///
57/// Now this code contains an illegal copy from a VGPR to an SGPR.
58///
59/// In order to avoid this problem, this pass searches for PHI instructions
60/// which define a <vsrc> register and constrains its definition class to
61/// <vgpr> if the user of the PHI's definition register is a vector instruction.
62/// If the PHI's definition class is constrained to <vgpr> then the coalescer
63/// will be unable to perform the COPY removal from the above example which
64/// ultimately led to the creation of an illegal COPY.
65//===----------------------------------------------------------------------===//
66
67#include "SIFixSGPRCopies.h"
68#include "AMDGPU.h"
69#include "AMDGPULaneMaskUtils.h"
70#include "GCNSubtarget.h"
71#include "llvm/CodeGen/MachineDominators.h"
72#include "llvm/InitializePasses.h"
73#include "llvm/Target/TargetMachine.h"
74
75using namespace llvm;
76
77#define DEBUG_TYPE "si-fix-sgpr-copies"
78
79namespace {
80
81class V2SCopyInfo {
82public:
83 // VGPR to SGPR copy being processed
84 MachineInstr *Copy;
85 // All SALU instructions reachable from this copy in SSA graph
86 SetVector<MachineInstr *> SChain;
87 // Number of SGPR to VGPR copies that are used to put the SALU computation
88 // results back to VALU.
89 unsigned NumSVCopies = 0;
90
91 unsigned Score = 0;
92 // Actual count of v_readfirstlane_b32
93 // which need to be inserted to keep SChain SALU
94 unsigned NumReadfirstlanes = 0;
95 // Current score state. To speedup selection V2SCopyInfos for processing
96 bool NeedToBeConvertedToVALU = false;
97 // Marks entries lowered to VALU for bulk removal from V2SCopies.
98 bool Erased = false;
99 // Unique ID. Used as a key for mapping to keep permanent order.
100 unsigned ID;
101
102 // Count of another VGPR to SGPR copies that contribute to the
103 // current copy SChain
104 unsigned SiblingPenalty = 0;
105 SetVector<unsigned> Siblings;
106 V2SCopyInfo() : Copy(nullptr), ID(0){};
107 V2SCopyInfo(unsigned Id, MachineInstr *C, unsigned Width)
108 : Copy(C), NumReadfirstlanes(Width / 32), ID(Id){};
109#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
110 void dump() const {
111 dbgs() << ID << " : " << *Copy << "\n\tS:" << SChain.size()
112 << "\n\tSV:" << NumSVCopies << "\n\tSP: " << SiblingPenalty
113 << "\nScore: " << Score << "\n";
114 }
115#endif
116};
117
118class SIFixSGPRCopies {
119 MachineDominatorTree *MDT;
120 SmallVector<MachineInstr*, 4> SCCCopies;
121 SmallVector<MachineInstr*, 4> RegSequences;
122 SmallVector<MachineInstr*, 4> PHINodes;
123 SmallVector<MachineInstr*, 4> S2VCopies;
124 unsigned NextVGPRToSGPRCopyID = 0;
125 MapVector<unsigned, V2SCopyInfo> V2SCopies;
126 DenseMap<MachineInstr *, SetVector<unsigned>> SiblingPenalty;
127 DenseSet<MachineInstr *> PHISources;
128
129public:
130 MachineRegisterInfo *MRI;
131 const SIRegisterInfo *TRI;
132 const SIInstrInfo *TII;
133
134 SIFixSGPRCopies(MachineDominatorTree *MDT) : MDT(MDT) {}
135
136 bool run(MachineFunction &MF);
137 void fixSCCCopies(MachineFunction &MF);
138 unsigned getNextVGPRToSGPRCopyId() { return ++NextVGPRToSGPRCopyID; }
139 bool needToBeConvertedToVALU(V2SCopyInfo *I);
140 void analyzeVGPRToSGPRCopy(MachineInstr *MI);
141 void lowerVGPR2SGPRCopies(MachineFunction &MF);
142 // Handles copies which source register is:
143 // 1. Physical register
144 // 2. AGPR
145 // 3. Defined by the instruction the merely moves the immediate
146 bool lowerSpecialCase(MachineInstr &MI, MachineBasicBlock::iterator &I);
147
148 void processPHINode(MachineInstr &MI);
149
150 // Check if MO is an immediate materialized into a VGPR, and if so replace it
151 // with an SGPR immediate. The VGPR immediate is also deleted if it does not
152 // have any other uses.
153 bool tryMoveVGPRConstToSGPR(MachineOperand &MO, Register NewDst,
154 MachineBasicBlock *BlockToInsertTo,
155 MachineBasicBlock::iterator PointToInsertTo,
156 const DebugLoc &DL);
157};
158
159class SIFixSGPRCopiesLegacy : public MachineFunctionPass {
160public:
161 static char ID;
162
163 SIFixSGPRCopiesLegacy() : MachineFunctionPass(ID) {}
164
165 bool runOnMachineFunction(MachineFunction &MF) override {
166 MachineDominatorTree *MDT =
167 &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
168 SIFixSGPRCopies Impl(MDT);
169 return Impl.run(MF);
170 }
171
172 StringRef getPassName() const override { return "SI Fix SGPR copies"; }
173
174 void getAnalysisUsage(AnalysisUsage &AU) const override {
175 AU.addRequired<MachineDominatorTreeWrapperPass>();
176 AU.setPreservesCFG();
177 MachineFunctionPass::getAnalysisUsage(AU);
178 }
179
180 // Waterfall expansion may introduce Phi nodes and -verify-machineinstrs will
181 // fail.
182 MachineFunctionProperties getClearedProperties() const override {
183 return MachineFunctionProperties().setNoPHIs();
184 }
185};
186
187} // end anonymous namespace
188
189INITIALIZE_PASS_BEGIN(SIFixSGPRCopiesLegacy, DEBUG_TYPE, "SI Fix SGPR copies",
190 false, false)
191INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
192INITIALIZE_PASS_END(SIFixSGPRCopiesLegacy, DEBUG_TYPE, "SI Fix SGPR copies",
193 false, false)
194
195char SIFixSGPRCopiesLegacy::ID = 0;
196
197char &llvm::SIFixSGPRCopiesLegacyID = SIFixSGPRCopiesLegacy::ID;
198
199static std::pair<const TargetRegisterClass *, const TargetRegisterClass *>
200getCopyRegClasses(const MachineInstr &Copy,
201 const SIRegisterInfo &TRI,
202 const MachineRegisterInfo &MRI) {
203 Register DstReg = Copy.getOperand(i: 0).getReg();
204 Register SrcReg = Copy.getOperand(i: 1).getReg();
205
206 const TargetRegisterClass *SrcRC = SrcReg.isVirtual()
207 ? MRI.getRegClass(Reg: SrcReg)
208 : TRI.getPhysRegBaseClass(Reg: SrcReg);
209
210 // We don't really care about the subregister here.
211 // SrcRC = TRI.getSubRegClass(SrcRC, Copy.getOperand(1).getSubReg());
212
213 const TargetRegisterClass *DstRC = DstReg.isVirtual()
214 ? MRI.getRegClass(Reg: DstReg)
215 : TRI.getPhysRegBaseClass(Reg: DstReg);
216
217 return std::pair(SrcRC, DstRC);
218}
219
220static bool isVGPRToSGPRCopy(const TargetRegisterClass *SrcRC,
221 const TargetRegisterClass *DstRC,
222 const SIRegisterInfo &TRI) {
223 return SrcRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(RC: DstRC) &&
224 TRI.hasVectorRegisters(RC: SrcRC);
225}
226
227static bool isSGPRToVGPRCopy(const TargetRegisterClass *SrcRC,
228 const TargetRegisterClass *DstRC,
229 const SIRegisterInfo &TRI) {
230 return DstRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(RC: SrcRC) &&
231 TRI.hasVectorRegisters(RC: DstRC);
232}
233
234static bool tryChangeVGPRtoSGPRinCopy(MachineInstr &MI,
235 const SIRegisterInfo *TRI,
236 const SIInstrInfo *TII) {
237 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
238 auto &Src = MI.getOperand(i: 1);
239 Register DstReg = MI.getOperand(i: 0).getReg();
240 Register SrcReg = Src.getReg();
241 if (!SrcReg.isVirtual() || !DstReg.isVirtual())
242 return false;
243
244 for (const auto &MO : MRI.reg_nodbg_operands(Reg: DstReg)) {
245 const auto *UseMI = MO.getParent();
246 if (UseMI == &MI)
247 continue;
248 if (MO.isDef() || UseMI->getParent() != MI.getParent() ||
249 UseMI->getOpcode() <= TargetOpcode::GENERIC_OP_END)
250 return false;
251
252 unsigned OpIdx = MO.getOperandNo();
253 if (OpIdx >= UseMI->getDesc().getNumOperands() ||
254 !TII->isOperandLegal(MI: *UseMI, OpIdx, MO: &Src))
255 return false;
256 }
257 // Change VGPR to SGPR destination.
258 MRI.setRegClass(Reg: DstReg, RC: TRI->getEquivalentSGPRClass(VRC: MRI.getRegClass(Reg: DstReg)));
259 return true;
260}
261
262// Distribute an SGPR->VGPR copy of a REG_SEQUENCE into a VGPR REG_SEQUENCE.
263//
264// SGPRx = ...
265// SGPRy = REG_SEQUENCE SGPRx, sub0 ...
266// VGPRz = COPY SGPRy
267//
268// ==>
269//
270// VGPRx = COPY SGPRx
271// VGPRz = REG_SEQUENCE VGPRx, sub0
272//
273// This exposes immediate folding opportunities when materializing 64-bit
274// immediates.
275static bool foldVGPRCopyIntoRegSequence(MachineInstr &MI,
276 const SIRegisterInfo *TRI,
277 const SIInstrInfo *TII,
278 MachineRegisterInfo &MRI) {
279 assert(MI.isRegSequence());
280
281 Register DstReg = MI.getOperand(i: 0).getReg();
282 if (!TRI->isSGPRClass(RC: MRI.getRegClass(Reg: DstReg)))
283 return false;
284
285 if (!MRI.hasOneUse(RegNo: DstReg))
286 return false;
287
288 MachineInstr &CopyUse = *MRI.use_instr_begin(RegNo: DstReg);
289 if (!CopyUse.isCopy())
290 return false;
291
292 // It is illegal to have vreg inputs to a physreg defining reg_sequence.
293 if (CopyUse.getOperand(i: 0).getReg().isPhysical())
294 return false;
295
296 const TargetRegisterClass *SrcRC, *DstRC;
297 std::tie(args&: SrcRC, args&: DstRC) = getCopyRegClasses(Copy: CopyUse, TRI: *TRI, MRI);
298
299 if (!isSGPRToVGPRCopy(SrcRC, DstRC, TRI: *TRI))
300 return false;
301
302 if (tryChangeVGPRtoSGPRinCopy(MI&: CopyUse, TRI, TII))
303 return true;
304
305 // TODO: Could have multiple extracts?
306 unsigned SubReg = CopyUse.getOperand(i: 1).getSubReg();
307 if (SubReg != AMDGPU::NoSubRegister)
308 return false;
309
310 MRI.setRegClass(Reg: DstReg, RC: DstRC);
311
312 // SGPRx = ...
313 // SGPRy = REG_SEQUENCE SGPRx, sub0 ...
314 // VGPRz = COPY SGPRy
315
316 // =>
317 // VGPRx = COPY SGPRx
318 // VGPRz = REG_SEQUENCE VGPRx, sub0
319
320 MI.getOperand(i: 0).setReg(CopyUse.getOperand(i: 0).getReg());
321 bool IsAGPR = TRI->isAGPRClass(RC: DstRC);
322
323 for (unsigned I = 1, N = MI.getNumOperands(); I != N; I += 2) {
324 const TargetRegisterClass *SrcRC =
325 TRI->getRegClassForOperandReg(MRI, MO: MI.getOperand(i: I));
326 assert(TRI->isSGPRClass(SrcRC) &&
327 "Expected SGPR REG_SEQUENCE to only have SGPR inputs");
328 const TargetRegisterClass *NewSrcRC = TRI->getEquivalentVGPRClass(SRC: SrcRC);
329
330 Register TmpReg = MRI.createVirtualRegister(RegClass: NewSrcRC);
331
332 BuildMI(BB&: *MI.getParent(), I: &MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY),
333 DestReg: TmpReg)
334 .add(MO: MI.getOperand(i: I));
335
336 if (IsAGPR) {
337 const TargetRegisterClass *NewSrcRC = TRI->getEquivalentAGPRClass(SRC: SrcRC);
338 Register TmpAReg = MRI.createVirtualRegister(RegClass: NewSrcRC);
339 unsigned Opc = NewSrcRC == &AMDGPU::AGPR_32RegClass ?
340 AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::COPY;
341 BuildMI(BB&: *MI.getParent(), I: &MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc), DestReg: TmpAReg)
342 .addReg(RegNo: TmpReg);
343 TmpReg = TmpAReg;
344 }
345
346 MI.getOperand(i: I).setReg(TmpReg);
347 }
348
349 CopyUse.eraseFromParent();
350 return true;
351}
352
353static bool isSafeToFoldImmIntoCopy(const MachineInstr *Copy,
354 const MachineInstr *MoveImm,
355 const SIInstrInfo *TII,
356 unsigned &SMovOp,
357 int64_t &Imm) {
358 if (Copy->getOpcode() != AMDGPU::COPY)
359 return false;
360
361 if (!MoveImm || !MoveImm->isMoveImmediate())
362 return false;
363
364 const MachineOperand *ImmOp =
365 TII->getNamedOperand(MI: *MoveImm, OperandName: AMDGPU::OpName::src0);
366 if (!ImmOp->isImm())
367 return false;
368
369 // FIXME: Handle copies with sub-regs.
370 if (Copy->getOperand(i: 1).getSubReg())
371 return false;
372
373 switch (MoveImm->getOpcode()) {
374 default:
375 return false;
376 case AMDGPU::V_MOV_B32_e32:
377 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
378 SMovOp = AMDGPU::S_MOV_B32;
379 break;
380 case AMDGPU::V_MOV_B64_e32:
381 case AMDGPU::V_MOV_B64_PSEUDO:
382 SMovOp = AMDGPU::S_MOV_B64_IMM_PSEUDO;
383 break;
384 }
385 Imm = ImmOp->getImm();
386 return true;
387}
388
389template <class UnaryPredicate>
390bool searchPredecessors(const MachineBasicBlock *MBB,
391 const MachineBasicBlock *CutOff,
392 UnaryPredicate Predicate) {
393 if (MBB == CutOff)
394 return false;
395
396 DenseSet<const MachineBasicBlock *> Visited;
397 SmallVector<MachineBasicBlock *, 4> Worklist(MBB->predecessors());
398
399 while (!Worklist.empty()) {
400 MachineBasicBlock *MBB = Worklist.pop_back_val();
401
402 if (!Visited.insert(V: MBB).second)
403 continue;
404 if (MBB == CutOff)
405 continue;
406 if (Predicate(MBB))
407 return true;
408
409 Worklist.append(in_start: MBB->pred_begin(), in_end: MBB->pred_end());
410 }
411
412 return false;
413}
414
415// Checks if there is potential path From instruction To instruction.
416// If CutOff is specified and it sits in between of that path we ignore
417// a higher portion of the path and report it is not reachable.
418static bool isReachable(const MachineInstr *From,
419 const MachineInstr *To,
420 const MachineBasicBlock *CutOff,
421 MachineDominatorTree &MDT) {
422 if (MDT.dominates(A: From, B: To))
423 return true;
424
425 const MachineBasicBlock *MBBFrom = From->getParent();
426 const MachineBasicBlock *MBBTo = To->getParent();
427
428 // Do predecessor search.
429 // We should almost never get here since we do not usually produce M0 stores
430 // other than -1.
431 return searchPredecessors(MBB: MBBTo, CutOff, Predicate: [MBBFrom]
432 (const MachineBasicBlock *MBB) { return MBB == MBBFrom; });
433}
434
435// Return the first non-prologue instruction in the block.
436static MachineBasicBlock::iterator
437getFirstNonPrologue(MachineBasicBlock *MBB, const TargetInstrInfo *TII) {
438 MachineBasicBlock::iterator I = MBB->getFirstNonPHI();
439 while (I != MBB->end() && TII->isBasicBlockPrologue(MI: *I))
440 ++I;
441
442 return I;
443}
444
445// Hoist and merge identical SGPR initializations into a common predecessor.
446// This is intended to combine M0 initializations, but can work with any
447// SGPR. A VGPR cannot be processed since we cannot guarantee vector
448// executioon.
449static bool hoistAndMergeSGPRInits(unsigned Reg,
450 ArrayRef<MachineInstr *> RegMaskInstrs,
451 const MachineRegisterInfo &MRI,
452 const TargetRegisterInfo *TRI,
453 MachineDominatorTree &MDT,
454 const TargetInstrInfo *TII) {
455 // List of inits by immediate value.
456 using InitListMap = std::map<unsigned, std::list<MachineInstr *>>;
457 InitListMap Inits;
458 // List of clobbering instructions.
459 SmallVector<MachineInstr*, 8> Clobbers;
460 // List of instructions marked for deletion.
461 SmallPtrSet<MachineInstr *, 8> MergedInstrs;
462
463 bool Changed = false;
464
465 for (auto &MI : MRI.def_instructions(Reg)) {
466 MachineOperand *Imm = nullptr;
467 for (auto &MO : MI.operands()) {
468 if ((MO.isReg() && ((MO.isDef() && MO.getReg() != Reg) || !MO.isDef())) ||
469 (!MO.isImm() && !MO.isReg()) || (MO.isImm() && Imm)) {
470 Imm = nullptr;
471 break;
472 }
473 if (MO.isImm())
474 Imm = &MO;
475 }
476 if (Imm)
477 Inits[Imm->getImm()].push_front(x: &MI);
478 else
479 Clobbers.push_back(Elt: &MI);
480 }
481
482 // A regmask clobbers Reg instead of explicitly defining it, so these are not
483 // on the def list of Reg.
484 for (MachineInstr *MI : RegMaskInstrs)
485 if (MI->modifiesRegister(Reg, TRI))
486 Clobbers.push_back(Elt: MI);
487
488 for (auto &Init : Inits) {
489 auto &Defs = Init.second;
490
491 for (auto I1 = Defs.begin(), E = Defs.end(); I1 != E; ) {
492 MachineInstr *MI1 = *I1;
493
494 for (auto I2 = std::next(x: I1); I2 != E; ) {
495 MachineInstr *MI2 = *I2;
496
497 // Check any possible interference
498 auto interferes = [&](MachineBasicBlock::iterator From,
499 MachineBasicBlock::iterator To) -> bool {
500
501 assert(MDT.dominates(&*To, &*From));
502
503 auto interferes = [&MDT, From, To](MachineInstr* &Clobber) -> bool {
504 const MachineBasicBlock *MBBFrom = From->getParent();
505 const MachineBasicBlock *MBBTo = To->getParent();
506 bool MayClobberFrom = isReachable(From: Clobber, To: &*From, CutOff: MBBTo, MDT);
507 bool MayClobberTo = isReachable(From: Clobber, To: &*To, CutOff: MBBTo, MDT);
508 if (!MayClobberFrom && !MayClobberTo)
509 return false;
510 if ((MayClobberFrom && !MayClobberTo) ||
511 (!MayClobberFrom && MayClobberTo))
512 return true;
513 // Both can clobber, this is not an interference only if both are
514 // dominated by Clobber and belong to the same block or if Clobber
515 // properly dominates To, given that To >> From, so it dominates
516 // both and located in a common dominator.
517 return !((MBBFrom == MBBTo &&
518 MDT.dominates(A: Clobber, B: &*From) &&
519 MDT.dominates(A: Clobber, B: &*To)) ||
520 MDT.properlyDominates(A: Clobber->getParent(), B: MBBTo));
521 };
522
523 return (llvm::any_of(Range&: Clobbers, P: interferes)) ||
524 (llvm::any_of(Range&: Inits, P: [&](InitListMap::value_type &C) {
525 return C.first != Init.first &&
526 llvm::any_of(Range&: C.second, P: interferes);
527 }));
528 };
529
530 if (MDT.dominates(A: MI1, B: MI2)) {
531 if (!interferes(MI2, MI1)) {
532 LLVM_DEBUG(dbgs()
533 << "Erasing from "
534 << printMBBReference(*MI2->getParent()) << " " << *MI2);
535 MergedInstrs.insert(Ptr: MI2);
536 Changed = true;
537 ++I2;
538 continue;
539 }
540 } else if (MDT.dominates(A: MI2, B: MI1)) {
541 if (!interferes(MI1, MI2)) {
542 LLVM_DEBUG(dbgs()
543 << "Erasing from "
544 << printMBBReference(*MI1->getParent()) << " " << *MI1);
545 MergedInstrs.insert(Ptr: MI1);
546 Changed = true;
547 ++I1;
548 break;
549 }
550 } else {
551 auto *MBB = MDT.findNearestCommonDominator(A: MI1->getParent(),
552 B: MI2->getParent());
553 if (!MBB) {
554 ++I2;
555 continue;
556 }
557
558 MachineBasicBlock::iterator I = getFirstNonPrologue(MBB, TII);
559 if (!interferes(MI1, I) && !interferes(MI2, I)) {
560 LLVM_DEBUG(dbgs()
561 << "Erasing from "
562 << printMBBReference(*MI1->getParent()) << " " << *MI1
563 << "and moving from "
564 << printMBBReference(*MI2->getParent()) << " to "
565 << printMBBReference(*I->getParent()) << " " << *MI2);
566 I->getParent()->splice(Where: I, Other: MI2->getParent(), From: MI2);
567 MergedInstrs.insert(Ptr: MI1);
568 Changed = true;
569 ++I1;
570 break;
571 }
572 }
573 ++I2;
574 }
575 ++I1;
576 }
577 }
578
579 // Remove initializations that were merged into another.
580 for (auto &Init : Inits) {
581 auto &Defs = Init.second;
582 auto I = Defs.begin();
583 while (I != Defs.end()) {
584 if (MergedInstrs.count(Ptr: *I)) {
585 (*I)->eraseFromParent();
586 I = Defs.erase(position: I);
587 } else
588 ++I;
589 }
590 }
591
592 // Try to schedule SGPR initializations as early as possible in the MBB.
593 for (auto &Init : Inits) {
594 auto &Defs = Init.second;
595 for (auto *MI : Defs) {
596 auto *MBB = MI->getParent();
597 MachineInstr &BoundaryMI = *getFirstNonPrologue(MBB, TII);
598 MachineBasicBlock::reverse_iterator B(BoundaryMI);
599 // Check if B should actually be a boundary. If not set the previous
600 // instruction as the boundary instead.
601 if (!TII->isBasicBlockPrologue(MI: *B))
602 B++;
603
604 auto R = std::next(x: MI->getReverseIterator());
605 const unsigned Threshold = 50;
606 // Search until B or Threshold for a place to insert the initialization.
607 for (unsigned I = 0; R != B && I < Threshold; ++R, ++I)
608 if (R->readsRegister(Reg, TRI) || R->modifiesRegister(Reg, TRI) ||
609 TII->isSchedulingBoundary(MI: *R, MBB, MF: *MBB->getParent()))
610 break;
611
612 // Move to directly after R.
613 if (&*--R != MI)
614 MBB->splice(Where: *R, Other: MBB, From: MI);
615 }
616 }
617
618 if (Changed)
619 MRI.clearKillFlags(Reg);
620
621 return Changed;
622}
623
624bool SIFixSGPRCopies::run(MachineFunction &MF) {
625 // Only need to run this in SelectionDAG path.
626 if (MF.getProperties().hasSelected())
627 return false;
628
629 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
630 MRI = &MF.getRegInfo();
631 TRI = ST.getRegisterInfo();
632 TII = ST.getInstrInfo();
633
634 // Instructions to re-legalize after changing register classes
635 SmallVector<MachineInstr *, 8> Relegalize;
636 SmallVector<MachineInstr *, 4> RegMaskInstrs;
637
638 for (MachineBasicBlock &MBB : MF) {
639 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E;
640 ++I) {
641 MachineInstr &MI = *I;
642
643 // Regmask operands clobber registers without an explicit def, so record
644 // their instructions for hoistAndMergeSGPRInits.
645 if (llvm::any_of(Range: MI.operands(),
646 P: [](const MachineOperand &MO) { return MO.isRegMask(); }))
647 RegMaskInstrs.push_back(Elt: &MI);
648
649 switch (MI.getOpcode()) {
650 default:
651 // scale_src has a register class restricted to low 256 VGPRs, changing
652 // registers to VGPR may not take it into acount.
653 if (TII->isWMMA(MI) &&
654 AMDGPU::hasNamedOperand(Opcode: MI.getOpcode(), NamedIdx: AMDGPU::OpName::scale_src0))
655 Relegalize.push_back(Elt: &MI);
656 continue;
657 case AMDGPU::COPY: {
658 const TargetRegisterClass *SrcRC, *DstRC;
659 std::tie(args&: SrcRC, args&: DstRC) = getCopyRegClasses(Copy: MI, TRI: *TRI, MRI: *MRI);
660
661 if (isSGPRToVGPRCopy(SrcRC, DstRC, TRI: *TRI)) {
662 // Since VGPR to SGPR copies affect VGPR to SGPR copy
663 // score and, hence the lowering decision, let's try to get rid of
664 // them as early as possible
665 if (tryChangeVGPRtoSGPRinCopy(MI, TRI, TII))
666 continue;
667
668 // Collect those not changed to try them after VGPR to SGPR copies
669 // lowering as there will be more opportunities.
670 S2VCopies.push_back(Elt: &MI);
671 }
672 if (!isVGPRToSGPRCopy(SrcRC, DstRC, TRI: *TRI))
673 continue;
674 if (lowerSpecialCase(MI, I))
675 continue;
676
677 analyzeVGPRToSGPRCopy(MI: &MI);
678
679 break;
680 }
681 case AMDGPU::WQM:
682 case AMDGPU::STRICT_WQM:
683 case AMDGPU::SOFT_WQM:
684 case AMDGPU::STRICT_WWM:
685 case AMDGPU::INSERT_SUBREG:
686 case AMDGPU::PHI:
687 case AMDGPU::REG_SEQUENCE: {
688 if (TRI->isSGPRClass(RC: TII->getOpRegClass(MI, OpNo: 0))) {
689 for (MachineOperand &MO : MI.operands()) {
690 if (!MO.isReg() || !MO.getReg().isVirtual())
691 continue;
692 const TargetRegisterClass *SrcRC = MRI->getRegClass(Reg: MO.getReg());
693 if (SrcRC == &AMDGPU::VReg_1RegClass)
694 continue;
695
696 if (TRI->hasVectorRegisters(RC: SrcRC)) {
697 const TargetRegisterClass *DestRC =
698 TRI->getEquivalentSGPRClass(VRC: SrcRC);
699 Register NewDst = MRI->createVirtualRegister(RegClass: DestRC);
700 MachineBasicBlock *BlockToInsertCopy =
701 MI.isPHI() ? MI.getOperand(i: MO.getOperandNo() + 1).getMBB()
702 : &MBB;
703 MachineBasicBlock::iterator PointToInsertCopy =
704 MI.isPHI() ? BlockToInsertCopy->getFirstInstrTerminator() : I;
705
706 const DebugLoc &DL = MI.getDebugLoc();
707 if (!tryMoveVGPRConstToSGPR(MO, NewDst, BlockToInsertTo: BlockToInsertCopy,
708 PointToInsertTo: PointToInsertCopy, DL)) {
709 MachineInstr *NewCopy =
710 BuildMI(BB&: *BlockToInsertCopy, I: PointToInsertCopy, MIMD: DL,
711 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: NewDst)
712 .addReg(RegNo: MO.getReg());
713 MO.setReg(NewDst);
714 analyzeVGPRToSGPRCopy(MI: NewCopy);
715 PHISources.insert(V: NewCopy);
716 }
717 }
718 }
719 }
720
721 if (MI.isPHI())
722 PHINodes.push_back(Elt: &MI);
723 else if (MI.isRegSequence())
724 RegSequences.push_back(Elt: &MI);
725
726 break;
727 }
728 case AMDGPU::V_WRITELANE_B32: {
729 // Some architectures allow more than one constant bus access without
730 // SGPR restriction
731 if (ST.getConstantBusLimit(Opcode: MI.getOpcode()) != 1)
732 break;
733
734 // Writelane is special in that it can use SGPR and M0 (which would
735 // normally count as using the constant bus twice - but in this case it
736 // is allowed since the lane selector doesn't count as a use of the
737 // constant bus). However, it is still required to abide by the 1 SGPR
738 // rule. Apply a fix here as we might have multiple SGPRs after
739 // legalizing VGPRs to SGPRs
740 int Src0Idx =
741 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src0);
742 int Src1Idx =
743 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src1);
744 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
745 MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
746
747 // Check to see if the instruction violates the 1 SGPR rule
748 if ((Src0.isReg() && TRI->isSGPRReg(MRI: *MRI, Reg: Src0.getReg()) &&
749 Src0.getReg() != AMDGPU::M0) &&
750 (Src1.isReg() && TRI->isSGPRReg(MRI: *MRI, Reg: Src1.getReg()) &&
751 Src1.getReg() != AMDGPU::M0)) {
752
753 // Check for trivially easy constant prop into one of the operands
754 // If this is the case then perform the operation now to resolve SGPR
755 // issue. If we don't do that here we will always insert a mov to m0
756 // that can't be resolved in later operand folding pass
757 bool Resolved = false;
758 for (MachineOperand *MO : {&Src0, &Src1}) {
759 if (MO->getReg().isVirtual()) {
760 MachineInstr *DefMI = MRI->getVRegDef(Reg: MO->getReg());
761 if (DefMI && TII->isFoldableCopy(MI: *DefMI)) {
762 const MachineOperand &Def = DefMI->getOperand(i: 0);
763 if (Def.isReg() &&
764 MO->getReg() == Def.getReg() &&
765 MO->getSubReg() == Def.getSubReg()) {
766 const MachineOperand &Copied = DefMI->getOperand(i: 1);
767 if (Copied.isImm() &&
768 TII->isInlineConstant(Imm: APInt(64, Copied.getImm(), true))) {
769 MO->ChangeToImmediate(ImmVal: Copied.getImm());
770 Resolved = true;
771 break;
772 }
773 }
774 }
775 }
776 }
777
778 if (!Resolved) {
779 // Haven't managed to resolve by replacing an SGPR with an immediate
780 // Move src1 to be in M0
781 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
782 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
783 .add(MO: Src1);
784 Src1.ChangeToRegister(Reg: AMDGPU::M0, isDef: false);
785 }
786 }
787 break;
788 }
789 }
790 }
791 }
792
793 lowerVGPR2SGPRCopies(MF);
794 // Postprocessing
795 fixSCCCopies(MF);
796 for (auto *MI : S2VCopies) {
797 // Check if it is still valid
798 if (MI->isCopy()) {
799 const TargetRegisterClass *SrcRC, *DstRC;
800 std::tie(args&: SrcRC, args&: DstRC) = getCopyRegClasses(Copy: *MI, TRI: *TRI, MRI: *MRI);
801 if (isSGPRToVGPRCopy(SrcRC, DstRC, TRI: *TRI))
802 tryChangeVGPRtoSGPRinCopy(MI&: *MI, TRI, TII);
803 }
804 }
805 for (auto *MI : RegSequences) {
806 // Check if it is still valid
807 if (MI->isRegSequence())
808 foldVGPRCopyIntoRegSequence(MI&: *MI, TRI, TII, MRI&: *MRI);
809 }
810 for (auto *MI : PHINodes) {
811 processPHINode(MI&: *MI);
812 }
813 while (!Relegalize.empty())
814 TII->legalizeOperands(MI&: *Relegalize.pop_back_val(), MDT);
815
816 if (MF.getTarget().getOptLevel() > CodeGenOptLevel::None &&
817 ST.getCLOpts().enable_merge_m0)
818 hoistAndMergeSGPRInits(Reg: AMDGPU::M0, RegMaskInstrs, MRI: *MRI, TRI, MDT&: *MDT, TII);
819
820 SiblingPenalty.clear();
821 V2SCopies.clear();
822 SCCCopies.clear();
823 RegSequences.clear();
824 PHINodes.clear();
825 S2VCopies.clear();
826 PHISources.clear();
827
828 return true;
829}
830
831void SIFixSGPRCopies::processPHINode(MachineInstr &MI) {
832 bool AllAGPRUses = true;
833 SetVector<const MachineInstr *> worklist;
834 SmallPtrSet<const MachineInstr *, 4> Visited;
835 SetVector<MachineInstr *> PHIOperands;
836 worklist.insert(X: &MI);
837 Visited.insert(Ptr: &MI);
838 // HACK to make MIR tests with no uses happy
839 bool HasUses = false;
840 while (!worklist.empty()) {
841 const MachineInstr *Instr = worklist.pop_back_val();
842 Register Reg = Instr->getOperand(i: 0).getReg();
843 for (const auto &Use : MRI->use_operands(Reg)) {
844 HasUses = true;
845 const MachineInstr *UseMI = Use.getParent();
846 AllAGPRUses &= (UseMI->isCopy() &&
847 TRI->isAGPR(MRI: *MRI, Reg: UseMI->getOperand(i: 0).getReg())) ||
848 TRI->isAGPR(MRI: *MRI, Reg: Use.getReg());
849 if (UseMI->isCopy() || UseMI->isRegSequence()) {
850 if (Visited.insert(Ptr: UseMI).second)
851 worklist.insert(X: UseMI);
852
853 continue;
854 }
855 }
856 }
857
858 Register PHIRes = MI.getOperand(i: 0).getReg();
859 const TargetRegisterClass *RC0 = MRI->getRegClass(Reg: PHIRes);
860 if (HasUses && AllAGPRUses && !TRI->isAGPRClass(RC: RC0)) {
861 LLVM_DEBUG(dbgs() << "Moving PHI to AGPR: " << MI);
862 MRI->setRegClass(Reg: PHIRes, RC: TRI->getEquivalentAGPRClass(SRC: RC0));
863 for (unsigned I = 1, N = MI.getNumOperands(); I != N; I += 2) {
864 MachineInstr *DefMI = MRI->getVRegDef(Reg: MI.getOperand(i: I).getReg());
865 if (DefMI && DefMI->isPHI() &&
866 TRI->isSGPRReg(MRI: *MRI, Reg: DefMI->getOperand(i: 0).getReg()))
867 PHIOperands.insert(X: DefMI);
868 }
869 }
870
871 // An SGPR input of a vector PHI may be temporally divergent, e.g. used after
872 // a divergent loop. Copy it at the end of its incoming block, so each lane
873 // keeps the value from its last iteration.
874 // FIXME: SelectionDAG should handle temporal divergence, as GlobalISel does.
875 const TargetRegisterClass *RC = MRI->getRegClass(Reg: PHIRes);
876 if (TRI->hasVectorRegisters(RC)) {
877 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
878 MachineOperand &Op = MI.getOperand(i: I);
879 if (!Op.isReg() || !Op.getReg().isVirtual() ||
880 !TRI->isSGPRReg(MRI: *MRI, Reg: Op.getReg()))
881 continue;
882
883 LLVM_DEBUG(dbgs() << "Copying SGPR input " << printReg(Op.getReg(), TRI)
884 << " of PHI: " << MI);
885 MachineBasicBlock *InsertBB = MI.getOperand(i: I + 1).getMBB();
886 TII->legalizeGenericOperand(InsertMBB&: *InsertBB, I: InsertBB->getFirstTerminator(), DstRC: RC,
887 Op, MRI&: *MRI, DL: MI.getDebugLoc());
888 }
889 }
890
891 // Propagate the AGPR class back to PHI operands that are SGPR PHIs, which
892 // are the only inputs copied above.
893 while (!PHIOperands.empty()) {
894 processPHINode(MI&: *PHIOperands.pop_back_val());
895 }
896}
897
898bool SIFixSGPRCopies::tryMoveVGPRConstToSGPR(
899 MachineOperand &MaybeVGPRConstMO, Register DstReg,
900 MachineBasicBlock *BlockToInsertTo,
901 MachineBasicBlock::iterator PointToInsertTo, const DebugLoc &DL) {
902
903 MachineInstr *DefMI = MRI->getVRegDef(Reg: MaybeVGPRConstMO.getReg());
904 if (!DefMI || !DefMI->isMoveImmediate())
905 return false;
906
907 MachineOperand *SrcConst = TII->getNamedOperand(MI&: *DefMI, OperandName: AMDGPU::OpName::src0);
908 if (SrcConst->isReg())
909 return false;
910
911 const TargetRegisterClass *SrcRC =
912 MRI->getRegClass(Reg: MaybeVGPRConstMO.getReg());
913 unsigned MoveSize = TRI->getRegSizeInBits(RC: *SrcRC);
914 unsigned MoveOp =
915 MoveSize == 64 ? AMDGPU::S_MOV_B64_IMM_PSEUDO : AMDGPU::S_MOV_B32;
916 BuildMI(BB&: *BlockToInsertTo, I: PointToInsertTo, MIMD: DL, MCID: TII->get(Opcode: MoveOp), DestReg: DstReg)
917 .add(MO: *SrcConst);
918 if (MRI->hasOneUse(RegNo: MaybeVGPRConstMO.getReg()))
919 DefMI->eraseFromParent();
920 MaybeVGPRConstMO.setReg(DstReg);
921 return true;
922}
923
924bool SIFixSGPRCopies::lowerSpecialCase(MachineInstr &MI,
925 MachineBasicBlock::iterator &I) {
926 Register DstReg = MI.getOperand(i: 0).getReg();
927 Register SrcReg = MI.getOperand(i: 1).getReg();
928 if (!DstReg.isVirtual()) {
929 // moveToVALU lowers a VGPR copied into M0: each movrel access that reads it
930 // gets a waterfall loop, and every other reader the first lane.
931 const TargetRegisterClass *SrcRC = MRI->getRegClass(Reg: SrcReg);
932 if (DstReg == AMDGPU::M0 && TRI->hasVectorRegisters(RC: SrcRC)) {
933 const TargetRegisterClass *OpRC =
934 TII->getRegClass(MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), OpNum: 1);
935 unsigned SubReg = MI.getOperand(i: 1).getSubReg();
936 const TargetRegisterClass *ConstrainRC =
937 SubReg == AMDGPU::NoSubRegister
938 ? OpRC
939 : TRI->getMatchingSuperRegClass(A: SrcRC, B: OpRC, Idx: SubReg);
940
941 if (!MRI->constrainRegClass(Reg: SrcReg, RC: ConstrainRC))
942 llvm_unreachable("failed to constrain register");
943 return false;
944 }
945
946 if (tryMoveVGPRConstToSGPR(MaybeVGPRConstMO&: MI.getOperand(i: 1), DstReg, BlockToInsertTo: MI.getParent(), PointToInsertTo: MI,
947 DL: MI.getDebugLoc())) {
948 I = MI.eraseFromParent();
949 return true;
950 }
951
952 if (!SrcReg.isVirtual())
953 return true;
954 }
955 if (!SrcReg.isVirtual() || TRI->isAGPR(MRI: *MRI, Reg: SrcReg)) {
956 SIInstrWorklist worklist;
957 worklist.insert(MI: &MI);
958 TII->moveToVALU(Worklist&: worklist, MDT);
959 return true;
960 }
961
962 unsigned SMovOp;
963 int64_t Imm;
964 // If we are just copying an immediate, we can replace the copy with
965 // s_mov_b32.
966 if (isSafeToFoldImmIntoCopy(Copy: &MI, MoveImm: MRI->getVRegDef(Reg: SrcReg), TII, SMovOp, Imm)) {
967 MI.getOperand(i: 1).ChangeToImmediate(ImmVal: Imm);
968 MI.addImplicitDefUseOperands(MF&: *MI.getMF());
969 MI.setDesc(TII->get(Opcode: SMovOp));
970 return true;
971 }
972 return false;
973}
974
975void SIFixSGPRCopies::analyzeVGPRToSGPRCopy(MachineInstr* MI) {
976 if (PHISources.contains(V: MI))
977 return;
978 Register DstReg = MI->getOperand(i: 0).getReg();
979 const TargetRegisterClass *DstRC = TRI->getRegClassForReg(MRI: *MRI, Reg: DstReg);
980
981 V2SCopyInfo Info(getNextVGPRToSGPRCopyId(), MI,
982 TRI->getRegSizeInBits(RC: *DstRC));
983 SmallVector<MachineInstr *, 8> AnalysisWorklist;
984 // Needed because the SSA is not a tree but a graph and may have
985 // forks and joins. We should not then go same way twice.
986 DenseSet<MachineInstr *> Visited;
987 AnalysisWorklist.push_back(Elt: Info.Copy);
988 while (!AnalysisWorklist.empty()) {
989
990 MachineInstr *Inst = AnalysisWorklist.pop_back_val();
991
992 if (!Visited.insert(V: Inst).second)
993 continue;
994
995 // Copies and REG_SEQUENCE do not contribute to the final assembly
996 // So, skip them but take care of the SGPR to VGPR copies bookkeeping.
997 if (Inst->isRegSequence() &&
998 TRI->isVGPR(MRI: *MRI, Reg: Inst->getOperand(i: 0).getReg())) {
999 Info.NumSVCopies++;
1000 continue;
1001 }
1002 if (Inst->isCopy()) {
1003 const TargetRegisterClass *SrcRC, *DstRC;
1004 std::tie(args&: SrcRC, args&: DstRC) = getCopyRegClasses(Copy: *Inst, TRI: *TRI, MRI: *MRI);
1005 if (isSGPRToVGPRCopy(SrcRC, DstRC, TRI: *TRI) &&
1006 !tryChangeVGPRtoSGPRinCopy(MI&: *Inst, TRI, TII)) {
1007 Info.NumSVCopies++;
1008 continue;
1009 }
1010 }
1011
1012 SiblingPenalty[Inst].insert(X: Info.ID);
1013
1014 SmallVector<MachineInstr *, 4> Users;
1015 if ((TII->isSALU(MI: *Inst) && Inst->isCompare()) ||
1016 (Inst->isCopy() && Inst->getOperand(i: 0).getReg() == AMDGPU::SCC)) {
1017 auto I = Inst->getIterator();
1018 auto E = Inst->getParent()->end();
1019 while (++I != E &&
1020 !I->findRegisterDefOperand(Reg: AMDGPU::SCC, /*TRI=*/nullptr)) {
1021 if (I->readsRegister(Reg: AMDGPU::SCC, /*TRI=*/nullptr))
1022 Users.push_back(Elt: &*I);
1023 }
1024 } else if (Inst->getNumExplicitDefs() != 0) {
1025 Register Reg = Inst->getOperand(i: 0).getReg();
1026 if (Reg.isVirtual() && TRI->isSGPRReg(MRI: *MRI, Reg) &&
1027 !TII->isVALU(MI: *Inst, /*AllowLDSDMA=*/false)) {
1028 for (auto &U : MRI->use_instructions(Reg))
1029 Users.push_back(Elt: &U);
1030 }
1031 }
1032 for (auto *U : Users) {
1033 if (TII->isSALU(MI: *U))
1034 Info.SChain.insert(X: U);
1035 AnalysisWorklist.push_back(Elt: U);
1036 }
1037 }
1038 V2SCopies[Info.ID] = std::move(Info);
1039}
1040
1041// The main function that computes the VGPR to SGPR copy score
1042// and determines copy further lowering way: v_readfirstlane_b32 or moveToVALU
1043bool SIFixSGPRCopies::needToBeConvertedToVALU(V2SCopyInfo *Info) {
1044 if (Info->SChain.empty()) {
1045 Info->Score = 0;
1046 return true;
1047 }
1048 Info->Siblings = SiblingPenalty[*llvm::max_element(
1049 Range&: Info->SChain, C: [&](MachineInstr *A, MachineInstr *B) -> bool {
1050 return SiblingPenalty[A].size() < SiblingPenalty[B].size();
1051 })];
1052 Info->Siblings.remove_if(P: [&](unsigned ID) { return ID == Info->ID; });
1053 // The loop below computes the number of another VGPR to SGPR V2SCopies
1054 // which contribute to the current copy SALU chain. We assume that all the
1055 // V2SCopies with the same source virtual register will be squashed to one
1056 // by regalloc. Also we take care of the V2SCopies of the differnt subregs
1057 // of the same register.
1058 SmallSet<std::pair<Register, unsigned>, 4> SrcRegs;
1059 for (auto J : Info->Siblings) {
1060 auto *InfoIt = V2SCopies.find(Key: J);
1061 if (InfoIt != V2SCopies.end()) {
1062 MachineInstr *SiblingCopy = InfoIt->second.Copy;
1063 if (SiblingCopy->isImplicitDef())
1064 // the COPY has already been MoveToVALUed
1065 continue;
1066
1067 SrcRegs.insert(V: std::pair(SiblingCopy->getOperand(i: 1).getReg(),
1068 SiblingCopy->getOperand(i: 1).getSubReg()));
1069 }
1070 }
1071 Info->SiblingPenalty = SrcRegs.size();
1072
1073 unsigned Penalty =
1074 Info->NumSVCopies + Info->SiblingPenalty + Info->NumReadfirstlanes;
1075 unsigned Profit = Info->SChain.size();
1076 Info->Score = Penalty > Profit ? 0 : Profit - Penalty;
1077 Info->NeedToBeConvertedToVALU = Info->Score < 3;
1078 return Info->NeedToBeConvertedToVALU;
1079}
1080
1081void SIFixSGPRCopies::lowerVGPR2SGPRCopies(MachineFunction &MF) {
1082
1083 SmallVector<unsigned, 8> LoweringWorklist;
1084 for (auto &C : V2SCopies) {
1085 if (needToBeConvertedToVALU(Info: &C.second))
1086 LoweringWorklist.push_back(Elt: C.second.ID);
1087 }
1088
1089 // Store all the V2S copy instructions that need to be moved to VALU
1090 // in the Copies worklist.
1091 SIInstrWorklist Copies;
1092
1093 while (!LoweringWorklist.empty()) {
1094 unsigned CurID = LoweringWorklist.pop_back_val();
1095 auto *CurInfoIt = V2SCopies.find(Key: CurID);
1096 if (CurInfoIt != V2SCopies.end() && !CurInfoIt->second.Erased) {
1097 V2SCopyInfo &C = CurInfoIt->second;
1098 LLVM_DEBUG(dbgs() << "Processing ...\n"; C.dump());
1099 for (auto S : C.Siblings) {
1100 auto *SibInfoIt = V2SCopies.find(Key: S);
1101 if (SibInfoIt != V2SCopies.end() && !SibInfoIt->second.Erased) {
1102 V2SCopyInfo &SI = SibInfoIt->second;
1103 LLVM_DEBUG(dbgs() << "Sibling:\n"; SI.dump());
1104 if (!SI.NeedToBeConvertedToVALU) {
1105 SI.SChain.set_subtract(C.SChain);
1106 if (needToBeConvertedToVALU(Info: &SI))
1107 LoweringWorklist.push_back(Elt: SI.ID);
1108 }
1109 SI.Siblings.remove_if(P: [&](unsigned ID) { return ID == C.ID; });
1110 }
1111 }
1112 LLVM_DEBUG(dbgs() << "V2S copy " << *C.Copy
1113 << " is being turned to VALU\n");
1114 Copies.insert(MI: C.Copy);
1115 C.Erased = true;
1116 }
1117 }
1118 V2SCopies.remove_if(Pred: [](const auto &P) { return P.second.Erased; });
1119
1120 TII->moveToVALU(Worklist&: Copies, MDT);
1121 Copies.clear();
1122
1123 // Now do actual lowering
1124 for (auto C : V2SCopies) {
1125 MachineInstr *MI = C.second.Copy;
1126 MachineBasicBlock *MBB = MI->getParent();
1127 // We decide to turn V2S copy to v_readfirstlane_b32
1128 // remove it from the V2SCopies and remove it from all its siblings
1129 LLVM_DEBUG(dbgs() << "V2S copy " << *MI
1130 << " is being turned to v_readfirstlane_b32"
1131 << " Score: " << C.second.Score << "\n");
1132 Register DstReg = MI->getOperand(i: 0).getReg();
1133 MRI->constrainRegClass(Reg: DstReg, RC: &AMDGPU::SReg_32_XM0RegClass);
1134
1135 Register SrcReg = MI->getOperand(i: 1).getReg();
1136 unsigned SubReg = MI->getOperand(i: 1).getSubReg();
1137 const TargetRegisterClass *SrcRC =
1138 TRI->getRegClassForOperandReg(MRI: *MRI, MO: MI->getOperand(i: 1));
1139 size_t SrcSize = TRI->getRegSizeInBits(RC: *SrcRC);
1140 if (SrcSize == 16) {
1141 assert(MF.getSubtarget<GCNSubtarget>().useRealTrue16Insts() &&
1142 "We do not expect to see 16-bit copies from VGPR to SGPR unless "
1143 "we have 16-bit VGPRs");
1144 assert(MRI->getRegClass(DstReg) == &AMDGPU::SReg_32RegClass ||
1145 MRI->getRegClass(DstReg) == &AMDGPU::SReg_32_XM0RegClass);
1146 // There is no V_READFIRSTLANE_B16, so legalize the dst/src reg to 32 bits
1147 MRI->setRegClass(Reg: DstReg, RC: &AMDGPU::SReg_32_XM0RegClass);
1148 const DebugLoc &DL = MI->getDebugLoc();
1149 Register VReg32 = TII->buildVGPR32FromLo16(MBB&: *MBB, I: MI, DL, Src: SrcReg, SrcSubReg: SubReg);
1150 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
1151 .addReg(RegNo: VReg32);
1152 } else if (SrcSize == 32) {
1153 const MCInstrDesc &ReadFirstLaneDesc =
1154 TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32);
1155 const TargetRegisterClass *OpRC = TII->getRegClass(MCID: ReadFirstLaneDesc, OpNum: 1);
1156 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: ReadFirstLaneDesc, DestReg: DstReg)
1157 .addReg(RegNo: SrcReg, Flags: {}, SubReg);
1158
1159 const TargetRegisterClass *ConstrainRC =
1160 SubReg == AMDGPU::NoSubRegister
1161 ? OpRC
1162 : TRI->getMatchingSuperRegClass(A: MRI->getRegClass(Reg: SrcReg), B: OpRC,
1163 Idx: SubReg);
1164
1165 if (!MRI->constrainRegClass(Reg: SrcReg, RC: ConstrainRC))
1166 llvm_unreachable("failed to constrain register");
1167 } else {
1168 auto Result = BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(),
1169 MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
1170 int N = TRI->getRegSizeInBits(RC: *SrcRC) / 32;
1171 for (int i = 0; i < N; i++) {
1172 Register PartialSrc = TII->buildExtractSubReg(
1173 MI: Result, MRI&: *MRI, SuperReg: MI->getOperand(i: 1), SuperRC: SrcRC,
1174 SubIdx: TRI->getSubRegFromChannel(Channel: i), SubRC: &AMDGPU::VGPR_32RegClass);
1175 Register PartialDst =
1176 MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
1177 BuildMI(BB&: *MBB, I&: *Result, MIMD: Result->getDebugLoc(),
1178 MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: PartialDst)
1179 .addReg(RegNo: PartialSrc);
1180 Result.addReg(RegNo: PartialDst).addImm(Val: TRI->getSubRegFromChannel(Channel: i));
1181 }
1182 }
1183 MI->eraseFromParent();
1184 }
1185}
1186
1187void SIFixSGPRCopies::fixSCCCopies(MachineFunction &MF) {
1188 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1189 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
1190 // S_CMP_LG_U64 is only available on GFX8+. S_CMP_LG_U32 always is, and
1191 // wave32 implies GFX10+ anyway.
1192 bool HasCmp = ST.isWave32() || ST.hasScalarCompareEq64();
1193 for (MachineBasicBlock &MBB : MF) {
1194 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E;
1195 ++I) {
1196 MachineInstr &MI = *I;
1197 // May already have been lowered.
1198 if (!MI.isCopy())
1199 continue;
1200 const MachineOperand &Src = MI.getOperand(i: 1);
1201 Register SrcReg = Src.getReg();
1202 Register DstReg = MI.getOperand(i: 0).getReg();
1203 if (SrcReg == AMDGPU::SCC) {
1204 Register SCCCopy =
1205 MRI->createVirtualRegister(RegClass: TRI->getWaveMaskRegClass());
1206 I = BuildMI(BB&: *MI.getParent(), I: std::next(x: MachineBasicBlock::iterator(MI)),
1207 MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: LMC.CSelectOpc), DestReg: SCCCopy)
1208 .addImm(Val: -1)
1209 .addImm(Val: 0);
1210 I = BuildMI(BB&: *MI.getParent(), I: std::next(x: I), MIMD: I->getDebugLoc(),
1211 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: DstReg)
1212 .addReg(RegNo: SCCCopy);
1213 MI.eraseFromParent();
1214 continue;
1215 }
1216 if (DstReg == AMDGPU::SCC) {
1217 MachineBasicBlock::iterator InsPt =
1218 std::next(x: MachineBasicBlock::iterator(MI));
1219 if (HasCmp && !Src.getSubReg() &&
1220 TII->isMaskedByExec(Reg: SrcReg, Use: MI, MRI: *MRI)) {
1221 // The source already has 0 in the bits of all inactive lanes, so
1222 // SCC is just "source is non-zero". S_CMP computes that without
1223 // needing a destination register.
1224 I = BuildMI(BB&: *MI.getParent(), I: InsPt, MIMD: MI.getDebugLoc(),
1225 MCID: TII->get(Opcode: LMC.CmpLgOpc))
1226 .add(MO: Src)
1227 .addImm(Val: 0);
1228 } else {
1229 Register Tmp = MRI->createVirtualRegister(RegClass: TRI->getBoolRC());
1230 I = BuildMI(BB&: *MI.getParent(), I: InsPt, MIMD: MI.getDebugLoc(),
1231 MCID: TII->get(Opcode: LMC.AndOpc))
1232 .addReg(RegNo: Tmp, Flags: getDefRegState(B: true))
1233 .add(MO: Src)
1234 .addReg(RegNo: LMC.ExecReg);
1235 }
1236 MI.eraseFromParent();
1237 }
1238 }
1239 }
1240}
1241
1242PreservedAnalyses
1243SIFixSGPRCopiesPass::run(MachineFunction &MF,
1244 MachineFunctionAnalysisManager &MFAM) {
1245 MFPropsModifier _(*this, MF);
1246 MachineDominatorTree &MDT = MFAM.getResult<MachineDominatorTreeAnalysis>(IR&: MF);
1247 SIFixSGPRCopies Impl(&MDT);
1248 bool Changed = Impl.run(MF);
1249 if (!Changed)
1250 return PreservedAnalyses::all();
1251
1252 // TODO: We could detect CFG changed.
1253 auto PA = getMachineFunctionPassPreservedAnalyses();
1254 return PA;
1255}
1256