1//===- SIFixSGPRCopies.cpp - Remove potential VGPR => SGPR copies ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Copies from VGPR to SGPR registers are illegal and the register coalescer
11/// will sometimes generate these illegal copies in situations like this:
12///
13/// Register Class <vsrc> is the union of <vgpr> and <sgpr>
14///
15/// BB0:
16/// %0 <sgpr> = SCALAR_INST
17/// %1 <vsrc> = COPY %0 <sgpr>
18/// ...
19/// BRANCH %cond BB1, BB2
20/// BB1:
21/// %2 <vgpr> = VECTOR_INST
22/// %3 <vsrc> = COPY %2 <vgpr>
23/// BB2:
24/// %4 <vsrc> = PHI %1 <vsrc>, <%bb.0>, %3 <vrsc>, <%bb.1>
25/// %5 <vgpr> = VECTOR_INST %4 <vsrc>
26///
27///
28/// The coalescer will begin at BB0 and eliminate its copy, then the resulting
29/// code will look like this:
30///
31/// BB0:
32/// %0 <sgpr> = SCALAR_INST
33/// ...
34/// BRANCH %cond BB1, BB2
35/// BB1:
36/// %2 <vgpr> = VECTOR_INST
37/// %3 <vsrc> = COPY %2 <vgpr>
38/// BB2:
39/// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <vsrc>, <%bb.1>
40/// %5 <vgpr> = VECTOR_INST %4 <sgpr>
41///
42/// Now that the result of the PHI instruction is an SGPR, the register
43/// allocator is now forced to constrain the register class of %3 to
44/// <sgpr> so we end up with final code like this:
45///
46/// BB0:
47/// %0 <sgpr> = SCALAR_INST
48/// ...
49/// BRANCH %cond BB1, BB2
50/// BB1:
51/// %2 <vgpr> = VECTOR_INST
52/// %3 <sgpr> = COPY %2 <vgpr>
53/// BB2:
54/// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <sgpr>, <%bb.1>
55/// %5 <vgpr> = VECTOR_INST %4 <sgpr>
56///
57/// Now this code contains an illegal copy from a VGPR to an SGPR.
58///
59/// In order to avoid this problem, this pass searches for PHI instructions
60/// which define a <vsrc> register and constrains its definition class to
61/// <vgpr> if the user of the PHI's definition register is a vector instruction.
62/// If the PHI's definition class is constrained to <vgpr> then the coalescer
63/// will be unable to perform the COPY removal from the above example which
64/// ultimately led to the creation of an illegal COPY.
65//===----------------------------------------------------------------------===//
66
67#include "SIFixSGPRCopies.h"
68#include "AMDGPU.h"
69#include "AMDGPULaneMaskUtils.h"
70#include "GCNSubtarget.h"
71#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
72#include "llvm/CodeGen/MachineDominators.h"
73#include "llvm/InitializePasses.h"
74#include "llvm/Target/TargetMachine.h"
75
76using namespace llvm;
77
78#define DEBUG_TYPE "si-fix-sgpr-copies"
79
80static cl::opt<bool> EnableM0Merge(
81 "amdgpu-enable-merge-m0",
82 cl::desc("Merge and hoist M0 initializations"),
83 cl::init(Val: true));
84
85namespace {
86
87class V2SCopyInfo {
88public:
89 // VGPR to SGPR copy being processed
90 MachineInstr *Copy;
91 // All SALU instructions reachable from this copy in SSA graph
92 SetVector<MachineInstr *> SChain;
93 // Number of SGPR to VGPR copies that are used to put the SALU computation
94 // results back to VALU.
95 unsigned NumSVCopies = 0;
96
97 unsigned Score = 0;
98 // Actual count of v_readfirstlane_b32
99 // which need to be inserted to keep SChain SALU
100 unsigned NumReadfirstlanes = 0;
101 // Current score state. To speedup selection V2SCopyInfos for processing
102 bool NeedToBeConvertedToVALU = false;
103 // Unique ID. Used as a key for mapping to keep permanent order.
104 unsigned ID;
105
106 // Count of another VGPR to SGPR copies that contribute to the
107 // current copy SChain
108 unsigned SiblingPenalty = 0;
109 SetVector<unsigned> Siblings;
110 V2SCopyInfo() : Copy(nullptr), ID(0){};
111 V2SCopyInfo(unsigned Id, MachineInstr *C, unsigned Width)
112 : Copy(C), NumReadfirstlanes(Width / 32), ID(Id){};
113#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
114 void dump() const {
115 dbgs() << ID << " : " << *Copy << "\n\tS:" << SChain.size()
116 << "\n\tSV:" << NumSVCopies << "\n\tSP: " << SiblingPenalty
117 << "\nScore: " << Score << "\n";
118 }
119#endif
120};
121
122class SIFixSGPRCopies {
123 MachineDominatorTree *MDT;
124 SmallVector<MachineInstr*, 4> SCCCopies;
125 SmallVector<MachineInstr*, 4> RegSequences;
126 SmallVector<MachineInstr*, 4> PHINodes;
127 SmallVector<MachineInstr*, 4> S2VCopies;
128 unsigned NextVGPRToSGPRCopyID = 0;
129 MapVector<unsigned, V2SCopyInfo> V2SCopies;
130 DenseMap<MachineInstr *, SetVector<unsigned>> SiblingPenalty;
131 DenseSet<MachineInstr *> PHISources;
132
133public:
134 MachineRegisterInfo *MRI;
135 const SIRegisterInfo *TRI;
136 const SIInstrInfo *TII;
137
138 SIFixSGPRCopies(MachineDominatorTree *MDT) : MDT(MDT) {}
139
140 bool run(MachineFunction &MF);
141 void fixSCCCopies(MachineFunction &MF);
142 void prepareRegSequenceAndPHIs(MachineFunction &MF);
143 unsigned getNextVGPRToSGPRCopyId() { return ++NextVGPRToSGPRCopyID; }
144 bool needToBeConvertedToVALU(V2SCopyInfo *I);
145 void analyzeVGPRToSGPRCopy(MachineInstr *MI);
146 void lowerVGPR2SGPRCopies(MachineFunction &MF);
147 // Handles copies which source register is:
148 // 1. Physical register
149 // 2. AGPR
150 // 3. Defined by the instruction the merely moves the immediate
151 bool lowerSpecialCase(MachineInstr &MI, MachineBasicBlock::iterator &I);
152
153 void processPHINode(MachineInstr &MI);
154
155 // Check if MO is an immediate materialized into a VGPR, and if so replace it
156 // with an SGPR immediate. The VGPR immediate is also deleted if it does not
157 // have any other uses.
158 bool tryMoveVGPRConstToSGPR(MachineOperand &MO, Register NewDst,
159 MachineBasicBlock *BlockToInsertTo,
160 MachineBasicBlock::iterator PointToInsertTo,
161 const DebugLoc &DL);
162};
163
164class SIFixSGPRCopiesLegacy : public MachineFunctionPass {
165public:
166 static char ID;
167
168 SIFixSGPRCopiesLegacy() : MachineFunctionPass(ID) {}
169
170 bool runOnMachineFunction(MachineFunction &MF) override {
171 MachineDominatorTree *MDT =
172 &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
173 SIFixSGPRCopies Impl(MDT);
174 return Impl.run(MF);
175 }
176
177 StringRef getPassName() const override { return "SI Fix SGPR copies"; }
178
179 void getAnalysisUsage(AnalysisUsage &AU) const override {
180 AU.addRequired<MachineDominatorTreeWrapperPass>();
181 AU.addPreserved<MachineDominatorTreeWrapperPass>();
182 AU.setPreservesCFG();
183 MachineFunctionPass::getAnalysisUsage(AU);
184 }
185
186 // Waterfall expansion may introduce Phi nodes and -verify-machineinstrs will
187 // fail.
188 MachineFunctionProperties getClearedProperties() const override {
189 return MachineFunctionProperties().setNoPHIs();
190 }
191};
192
193} // end anonymous namespace
194
195INITIALIZE_PASS_BEGIN(SIFixSGPRCopiesLegacy, DEBUG_TYPE, "SI Fix SGPR copies",
196 false, false)
197INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
198INITIALIZE_PASS_END(SIFixSGPRCopiesLegacy, DEBUG_TYPE, "SI Fix SGPR copies",
199 false, false)
200
201char SIFixSGPRCopiesLegacy::ID = 0;
202
203char &llvm::SIFixSGPRCopiesLegacyID = SIFixSGPRCopiesLegacy::ID;
204
205FunctionPass *llvm::createSIFixSGPRCopiesLegacyPass() {
206 return new SIFixSGPRCopiesLegacy();
207}
208
209static std::pair<const TargetRegisterClass *, const TargetRegisterClass *>
210getCopyRegClasses(const MachineInstr &Copy,
211 const SIRegisterInfo &TRI,
212 const MachineRegisterInfo &MRI) {
213 Register DstReg = Copy.getOperand(i: 0).getReg();
214 Register SrcReg = Copy.getOperand(i: 1).getReg();
215
216 const TargetRegisterClass *SrcRC = SrcReg.isVirtual()
217 ? MRI.getRegClass(Reg: SrcReg)
218 : TRI.getPhysRegBaseClass(Reg: SrcReg);
219
220 // We don't really care about the subregister here.
221 // SrcRC = TRI.getSubRegClass(SrcRC, Copy.getOperand(1).getSubReg());
222
223 const TargetRegisterClass *DstRC = DstReg.isVirtual()
224 ? MRI.getRegClass(Reg: DstReg)
225 : TRI.getPhysRegBaseClass(Reg: DstReg);
226
227 return std::pair(SrcRC, DstRC);
228}
229
230static bool isVGPRToSGPRCopy(const TargetRegisterClass *SrcRC,
231 const TargetRegisterClass *DstRC,
232 const SIRegisterInfo &TRI) {
233 return SrcRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(RC: DstRC) &&
234 TRI.hasVectorRegisters(RC: SrcRC);
235}
236
237static bool isSGPRToVGPRCopy(const TargetRegisterClass *SrcRC,
238 const TargetRegisterClass *DstRC,
239 const SIRegisterInfo &TRI) {
240 return DstRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(RC: SrcRC) &&
241 TRI.hasVectorRegisters(RC: DstRC);
242}
243
244static bool tryChangeVGPRtoSGPRinCopy(MachineInstr &MI,
245 const SIRegisterInfo *TRI,
246 const SIInstrInfo *TII) {
247 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
248 auto &Src = MI.getOperand(i: 1);
249 Register DstReg = MI.getOperand(i: 0).getReg();
250 Register SrcReg = Src.getReg();
251 if (!SrcReg.isVirtual() || !DstReg.isVirtual())
252 return false;
253
254 for (const auto &MO : MRI.reg_nodbg_operands(Reg: DstReg)) {
255 const auto *UseMI = MO.getParent();
256 if (UseMI == &MI)
257 continue;
258 if (MO.isDef() || UseMI->getParent() != MI.getParent() ||
259 UseMI->getOpcode() <= TargetOpcode::GENERIC_OP_END)
260 return false;
261
262 unsigned OpIdx = MO.getOperandNo();
263 if (OpIdx >= UseMI->getDesc().getNumOperands() ||
264 !TII->isOperandLegal(MI: *UseMI, OpIdx, MO: &Src))
265 return false;
266 }
267 // Change VGPR to SGPR destination.
268 MRI.setRegClass(Reg: DstReg, RC: TRI->getEquivalentSGPRClass(VRC: MRI.getRegClass(Reg: DstReg)));
269 return true;
270}
271
272// Distribute an SGPR->VGPR copy of a REG_SEQUENCE into a VGPR REG_SEQUENCE.
273//
274// SGPRx = ...
275// SGPRy = REG_SEQUENCE SGPRx, sub0 ...
276// VGPRz = COPY SGPRy
277//
278// ==>
279//
280// VGPRx = COPY SGPRx
281// VGPRz = REG_SEQUENCE VGPRx, sub0
282//
283// This exposes immediate folding opportunities when materializing 64-bit
284// immediates.
285static bool foldVGPRCopyIntoRegSequence(MachineInstr &MI,
286 const SIRegisterInfo *TRI,
287 const SIInstrInfo *TII,
288 MachineRegisterInfo &MRI) {
289 assert(MI.isRegSequence());
290
291 Register DstReg = MI.getOperand(i: 0).getReg();
292 if (!TRI->isSGPRClass(RC: MRI.getRegClass(Reg: DstReg)))
293 return false;
294
295 if (!MRI.hasOneUse(RegNo: DstReg))
296 return false;
297
298 MachineInstr &CopyUse = *MRI.use_instr_begin(RegNo: DstReg);
299 if (!CopyUse.isCopy())
300 return false;
301
302 // It is illegal to have vreg inputs to a physreg defining reg_sequence.
303 if (CopyUse.getOperand(i: 0).getReg().isPhysical())
304 return false;
305
306 const TargetRegisterClass *SrcRC, *DstRC;
307 std::tie(args&: SrcRC, args&: DstRC) = getCopyRegClasses(Copy: CopyUse, TRI: *TRI, MRI);
308
309 if (!isSGPRToVGPRCopy(SrcRC, DstRC, TRI: *TRI))
310 return false;
311
312 if (tryChangeVGPRtoSGPRinCopy(MI&: CopyUse, TRI, TII))
313 return true;
314
315 // TODO: Could have multiple extracts?
316 unsigned SubReg = CopyUse.getOperand(i: 1).getSubReg();
317 if (SubReg != AMDGPU::NoSubRegister)
318 return false;
319
320 MRI.setRegClass(Reg: DstReg, RC: DstRC);
321
322 // SGPRx = ...
323 // SGPRy = REG_SEQUENCE SGPRx, sub0 ...
324 // VGPRz = COPY SGPRy
325
326 // =>
327 // VGPRx = COPY SGPRx
328 // VGPRz = REG_SEQUENCE VGPRx, sub0
329
330 MI.getOperand(i: 0).setReg(CopyUse.getOperand(i: 0).getReg());
331 bool IsAGPR = TRI->isAGPRClass(RC: DstRC);
332
333 for (unsigned I = 1, N = MI.getNumOperands(); I != N; I += 2) {
334 const TargetRegisterClass *SrcRC =
335 TRI->getRegClassForOperandReg(MRI, MO: MI.getOperand(i: I));
336 assert(TRI->isSGPRClass(SrcRC) &&
337 "Expected SGPR REG_SEQUENCE to only have SGPR inputs");
338 const TargetRegisterClass *NewSrcRC = TRI->getEquivalentVGPRClass(SRC: SrcRC);
339
340 Register TmpReg = MRI.createVirtualRegister(RegClass: NewSrcRC);
341
342 BuildMI(BB&: *MI.getParent(), I: &MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY),
343 DestReg: TmpReg)
344 .add(MO: MI.getOperand(i: I));
345
346 if (IsAGPR) {
347 const TargetRegisterClass *NewSrcRC = TRI->getEquivalentAGPRClass(SRC: SrcRC);
348 Register TmpAReg = MRI.createVirtualRegister(RegClass: NewSrcRC);
349 unsigned Opc = NewSrcRC == &AMDGPU::AGPR_32RegClass ?
350 AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::COPY;
351 BuildMI(BB&: *MI.getParent(), I: &MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc),
352 DestReg: TmpAReg)
353 .addReg(RegNo: TmpReg, Flags: RegState::Kill);
354 TmpReg = TmpAReg;
355 }
356
357 MI.getOperand(i: I).setReg(TmpReg);
358 }
359
360 CopyUse.eraseFromParent();
361 return true;
362}
363
364static bool isSafeToFoldImmIntoCopy(const MachineInstr *Copy,
365 const MachineInstr *MoveImm,
366 const SIInstrInfo *TII,
367 unsigned &SMovOp,
368 int64_t &Imm) {
369 if (Copy->getOpcode() != AMDGPU::COPY)
370 return false;
371
372 if (!MoveImm->isMoveImmediate())
373 return false;
374
375 const MachineOperand *ImmOp =
376 TII->getNamedOperand(MI: *MoveImm, OperandName: AMDGPU::OpName::src0);
377 if (!ImmOp->isImm())
378 return false;
379
380 // FIXME: Handle copies with sub-regs.
381 if (Copy->getOperand(i: 1).getSubReg())
382 return false;
383
384 switch (MoveImm->getOpcode()) {
385 default:
386 return false;
387 case AMDGPU::V_MOV_B32_e32:
388 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
389 SMovOp = AMDGPU::S_MOV_B32;
390 break;
391 case AMDGPU::V_MOV_B64_PSEUDO:
392 SMovOp = AMDGPU::S_MOV_B64_IMM_PSEUDO;
393 break;
394 }
395 Imm = ImmOp->getImm();
396 return true;
397}
398
399template <class UnaryPredicate>
400bool searchPredecessors(const MachineBasicBlock *MBB,
401 const MachineBasicBlock *CutOff,
402 UnaryPredicate Predicate) {
403 if (MBB == CutOff)
404 return false;
405
406 DenseSet<const MachineBasicBlock *> Visited;
407 SmallVector<MachineBasicBlock *, 4> Worklist(MBB->predecessors());
408
409 while (!Worklist.empty()) {
410 MachineBasicBlock *MBB = Worklist.pop_back_val();
411
412 if (!Visited.insert(V: MBB).second)
413 continue;
414 if (MBB == CutOff)
415 continue;
416 if (Predicate(MBB))
417 return true;
418
419 Worklist.append(in_start: MBB->pred_begin(), in_end: MBB->pred_end());
420 }
421
422 return false;
423}
424
425// Checks if there is potential path From instruction To instruction.
426// If CutOff is specified and it sits in between of that path we ignore
427// a higher portion of the path and report it is not reachable.
428static bool isReachable(const MachineInstr *From,
429 const MachineInstr *To,
430 const MachineBasicBlock *CutOff,
431 MachineDominatorTree &MDT) {
432 if (MDT.dominates(A: From, B: To))
433 return true;
434
435 const MachineBasicBlock *MBBFrom = From->getParent();
436 const MachineBasicBlock *MBBTo = To->getParent();
437
438 // Do predecessor search.
439 // We should almost never get here since we do not usually produce M0 stores
440 // other than -1.
441 return searchPredecessors(MBB: MBBTo, CutOff, Predicate: [MBBFrom]
442 (const MachineBasicBlock *MBB) { return MBB == MBBFrom; });
443}
444
445// Return the first non-prologue instruction in the block.
446static MachineBasicBlock::iterator
447getFirstNonPrologue(MachineBasicBlock *MBB, const TargetInstrInfo *TII) {
448 MachineBasicBlock::iterator I = MBB->getFirstNonPHI();
449 while (I != MBB->end() && TII->isBasicBlockPrologue(MI: *I))
450 ++I;
451
452 return I;
453}
454
455// Hoist and merge identical SGPR initializations into a common predecessor.
456// This is intended to combine M0 initializations, but can work with any
457// SGPR. A VGPR cannot be processed since we cannot guarantee vector
458// executioon.
459static bool hoistAndMergeSGPRInits(unsigned Reg,
460 const MachineRegisterInfo &MRI,
461 const TargetRegisterInfo *TRI,
462 MachineDominatorTree &MDT,
463 const TargetInstrInfo *TII) {
464 // List of inits by immediate value.
465 using InitListMap = std::map<unsigned, std::list<MachineInstr *>>;
466 InitListMap Inits;
467 // List of clobbering instructions.
468 SmallVector<MachineInstr*, 8> Clobbers;
469 // List of instructions marked for deletion.
470 SmallPtrSet<MachineInstr *, 8> MergedInstrs;
471
472 bool Changed = false;
473
474 for (auto &MI : MRI.def_instructions(Reg)) {
475 MachineOperand *Imm = nullptr;
476 for (auto &MO : MI.operands()) {
477 if ((MO.isReg() && ((MO.isDef() && MO.getReg() != Reg) || !MO.isDef())) ||
478 (!MO.isImm() && !MO.isReg()) || (MO.isImm() && Imm)) {
479 Imm = nullptr;
480 break;
481 }
482 if (MO.isImm())
483 Imm = &MO;
484 }
485 if (Imm)
486 Inits[Imm->getImm()].push_front(x: &MI);
487 else
488 Clobbers.push_back(Elt: &MI);
489 }
490
491 for (auto &Init : Inits) {
492 auto &Defs = Init.second;
493
494 for (auto I1 = Defs.begin(), E = Defs.end(); I1 != E; ) {
495 MachineInstr *MI1 = *I1;
496
497 for (auto I2 = std::next(x: I1); I2 != E; ) {
498 MachineInstr *MI2 = *I2;
499
500 // Check any possible interference
501 auto interferes = [&](MachineBasicBlock::iterator From,
502 MachineBasicBlock::iterator To) -> bool {
503
504 assert(MDT.dominates(&*To, &*From));
505
506 auto interferes = [&MDT, From, To](MachineInstr* &Clobber) -> bool {
507 const MachineBasicBlock *MBBFrom = From->getParent();
508 const MachineBasicBlock *MBBTo = To->getParent();
509 bool MayClobberFrom = isReachable(From: Clobber, To: &*From, CutOff: MBBTo, MDT);
510 bool MayClobberTo = isReachable(From: Clobber, To: &*To, CutOff: MBBTo, MDT);
511 if (!MayClobberFrom && !MayClobberTo)
512 return false;
513 if ((MayClobberFrom && !MayClobberTo) ||
514 (!MayClobberFrom && MayClobberTo))
515 return true;
516 // Both can clobber, this is not an interference only if both are
517 // dominated by Clobber and belong to the same block or if Clobber
518 // properly dominates To, given that To >> From, so it dominates
519 // both and located in a common dominator.
520 return !((MBBFrom == MBBTo &&
521 MDT.dominates(A: Clobber, B: &*From) &&
522 MDT.dominates(A: Clobber, B: &*To)) ||
523 MDT.properlyDominates(A: Clobber->getParent(), B: MBBTo));
524 };
525
526 return (llvm::any_of(Range&: Clobbers, P: interferes)) ||
527 (llvm::any_of(Range&: Inits, P: [&](InitListMap::value_type &C) {
528 return C.first != Init.first &&
529 llvm::any_of(Range&: C.second, P: interferes);
530 }));
531 };
532
533 if (MDT.dominates(A: MI1, B: MI2)) {
534 if (!interferes(MI2, MI1)) {
535 LLVM_DEBUG(dbgs()
536 << "Erasing from "
537 << printMBBReference(*MI2->getParent()) << " " << *MI2);
538 MergedInstrs.insert(Ptr: MI2);
539 Changed = true;
540 ++I2;
541 continue;
542 }
543 } else if (MDT.dominates(A: MI2, B: MI1)) {
544 if (!interferes(MI1, MI2)) {
545 LLVM_DEBUG(dbgs()
546 << "Erasing from "
547 << printMBBReference(*MI1->getParent()) << " " << *MI1);
548 MergedInstrs.insert(Ptr: MI1);
549 Changed = true;
550 ++I1;
551 break;
552 }
553 } else {
554 auto *MBB = MDT.findNearestCommonDominator(A: MI1->getParent(),
555 B: MI2->getParent());
556 if (!MBB) {
557 ++I2;
558 continue;
559 }
560
561 MachineBasicBlock::iterator I = getFirstNonPrologue(MBB, TII);
562 if (!interferes(MI1, I) && !interferes(MI2, I)) {
563 LLVM_DEBUG(dbgs()
564 << "Erasing from "
565 << printMBBReference(*MI1->getParent()) << " " << *MI1
566 << "and moving from "
567 << printMBBReference(*MI2->getParent()) << " to "
568 << printMBBReference(*I->getParent()) << " " << *MI2);
569 I->getParent()->splice(Where: I, Other: MI2->getParent(), From: MI2);
570 MergedInstrs.insert(Ptr: MI1);
571 Changed = true;
572 ++I1;
573 break;
574 }
575 }
576 ++I2;
577 }
578 ++I1;
579 }
580 }
581
582 // Remove initializations that were merged into another.
583 for (auto &Init : Inits) {
584 auto &Defs = Init.second;
585 auto I = Defs.begin();
586 while (I != Defs.end()) {
587 if (MergedInstrs.count(Ptr: *I)) {
588 (*I)->eraseFromParent();
589 I = Defs.erase(position: I);
590 } else
591 ++I;
592 }
593 }
594
595 // Try to schedule SGPR initializations as early as possible in the MBB.
596 for (auto &Init : Inits) {
597 auto &Defs = Init.second;
598 for (auto *MI : Defs) {
599 auto *MBB = MI->getParent();
600 MachineInstr &BoundaryMI = *getFirstNonPrologue(MBB, TII);
601 MachineBasicBlock::reverse_iterator B(BoundaryMI);
602 // Check if B should actually be a boundary. If not set the previous
603 // instruction as the boundary instead.
604 if (!TII->isBasicBlockPrologue(MI: *B))
605 B++;
606
607 auto R = std::next(x: MI->getReverseIterator());
608 const unsigned Threshold = 50;
609 // Search until B or Threshold for a place to insert the initialization.
610 for (unsigned I = 0; R != B && I < Threshold; ++R, ++I)
611 if (R->readsRegister(Reg, TRI) || R->definesRegister(Reg, TRI) ||
612 TII->isSchedulingBoundary(MI: *R, MBB, MF: *MBB->getParent()))
613 break;
614
615 // Move to directly after R.
616 if (&*--R != MI)
617 MBB->splice(Where: *R, Other: MBB, From: MI);
618 }
619 }
620
621 if (Changed)
622 MRI.clearKillFlags(Reg);
623
624 return Changed;
625}
626
627bool SIFixSGPRCopies::run(MachineFunction &MF) {
628 // Only need to run this in SelectionDAG path.
629 if (MF.getProperties().hasSelected())
630 return false;
631
632 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
633 MRI = &MF.getRegInfo();
634 TRI = ST.getRegisterInfo();
635 TII = ST.getInstrInfo();
636
637 // Instructions to re-legalize after changing register classes
638 SmallVector<MachineInstr *, 8> Relegalize;
639
640 for (MachineBasicBlock &MBB : MF) {
641 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E;
642 ++I) {
643 MachineInstr &MI = *I;
644
645 switch (MI.getOpcode()) {
646 default:
647 // scale_src has a register class restricted to low 256 VGPRs, changing
648 // registers to VGPR may not take it into acount.
649 if (TII->isWMMA(MI) &&
650 AMDGPU::hasNamedOperand(Opcode: MI.getOpcode(), NamedIdx: AMDGPU::OpName::scale_src0))
651 Relegalize.push_back(Elt: &MI);
652 continue;
653 case AMDGPU::COPY: {
654 const TargetRegisterClass *SrcRC, *DstRC;
655 std::tie(args&: SrcRC, args&: DstRC) = getCopyRegClasses(Copy: MI, TRI: *TRI, MRI: *MRI);
656
657 if (isSGPRToVGPRCopy(SrcRC, DstRC, TRI: *TRI)) {
658 // Since VGPR to SGPR copies affect VGPR to SGPR copy
659 // score and, hence the lowering decision, let's try to get rid of
660 // them as early as possible
661 if (tryChangeVGPRtoSGPRinCopy(MI, TRI, TII))
662 continue;
663
664 // Collect those not changed to try them after VGPR to SGPR copies
665 // lowering as there will be more opportunities.
666 S2VCopies.push_back(Elt: &MI);
667 }
668 if (!isVGPRToSGPRCopy(SrcRC, DstRC, TRI: *TRI))
669 continue;
670 if (lowerSpecialCase(MI, I))
671 continue;
672
673 analyzeVGPRToSGPRCopy(MI: &MI);
674
675 break;
676 }
677 case AMDGPU::WQM:
678 case AMDGPU::STRICT_WQM:
679 case AMDGPU::SOFT_WQM:
680 case AMDGPU::STRICT_WWM:
681 case AMDGPU::INSERT_SUBREG:
682 case AMDGPU::PHI:
683 case AMDGPU::REG_SEQUENCE: {
684 if (TRI->isSGPRClass(RC: TII->getOpRegClass(MI, OpNo: 0))) {
685 for (MachineOperand &MO : MI.operands()) {
686 if (!MO.isReg() || !MO.getReg().isVirtual())
687 continue;
688 const TargetRegisterClass *SrcRC = MRI->getRegClass(Reg: MO.getReg());
689 if (SrcRC == &AMDGPU::VReg_1RegClass)
690 continue;
691
692 if (TRI->hasVectorRegisters(RC: SrcRC)) {
693 const TargetRegisterClass *DestRC =
694 TRI->getEquivalentSGPRClass(VRC: SrcRC);
695 Register NewDst = MRI->createVirtualRegister(RegClass: DestRC);
696 MachineBasicBlock *BlockToInsertCopy =
697 MI.isPHI() ? MI.getOperand(i: MO.getOperandNo() + 1).getMBB()
698 : &MBB;
699 MachineBasicBlock::iterator PointToInsertCopy =
700 MI.isPHI() ? BlockToInsertCopy->getFirstInstrTerminator() : I;
701
702 const DebugLoc &DL = MI.getDebugLoc();
703 if (!tryMoveVGPRConstToSGPR(MO, NewDst, BlockToInsertTo: BlockToInsertCopy,
704 PointToInsertTo: PointToInsertCopy, DL)) {
705 MachineInstr *NewCopy =
706 BuildMI(BB&: *BlockToInsertCopy, I: PointToInsertCopy, MIMD: DL,
707 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: NewDst)
708 .addReg(RegNo: MO.getReg());
709 MO.setReg(NewDst);
710 analyzeVGPRToSGPRCopy(MI: NewCopy);
711 PHISources.insert(V: NewCopy);
712 }
713 }
714 }
715 }
716
717 if (MI.isPHI())
718 PHINodes.push_back(Elt: &MI);
719 else if (MI.isRegSequence())
720 RegSequences.push_back(Elt: &MI);
721
722 break;
723 }
724 case AMDGPU::V_WRITELANE_B32: {
725 // Some architectures allow more than one constant bus access without
726 // SGPR restriction
727 if (ST.getConstantBusLimit(Opcode: MI.getOpcode()) != 1)
728 break;
729
730 // Writelane is special in that it can use SGPR and M0 (which would
731 // normally count as using the constant bus twice - but in this case it
732 // is allowed since the lane selector doesn't count as a use of the
733 // constant bus). However, it is still required to abide by the 1 SGPR
734 // rule. Apply a fix here as we might have multiple SGPRs after
735 // legalizing VGPRs to SGPRs
736 int Src0Idx =
737 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src0);
738 int Src1Idx =
739 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src1);
740 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
741 MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
742
743 // Check to see if the instruction violates the 1 SGPR rule
744 if ((Src0.isReg() && TRI->isSGPRReg(MRI: *MRI, Reg: Src0.getReg()) &&
745 Src0.getReg() != AMDGPU::M0) &&
746 (Src1.isReg() && TRI->isSGPRReg(MRI: *MRI, Reg: Src1.getReg()) &&
747 Src1.getReg() != AMDGPU::M0)) {
748
749 // Check for trivially easy constant prop into one of the operands
750 // If this is the case then perform the operation now to resolve SGPR
751 // issue. If we don't do that here we will always insert a mov to m0
752 // that can't be resolved in later operand folding pass
753 bool Resolved = false;
754 for (MachineOperand *MO : {&Src0, &Src1}) {
755 if (MO->getReg().isVirtual()) {
756 MachineInstr *DefMI = MRI->getVRegDef(Reg: MO->getReg());
757 if (DefMI && TII->isFoldableCopy(MI: *DefMI)) {
758 const MachineOperand &Def = DefMI->getOperand(i: 0);
759 if (Def.isReg() &&
760 MO->getReg() == Def.getReg() &&
761 MO->getSubReg() == Def.getSubReg()) {
762 const MachineOperand &Copied = DefMI->getOperand(i: 1);
763 if (Copied.isImm() &&
764 TII->isInlineConstant(Imm: APInt(64, Copied.getImm(), true))) {
765 MO->ChangeToImmediate(ImmVal: Copied.getImm());
766 Resolved = true;
767 break;
768 }
769 }
770 }
771 }
772 }
773
774 if (!Resolved) {
775 // Haven't managed to resolve by replacing an SGPR with an immediate
776 // Move src1 to be in M0
777 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
778 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
779 .add(MO: Src1);
780 Src1.ChangeToRegister(Reg: AMDGPU::M0, isDef: false);
781 }
782 }
783 break;
784 }
785 }
786 }
787 }
788
789 lowerVGPR2SGPRCopies(MF);
790 // Postprocessing
791 fixSCCCopies(MF);
792 for (auto *MI : S2VCopies) {
793 // Check if it is still valid
794 if (MI->isCopy()) {
795 const TargetRegisterClass *SrcRC, *DstRC;
796 std::tie(args&: SrcRC, args&: DstRC) = getCopyRegClasses(Copy: *MI, TRI: *TRI, MRI: *MRI);
797 if (isSGPRToVGPRCopy(SrcRC, DstRC, TRI: *TRI))
798 tryChangeVGPRtoSGPRinCopy(MI&: *MI, TRI, TII);
799 }
800 }
801 for (auto *MI : RegSequences) {
802 // Check if it is still valid
803 if (MI->isRegSequence())
804 foldVGPRCopyIntoRegSequence(MI&: *MI, TRI, TII, MRI&: *MRI);
805 }
806 for (auto *MI : PHINodes) {
807 processPHINode(MI&: *MI);
808 }
809 while (!Relegalize.empty())
810 TII->legalizeOperands(MI&: *Relegalize.pop_back_val(), MDT);
811
812 if (MF.getTarget().getOptLevel() > CodeGenOptLevel::None && EnableM0Merge)
813 hoistAndMergeSGPRInits(Reg: AMDGPU::M0, MRI: *MRI, TRI, MDT&: *MDT, TII);
814
815 SiblingPenalty.clear();
816 V2SCopies.clear();
817 SCCCopies.clear();
818 RegSequences.clear();
819 PHINodes.clear();
820 S2VCopies.clear();
821 PHISources.clear();
822
823 return true;
824}
825
826void SIFixSGPRCopies::processPHINode(MachineInstr &MI) {
827 bool AllAGPRUses = true;
828 SetVector<const MachineInstr *> worklist;
829 SmallPtrSet<const MachineInstr *, 4> Visited;
830 SetVector<MachineInstr *> PHIOperands;
831 worklist.insert(X: &MI);
832 Visited.insert(Ptr: &MI);
833 // HACK to make MIR tests with no uses happy
834 bool HasUses = false;
835 while (!worklist.empty()) {
836 const MachineInstr *Instr = worklist.pop_back_val();
837 Register Reg = Instr->getOperand(i: 0).getReg();
838 for (const auto &Use : MRI->use_operands(Reg)) {
839 HasUses = true;
840 const MachineInstr *UseMI = Use.getParent();
841 AllAGPRUses &= (UseMI->isCopy() &&
842 TRI->isAGPR(MRI: *MRI, Reg: UseMI->getOperand(i: 0).getReg())) ||
843 TRI->isAGPR(MRI: *MRI, Reg: Use.getReg());
844 if (UseMI->isCopy() || UseMI->isRegSequence()) {
845 if (Visited.insert(Ptr: UseMI).second)
846 worklist.insert(X: UseMI);
847
848 continue;
849 }
850 }
851 }
852
853 Register PHIRes = MI.getOperand(i: 0).getReg();
854 const TargetRegisterClass *RC0 = MRI->getRegClass(Reg: PHIRes);
855 if (HasUses && AllAGPRUses && !TRI->isAGPRClass(RC: RC0)) {
856 LLVM_DEBUG(dbgs() << "Moving PHI to AGPR: " << MI);
857 MRI->setRegClass(Reg: PHIRes, RC: TRI->getEquivalentAGPRClass(SRC: RC0));
858 for (unsigned I = 1, N = MI.getNumOperands(); I != N; I += 2) {
859 MachineInstr *DefMI = MRI->getVRegDef(Reg: MI.getOperand(i: I).getReg());
860 if (DefMI && DefMI->isPHI())
861 PHIOperands.insert(X: DefMI);
862 }
863 }
864
865 if (TRI->hasVectorRegisters(RC: MRI->getRegClass(Reg: PHIRes)) ||
866 RC0 == &AMDGPU::VReg_1RegClass) {
867 LLVM_DEBUG(dbgs() << "Legalizing PHI: " << MI);
868 TII->legalizeOperands(MI, MDT);
869 }
870
871 // Propagate register class back to PHI operands which are PHI themselves.
872 while (!PHIOperands.empty()) {
873 processPHINode(MI&: *PHIOperands.pop_back_val());
874 }
875}
876
877bool SIFixSGPRCopies::tryMoveVGPRConstToSGPR(
878 MachineOperand &MaybeVGPRConstMO, Register DstReg,
879 MachineBasicBlock *BlockToInsertTo,
880 MachineBasicBlock::iterator PointToInsertTo, const DebugLoc &DL) {
881
882 MachineInstr *DefMI = MRI->getVRegDef(Reg: MaybeVGPRConstMO.getReg());
883 if (!DefMI || !DefMI->isMoveImmediate())
884 return false;
885
886 MachineOperand *SrcConst = TII->getNamedOperand(MI&: *DefMI, OperandName: AMDGPU::OpName::src0);
887 if (SrcConst->isReg())
888 return false;
889
890 const TargetRegisterClass *SrcRC =
891 MRI->getRegClass(Reg: MaybeVGPRConstMO.getReg());
892 unsigned MoveSize = TRI->getRegSizeInBits(RC: *SrcRC);
893 unsigned MoveOp =
894 MoveSize == 64 ? AMDGPU::S_MOV_B64_IMM_PSEUDO : AMDGPU::S_MOV_B32;
895 BuildMI(BB&: *BlockToInsertTo, I: PointToInsertTo, MIMD: DL, MCID: TII->get(Opcode: MoveOp), DestReg: DstReg)
896 .add(MO: *SrcConst);
897 if (MRI->hasOneUse(RegNo: MaybeVGPRConstMO.getReg()))
898 DefMI->eraseFromParent();
899 MaybeVGPRConstMO.setReg(DstReg);
900 return true;
901}
902
903bool SIFixSGPRCopies::lowerSpecialCase(MachineInstr &MI,
904 MachineBasicBlock::iterator &I) {
905 Register DstReg = MI.getOperand(i: 0).getReg();
906 Register SrcReg = MI.getOperand(i: 1).getReg();
907 if (!DstReg.isVirtual()) {
908 // If the destination register is a physical register there isn't
909 // really much we can do to fix this.
910 // Some special instructions use M0 as an input. Some even only use
911 // the first lane. Insert a readfirstlane and hope for the best.
912 const TargetRegisterClass *SrcRC = MRI->getRegClass(Reg: SrcReg);
913 if (DstReg == AMDGPU::M0 && TRI->hasVectorRegisters(RC: SrcRC)) {
914 Register TmpReg =
915 MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
916
917 const MCInstrDesc &ReadFirstLaneDesc =
918 TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32);
919 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: ReadFirstLaneDesc, DestReg: TmpReg)
920 .add(MO: MI.getOperand(i: 1));
921
922 unsigned SubReg = MI.getOperand(i: 1).getSubReg();
923 MI.getOperand(i: 1).setReg(TmpReg);
924 MI.getOperand(i: 1).setSubReg(AMDGPU::NoSubRegister);
925
926 const TargetRegisterClass *OpRC = TII->getRegClass(MCID: ReadFirstLaneDesc, OpNum: 1);
927 const TargetRegisterClass *ConstrainRC =
928 SubReg == AMDGPU::NoSubRegister
929 ? OpRC
930 : TRI->getMatchingSuperRegClass(A: SrcRC, B: OpRC, Idx: SubReg);
931
932 if (!MRI->constrainRegClass(Reg: SrcReg, RC: ConstrainRC))
933 llvm_unreachable("failed to constrain register");
934 return true;
935 }
936
937 if (tryMoveVGPRConstToSGPR(MaybeVGPRConstMO&: MI.getOperand(i: 1), DstReg, BlockToInsertTo: MI.getParent(), PointToInsertTo: MI,
938 DL: MI.getDebugLoc())) {
939 I = MI.eraseFromParent();
940 return true;
941 }
942
943 if (!SrcReg.isVirtual())
944 return true;
945 }
946 if (!SrcReg.isVirtual() || TRI->isAGPR(MRI: *MRI, Reg: SrcReg)) {
947 SIInstrWorklist worklist;
948 worklist.insert(MI: &MI);
949 TII->moveToVALU(Worklist&: worklist, MDT);
950 return true;
951 }
952
953 unsigned SMovOp;
954 int64_t Imm;
955 // If we are just copying an immediate, we can replace the copy with
956 // s_mov_b32.
957 if (isSafeToFoldImmIntoCopy(Copy: &MI, MoveImm: MRI->getVRegDef(Reg: SrcReg), TII, SMovOp, Imm)) {
958 MI.getOperand(i: 1).ChangeToImmediate(ImmVal: Imm);
959 MI.addImplicitDefUseOperands(MF&: *MI.getMF());
960 MI.setDesc(TII->get(Opcode: SMovOp));
961 return true;
962 }
963 return false;
964}
965
966void SIFixSGPRCopies::analyzeVGPRToSGPRCopy(MachineInstr* MI) {
967 if (PHISources.contains(V: MI))
968 return;
969 Register DstReg = MI->getOperand(i: 0).getReg();
970 const TargetRegisterClass *DstRC = TRI->getRegClassForReg(MRI: *MRI, Reg: DstReg);
971
972 V2SCopyInfo Info(getNextVGPRToSGPRCopyId(), MI,
973 TRI->getRegSizeInBits(RC: *DstRC));
974 SmallVector<MachineInstr *, 8> AnalysisWorklist;
975 // Needed because the SSA is not a tree but a graph and may have
976 // forks and joins. We should not then go same way twice.
977 DenseSet<MachineInstr *> Visited;
978 AnalysisWorklist.push_back(Elt: Info.Copy);
979 while (!AnalysisWorklist.empty()) {
980
981 MachineInstr *Inst = AnalysisWorklist.pop_back_val();
982
983 if (!Visited.insert(V: Inst).second)
984 continue;
985
986 // Copies and REG_SEQUENCE do not contribute to the final assembly
987 // So, skip them but take care of the SGPR to VGPR copies bookkeeping.
988 if (Inst->isRegSequence() &&
989 TRI->isVGPR(MRI: *MRI, Reg: Inst->getOperand(i: 0).getReg())) {
990 Info.NumSVCopies++;
991 continue;
992 }
993 if (Inst->isCopy()) {
994 const TargetRegisterClass *SrcRC, *DstRC;
995 std::tie(args&: SrcRC, args&: DstRC) = getCopyRegClasses(Copy: *Inst, TRI: *TRI, MRI: *MRI);
996 if (isSGPRToVGPRCopy(SrcRC, DstRC, TRI: *TRI) &&
997 !tryChangeVGPRtoSGPRinCopy(MI&: *Inst, TRI, TII)) {
998 Info.NumSVCopies++;
999 continue;
1000 }
1001 }
1002
1003 SiblingPenalty[Inst].insert(X: Info.ID);
1004
1005 SmallVector<MachineInstr *, 4> Users;
1006 if ((TII->isSALU(MI: *Inst) && Inst->isCompare()) ||
1007 (Inst->isCopy() && Inst->getOperand(i: 0).getReg() == AMDGPU::SCC)) {
1008 auto I = Inst->getIterator();
1009 auto E = Inst->getParent()->end();
1010 while (++I != E &&
1011 !I->findRegisterDefOperand(Reg: AMDGPU::SCC, /*TRI=*/nullptr)) {
1012 if (I->readsRegister(Reg: AMDGPU::SCC, /*TRI=*/nullptr))
1013 Users.push_back(Elt: &*I);
1014 }
1015 } else if (Inst->getNumExplicitDefs() != 0) {
1016 Register Reg = Inst->getOperand(i: 0).getReg();
1017 if (Reg.isVirtual() && TRI->isSGPRReg(MRI: *MRI, Reg) &&
1018 !TII->isVALU(MI: *Inst, /*AllowLDSDMA=*/true)) {
1019 for (auto &U : MRI->use_instructions(Reg))
1020 Users.push_back(Elt: &U);
1021 }
1022 }
1023 for (auto *U : Users) {
1024 if (TII->isSALU(MI: *U))
1025 Info.SChain.insert(X: U);
1026 AnalysisWorklist.push_back(Elt: U);
1027 }
1028 }
1029 V2SCopies[Info.ID] = std::move(Info);
1030}
1031
1032// The main function that computes the VGPR to SGPR copy score
1033// and determines copy further lowering way: v_readfirstlane_b32 or moveToVALU
1034bool SIFixSGPRCopies::needToBeConvertedToVALU(V2SCopyInfo *Info) {
1035 if (Info->SChain.empty()) {
1036 Info->Score = 0;
1037 return true;
1038 }
1039 Info->Siblings = SiblingPenalty[*llvm::max_element(
1040 Range&: Info->SChain, C: [&](MachineInstr *A, MachineInstr *B) -> bool {
1041 return SiblingPenalty[A].size() < SiblingPenalty[B].size();
1042 })];
1043 Info->Siblings.remove_if(P: [&](unsigned ID) { return ID == Info->ID; });
1044 // The loop below computes the number of another VGPR to SGPR V2SCopies
1045 // which contribute to the current copy SALU chain. We assume that all the
1046 // V2SCopies with the same source virtual register will be squashed to one
1047 // by regalloc. Also we take care of the V2SCopies of the differnt subregs
1048 // of the same register.
1049 SmallSet<std::pair<Register, unsigned>, 4> SrcRegs;
1050 for (auto J : Info->Siblings) {
1051 auto *InfoIt = V2SCopies.find(Key: J);
1052 if (InfoIt != V2SCopies.end()) {
1053 MachineInstr *SiblingCopy = InfoIt->second.Copy;
1054 if (SiblingCopy->isImplicitDef())
1055 // the COPY has already been MoveToVALUed
1056 continue;
1057
1058 SrcRegs.insert(V: std::pair(SiblingCopy->getOperand(i: 1).getReg(),
1059 SiblingCopy->getOperand(i: 1).getSubReg()));
1060 }
1061 }
1062 Info->SiblingPenalty = SrcRegs.size();
1063
1064 unsigned Penalty =
1065 Info->NumSVCopies + Info->SiblingPenalty + Info->NumReadfirstlanes;
1066 unsigned Profit = Info->SChain.size();
1067 Info->Score = Penalty > Profit ? 0 : Profit - Penalty;
1068 Info->NeedToBeConvertedToVALU = Info->Score < 3;
1069 return Info->NeedToBeConvertedToVALU;
1070}
1071
1072void SIFixSGPRCopies::lowerVGPR2SGPRCopies(MachineFunction &MF) {
1073
1074 SmallVector<unsigned, 8> LoweringWorklist;
1075 for (auto &C : V2SCopies) {
1076 if (needToBeConvertedToVALU(Info: &C.second))
1077 LoweringWorklist.push_back(Elt: C.second.ID);
1078 }
1079
1080 // Store all the V2S copy instructions that need to be moved to VALU
1081 // in the Copies worklist.
1082 SIInstrWorklist Copies;
1083
1084 while (!LoweringWorklist.empty()) {
1085 unsigned CurID = LoweringWorklist.pop_back_val();
1086 auto *CurInfoIt = V2SCopies.find(Key: CurID);
1087 if (CurInfoIt != V2SCopies.end()) {
1088 const V2SCopyInfo &C = CurInfoIt->second;
1089 LLVM_DEBUG(dbgs() << "Processing ...\n"; C.dump());
1090 for (auto S : C.Siblings) {
1091 auto *SibInfoIt = V2SCopies.find(Key: S);
1092 if (SibInfoIt != V2SCopies.end()) {
1093 V2SCopyInfo &SI = SibInfoIt->second;
1094 LLVM_DEBUG(dbgs() << "Sibling:\n"; SI.dump());
1095 if (!SI.NeedToBeConvertedToVALU) {
1096 SI.SChain.set_subtract(C.SChain);
1097 if (needToBeConvertedToVALU(Info: &SI))
1098 LoweringWorklist.push_back(Elt: SI.ID);
1099 }
1100 SI.Siblings.remove_if(P: [&](unsigned ID) { return ID == C.ID; });
1101 }
1102 }
1103 LLVM_DEBUG(dbgs() << "V2S copy " << *C.Copy
1104 << " is being turned to VALU\n");
1105 Copies.insert(MI: C.Copy);
1106 // TODO: MapVector::erase is inefficient. Do bulk removal with remove_if
1107 // instead.
1108 V2SCopies.erase(Key: C.ID);
1109 }
1110 }
1111
1112 TII->moveToVALU(Worklist&: Copies, MDT);
1113 Copies.clear();
1114
1115 // Now do actual lowering
1116 for (auto C : V2SCopies) {
1117 MachineInstr *MI = C.second.Copy;
1118 MachineBasicBlock *MBB = MI->getParent();
1119 // We decide to turn V2S copy to v_readfirstlane_b32
1120 // remove it from the V2SCopies and remove it from all its siblings
1121 LLVM_DEBUG(dbgs() << "V2S copy " << *MI
1122 << " is being turned to v_readfirstlane_b32"
1123 << " Score: " << C.second.Score << "\n");
1124 Register DstReg = MI->getOperand(i: 0).getReg();
1125 MRI->constrainRegClass(Reg: DstReg, RC: &AMDGPU::SReg_32_XM0RegClass);
1126
1127 Register SrcReg = MI->getOperand(i: 1).getReg();
1128 unsigned SubReg = MI->getOperand(i: 1).getSubReg();
1129 const TargetRegisterClass *SrcRC =
1130 TRI->getRegClassForOperandReg(MRI: *MRI, MO: MI->getOperand(i: 1));
1131 size_t SrcSize = TRI->getRegSizeInBits(RC: *SrcRC);
1132 if (SrcSize == 16) {
1133 assert(MF.getSubtarget<GCNSubtarget>().useRealTrue16Insts() &&
1134 "We do not expect to see 16-bit copies from VGPR to SGPR unless "
1135 "we have 16-bit VGPRs");
1136 assert(MRI->getRegClass(DstReg) == &AMDGPU::SReg_32RegClass ||
1137 MRI->getRegClass(DstReg) == &AMDGPU::SReg_32_XM0RegClass);
1138 // There is no V_READFIRSTLANE_B16, so legalize the dst/src reg to 32 bits
1139 MRI->setRegClass(Reg: DstReg, RC: &AMDGPU::SReg_32_XM0RegClass);
1140 Register VReg32 = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1141 const DebugLoc &DL = MI->getDebugLoc();
1142 Register Undef = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_16RegClass);
1143 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Undef);
1144 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: VReg32)
1145 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
1146 .addImm(Val: AMDGPU::lo16)
1147 .addReg(RegNo: Undef)
1148 .addImm(Val: AMDGPU::hi16);
1149 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
1150 .addReg(RegNo: VReg32);
1151 } else if (SrcSize == 32) {
1152 const MCInstrDesc &ReadFirstLaneDesc =
1153 TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32);
1154 const TargetRegisterClass *OpRC = TII->getRegClass(MCID: ReadFirstLaneDesc, OpNum: 1);
1155 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: ReadFirstLaneDesc, DestReg: DstReg)
1156 .addReg(RegNo: SrcReg, Flags: {}, SubReg);
1157
1158 const TargetRegisterClass *ConstrainRC =
1159 SubReg == AMDGPU::NoSubRegister
1160 ? OpRC
1161 : TRI->getMatchingSuperRegClass(A: MRI->getRegClass(Reg: SrcReg), B: OpRC,
1162 Idx: SubReg);
1163
1164 if (!MRI->constrainRegClass(Reg: SrcReg, RC: ConstrainRC))
1165 llvm_unreachable("failed to constrain register");
1166 } else {
1167 auto Result = BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(),
1168 MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
1169 int N = TRI->getRegSizeInBits(RC: *SrcRC) / 32;
1170 for (int i = 0; i < N; i++) {
1171 Register PartialSrc = TII->buildExtractSubReg(
1172 MI: Result, MRI&: *MRI, SuperReg: MI->getOperand(i: 1), SuperRC: SrcRC,
1173 SubIdx: TRI->getSubRegFromChannel(Channel: i), SubRC: &AMDGPU::VGPR_32RegClass);
1174 Register PartialDst =
1175 MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
1176 BuildMI(BB&: *MBB, I&: *Result, MIMD: Result->getDebugLoc(),
1177 MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: PartialDst)
1178 .addReg(RegNo: PartialSrc);
1179 Result.addReg(RegNo: PartialDst).addImm(Val: TRI->getSubRegFromChannel(Channel: i));
1180 }
1181 }
1182 MI->eraseFromParent();
1183 }
1184}
1185
1186void SIFixSGPRCopies::fixSCCCopies(MachineFunction &MF) {
1187 const AMDGPU::LaneMaskConstants &LMC =
1188 AMDGPU::LaneMaskConstants::get(ST: MF.getSubtarget<GCNSubtarget>());
1189 for (MachineBasicBlock &MBB : MF) {
1190 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E;
1191 ++I) {
1192 MachineInstr &MI = *I;
1193 // May already have been lowered.
1194 if (!MI.isCopy())
1195 continue;
1196 Register SrcReg = MI.getOperand(i: 1).getReg();
1197 Register DstReg = MI.getOperand(i: 0).getReg();
1198 if (SrcReg == AMDGPU::SCC) {
1199 Register SCCCopy =
1200 MRI->createVirtualRegister(RegClass: TRI->getWaveMaskRegClass());
1201 I = BuildMI(BB&: *MI.getParent(), I: std::next(x: MachineBasicBlock::iterator(MI)),
1202 MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: LMC.CSelectOpc), DestReg: SCCCopy)
1203 .addImm(Val: -1)
1204 .addImm(Val: 0);
1205 I = BuildMI(BB&: *MI.getParent(), I: std::next(x: I), MIMD: I->getDebugLoc(),
1206 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: DstReg)
1207 .addReg(RegNo: SCCCopy);
1208 MI.eraseFromParent();
1209 continue;
1210 }
1211 if (DstReg == AMDGPU::SCC) {
1212 Register Tmp = MRI->createVirtualRegister(RegClass: TRI->getBoolRC());
1213 I = BuildMI(BB&: *MI.getParent(), I: std::next(x: MachineBasicBlock::iterator(MI)),
1214 MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: LMC.AndOpc))
1215 .addReg(RegNo: Tmp, Flags: getDefRegState(B: true))
1216 .addReg(RegNo: SrcReg)
1217 .addReg(RegNo: LMC.ExecReg);
1218 MI.eraseFromParent();
1219 }
1220 }
1221 }
1222}
1223
1224PreservedAnalyses
1225SIFixSGPRCopiesPass::run(MachineFunction &MF,
1226 MachineFunctionAnalysisManager &MFAM) {
1227 MachineDominatorTree &MDT = MFAM.getResult<MachineDominatorTreeAnalysis>(IR&: MF);
1228 SIFixSGPRCopies Impl(&MDT);
1229 bool Changed = Impl.run(MF);
1230 if (!Changed)
1231 return PreservedAnalyses::all();
1232
1233 // TODO: We could detect CFG changed.
1234 auto PA = getMachineFunctionPassPreservedAnalyses();
1235 return PA;
1236}
1237