1//===- SIFixSGPRCopies.cpp - Remove potential VGPR => SGPR copies ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Copies from VGPR to SGPR registers are illegal and the register coalescer
11/// will sometimes generate these illegal copies in situations like this:
12///
13/// Register Class <vsrc> is the union of <vgpr> and <sgpr>
14///
15/// BB0:
16/// %0 <sgpr> = SCALAR_INST
17/// %1 <vsrc> = COPY %0 <sgpr>
18/// ...
19/// BRANCH %cond BB1, BB2
20/// BB1:
21/// %2 <vgpr> = VECTOR_INST
22/// %3 <vsrc> = COPY %2 <vgpr>
23/// BB2:
24/// %4 <vsrc> = PHI %1 <vsrc>, <%bb.0>, %3 <vrsc>, <%bb.1>
25/// %5 <vgpr> = VECTOR_INST %4 <vsrc>
26///
27///
28/// The coalescer will begin at BB0 and eliminate its copy, then the resulting
29/// code will look like this:
30///
31/// BB0:
32/// %0 <sgpr> = SCALAR_INST
33/// ...
34/// BRANCH %cond BB1, BB2
35/// BB1:
36/// %2 <vgpr> = VECTOR_INST
37/// %3 <vsrc> = COPY %2 <vgpr>
38/// BB2:
39/// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <vsrc>, <%bb.1>
40/// %5 <vgpr> = VECTOR_INST %4 <sgpr>
41///
42/// Now that the result of the PHI instruction is an SGPR, the register
43/// allocator is now forced to constrain the register class of %3 to
44/// <sgpr> so we end up with final code like this:
45///
46/// BB0:
47/// %0 <sgpr> = SCALAR_INST
48/// ...
49/// BRANCH %cond BB1, BB2
50/// BB1:
51/// %2 <vgpr> = VECTOR_INST
52/// %3 <sgpr> = COPY %2 <vgpr>
53/// BB2:
54/// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <sgpr>, <%bb.1>
55/// %5 <vgpr> = VECTOR_INST %4 <sgpr>
56///
57/// Now this code contains an illegal copy from a VGPR to an SGPR.
58///
59/// In order to avoid this problem, this pass searches for PHI instructions
60/// which define a <vsrc> register and constrains its definition class to
61/// <vgpr> if the user of the PHI's definition register is a vector instruction.
62/// If the PHI's definition class is constrained to <vgpr> then the coalescer
63/// will be unable to perform the COPY removal from the above example which
64/// ultimately led to the creation of an illegal COPY.
65//===----------------------------------------------------------------------===//
66
67#include "SIFixSGPRCopies.h"
68#include "AMDGPU.h"
69#include "AMDGPULaneMaskUtils.h"
70#include "GCNSubtarget.h"
71#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
72#include "llvm/CodeGen/MachineDominators.h"
73#include "llvm/InitializePasses.h"
74#include "llvm/Target/TargetMachine.h"
75
76using namespace llvm;
77
78#define DEBUG_TYPE "si-fix-sgpr-copies"
79
80static cl::opt<bool> EnableM0Merge(
81 "amdgpu-enable-merge-m0",
82 cl::desc("Merge and hoist M0 initializations"),
83 cl::init(Val: true));
84
85namespace {
86
87class V2SCopyInfo {
88public:
89 // VGPR to SGPR copy being processed
90 MachineInstr *Copy;
91 // All SALU instructions reachable from this copy in SSA graph
92 SetVector<MachineInstr *> SChain;
93 // Number of SGPR to VGPR copies that are used to put the SALU computation
94 // results back to VALU.
95 unsigned NumSVCopies = 0;
96
97 unsigned Score = 0;
98 // Actual count of v_readfirstlane_b32
99 // which need to be inserted to keep SChain SALU
100 unsigned NumReadfirstlanes = 0;
101 // Current score state. To speedup selection V2SCopyInfos for processing
102 bool NeedToBeConvertedToVALU = false;
103 // Marks entries lowered to VALU for bulk removal from V2SCopies.
104 bool Erased = false;
105 // Unique ID. Used as a key for mapping to keep permanent order.
106 unsigned ID;
107
108 // Count of another VGPR to SGPR copies that contribute to the
109 // current copy SChain
110 unsigned SiblingPenalty = 0;
111 SetVector<unsigned> Siblings;
112 V2SCopyInfo() : Copy(nullptr), ID(0){};
113 V2SCopyInfo(unsigned Id, MachineInstr *C, unsigned Width)
114 : Copy(C), NumReadfirstlanes(Width / 32), ID(Id){};
115#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
116 void dump() const {
117 dbgs() << ID << " : " << *Copy << "\n\tS:" << SChain.size()
118 << "\n\tSV:" << NumSVCopies << "\n\tSP: " << SiblingPenalty
119 << "\nScore: " << Score << "\n";
120 }
121#endif
122};
123
124class SIFixSGPRCopies {
125 MachineDominatorTree *MDT;
126 SmallVector<MachineInstr*, 4> SCCCopies;
127 SmallVector<MachineInstr*, 4> RegSequences;
128 SmallVector<MachineInstr*, 4> PHINodes;
129 SmallVector<MachineInstr*, 4> S2VCopies;
130 unsigned NextVGPRToSGPRCopyID = 0;
131 MapVector<unsigned, V2SCopyInfo> V2SCopies;
132 DenseMap<MachineInstr *, SetVector<unsigned>> SiblingPenalty;
133 DenseSet<MachineInstr *> PHISources;
134
135public:
136 MachineRegisterInfo *MRI;
137 const SIRegisterInfo *TRI;
138 const SIInstrInfo *TII;
139
140 SIFixSGPRCopies(MachineDominatorTree *MDT) : MDT(MDT) {}
141
142 bool run(MachineFunction &MF);
143 void fixSCCCopies(MachineFunction &MF);
144 void prepareRegSequenceAndPHIs(MachineFunction &MF);
145 unsigned getNextVGPRToSGPRCopyId() { return ++NextVGPRToSGPRCopyID; }
146 bool needToBeConvertedToVALU(V2SCopyInfo *I);
147 void analyzeVGPRToSGPRCopy(MachineInstr *MI);
148 void lowerVGPR2SGPRCopies(MachineFunction &MF);
149 // Handles copies which source register is:
150 // 1. Physical register
151 // 2. AGPR
152 // 3. Defined by the instruction the merely moves the immediate
153 bool lowerSpecialCase(MachineInstr &MI, MachineBasicBlock::iterator &I);
154
155 void processPHINode(MachineInstr &MI);
156
157 // Check if MO is an immediate materialized into a VGPR, and if so replace it
158 // with an SGPR immediate. The VGPR immediate is also deleted if it does not
159 // have any other uses.
160 bool tryMoveVGPRConstToSGPR(MachineOperand &MO, Register NewDst,
161 MachineBasicBlock *BlockToInsertTo,
162 MachineBasicBlock::iterator PointToInsertTo,
163 const DebugLoc &DL);
164};
165
166class SIFixSGPRCopiesLegacy : public MachineFunctionPass {
167public:
168 static char ID;
169
170 SIFixSGPRCopiesLegacy() : MachineFunctionPass(ID) {}
171
172 bool runOnMachineFunction(MachineFunction &MF) override {
173 MachineDominatorTree *MDT =
174 &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
175 SIFixSGPRCopies Impl(MDT);
176 return Impl.run(MF);
177 }
178
179 StringRef getPassName() const override { return "SI Fix SGPR copies"; }
180
181 void getAnalysisUsage(AnalysisUsage &AU) const override {
182 AU.addRequired<MachineDominatorTreeWrapperPass>();
183 AU.setPreservesCFG();
184 MachineFunctionPass::getAnalysisUsage(AU);
185 }
186
187 // Waterfall expansion may introduce Phi nodes and -verify-machineinstrs will
188 // fail.
189 MachineFunctionProperties getClearedProperties() const override {
190 return MachineFunctionProperties().setNoPHIs();
191 }
192};
193
194} // end anonymous namespace
195
196INITIALIZE_PASS_BEGIN(SIFixSGPRCopiesLegacy, DEBUG_TYPE, "SI Fix SGPR copies",
197 false, false)
198INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
199INITIALIZE_PASS_END(SIFixSGPRCopiesLegacy, DEBUG_TYPE, "SI Fix SGPR copies",
200 false, false)
201
202char SIFixSGPRCopiesLegacy::ID = 0;
203
204char &llvm::SIFixSGPRCopiesLegacyID = SIFixSGPRCopiesLegacy::ID;
205
206FunctionPass *llvm::createSIFixSGPRCopiesLegacyPass() {
207 return new SIFixSGPRCopiesLegacy();
208}
209
210static std::pair<const TargetRegisterClass *, const TargetRegisterClass *>
211getCopyRegClasses(const MachineInstr &Copy,
212 const SIRegisterInfo &TRI,
213 const MachineRegisterInfo &MRI) {
214 Register DstReg = Copy.getOperand(i: 0).getReg();
215 Register SrcReg = Copy.getOperand(i: 1).getReg();
216
217 const TargetRegisterClass *SrcRC = SrcReg.isVirtual()
218 ? MRI.getRegClass(Reg: SrcReg)
219 : TRI.getPhysRegBaseClass(Reg: SrcReg);
220
221 // We don't really care about the subregister here.
222 // SrcRC = TRI.getSubRegClass(SrcRC, Copy.getOperand(1).getSubReg());
223
224 const TargetRegisterClass *DstRC = DstReg.isVirtual()
225 ? MRI.getRegClass(Reg: DstReg)
226 : TRI.getPhysRegBaseClass(Reg: DstReg);
227
228 return std::pair(SrcRC, DstRC);
229}
230
231static bool isVGPRToSGPRCopy(const TargetRegisterClass *SrcRC,
232 const TargetRegisterClass *DstRC,
233 const SIRegisterInfo &TRI) {
234 return SrcRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(RC: DstRC) &&
235 TRI.hasVectorRegisters(RC: SrcRC);
236}
237
238static bool isSGPRToVGPRCopy(const TargetRegisterClass *SrcRC,
239 const TargetRegisterClass *DstRC,
240 const SIRegisterInfo &TRI) {
241 return DstRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(RC: SrcRC) &&
242 TRI.hasVectorRegisters(RC: DstRC);
243}
244
245static bool tryChangeVGPRtoSGPRinCopy(MachineInstr &MI,
246 const SIRegisterInfo *TRI,
247 const SIInstrInfo *TII) {
248 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
249 auto &Src = MI.getOperand(i: 1);
250 Register DstReg = MI.getOperand(i: 0).getReg();
251 Register SrcReg = Src.getReg();
252 if (!SrcReg.isVirtual() || !DstReg.isVirtual())
253 return false;
254
255 for (const auto &MO : MRI.reg_nodbg_operands(Reg: DstReg)) {
256 const auto *UseMI = MO.getParent();
257 if (UseMI == &MI)
258 continue;
259 if (MO.isDef() || UseMI->getParent() != MI.getParent() ||
260 UseMI->getOpcode() <= TargetOpcode::GENERIC_OP_END)
261 return false;
262
263 unsigned OpIdx = MO.getOperandNo();
264 if (OpIdx >= UseMI->getDesc().getNumOperands() ||
265 !TII->isOperandLegal(MI: *UseMI, OpIdx, MO: &Src))
266 return false;
267 }
268 // Change VGPR to SGPR destination.
269 MRI.setRegClass(Reg: DstReg, RC: TRI->getEquivalentSGPRClass(VRC: MRI.getRegClass(Reg: DstReg)));
270 return true;
271}
272
273// Distribute an SGPR->VGPR copy of a REG_SEQUENCE into a VGPR REG_SEQUENCE.
274//
275// SGPRx = ...
276// SGPRy = REG_SEQUENCE SGPRx, sub0 ...
277// VGPRz = COPY SGPRy
278//
279// ==>
280//
281// VGPRx = COPY SGPRx
282// VGPRz = REG_SEQUENCE VGPRx, sub0
283//
284// This exposes immediate folding opportunities when materializing 64-bit
285// immediates.
286static bool foldVGPRCopyIntoRegSequence(MachineInstr &MI,
287 const SIRegisterInfo *TRI,
288 const SIInstrInfo *TII,
289 MachineRegisterInfo &MRI) {
290 assert(MI.isRegSequence());
291
292 Register DstReg = MI.getOperand(i: 0).getReg();
293 if (!TRI->isSGPRClass(RC: MRI.getRegClass(Reg: DstReg)))
294 return false;
295
296 if (!MRI.hasOneUse(RegNo: DstReg))
297 return false;
298
299 MachineInstr &CopyUse = *MRI.use_instr_begin(RegNo: DstReg);
300 if (!CopyUse.isCopy())
301 return false;
302
303 // It is illegal to have vreg inputs to a physreg defining reg_sequence.
304 if (CopyUse.getOperand(i: 0).getReg().isPhysical())
305 return false;
306
307 const TargetRegisterClass *SrcRC, *DstRC;
308 std::tie(args&: SrcRC, args&: DstRC) = getCopyRegClasses(Copy: CopyUse, TRI: *TRI, MRI);
309
310 if (!isSGPRToVGPRCopy(SrcRC, DstRC, TRI: *TRI))
311 return false;
312
313 if (tryChangeVGPRtoSGPRinCopy(MI&: CopyUse, TRI, TII))
314 return true;
315
316 // TODO: Could have multiple extracts?
317 unsigned SubReg = CopyUse.getOperand(i: 1).getSubReg();
318 if (SubReg != AMDGPU::NoSubRegister)
319 return false;
320
321 MRI.setRegClass(Reg: DstReg, RC: DstRC);
322
323 // SGPRx = ...
324 // SGPRy = REG_SEQUENCE SGPRx, sub0 ...
325 // VGPRz = COPY SGPRy
326
327 // =>
328 // VGPRx = COPY SGPRx
329 // VGPRz = REG_SEQUENCE VGPRx, sub0
330
331 MI.getOperand(i: 0).setReg(CopyUse.getOperand(i: 0).getReg());
332 bool IsAGPR = TRI->isAGPRClass(RC: DstRC);
333
334 for (unsigned I = 1, N = MI.getNumOperands(); I != N; I += 2) {
335 const TargetRegisterClass *SrcRC =
336 TRI->getRegClassForOperandReg(MRI, MO: MI.getOperand(i: I));
337 assert(TRI->isSGPRClass(SrcRC) &&
338 "Expected SGPR REG_SEQUENCE to only have SGPR inputs");
339 const TargetRegisterClass *NewSrcRC = TRI->getEquivalentVGPRClass(SRC: SrcRC);
340
341 Register TmpReg = MRI.createVirtualRegister(RegClass: NewSrcRC);
342
343 BuildMI(BB&: *MI.getParent(), I: &MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY),
344 DestReg: TmpReg)
345 .add(MO: MI.getOperand(i: I));
346
347 if (IsAGPR) {
348 const TargetRegisterClass *NewSrcRC = TRI->getEquivalentAGPRClass(SRC: SrcRC);
349 Register TmpAReg = MRI.createVirtualRegister(RegClass: NewSrcRC);
350 unsigned Opc = NewSrcRC == &AMDGPU::AGPR_32RegClass ?
351 AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::COPY;
352 BuildMI(BB&: *MI.getParent(), I: &MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc),
353 DestReg: TmpAReg)
354 .addReg(RegNo: TmpReg, Flags: RegState::Kill);
355 TmpReg = TmpAReg;
356 }
357
358 MI.getOperand(i: I).setReg(TmpReg);
359 }
360
361 CopyUse.eraseFromParent();
362 return true;
363}
364
365static bool isSafeToFoldImmIntoCopy(const MachineInstr *Copy,
366 const MachineInstr *MoveImm,
367 const SIInstrInfo *TII,
368 unsigned &SMovOp,
369 int64_t &Imm) {
370 if (Copy->getOpcode() != AMDGPU::COPY)
371 return false;
372
373 if (!MoveImm || !MoveImm->isMoveImmediate())
374 return false;
375
376 const MachineOperand *ImmOp =
377 TII->getNamedOperand(MI: *MoveImm, OperandName: AMDGPU::OpName::src0);
378 if (!ImmOp->isImm())
379 return false;
380
381 // FIXME: Handle copies with sub-regs.
382 if (Copy->getOperand(i: 1).getSubReg())
383 return false;
384
385 switch (MoveImm->getOpcode()) {
386 default:
387 return false;
388 case AMDGPU::V_MOV_B32_e32:
389 case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
390 SMovOp = AMDGPU::S_MOV_B32;
391 break;
392 case AMDGPU::V_MOV_B64_e32:
393 case AMDGPU::V_MOV_B64_PSEUDO:
394 SMovOp = AMDGPU::S_MOV_B64_IMM_PSEUDO;
395 break;
396 }
397 Imm = ImmOp->getImm();
398 return true;
399}
400
401template <class UnaryPredicate>
402bool searchPredecessors(const MachineBasicBlock *MBB,
403 const MachineBasicBlock *CutOff,
404 UnaryPredicate Predicate) {
405 if (MBB == CutOff)
406 return false;
407
408 DenseSet<const MachineBasicBlock *> Visited;
409 SmallVector<MachineBasicBlock *, 4> Worklist(MBB->predecessors());
410
411 while (!Worklist.empty()) {
412 MachineBasicBlock *MBB = Worklist.pop_back_val();
413
414 if (!Visited.insert(V: MBB).second)
415 continue;
416 if (MBB == CutOff)
417 continue;
418 if (Predicate(MBB))
419 return true;
420
421 Worklist.append(in_start: MBB->pred_begin(), in_end: MBB->pred_end());
422 }
423
424 return false;
425}
426
427// Checks if there is potential path From instruction To instruction.
428// If CutOff is specified and it sits in between of that path we ignore
429// a higher portion of the path and report it is not reachable.
430static bool isReachable(const MachineInstr *From,
431 const MachineInstr *To,
432 const MachineBasicBlock *CutOff,
433 MachineDominatorTree &MDT) {
434 if (MDT.dominates(A: From, B: To))
435 return true;
436
437 const MachineBasicBlock *MBBFrom = From->getParent();
438 const MachineBasicBlock *MBBTo = To->getParent();
439
440 // Do predecessor search.
441 // We should almost never get here since we do not usually produce M0 stores
442 // other than -1.
443 return searchPredecessors(MBB: MBBTo, CutOff, Predicate: [MBBFrom]
444 (const MachineBasicBlock *MBB) { return MBB == MBBFrom; });
445}
446
447// Return the first non-prologue instruction in the block.
448static MachineBasicBlock::iterator
449getFirstNonPrologue(MachineBasicBlock *MBB, const TargetInstrInfo *TII) {
450 MachineBasicBlock::iterator I = MBB->getFirstNonPHI();
451 while (I != MBB->end() && TII->isBasicBlockPrologue(MI: *I))
452 ++I;
453
454 return I;
455}
456
457// Hoist and merge identical SGPR initializations into a common predecessor.
458// This is intended to combine M0 initializations, but can work with any
459// SGPR. A VGPR cannot be processed since we cannot guarantee vector
460// executioon.
461static bool hoistAndMergeSGPRInits(unsigned Reg,
462 const MachineRegisterInfo &MRI,
463 const TargetRegisterInfo *TRI,
464 MachineDominatorTree &MDT,
465 const TargetInstrInfo *TII) {
466 // List of inits by immediate value.
467 using InitListMap = std::map<unsigned, std::list<MachineInstr *>>;
468 InitListMap Inits;
469 // List of clobbering instructions.
470 SmallVector<MachineInstr*, 8> Clobbers;
471 // List of instructions marked for deletion.
472 SmallPtrSet<MachineInstr *, 8> MergedInstrs;
473
474 bool Changed = false;
475
476 for (auto &MI : MRI.def_instructions(Reg)) {
477 MachineOperand *Imm = nullptr;
478 for (auto &MO : MI.operands()) {
479 if ((MO.isReg() && ((MO.isDef() && MO.getReg() != Reg) || !MO.isDef())) ||
480 (!MO.isImm() && !MO.isReg()) || (MO.isImm() && Imm)) {
481 Imm = nullptr;
482 break;
483 }
484 if (MO.isImm())
485 Imm = &MO;
486 }
487 if (Imm)
488 Inits[Imm->getImm()].push_front(x: &MI);
489 else
490 Clobbers.push_back(Elt: &MI);
491 }
492
493 for (auto &Init : Inits) {
494 auto &Defs = Init.second;
495
496 for (auto I1 = Defs.begin(), E = Defs.end(); I1 != E; ) {
497 MachineInstr *MI1 = *I1;
498
499 for (auto I2 = std::next(x: I1); I2 != E; ) {
500 MachineInstr *MI2 = *I2;
501
502 // Check any possible interference
503 auto interferes = [&](MachineBasicBlock::iterator From,
504 MachineBasicBlock::iterator To) -> bool {
505
506 assert(MDT.dominates(&*To, &*From));
507
508 auto interferes = [&MDT, From, To](MachineInstr* &Clobber) -> bool {
509 const MachineBasicBlock *MBBFrom = From->getParent();
510 const MachineBasicBlock *MBBTo = To->getParent();
511 bool MayClobberFrom = isReachable(From: Clobber, To: &*From, CutOff: MBBTo, MDT);
512 bool MayClobberTo = isReachable(From: Clobber, To: &*To, CutOff: MBBTo, MDT);
513 if (!MayClobberFrom && !MayClobberTo)
514 return false;
515 if ((MayClobberFrom && !MayClobberTo) ||
516 (!MayClobberFrom && MayClobberTo))
517 return true;
518 // Both can clobber, this is not an interference only if both are
519 // dominated by Clobber and belong to the same block or if Clobber
520 // properly dominates To, given that To >> From, so it dominates
521 // both and located in a common dominator.
522 return !((MBBFrom == MBBTo &&
523 MDT.dominates(A: Clobber, B: &*From) &&
524 MDT.dominates(A: Clobber, B: &*To)) ||
525 MDT.properlyDominates(A: Clobber->getParent(), B: MBBTo));
526 };
527
528 return (llvm::any_of(Range&: Clobbers, P: interferes)) ||
529 (llvm::any_of(Range&: Inits, P: [&](InitListMap::value_type &C) {
530 return C.first != Init.first &&
531 llvm::any_of(Range&: C.second, P: interferes);
532 }));
533 };
534
535 if (MDT.dominates(A: MI1, B: MI2)) {
536 if (!interferes(MI2, MI1)) {
537 LLVM_DEBUG(dbgs()
538 << "Erasing from "
539 << printMBBReference(*MI2->getParent()) << " " << *MI2);
540 MergedInstrs.insert(Ptr: MI2);
541 Changed = true;
542 ++I2;
543 continue;
544 }
545 } else if (MDT.dominates(A: MI2, B: MI1)) {
546 if (!interferes(MI1, MI2)) {
547 LLVM_DEBUG(dbgs()
548 << "Erasing from "
549 << printMBBReference(*MI1->getParent()) << " " << *MI1);
550 MergedInstrs.insert(Ptr: MI1);
551 Changed = true;
552 ++I1;
553 break;
554 }
555 } else {
556 auto *MBB = MDT.findNearestCommonDominator(A: MI1->getParent(),
557 B: MI2->getParent());
558 if (!MBB) {
559 ++I2;
560 continue;
561 }
562
563 MachineBasicBlock::iterator I = getFirstNonPrologue(MBB, TII);
564 if (!interferes(MI1, I) && !interferes(MI2, I)) {
565 LLVM_DEBUG(dbgs()
566 << "Erasing from "
567 << printMBBReference(*MI1->getParent()) << " " << *MI1
568 << "and moving from "
569 << printMBBReference(*MI2->getParent()) << " to "
570 << printMBBReference(*I->getParent()) << " " << *MI2);
571 I->getParent()->splice(Where: I, Other: MI2->getParent(), From: MI2);
572 MergedInstrs.insert(Ptr: MI1);
573 Changed = true;
574 ++I1;
575 break;
576 }
577 }
578 ++I2;
579 }
580 ++I1;
581 }
582 }
583
584 // Remove initializations that were merged into another.
585 for (auto &Init : Inits) {
586 auto &Defs = Init.second;
587 auto I = Defs.begin();
588 while (I != Defs.end()) {
589 if (MergedInstrs.count(Ptr: *I)) {
590 (*I)->eraseFromParent();
591 I = Defs.erase(position: I);
592 } else
593 ++I;
594 }
595 }
596
597 // Try to schedule SGPR initializations as early as possible in the MBB.
598 for (auto &Init : Inits) {
599 auto &Defs = Init.second;
600 for (auto *MI : Defs) {
601 auto *MBB = MI->getParent();
602 MachineInstr &BoundaryMI = *getFirstNonPrologue(MBB, TII);
603 MachineBasicBlock::reverse_iterator B(BoundaryMI);
604 // Check if B should actually be a boundary. If not set the previous
605 // instruction as the boundary instead.
606 if (!TII->isBasicBlockPrologue(MI: *B))
607 B++;
608
609 auto R = std::next(x: MI->getReverseIterator());
610 const unsigned Threshold = 50;
611 // Search until B or Threshold for a place to insert the initialization.
612 for (unsigned I = 0; R != B && I < Threshold; ++R, ++I)
613 if (R->readsRegister(Reg, TRI) || R->definesRegister(Reg, TRI) ||
614 TII->isSchedulingBoundary(MI: *R, MBB, MF: *MBB->getParent()))
615 break;
616
617 // Move to directly after R.
618 if (&*--R != MI)
619 MBB->splice(Where: *R, Other: MBB, From: MI);
620 }
621 }
622
623 if (Changed)
624 MRI.clearKillFlags(Reg);
625
626 return Changed;
627}
628
629bool SIFixSGPRCopies::run(MachineFunction &MF) {
630 // Only need to run this in SelectionDAG path.
631 if (MF.getProperties().hasSelected())
632 return false;
633
634 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
635 MRI = &MF.getRegInfo();
636 TRI = ST.getRegisterInfo();
637 TII = ST.getInstrInfo();
638
639 // Instructions to re-legalize after changing register classes
640 SmallVector<MachineInstr *, 8> Relegalize;
641
642 for (MachineBasicBlock &MBB : MF) {
643 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E;
644 ++I) {
645 MachineInstr &MI = *I;
646
647 switch (MI.getOpcode()) {
648 default:
649 // scale_src has a register class restricted to low 256 VGPRs, changing
650 // registers to VGPR may not take it into acount.
651 if (TII->isWMMA(MI) &&
652 AMDGPU::hasNamedOperand(Opcode: MI.getOpcode(), NamedIdx: AMDGPU::OpName::scale_src0))
653 Relegalize.push_back(Elt: &MI);
654 continue;
655 case AMDGPU::COPY: {
656 const TargetRegisterClass *SrcRC, *DstRC;
657 std::tie(args&: SrcRC, args&: DstRC) = getCopyRegClasses(Copy: MI, TRI: *TRI, MRI: *MRI);
658
659 if (isSGPRToVGPRCopy(SrcRC, DstRC, TRI: *TRI)) {
660 // Since VGPR to SGPR copies affect VGPR to SGPR copy
661 // score and, hence the lowering decision, let's try to get rid of
662 // them as early as possible
663 if (tryChangeVGPRtoSGPRinCopy(MI, TRI, TII))
664 continue;
665
666 // Collect those not changed to try them after VGPR to SGPR copies
667 // lowering as there will be more opportunities.
668 S2VCopies.push_back(Elt: &MI);
669 }
670 if (!isVGPRToSGPRCopy(SrcRC, DstRC, TRI: *TRI))
671 continue;
672 if (lowerSpecialCase(MI, I))
673 continue;
674
675 analyzeVGPRToSGPRCopy(MI: &MI);
676
677 break;
678 }
679 case AMDGPU::WQM:
680 case AMDGPU::STRICT_WQM:
681 case AMDGPU::SOFT_WQM:
682 case AMDGPU::STRICT_WWM:
683 case AMDGPU::INSERT_SUBREG:
684 case AMDGPU::PHI:
685 case AMDGPU::REG_SEQUENCE: {
686 if (TRI->isSGPRClass(RC: TII->getOpRegClass(MI, OpNo: 0))) {
687 for (MachineOperand &MO : MI.operands()) {
688 if (!MO.isReg() || !MO.getReg().isVirtual())
689 continue;
690 const TargetRegisterClass *SrcRC = MRI->getRegClass(Reg: MO.getReg());
691 if (SrcRC == &AMDGPU::VReg_1RegClass)
692 continue;
693
694 if (TRI->hasVectorRegisters(RC: SrcRC)) {
695 const TargetRegisterClass *DestRC =
696 TRI->getEquivalentSGPRClass(VRC: SrcRC);
697 Register NewDst = MRI->createVirtualRegister(RegClass: DestRC);
698 MachineBasicBlock *BlockToInsertCopy =
699 MI.isPHI() ? MI.getOperand(i: MO.getOperandNo() + 1).getMBB()
700 : &MBB;
701 MachineBasicBlock::iterator PointToInsertCopy =
702 MI.isPHI() ? BlockToInsertCopy->getFirstInstrTerminator() : I;
703
704 const DebugLoc &DL = MI.getDebugLoc();
705 if (!tryMoveVGPRConstToSGPR(MO, NewDst, BlockToInsertTo: BlockToInsertCopy,
706 PointToInsertTo: PointToInsertCopy, DL)) {
707 MachineInstr *NewCopy =
708 BuildMI(BB&: *BlockToInsertCopy, I: PointToInsertCopy, MIMD: DL,
709 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: NewDst)
710 .addReg(RegNo: MO.getReg());
711 MO.setReg(NewDst);
712 analyzeVGPRToSGPRCopy(MI: NewCopy);
713 PHISources.insert(V: NewCopy);
714 }
715 }
716 }
717 }
718
719 if (MI.isPHI())
720 PHINodes.push_back(Elt: &MI);
721 else if (MI.isRegSequence())
722 RegSequences.push_back(Elt: &MI);
723
724 break;
725 }
726 case AMDGPU::V_WRITELANE_B32: {
727 // Some architectures allow more than one constant bus access without
728 // SGPR restriction
729 if (ST.getConstantBusLimit(Opcode: MI.getOpcode()) != 1)
730 break;
731
732 // Writelane is special in that it can use SGPR and M0 (which would
733 // normally count as using the constant bus twice - but in this case it
734 // is allowed since the lane selector doesn't count as a use of the
735 // constant bus). However, it is still required to abide by the 1 SGPR
736 // rule. Apply a fix here as we might have multiple SGPRs after
737 // legalizing VGPRs to SGPRs
738 int Src0Idx =
739 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src0);
740 int Src1Idx =
741 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src1);
742 MachineOperand &Src0 = MI.getOperand(i: Src0Idx);
743 MachineOperand &Src1 = MI.getOperand(i: Src1Idx);
744
745 // Check to see if the instruction violates the 1 SGPR rule
746 if ((Src0.isReg() && TRI->isSGPRReg(MRI: *MRI, Reg: Src0.getReg()) &&
747 Src0.getReg() != AMDGPU::M0) &&
748 (Src1.isReg() && TRI->isSGPRReg(MRI: *MRI, Reg: Src1.getReg()) &&
749 Src1.getReg() != AMDGPU::M0)) {
750
751 // Check for trivially easy constant prop into one of the operands
752 // If this is the case then perform the operation now to resolve SGPR
753 // issue. If we don't do that here we will always insert a mov to m0
754 // that can't be resolved in later operand folding pass
755 bool Resolved = false;
756 for (MachineOperand *MO : {&Src0, &Src1}) {
757 if (MO->getReg().isVirtual()) {
758 MachineInstr *DefMI = MRI->getVRegDef(Reg: MO->getReg());
759 if (DefMI && TII->isFoldableCopy(MI: *DefMI)) {
760 const MachineOperand &Def = DefMI->getOperand(i: 0);
761 if (Def.isReg() &&
762 MO->getReg() == Def.getReg() &&
763 MO->getSubReg() == Def.getSubReg()) {
764 const MachineOperand &Copied = DefMI->getOperand(i: 1);
765 if (Copied.isImm() &&
766 TII->isInlineConstant(Imm: APInt(64, Copied.getImm(), true))) {
767 MO->ChangeToImmediate(ImmVal: Copied.getImm());
768 Resolved = true;
769 break;
770 }
771 }
772 }
773 }
774 }
775
776 if (!Resolved) {
777 // Haven't managed to resolve by replacing an SGPR with an immediate
778 // Move src1 to be in M0
779 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(),
780 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::M0)
781 .add(MO: Src1);
782 Src1.ChangeToRegister(Reg: AMDGPU::M0, isDef: false);
783 }
784 }
785 break;
786 }
787 }
788 }
789 }
790
791 lowerVGPR2SGPRCopies(MF);
792 // Postprocessing
793 fixSCCCopies(MF);
794 for (auto *MI : S2VCopies) {
795 // Check if it is still valid
796 if (MI->isCopy()) {
797 const TargetRegisterClass *SrcRC, *DstRC;
798 std::tie(args&: SrcRC, args&: DstRC) = getCopyRegClasses(Copy: *MI, TRI: *TRI, MRI: *MRI);
799 if (isSGPRToVGPRCopy(SrcRC, DstRC, TRI: *TRI))
800 tryChangeVGPRtoSGPRinCopy(MI&: *MI, TRI, TII);
801 }
802 }
803 for (auto *MI : RegSequences) {
804 // Check if it is still valid
805 if (MI->isRegSequence())
806 foldVGPRCopyIntoRegSequence(MI&: *MI, TRI, TII, MRI&: *MRI);
807 }
808 for (auto *MI : PHINodes) {
809 processPHINode(MI&: *MI);
810 }
811 while (!Relegalize.empty())
812 TII->legalizeOperands(MI&: *Relegalize.pop_back_val(), MDT);
813
814 if (MF.getTarget().getOptLevel() > CodeGenOptLevel::None && EnableM0Merge)
815 hoistAndMergeSGPRInits(Reg: AMDGPU::M0, MRI: *MRI, TRI, MDT&: *MDT, TII);
816
817 SiblingPenalty.clear();
818 V2SCopies.clear();
819 SCCCopies.clear();
820 RegSequences.clear();
821 PHINodes.clear();
822 S2VCopies.clear();
823 PHISources.clear();
824
825 return true;
826}
827
828void SIFixSGPRCopies::processPHINode(MachineInstr &MI) {
829 bool AllAGPRUses = true;
830 SetVector<const MachineInstr *> worklist;
831 SmallPtrSet<const MachineInstr *, 4> Visited;
832 SetVector<MachineInstr *> PHIOperands;
833 worklist.insert(X: &MI);
834 Visited.insert(Ptr: &MI);
835 // HACK to make MIR tests with no uses happy
836 bool HasUses = false;
837 while (!worklist.empty()) {
838 const MachineInstr *Instr = worklist.pop_back_val();
839 Register Reg = Instr->getOperand(i: 0).getReg();
840 for (const auto &Use : MRI->use_operands(Reg)) {
841 HasUses = true;
842 const MachineInstr *UseMI = Use.getParent();
843 AllAGPRUses &= (UseMI->isCopy() &&
844 TRI->isAGPR(MRI: *MRI, Reg: UseMI->getOperand(i: 0).getReg())) ||
845 TRI->isAGPR(MRI: *MRI, Reg: Use.getReg());
846 if (UseMI->isCopy() || UseMI->isRegSequence()) {
847 if (Visited.insert(Ptr: UseMI).second)
848 worklist.insert(X: UseMI);
849
850 continue;
851 }
852 }
853 }
854
855 Register PHIRes = MI.getOperand(i: 0).getReg();
856 const TargetRegisterClass *RC0 = MRI->getRegClass(Reg: PHIRes);
857 if (HasUses && AllAGPRUses && !TRI->isAGPRClass(RC: RC0)) {
858 LLVM_DEBUG(dbgs() << "Moving PHI to AGPR: " << MI);
859 MRI->setRegClass(Reg: PHIRes, RC: TRI->getEquivalentAGPRClass(SRC: RC0));
860 for (unsigned I = 1, N = MI.getNumOperands(); I != N; I += 2) {
861 MachineInstr *DefMI = MRI->getVRegDef(Reg: MI.getOperand(i: I).getReg());
862 if (DefMI && DefMI->isPHI())
863 PHIOperands.insert(X: DefMI);
864 }
865 }
866
867 if (TRI->hasVectorRegisters(RC: MRI->getRegClass(Reg: PHIRes)) ||
868 RC0 == &AMDGPU::VReg_1RegClass) {
869 LLVM_DEBUG(dbgs() << "Legalizing PHI: " << MI);
870 TII->legalizeOperands(MI, MDT);
871 }
872
873 // Propagate register class back to PHI operands which are PHI themselves.
874 while (!PHIOperands.empty()) {
875 processPHINode(MI&: *PHIOperands.pop_back_val());
876 }
877}
878
879bool SIFixSGPRCopies::tryMoveVGPRConstToSGPR(
880 MachineOperand &MaybeVGPRConstMO, Register DstReg,
881 MachineBasicBlock *BlockToInsertTo,
882 MachineBasicBlock::iterator PointToInsertTo, const DebugLoc &DL) {
883
884 MachineInstr *DefMI = MRI->getVRegDef(Reg: MaybeVGPRConstMO.getReg());
885 if (!DefMI || !DefMI->isMoveImmediate())
886 return false;
887
888 MachineOperand *SrcConst = TII->getNamedOperand(MI&: *DefMI, OperandName: AMDGPU::OpName::src0);
889 if (SrcConst->isReg())
890 return false;
891
892 const TargetRegisterClass *SrcRC =
893 MRI->getRegClass(Reg: MaybeVGPRConstMO.getReg());
894 unsigned MoveSize = TRI->getRegSizeInBits(RC: *SrcRC);
895 unsigned MoveOp =
896 MoveSize == 64 ? AMDGPU::S_MOV_B64_IMM_PSEUDO : AMDGPU::S_MOV_B32;
897 BuildMI(BB&: *BlockToInsertTo, I: PointToInsertTo, MIMD: DL, MCID: TII->get(Opcode: MoveOp), DestReg: DstReg)
898 .add(MO: *SrcConst);
899 if (MRI->hasOneUse(RegNo: MaybeVGPRConstMO.getReg()))
900 DefMI->eraseFromParent();
901 MaybeVGPRConstMO.setReg(DstReg);
902 return true;
903}
904
905bool SIFixSGPRCopies::lowerSpecialCase(MachineInstr &MI,
906 MachineBasicBlock::iterator &I) {
907 Register DstReg = MI.getOperand(i: 0).getReg();
908 Register SrcReg = MI.getOperand(i: 1).getReg();
909 if (!DstReg.isVirtual()) {
910 // If the destination register is a physical register there isn't
911 // really much we can do to fix this.
912 // Some special instructions use M0 as an input. Some even only use
913 // the first lane. Insert a readfirstlane and hope for the best.
914 const TargetRegisterClass *SrcRC = MRI->getRegClass(Reg: SrcReg);
915 if (DstReg == AMDGPU::M0 && TRI->hasVectorRegisters(RC: SrcRC)) {
916 Register TmpReg =
917 MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
918
919 const MCInstrDesc &ReadFirstLaneDesc =
920 TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32);
921 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: ReadFirstLaneDesc, DestReg: TmpReg)
922 .add(MO: MI.getOperand(i: 1));
923
924 unsigned SubReg = MI.getOperand(i: 1).getSubReg();
925 MI.getOperand(i: 1).setReg(TmpReg);
926 MI.getOperand(i: 1).setSubReg(AMDGPU::NoSubRegister);
927
928 const TargetRegisterClass *OpRC = TII->getRegClass(MCID: ReadFirstLaneDesc, OpNum: 1);
929 const TargetRegisterClass *ConstrainRC =
930 SubReg == AMDGPU::NoSubRegister
931 ? OpRC
932 : TRI->getMatchingSuperRegClass(A: SrcRC, B: OpRC, Idx: SubReg);
933
934 if (!MRI->constrainRegClass(Reg: SrcReg, RC: ConstrainRC))
935 llvm_unreachable("failed to constrain register");
936 return true;
937 }
938
939 if (tryMoveVGPRConstToSGPR(MaybeVGPRConstMO&: MI.getOperand(i: 1), DstReg, BlockToInsertTo: MI.getParent(), PointToInsertTo: MI,
940 DL: MI.getDebugLoc())) {
941 I = MI.eraseFromParent();
942 return true;
943 }
944
945 if (!SrcReg.isVirtual())
946 return true;
947 }
948 if (!SrcReg.isVirtual() || TRI->isAGPR(MRI: *MRI, Reg: SrcReg)) {
949 SIInstrWorklist worklist;
950 worklist.insert(MI: &MI);
951 TII->moveToVALU(Worklist&: worklist, MDT);
952 return true;
953 }
954
955 unsigned SMovOp;
956 int64_t Imm;
957 // If we are just copying an immediate, we can replace the copy with
958 // s_mov_b32.
959 if (isSafeToFoldImmIntoCopy(Copy: &MI, MoveImm: MRI->getVRegDef(Reg: SrcReg), TII, SMovOp, Imm)) {
960 MI.getOperand(i: 1).ChangeToImmediate(ImmVal: Imm);
961 MI.addImplicitDefUseOperands(MF&: *MI.getMF());
962 MI.setDesc(TII->get(Opcode: SMovOp));
963 return true;
964 }
965 return false;
966}
967
968void SIFixSGPRCopies::analyzeVGPRToSGPRCopy(MachineInstr* MI) {
969 if (PHISources.contains(V: MI))
970 return;
971 Register DstReg = MI->getOperand(i: 0).getReg();
972 const TargetRegisterClass *DstRC = TRI->getRegClassForReg(MRI: *MRI, Reg: DstReg);
973
974 V2SCopyInfo Info(getNextVGPRToSGPRCopyId(), MI,
975 TRI->getRegSizeInBits(RC: *DstRC));
976 SmallVector<MachineInstr *, 8> AnalysisWorklist;
977 // Needed because the SSA is not a tree but a graph and may have
978 // forks and joins. We should not then go same way twice.
979 DenseSet<MachineInstr *> Visited;
980 AnalysisWorklist.push_back(Elt: Info.Copy);
981 while (!AnalysisWorklist.empty()) {
982
983 MachineInstr *Inst = AnalysisWorklist.pop_back_val();
984
985 if (!Visited.insert(V: Inst).second)
986 continue;
987
988 // Copies and REG_SEQUENCE do not contribute to the final assembly
989 // So, skip them but take care of the SGPR to VGPR copies bookkeeping.
990 if (Inst->isRegSequence() &&
991 TRI->isVGPR(MRI: *MRI, Reg: Inst->getOperand(i: 0).getReg())) {
992 Info.NumSVCopies++;
993 continue;
994 }
995 if (Inst->isCopy()) {
996 const TargetRegisterClass *SrcRC, *DstRC;
997 std::tie(args&: SrcRC, args&: DstRC) = getCopyRegClasses(Copy: *Inst, TRI: *TRI, MRI: *MRI);
998 if (isSGPRToVGPRCopy(SrcRC, DstRC, TRI: *TRI) &&
999 !tryChangeVGPRtoSGPRinCopy(MI&: *Inst, TRI, TII)) {
1000 Info.NumSVCopies++;
1001 continue;
1002 }
1003 }
1004
1005 SiblingPenalty[Inst].insert(X: Info.ID);
1006
1007 SmallVector<MachineInstr *, 4> Users;
1008 if ((TII->isSALU(MI: *Inst) && Inst->isCompare()) ||
1009 (Inst->isCopy() && Inst->getOperand(i: 0).getReg() == AMDGPU::SCC)) {
1010 auto I = Inst->getIterator();
1011 auto E = Inst->getParent()->end();
1012 while (++I != E &&
1013 !I->findRegisterDefOperand(Reg: AMDGPU::SCC, /*TRI=*/nullptr)) {
1014 if (I->readsRegister(Reg: AMDGPU::SCC, /*TRI=*/nullptr))
1015 Users.push_back(Elt: &*I);
1016 }
1017 } else if (Inst->getNumExplicitDefs() != 0) {
1018 Register Reg = Inst->getOperand(i: 0).getReg();
1019 if (Reg.isVirtual() && TRI->isSGPRReg(MRI: *MRI, Reg) &&
1020 !TII->isVALU(MI: *Inst, /*AllowLDSDMA=*/true)) {
1021 for (auto &U : MRI->use_instructions(Reg))
1022 Users.push_back(Elt: &U);
1023 }
1024 }
1025 for (auto *U : Users) {
1026 if (TII->isSALU(MI: *U))
1027 Info.SChain.insert(X: U);
1028 AnalysisWorklist.push_back(Elt: U);
1029 }
1030 }
1031 V2SCopies[Info.ID] = std::move(Info);
1032}
1033
1034// The main function that computes the VGPR to SGPR copy score
1035// and determines copy further lowering way: v_readfirstlane_b32 or moveToVALU
1036bool SIFixSGPRCopies::needToBeConvertedToVALU(V2SCopyInfo *Info) {
1037 if (Info->SChain.empty()) {
1038 Info->Score = 0;
1039 return true;
1040 }
1041 Info->Siblings = SiblingPenalty[*llvm::max_element(
1042 Range&: Info->SChain, C: [&](MachineInstr *A, MachineInstr *B) -> bool {
1043 return SiblingPenalty[A].size() < SiblingPenalty[B].size();
1044 })];
1045 Info->Siblings.remove_if(P: [&](unsigned ID) { return ID == Info->ID; });
1046 // The loop below computes the number of another VGPR to SGPR V2SCopies
1047 // which contribute to the current copy SALU chain. We assume that all the
1048 // V2SCopies with the same source virtual register will be squashed to one
1049 // by regalloc. Also we take care of the V2SCopies of the differnt subregs
1050 // of the same register.
1051 SmallSet<std::pair<Register, unsigned>, 4> SrcRegs;
1052 for (auto J : Info->Siblings) {
1053 auto *InfoIt = V2SCopies.find(Key: J);
1054 if (InfoIt != V2SCopies.end()) {
1055 MachineInstr *SiblingCopy = InfoIt->second.Copy;
1056 if (SiblingCopy->isImplicitDef())
1057 // the COPY has already been MoveToVALUed
1058 continue;
1059
1060 SrcRegs.insert(V: std::pair(SiblingCopy->getOperand(i: 1).getReg(),
1061 SiblingCopy->getOperand(i: 1).getSubReg()));
1062 }
1063 }
1064 Info->SiblingPenalty = SrcRegs.size();
1065
1066 unsigned Penalty =
1067 Info->NumSVCopies + Info->SiblingPenalty + Info->NumReadfirstlanes;
1068 unsigned Profit = Info->SChain.size();
1069 Info->Score = Penalty > Profit ? 0 : Profit - Penalty;
1070 Info->NeedToBeConvertedToVALU = Info->Score < 3;
1071 return Info->NeedToBeConvertedToVALU;
1072}
1073
1074void SIFixSGPRCopies::lowerVGPR2SGPRCopies(MachineFunction &MF) {
1075
1076 SmallVector<unsigned, 8> LoweringWorklist;
1077 for (auto &C : V2SCopies) {
1078 if (needToBeConvertedToVALU(Info: &C.second))
1079 LoweringWorklist.push_back(Elt: C.second.ID);
1080 }
1081
1082 // Store all the V2S copy instructions that need to be moved to VALU
1083 // in the Copies worklist.
1084 SIInstrWorklist Copies;
1085
1086 while (!LoweringWorklist.empty()) {
1087 unsigned CurID = LoweringWorklist.pop_back_val();
1088 auto *CurInfoIt = V2SCopies.find(Key: CurID);
1089 if (CurInfoIt != V2SCopies.end() && !CurInfoIt->second.Erased) {
1090 V2SCopyInfo &C = CurInfoIt->second;
1091 LLVM_DEBUG(dbgs() << "Processing ...\n"; C.dump());
1092 for (auto S : C.Siblings) {
1093 auto *SibInfoIt = V2SCopies.find(Key: S);
1094 if (SibInfoIt != V2SCopies.end() && !SibInfoIt->second.Erased) {
1095 V2SCopyInfo &SI = SibInfoIt->second;
1096 LLVM_DEBUG(dbgs() << "Sibling:\n"; SI.dump());
1097 if (!SI.NeedToBeConvertedToVALU) {
1098 SI.SChain.set_subtract(C.SChain);
1099 if (needToBeConvertedToVALU(Info: &SI))
1100 LoweringWorklist.push_back(Elt: SI.ID);
1101 }
1102 SI.Siblings.remove_if(P: [&](unsigned ID) { return ID == C.ID; });
1103 }
1104 }
1105 LLVM_DEBUG(dbgs() << "V2S copy " << *C.Copy
1106 << " is being turned to VALU\n");
1107 Copies.insert(MI: C.Copy);
1108 C.Erased = true;
1109 }
1110 }
1111 V2SCopies.remove_if(Pred: [](const auto &P) { return P.second.Erased; });
1112
1113 TII->moveToVALU(Worklist&: Copies, MDT);
1114 Copies.clear();
1115
1116 // Now do actual lowering
1117 for (auto C : V2SCopies) {
1118 MachineInstr *MI = C.second.Copy;
1119 MachineBasicBlock *MBB = MI->getParent();
1120 // We decide to turn V2S copy to v_readfirstlane_b32
1121 // remove it from the V2SCopies and remove it from all its siblings
1122 LLVM_DEBUG(dbgs() << "V2S copy " << *MI
1123 << " is being turned to v_readfirstlane_b32"
1124 << " Score: " << C.second.Score << "\n");
1125 Register DstReg = MI->getOperand(i: 0).getReg();
1126 MRI->constrainRegClass(Reg: DstReg, RC: &AMDGPU::SReg_32_XM0RegClass);
1127
1128 Register SrcReg = MI->getOperand(i: 1).getReg();
1129 unsigned SubReg = MI->getOperand(i: 1).getSubReg();
1130 const TargetRegisterClass *SrcRC =
1131 TRI->getRegClassForOperandReg(MRI: *MRI, MO: MI->getOperand(i: 1));
1132 size_t SrcSize = TRI->getRegSizeInBits(RC: *SrcRC);
1133 if (SrcSize == 16) {
1134 assert(MF.getSubtarget<GCNSubtarget>().useRealTrue16Insts() &&
1135 "We do not expect to see 16-bit copies from VGPR to SGPR unless "
1136 "we have 16-bit VGPRs");
1137 assert(MRI->getRegClass(DstReg) == &AMDGPU::SReg_32RegClass ||
1138 MRI->getRegClass(DstReg) == &AMDGPU::SReg_32_XM0RegClass);
1139 // There is no V_READFIRSTLANE_B16, so legalize the dst/src reg to 32 bits
1140 MRI->setRegClass(Reg: DstReg, RC: &AMDGPU::SReg_32_XM0RegClass);
1141 Register VReg32 = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
1142 const DebugLoc &DL = MI->getDebugLoc();
1143 Register Undef = MRI->createVirtualRegister(RegClass: &AMDGPU::VGPR_16RegClass);
1144 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Undef);
1145 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: VReg32)
1146 .addReg(RegNo: SrcReg, Flags: {}, SubReg)
1147 .addImm(Val: AMDGPU::lo16)
1148 .addReg(RegNo: Undef)
1149 .addImm(Val: AMDGPU::hi16);
1150 BuildMI(BB&: *MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: DstReg)
1151 .addReg(RegNo: VReg32);
1152 } else if (SrcSize == 32) {
1153 const MCInstrDesc &ReadFirstLaneDesc =
1154 TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32);
1155 const TargetRegisterClass *OpRC = TII->getRegClass(MCID: ReadFirstLaneDesc, OpNum: 1);
1156 BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(), MCID: ReadFirstLaneDesc, DestReg: DstReg)
1157 .addReg(RegNo: SrcReg, Flags: {}, SubReg);
1158
1159 const TargetRegisterClass *ConstrainRC =
1160 SubReg == AMDGPU::NoSubRegister
1161 ? OpRC
1162 : TRI->getMatchingSuperRegClass(A: MRI->getRegClass(Reg: SrcReg), B: OpRC,
1163 Idx: SubReg);
1164
1165 if (!MRI->constrainRegClass(Reg: SrcReg, RC: ConstrainRC))
1166 llvm_unreachable("failed to constrain register");
1167 } else {
1168 auto Result = BuildMI(BB&: *MBB, I: MI, MIMD: MI->getDebugLoc(),
1169 MCID: TII->get(Opcode: AMDGPU::REG_SEQUENCE), DestReg: DstReg);
1170 int N = TRI->getRegSizeInBits(RC: *SrcRC) / 32;
1171 for (int i = 0; i < N; i++) {
1172 Register PartialSrc = TII->buildExtractSubReg(
1173 MI: Result, MRI&: *MRI, SuperReg: MI->getOperand(i: 1), SuperRC: SrcRC,
1174 SubIdx: TRI->getSubRegFromChannel(Channel: i), SubRC: &AMDGPU::VGPR_32RegClass);
1175 Register PartialDst =
1176 MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
1177 BuildMI(BB&: *MBB, I&: *Result, MIMD: Result->getDebugLoc(),
1178 MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: PartialDst)
1179 .addReg(RegNo: PartialSrc);
1180 Result.addReg(RegNo: PartialDst).addImm(Val: TRI->getSubRegFromChannel(Channel: i));
1181 }
1182 }
1183 MI->eraseFromParent();
1184 }
1185}
1186
1187void SIFixSGPRCopies::fixSCCCopies(MachineFunction &MF) {
1188 const AMDGPU::LaneMaskConstants &LMC =
1189 AMDGPU::LaneMaskConstants::get(ST: MF.getSubtarget<GCNSubtarget>());
1190 for (MachineBasicBlock &MBB : MF) {
1191 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E;
1192 ++I) {
1193 MachineInstr &MI = *I;
1194 // May already have been lowered.
1195 if (!MI.isCopy())
1196 continue;
1197 Register SrcReg = MI.getOperand(i: 1).getReg();
1198 Register DstReg = MI.getOperand(i: 0).getReg();
1199 if (SrcReg == AMDGPU::SCC) {
1200 Register SCCCopy =
1201 MRI->createVirtualRegister(RegClass: TRI->getWaveMaskRegClass());
1202 I = BuildMI(BB&: *MI.getParent(), I: std::next(x: MachineBasicBlock::iterator(MI)),
1203 MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: LMC.CSelectOpc), DestReg: SCCCopy)
1204 .addImm(Val: -1)
1205 .addImm(Val: 0);
1206 I = BuildMI(BB&: *MI.getParent(), I: std::next(x: I), MIMD: I->getDebugLoc(),
1207 MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: DstReg)
1208 .addReg(RegNo: SCCCopy);
1209 MI.eraseFromParent();
1210 continue;
1211 }
1212 if (DstReg == AMDGPU::SCC) {
1213 Register Tmp = MRI->createVirtualRegister(RegClass: TRI->getBoolRC());
1214 I = BuildMI(BB&: *MI.getParent(), I: std::next(x: MachineBasicBlock::iterator(MI)),
1215 MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: LMC.AndOpc))
1216 .addReg(RegNo: Tmp, Flags: getDefRegState(B: true))
1217 .addReg(RegNo: SrcReg)
1218 .addReg(RegNo: LMC.ExecReg);
1219 MI.eraseFromParent();
1220 }
1221 }
1222 }
1223}
1224
1225PreservedAnalyses
1226SIFixSGPRCopiesPass::run(MachineFunction &MF,
1227 MachineFunctionAnalysisManager &MFAM) {
1228 MachineDominatorTree &MDT = MFAM.getResult<MachineDominatorTreeAnalysis>(IR&: MF);
1229 SIFixSGPRCopies Impl(&MDT);
1230 bool Changed = Impl.run(MF);
1231 if (!Changed)
1232 return PreservedAnalyses::all();
1233
1234 // TODO: We could detect CFG changed.
1235 auto PA = getMachineFunctionPassPreservedAnalyses();
1236 return PA;
1237}
1238