1//===-- SILowerWWMCopies.cpp - Lower Copies after regalloc ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Lowering the WWM_COPY instructions for various register classes.
11/// AMDGPU target generates WWM_COPY instruction to differentiate WWM
12/// copy from COPY. This pass generates the necessary exec mask manipulation
13/// instructions to replicate 'Whole Wave Mode' and lowers WWM_COPY back to
14/// COPY.
15//
16//===----------------------------------------------------------------------===//
17
18#include "SILowerWWMCopies.h"
19#include "AMDGPU.h"
20#include "GCNSubtarget.h"
21#include "SIMachineFunctionInfo.h"
22#include "llvm/CodeGen/LiveIntervals.h"
23#include "llvm/CodeGen/MachineFunctionPass.h"
24#include "llvm/CodeGen/VirtRegMap.h"
25#include "llvm/InitializePasses.h"
26
27using namespace llvm;
28
29#define DEBUG_TYPE "si-lower-wwm-copies"
30
31namespace {
32
33class SILowerWWMCopies {
34public:
35 SILowerWWMCopies(LiveIntervals *LIS, SlotIndexes *SI, VirtRegMap *VRM)
36 : LIS(LIS), Indexes(SI), VRM(VRM) {}
37 bool run(MachineFunction &MF);
38
39private:
40 bool isSCCLiveAtMI(const MachineInstr &MI);
41 void addToWWMSpills(MachineFunction &MF, Register Reg);
42
43 LiveIntervals *LIS;
44 SlotIndexes *Indexes;
45 VirtRegMap *VRM;
46 const SIRegisterInfo *TRI;
47 const MachineRegisterInfo *MRI;
48 SIMachineFunctionInfo *MFI;
49};
50
51class SILowerWWMCopiesLegacy : public MachineFunctionPass {
52public:
53 static char ID;
54
55 SILowerWWMCopiesLegacy() : MachineFunctionPass(ID) {}
56
57 bool runOnMachineFunction(MachineFunction &MF) override;
58
59 StringRef getPassName() const override { return "SI Lower WWM Copies"; }
60
61 void getAnalysisUsage(AnalysisUsage &AU) const override {
62 AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();
63 AU.addUsedIfAvailable<SlotIndexesWrapperPass>();
64 AU.addUsedIfAvailable<VirtRegMapWrapperLegacy>();
65 AU.setPreservesAll();
66 MachineFunctionPass::getAnalysisUsage(AU);
67 }
68};
69
70} // End anonymous namespace.
71
72INITIALIZE_PASS_BEGIN(SILowerWWMCopiesLegacy, DEBUG_TYPE, "SI Lower WWM Copies",
73 false, false)
74INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
75INITIALIZE_PASS_DEPENDENCY(VirtRegMapWrapperLegacy)
76INITIALIZE_PASS_END(SILowerWWMCopiesLegacy, DEBUG_TYPE, "SI Lower WWM Copies",
77 false, false)
78
79char SILowerWWMCopiesLegacy::ID = 0;
80
81char &llvm::SILowerWWMCopiesLegacyID = SILowerWWMCopiesLegacy::ID;
82
83bool SILowerWWMCopies::isSCCLiveAtMI(const MachineInstr &MI) {
84 // We can't determine the liveness info if LIS isn't available. Early return
85 // in that case and always assume SCC is live.
86 if (!LIS)
87 return true;
88
89 LiveRange &LR =
90 LIS->getRegUnit(Unit: *MCRegUnitIterator(MCRegister::from(Val: AMDGPU::SCC), TRI));
91 SlotIndex Idx = LIS->getInstructionIndex(Instr: MI);
92 return LR.liveAt(index: Idx);
93}
94
95// If \p Reg is assigned with a physical VGPR, add the latter into wwm-spills
96// for preserving its entire lanes at function prolog/epilog.
97void SILowerWWMCopies::addToWWMSpills(MachineFunction &MF, Register Reg) {
98 if (Reg.isPhysical())
99 return;
100
101 // FIXME: VRM may be null here.
102 MCRegister PhysReg = VRM->getPhys(virtReg: Reg);
103 assert(PhysReg && "should have allocated a physical register");
104
105 MFI->allocateWWMSpill(MF, VGPR: PhysReg);
106}
107
108bool SILowerWWMCopiesLegacy::runOnMachineFunction(MachineFunction &MF) {
109 auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
110 auto *LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
111
112 auto *SIWrapper = getAnalysisIfAvailable<SlotIndexesWrapperPass>();
113 auto *Indexes = SIWrapper ? &SIWrapper->getSI() : nullptr;
114
115 auto *VRMWrapper = getAnalysisIfAvailable<VirtRegMapWrapperLegacy>();
116 auto *VRM = VRMWrapper ? &VRMWrapper->getVRM() : nullptr;
117
118 SILowerWWMCopies Impl(LIS, Indexes, VRM);
119 return Impl.run(MF);
120}
121
122PreservedAnalyses
123SILowerWWMCopiesPass::run(MachineFunction &MF,
124 MachineFunctionAnalysisManager &MFAM) {
125 auto *LIS = MFAM.getCachedResult<LiveIntervalsAnalysis>(IR&: MF);
126 auto *Indexes = MFAM.getCachedResult<SlotIndexesAnalysis>(IR&: MF);
127 auto *VRM = MFAM.getCachedResult<VirtRegMapAnalysis>(IR&: MF);
128
129 SILowerWWMCopies Impl(LIS, Indexes, VRM);
130 Impl.run(MF);
131 return PreservedAnalyses::all();
132}
133
134bool SILowerWWMCopies::run(MachineFunction &MF) {
135 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
136 const SIInstrInfo *TII = ST.getInstrInfo();
137
138 MFI = MF.getInfo<SIMachineFunctionInfo>();
139 TRI = ST.getRegisterInfo();
140 MRI = &MF.getRegInfo();
141
142 if (!MFI->hasVRegFlags())
143 return false;
144
145 bool Changed = false;
146 for (MachineBasicBlock &MBB : MF) {
147 for (MachineInstr &MI : MBB) {
148 if (MI.getOpcode() != AMDGPU::WWM_COPY)
149 continue;
150
151 // TODO: Club adjacent WWM ops between same exec save/restore
152 assert(TII->isVGPRCopy(MI));
153
154 // For WWM vector copies, manipulate the exec mask around the copy
155 // instruction.
156 const DebugLoc &DL = MI.getDebugLoc();
157 MachineBasicBlock::iterator InsertPt = MI.getIterator();
158 Register RegForExecCopy = MFI->getSGPRForEXECCopy();
159 TII->insertScratchExecCopy(MF, MBB, MBBI: InsertPt, DL, Reg: RegForExecCopy,
160 IsSCCLive: isSCCLiveAtMI(MI), Indexes);
161 TII->restoreExec(MF, MBB, MBBI: ++InsertPt, DL, Reg: RegForExecCopy, Indexes);
162 addToWWMSpills(MF, Reg: MI.getOperand(i: 0).getReg());
163 LLVM_DEBUG(dbgs() << "WWM copy manipulation for " << MI);
164
165 // Lower WWM_COPY back to COPY
166 MI.setDesc(TII->get(Opcode: AMDGPU::COPY));
167 Changed |= true;
168 }
169 }
170
171 return Changed;
172}
173