1//===-- SILowerWWMCopies.cpp - Lower Copies after regalloc ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Lowering the WWM_COPY instructions for various register classes.
11/// AMDGPU target generates WWM_COPY instruction to differentiate WWM
12/// copy from COPY. This pass generates the necessary exec mask manipulation
13/// instructions to replicate 'Whole Wave Mode' and lowers WWM_COPY back to
14/// COPY.
15//
16//===----------------------------------------------------------------------===//
17
18#include "SILowerWWMCopies.h"
19#include "AMDGPU.h"
20#include "GCNSubtarget.h"
21#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
22#include "SIMachineFunctionInfo.h"
23#include "llvm/CodeGen/LiveIntervals.h"
24#include "llvm/CodeGen/MachineFunctionPass.h"
25#include "llvm/CodeGen/VirtRegMap.h"
26#include "llvm/InitializePasses.h"
27
28using namespace llvm;
29
30#define DEBUG_TYPE "si-lower-wwm-copies"
31
32namespace {
33
34class SILowerWWMCopies {
35public:
36 SILowerWWMCopies(LiveIntervals *LIS, SlotIndexes *SI, VirtRegMap *VRM)
37 : LIS(LIS), Indexes(SI), VRM(VRM) {}
38 bool run(MachineFunction &MF);
39
40private:
41 bool isSCCLiveAtMI(const MachineInstr &MI);
42 void addToWWMSpills(MachineFunction &MF, Register Reg);
43
44 LiveIntervals *LIS;
45 SlotIndexes *Indexes;
46 VirtRegMap *VRM;
47 const SIRegisterInfo *TRI;
48 const MachineRegisterInfo *MRI;
49 SIMachineFunctionInfo *MFI;
50};
51
52class SILowerWWMCopiesLegacy : public MachineFunctionPass {
53public:
54 static char ID;
55
56 SILowerWWMCopiesLegacy() : MachineFunctionPass(ID) {}
57
58 bool runOnMachineFunction(MachineFunction &MF) override;
59
60 StringRef getPassName() const override { return "SI Lower WWM Copies"; }
61
62 void getAnalysisUsage(AnalysisUsage &AU) const override {
63 AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();
64 AU.addUsedIfAvailable<SlotIndexesWrapperPass>();
65 AU.addUsedIfAvailable<VirtRegMapWrapperLegacy>();
66 AU.setPreservesAll();
67 MachineFunctionPass::getAnalysisUsage(AU);
68 }
69};
70
71} // End anonymous namespace.
72
73INITIALIZE_PASS_BEGIN(SILowerWWMCopiesLegacy, DEBUG_TYPE, "SI Lower WWM Copies",
74 false, false)
75INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
76INITIALIZE_PASS_DEPENDENCY(VirtRegMapWrapperLegacy)
77INITIALIZE_PASS_END(SILowerWWMCopiesLegacy, DEBUG_TYPE, "SI Lower WWM Copies",
78 false, false)
79
80char SILowerWWMCopiesLegacy::ID = 0;
81
82char &llvm::SILowerWWMCopiesLegacyID = SILowerWWMCopiesLegacy::ID;
83
84bool SILowerWWMCopies::isSCCLiveAtMI(const MachineInstr &MI) {
85 // We can't determine the liveness info if LIS isn't available. Early return
86 // in that case and always assume SCC is live.
87 if (!LIS)
88 return true;
89
90 LiveRange &LR =
91 LIS->getRegUnit(Unit: *MCRegUnitIterator(MCRegister::from(Val: AMDGPU::SCC), TRI));
92 SlotIndex Idx = LIS->getInstructionIndex(Instr: MI);
93 return LR.liveAt(index: Idx);
94}
95
96// If \p Reg is assigned with a physical VGPR, add the latter into wwm-spills
97// for preserving its entire lanes at function prolog/epilog.
98void SILowerWWMCopies::addToWWMSpills(MachineFunction &MF, Register Reg) {
99 if (Reg.isPhysical())
100 return;
101
102 // FIXME: VRM may be null here.
103 MCRegister PhysReg = VRM->getPhys(virtReg: Reg);
104 assert(PhysReg && "should have allocated a physical register");
105
106 MFI->allocateWWMSpill(MF, VGPR: PhysReg);
107}
108
109bool SILowerWWMCopiesLegacy::runOnMachineFunction(MachineFunction &MF) {
110 auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
111 auto *LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
112
113 auto *SIWrapper = getAnalysisIfAvailable<SlotIndexesWrapperPass>();
114 auto *Indexes = SIWrapper ? &SIWrapper->getSI() : nullptr;
115
116 auto *VRMWrapper = getAnalysisIfAvailable<VirtRegMapWrapperLegacy>();
117 auto *VRM = VRMWrapper ? &VRMWrapper->getVRM() : nullptr;
118
119 SILowerWWMCopies Impl(LIS, Indexes, VRM);
120 return Impl.run(MF);
121}
122
123PreservedAnalyses
124SILowerWWMCopiesPass::run(MachineFunction &MF,
125 MachineFunctionAnalysisManager &MFAM) {
126 auto *LIS = MFAM.getCachedResult<LiveIntervalsAnalysis>(IR&: MF);
127 auto *Indexes = MFAM.getCachedResult<SlotIndexesAnalysis>(IR&: MF);
128 auto *VRM = MFAM.getCachedResult<VirtRegMapAnalysis>(IR&: MF);
129
130 SILowerWWMCopies Impl(LIS, Indexes, VRM);
131 Impl.run(MF);
132 return PreservedAnalyses::all();
133}
134
135bool SILowerWWMCopies::run(MachineFunction &MF) {
136 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
137 const SIInstrInfo *TII = ST.getInstrInfo();
138
139 MFI = MF.getInfo<SIMachineFunctionInfo>();
140 TRI = ST.getRegisterInfo();
141 MRI = &MF.getRegInfo();
142
143 if (!MFI->hasVRegFlags())
144 return false;
145
146 bool Changed = false;
147 for (MachineBasicBlock &MBB : MF) {
148 for (MachineInstr &MI : MBB) {
149 if (MI.getOpcode() != AMDGPU::WWM_COPY)
150 continue;
151
152 // TODO: Club adjacent WWM ops between same exec save/restore
153 assert(TII->isVGPRCopy(MI));
154
155 // For WWM vector copies, manipulate the exec mask around the copy
156 // instruction.
157 const DebugLoc &DL = MI.getDebugLoc();
158 MachineBasicBlock::iterator InsertPt = MI.getIterator();
159 Register RegForExecCopy = MFI->getSGPRForEXECCopy();
160 TII->insertScratchExecCopy(MF, MBB, MBBI: InsertPt, DL, Reg: RegForExecCopy,
161 IsSCCLive: isSCCLiveAtMI(MI), Indexes);
162 TII->restoreExec(MF, MBB, MBBI: ++InsertPt, DL, Reg: RegForExecCopy, Indexes);
163 addToWWMSpills(MF, Reg: MI.getOperand(i: 0).getReg());
164 LLVM_DEBUG(dbgs() << "WWM copy manipulation for " << MI);
165
166 // Lower WWM_COPY back to COPY
167 MI.setDesc(TII->get(Opcode: AMDGPU::COPY));
168 Changed |= true;
169 }
170 }
171
172 return Changed;
173}
174