1//===-- SIWholeQuadMode.cpp - enter and suspend whole quad mode -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass adds instructions to enable whole quad mode (strict or non-strict)
11/// for pixel shaders, and strict whole wavefront mode for all programs.
12///
13/// The "strict" prefix indicates that inactive lanes do not take part in
14/// control flow, specifically an inactive lane enabled by a strict WQM/WWM will
15/// always be enabled irrespective of control flow decisions. Conversely in
16/// non-strict WQM inactive lanes may control flow decisions.
17///
18/// Whole quad mode is required for derivative computations, but it interferes
19/// with shader side effects (stores and atomics). It ensures that WQM is
20/// enabled when necessary, but disabled around stores and atomics.
21///
22/// When necessary, this pass creates a function prolog
23///
24/// S_MOV_B64 LiveMask, EXEC
25/// S_WQM_B64 EXEC, EXEC
26///
27/// to enter WQM at the top of the function and surrounds blocks of Exact
28/// instructions by
29///
30/// S_AND_SAVEEXEC_B64 Tmp, LiveMask
31/// ...
32/// S_MOV_B64 EXEC, Tmp
33///
34/// We also compute when a sequence of instructions requires strict whole
35/// wavefront mode (StrictWWM) and insert instructions to save and restore it:
36///
37/// S_OR_SAVEEXEC_B64 Tmp, -1
38/// ...
39/// S_MOV_B64 EXEC, Tmp
40///
41/// When a sequence of instructions requires strict whole quad mode (StrictWQM)
42/// we use a similar save and restore mechanism and force whole quad mode for
43/// those instructions:
44///
45/// S_MOV_B64 Tmp, EXEC
46/// S_WQM_B64 EXEC, EXEC
47/// ...
48/// S_MOV_B64 EXEC, Tmp
49///
50/// In order to avoid excessive switching during sequences of Exact
51/// instructions, the pass first analyzes which instructions must be run in WQM
52/// (aka which instructions produce values that lead to derivative
53/// computations).
54///
55/// Basic blocks are always exited in WQM as long as some successor needs WQM.
56///
57/// There is room for improvement given better control flow analysis:
58///
59/// (1) at the top level (outside of control flow statements, and as long as
60/// kill hasn't been used), one SGPR can be saved by recovering WQM from
61/// the LiveMask (this is implemented for the entry block).
62///
63/// (2) when entire regions (e.g. if-else blocks or entire loops) only
64/// consist of exact and don't-care instructions, the switch only has to
65/// be done at the entry and exit points rather than potentially in each
66/// block of the region.
67///
68//===----------------------------------------------------------------------===//
69
70#include "SIWholeQuadMode.h"
71#include "AMDGPU.h"
72#include "AMDGPULaneMaskUtils.h"
73#include "GCNSubtarget.h"
74#include "llvm/ADT/PostOrderIterator.h"
75#include "llvm/CodeGen/LiveIntervals.h"
76#include "llvm/CodeGen/MachineBasicBlock.h"
77#include "llvm/CodeGen/MachineDominators.h"
78#include "llvm/CodeGen/MachineFunctionPass.h"
79#include "llvm/CodeGen/MachineInstr.h"
80#include "llvm/CodeGen/MachinePostDominators.h"
81#include "llvm/IR/CallingConv.h"
82#include "llvm/InitializePasses.h"
83#include "llvm/Support/raw_ostream.h"
84
85using namespace llvm;
86
87#define DEBUG_TYPE "si-wqm"
88
89namespace {
90
91enum {
92 StateWQM = 0x1,
93 StateStrictWWM = 0x2,
94 StateStrictWQM = 0x4,
95 StateExact = 0x8,
96 StateStrict = StateStrictWWM | StateStrictWQM,
97};
98
99struct PrintState {
100public:
101 int State;
102
103 explicit PrintState(int State) : State(State) {}
104};
105
106#ifndef NDEBUG
107static raw_ostream &operator<<(raw_ostream &OS, const PrintState &PS) {
108
109 static const std::pair<char, const char *> Mapping[] = {
110 std::pair(StateWQM, "WQM"), std::pair(StateStrictWWM, "StrictWWM"),
111 std::pair(StateStrictWQM, "StrictWQM"), std::pair(StateExact, "Exact")};
112 char State = PS.State;
113 for (auto M : Mapping) {
114 if (State & M.first) {
115 OS << M.second;
116 State &= ~M.first;
117
118 if (State)
119 OS << '|';
120 }
121 }
122 assert(State == 0);
123 return OS;
124}
125#endif
126
127struct InstrInfo {
128 char Needs = 0;
129 char Disabled = 0;
130 char OutNeeds = 0;
131 char MarkedStates = 0;
132};
133
134struct BlockInfo {
135 char Needs = 0;
136 char InNeeds = 0;
137 char OutNeeds = 0;
138 char InitialState = 0;
139 bool NeedsLowering = false;
140};
141
142struct WorkItem {
143 MachineBasicBlock *MBB = nullptr;
144 MachineInstr *MI = nullptr;
145
146 WorkItem() = default;
147 WorkItem(MachineBasicBlock *MBB) : MBB(MBB) {}
148 WorkItem(MachineInstr *MI) : MI(MI) {}
149};
150
151class SIWholeQuadMode {
152public:
153 SIWholeQuadMode(MachineFunction &MF, LiveIntervals *LIS,
154 MachineDominatorTree *MDT, MachinePostDominatorTree *PDT)
155 : ST(&MF.getSubtarget<GCNSubtarget>()), TII(ST->getInstrInfo()),
156 TRI(&TII->getRegisterInfo()), MRI(&MF.getRegInfo()), LIS(LIS), MDT(MDT),
157 PDT(PDT), LMC(AMDGPU::LaneMaskConstants::get(ST: *ST)) {}
158 bool run(MachineFunction &MF);
159
160private:
161 const GCNSubtarget *ST;
162 const SIInstrInfo *TII;
163 const SIRegisterInfo *TRI;
164 MachineRegisterInfo *MRI;
165 LiveIntervals *LIS;
166 MachineDominatorTree *MDT;
167 MachinePostDominatorTree *PDT;
168 const AMDGPU::LaneMaskConstants &LMC;
169
170 Register LiveMaskReg;
171
172 DenseMap<const MachineInstr *, InstrInfo> Instructions;
173 MapVector<MachineBasicBlock *, BlockInfo> Blocks;
174
175 // Tracks state (WQM/StrictWWM/StrictWQM/Exact) after a given instruction
176 DenseMap<const MachineInstr *, char> StateTransition;
177
178 SmallVector<MachineInstr *, 2> LiveMaskQueries;
179 SmallVector<MachineInstr *, 4> LowerToMovInstrs;
180 SmallSetVector<MachineInstr *, 4> LowerToCopyInstrs;
181 SmallVector<MachineInstr *, 4> KillInstrs;
182 SmallVector<MachineInstr *, 4> InitExecInstrs;
183 SmallVector<MachineInstr *, 4> SetInactiveInstrs;
184
185 void printInfo();
186
187 void markInstruction(MachineInstr &MI, char Flag,
188 std::vector<WorkItem> &Worklist);
189 void markDefs(const MachineInstr &UseMI, LiveRange &LR,
190 VirtRegOrUnit VRegOrUnit, unsigned SubReg, char Flag,
191 std::vector<WorkItem> &Worklist);
192 void markOperand(const MachineInstr &MI, const MachineOperand &Op, char Flag,
193 std::vector<WorkItem> &Worklist);
194 void markInstructionUses(const MachineInstr &MI, char Flag,
195 std::vector<WorkItem> &Worklist);
196 char scanInstructions(MachineFunction &MF, std::vector<WorkItem> &Worklist,
197 SmallVector<MachineInstr *> &ExeczSideEffectInstrs);
198 void propagateInstruction(MachineInstr &MI, std::vector<WorkItem> &Worklist);
199 void propagateBlock(MachineBasicBlock &MBB, std::vector<WorkItem> &Worklist);
200 char analyzeFunction(MachineFunction &MF);
201
202 MachineBasicBlock::iterator saveSCC(MachineBasicBlock &MBB,
203 MachineBasicBlock::iterator Before);
204 MachineBasicBlock::iterator
205 prepareInsertion(MachineBasicBlock &MBB, MachineBasicBlock::iterator First,
206 MachineBasicBlock::iterator Last, bool PreferLast,
207 bool SaveSCC);
208 void toExact(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
209 Register SaveWQM);
210 void toWQM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
211 Register SavedWQM);
212 void toStrictMode(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
213 Register SaveOrig, char StrictStateNeeded);
214 void fromStrictMode(MachineBasicBlock &MBB,
215 MachineBasicBlock::iterator Before, Register SavedOrig,
216 char NonStrictState, char CurrentStrictState);
217
218 void splitBlock(MachineInstr *TermMI);
219 MachineInstr *lowerKillI1(MachineInstr &MI, bool IsWQM);
220 MachineInstr *lowerKillF32(MachineInstr &MI);
221
222 void lowerBlock(MachineBasicBlock &MBB, BlockInfo &BI);
223 void processBlock(MachineBasicBlock &MBB, BlockInfo &BI, bool IsEntry);
224
225 bool lowerLiveMaskQueries();
226 bool lowerCopyInstrs();
227 bool lowerKillInstrs(bool IsWQM);
228 void lowerInitExec(MachineInstr &MI);
229 MachineBasicBlock::iterator lowerInitExecInstrs(MachineBasicBlock &Entry,
230 bool &Changed);
231};
232
233class SIWholeQuadModeLegacy : public MachineFunctionPass {
234public:
235 static char ID;
236
237 SIWholeQuadModeLegacy() : MachineFunctionPass(ID) {}
238
239 bool runOnMachineFunction(MachineFunction &MF) override;
240
241 StringRef getPassName() const override { return "SI Whole Quad Mode"; }
242
243 void getAnalysisUsage(AnalysisUsage &AU) const override {
244 AU.addRequired<LiveIntervalsWrapperPass>();
245 AU.addPreserved<SlotIndexesWrapperPass>();
246 AU.addPreserved<LiveIntervalsWrapperPass>();
247 AU.addPreserved<MachineDominatorTreeWrapperPass>();
248 AU.addPreserved<MachinePostDominatorTreeWrapperPass>();
249 MachineFunctionPass::getAnalysisUsage(AU);
250 }
251
252 MachineFunctionProperties getClearedProperties() const override {
253 return MachineFunctionProperties().setIsSSA();
254 }
255};
256} // end anonymous namespace
257
258char SIWholeQuadModeLegacy::ID = 0;
259
260INITIALIZE_PASS_BEGIN(SIWholeQuadModeLegacy, DEBUG_TYPE, "SI Whole Quad Mode",
261 false, false)
262INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
263INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
264INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTreeWrapperPass)
265INITIALIZE_PASS_END(SIWholeQuadModeLegacy, DEBUG_TYPE, "SI Whole Quad Mode",
266 false, false)
267
268char &llvm::SIWholeQuadModeID = SIWholeQuadModeLegacy::ID;
269
270FunctionPass *llvm::createSIWholeQuadModeLegacyPass() {
271 return new SIWholeQuadModeLegacy;
272}
273
274#ifndef NDEBUG
275LLVM_DUMP_METHOD void SIWholeQuadMode::printInfo() {
276 for (const auto &BII : Blocks) {
277 dbgs() << "\n"
278 << printMBBReference(*BII.first) << ":\n"
279 << " InNeeds = " << PrintState(BII.second.InNeeds)
280 << ", Needs = " << PrintState(BII.second.Needs)
281 << ", OutNeeds = " << PrintState(BII.second.OutNeeds) << "\n\n";
282
283 for (const MachineInstr &MI : *BII.first) {
284 auto III = Instructions.find(&MI);
285 if (III != Instructions.end()) {
286 dbgs() << " " << MI << " Needs = " << PrintState(III->second.Needs)
287 << ", OutNeeds = " << PrintState(III->second.OutNeeds) << '\n';
288 }
289 }
290 }
291}
292#endif
293
294void SIWholeQuadMode::markInstruction(MachineInstr &MI, char Flag,
295 std::vector<WorkItem> &Worklist) {
296 InstrInfo &II = Instructions[&MI];
297
298 assert(!(Flag & StateExact) && Flag != 0);
299
300 // Capture all states requested in marking including disabled ones.
301 II.MarkedStates |= Flag;
302
303 // Remove any disabled states from the flag. The user that required it gets
304 // an undefined value in the helper lanes. For example, this can happen if
305 // the result of an atomic is used by instruction that requires WQM, where
306 // ignoring the request for WQM is correct as per the relevant specs.
307 Flag &= ~II.Disabled;
308
309 // Ignore if the flag is already encompassed by the existing needs, or we
310 // just disabled everything.
311 if ((II.Needs & Flag) == Flag)
312 return;
313
314 LLVM_DEBUG(dbgs() << "markInstruction " << PrintState(Flag) << ": " << MI);
315 II.Needs |= Flag;
316 Worklist.emplace_back(args: &MI);
317}
318
319/// Mark all relevant definitions of register \p Reg in usage \p UseMI.
320void SIWholeQuadMode::markDefs(const MachineInstr &UseMI, LiveRange &LR,
321 VirtRegOrUnit VRegOrUnit, unsigned SubReg,
322 char Flag, std::vector<WorkItem> &Worklist) {
323 LLVM_DEBUG(dbgs() << "markDefs " << PrintState(Flag) << ": " << UseMI);
324
325 LiveQueryResult UseLRQ = LR.Query(Idx: LIS->getInstructionIndex(Instr: UseMI));
326 const VNInfo *Value = UseLRQ.valueIn();
327 if (!Value)
328 return;
329
330 // Note: this code assumes that lane masks on AMDGPU completely
331 // cover registers.
332 const LaneBitmask UseLanes =
333 SubReg ? TRI->getSubRegIndexLaneMask(SubIdx: SubReg)
334 : (VRegOrUnit.isVirtualReg()
335 ? MRI->getMaxLaneMaskForVReg(Reg: VRegOrUnit.asVirtualReg())
336 : LaneBitmask::getNone());
337
338 // Perform a depth-first iteration of the LiveRange graph marking defs.
339 // Stop processing of a given branch when all use lanes have been defined.
340 // The first definition stops processing for a physical register.
341 struct PhiEntry {
342 const VNInfo *Phi;
343 unsigned PredIdx;
344 LaneBitmask DefinedLanes;
345
346 PhiEntry(const VNInfo *Phi, unsigned PredIdx, LaneBitmask DefinedLanes)
347 : Phi(Phi), PredIdx(PredIdx), DefinedLanes(DefinedLanes) {}
348 };
349 using VisitKey = std::pair<const VNInfo *, LaneBitmask>;
350 SmallVector<PhiEntry, 2> PhiStack;
351 SmallSet<VisitKey, 4> Visited;
352 LaneBitmask DefinedLanes;
353 unsigned NextPredIdx = 0; // Only used for processing phi nodes
354 do {
355 const VNInfo *NextValue = nullptr;
356 const VisitKey Key(Value, DefinedLanes);
357
358 if (Visited.insert(V: Key).second) {
359 // On first visit to a phi then start processing first predecessor
360 NextPredIdx = 0;
361 }
362
363 if (Value->isPHIDef()) {
364 // Each predecessor node in the phi must be processed as a subgraph
365 const MachineBasicBlock *MBB = LIS->getMBBFromIndex(index: Value->def);
366 assert(MBB && "Phi-def has no defining MBB");
367
368 // Find next predecessor to process
369 unsigned Idx = NextPredIdx;
370 const auto *PI = MBB->pred_begin() + Idx;
371 const auto *PE = MBB->pred_end();
372 for (; PI != PE && !NextValue; ++PI, ++Idx) {
373 if (const VNInfo *VN = LR.getVNInfoBefore(Idx: LIS->getMBBEndIdx(mbb: *PI))) {
374 if (!Visited.count(V: VisitKey(VN, DefinedLanes)))
375 NextValue = VN;
376 }
377 }
378
379 // If there are more predecessors to process; add phi to stack
380 if (PI != PE)
381 PhiStack.emplace_back(Args&: Value, Args&: Idx, Args&: DefinedLanes);
382 } else {
383 MachineInstr *MI = LIS->getInstructionFromIndex(index: Value->def);
384 assert(MI && "Def has no defining instruction");
385
386 if (VRegOrUnit.isVirtualReg()) {
387 // Iterate over all operands to find relevant definitions
388 bool HasDef = false;
389 for (const MachineOperand &Op : MI->all_defs()) {
390 if (Op.getReg() != VRegOrUnit.asVirtualReg())
391 continue;
392
393 // Compute lanes defined and overlap with use
394 LaneBitmask OpLanes =
395 Op.isUndef() ? LaneBitmask::getAll()
396 : TRI->getSubRegIndexLaneMask(SubIdx: Op.getSubReg());
397 LaneBitmask Overlap = (UseLanes & OpLanes);
398
399 // Record if this instruction defined any of use
400 HasDef |= Overlap.any();
401
402 // Mark any lanes defined
403 DefinedLanes |= OpLanes;
404 }
405
406 // Check if all lanes of use have been defined
407 if ((DefinedLanes & UseLanes) != UseLanes) {
408 // Definition not complete; need to process input value
409 LiveQueryResult LRQ = LR.Query(Idx: LIS->getInstructionIndex(Instr: *MI));
410 if (const VNInfo *VN = LRQ.valueIn()) {
411 if (!Visited.count(V: VisitKey(VN, DefinedLanes)))
412 NextValue = VN;
413 }
414 }
415
416 // Only mark the instruction if it defines some part of the use
417 if (HasDef)
418 markInstruction(MI&: *MI, Flag, Worklist);
419 } else {
420 // For physical registers simply mark the defining instruction
421 markInstruction(MI&: *MI, Flag, Worklist);
422 }
423 }
424
425 if (!NextValue && !PhiStack.empty()) {
426 // Reach end of chain; revert to processing last phi
427 PhiEntry &Entry = PhiStack.back();
428 NextValue = Entry.Phi;
429 NextPredIdx = Entry.PredIdx;
430 DefinedLanes = Entry.DefinedLanes;
431 PhiStack.pop_back();
432 }
433
434 Value = NextValue;
435 } while (Value);
436}
437
438void SIWholeQuadMode::markOperand(const MachineInstr &MI,
439 const MachineOperand &Op, char Flag,
440 std::vector<WorkItem> &Worklist) {
441 assert(Op.isReg());
442 Register Reg = Op.getReg();
443
444 // Ignore some hardware registers
445 switch (Reg) {
446 case AMDGPU::EXEC:
447 case AMDGPU::EXEC_LO:
448 return;
449 default:
450 break;
451 }
452
453 LLVM_DEBUG(dbgs() << "markOperand " << PrintState(Flag) << ": " << Op
454 << " for " << MI);
455 if (Reg.isVirtual()) {
456 LiveRange &LR = LIS->getInterval(Reg);
457 markDefs(UseMI: MI, LR, VRegOrUnit: VirtRegOrUnit(Reg), SubReg: Op.getSubReg(), Flag, Worklist);
458 } else {
459 // Handle physical registers that we need to track; this is mostly relevant
460 // for VCC, which can appear as the (implicit) input of a uniform branch,
461 // e.g. when a loop counter is stored in a VGPR.
462 for (MCRegUnit Unit : TRI->regunits(Reg: Reg.asMCReg())) {
463 LiveRange &LR = LIS->getRegUnit(Unit);
464 const VNInfo *Value = LR.Query(Idx: LIS->getInstructionIndex(Instr: MI)).valueIn();
465 if (Value)
466 markDefs(UseMI: MI, LR, VRegOrUnit: VirtRegOrUnit(Unit), SubReg: AMDGPU::NoSubRegister, Flag,
467 Worklist);
468 }
469 }
470}
471
472/// Mark all instructions defining the uses in \p MI with \p Flag.
473void SIWholeQuadMode::markInstructionUses(const MachineInstr &MI, char Flag,
474 std::vector<WorkItem> &Worklist) {
475 LLVM_DEBUG(dbgs() << "markInstructionUses " << PrintState(Flag) << ": "
476 << MI);
477
478 for (const MachineOperand &Use : MI.all_uses())
479 markOperand(MI, Op: Use, Flag, Worklist);
480}
481
482// Scan instructions to determine which ones require an Exact execmask and
483// which ones seed WQM requirements.
484char SIWholeQuadMode::scanInstructions(
485 MachineFunction &MF, std::vector<WorkItem> &Worklist,
486 SmallVector<MachineInstr *> &ExeczSideEffectInstrs) {
487 char GlobalFlags = 0;
488 bool WQMOutputs = MF.getFunction().hasFnAttribute(Kind: "amdgpu-ps-wqm-outputs");
489 SmallVector<MachineInstr *, 4> SoftWQMInstrs;
490 bool HasImplicitDerivatives =
491 MF.getFunction().getCallingConv() == CallingConv::AMDGPU_PS;
492
493 // We need to visit the basic blocks in reverse post-order so that we visit
494 // defs before uses, in particular so that we don't accidentally mark an
495 // instruction as needing e.g. WQM before visiting it and realizing it needs
496 // WQM disabled.
497 ReversePostOrderTraversal<MachineFunction *> RPOT(&MF);
498 for (MachineBasicBlock *MBB : RPOT) {
499 BlockInfo &BBI = Blocks[MBB];
500
501 for (MachineInstr &MI : *MBB) {
502 InstrInfo &III = Instructions[&MI];
503 unsigned Opcode = MI.getOpcode();
504 char Flags = 0;
505
506 if (TII->isWQM(Opcode)) {
507 // If LOD is not supported WQM is not needed.
508 // Only generate implicit WQM if implicit derivatives are required.
509 // This avoids inserting unintended WQM if a shader type without
510 // implicit derivatives uses an image sampling instruction.
511 if (ST->hasExtendedImageInsts() && HasImplicitDerivatives) {
512 // Sampling instructions don't need to produce results for all pixels
513 // in a quad, they just require all inputs of a quad to have been
514 // computed for derivatives.
515 markInstructionUses(MI, Flag: StateWQM, Worklist);
516 GlobalFlags |= StateWQM;
517 }
518 } else if (Opcode == AMDGPU::WQM) {
519 // The WQM intrinsic requires its output to have all the helper lanes
520 // correct, so we need it to be in WQM.
521 Flags = StateWQM;
522 LowerToCopyInstrs.insert(X: &MI);
523 } else if (Opcode == AMDGPU::SOFT_WQM) {
524 LowerToCopyInstrs.insert(X: &MI);
525 SoftWQMInstrs.push_back(Elt: &MI);
526 } else if (Opcode == AMDGPU::STRICT_WWM) {
527 // The STRICT_WWM intrinsic doesn't make the same guarantee, and plus
528 // it needs to be executed in WQM or Exact so that its copy doesn't
529 // clobber inactive lanes.
530 markInstructionUses(MI, Flag: StateStrictWWM, Worklist);
531 GlobalFlags |= StateStrictWWM;
532 LowerToMovInstrs.push_back(Elt: &MI);
533 } else if (Opcode == AMDGPU::STRICT_WQM ||
534 TII->isDualSourceBlendEXP(MI)) {
535 // STRICT_WQM is similar to STRICTWWM, but instead of enabling all
536 // threads of the wave like STRICTWWM, STRICT_WQM enables all threads in
537 // quads that have at least one active thread.
538 markInstructionUses(MI, Flag: StateStrictWQM, Worklist);
539 GlobalFlags |= StateStrictWQM;
540
541 if (Opcode == AMDGPU::STRICT_WQM) {
542 LowerToMovInstrs.push_back(Elt: &MI);
543 } else {
544 // Dual source blend export acts as implicit strict-wqm, its sources
545 // need to be shuffled in strict wqm, but the export itself needs to
546 // run in exact mode.
547 BBI.Needs |= StateExact;
548 if (!(BBI.InNeeds & StateExact)) {
549 BBI.InNeeds |= StateExact;
550 Worklist.emplace_back(args&: MBB);
551 }
552 GlobalFlags |= StateExact;
553 III.Disabled = StateWQM | StateStrict;
554 }
555 } else if (Opcode == AMDGPU::LDS_PARAM_LOAD ||
556 Opcode == AMDGPU::DS_PARAM_LOAD ||
557 Opcode == AMDGPU::LDS_DIRECT_LOAD ||
558 Opcode == AMDGPU::DS_DIRECT_LOAD) {
559 // Mark these STRICTWQM, but only for the instruction, not its operands.
560 // This avoid unnecessarily marking M0 as requiring WQM.
561 III.Needs |= StateStrictWQM;
562 GlobalFlags |= StateStrictWQM;
563 } else if (Opcode == AMDGPU::V_SET_INACTIVE_B32) {
564 // Disable strict states; StrictWQM will be added as required later.
565 III.Disabled = StateStrict;
566 MachineOperand &Inactive = MI.getOperand(i: 4);
567 if (Inactive.isReg()) {
568 if (Inactive.isUndef() && MI.getOperand(i: 3).getImm() == 0)
569 LowerToCopyInstrs.insert(X: &MI);
570 else
571 markOperand(MI, Op: Inactive, Flag: StateStrictWWM, Worklist);
572 }
573 SetInactiveInstrs.push_back(Elt: &MI);
574 BBI.NeedsLowering = true;
575 } else if (TII->isDisableWQM(MI)) {
576 BBI.Needs |= StateExact;
577 if (!(BBI.InNeeds & StateExact)) {
578 BBI.InNeeds |= StateExact;
579 Worklist.emplace_back(args&: MBB);
580 }
581 GlobalFlags |= StateExact;
582 III.Disabled = StateWQM | StateStrict;
583 } else if (Opcode == AMDGPU::SI_PS_LIVE ||
584 Opcode == AMDGPU::SI_LIVE_MASK) {
585 LiveMaskQueries.push_back(Elt: &MI);
586 } else if (Opcode == AMDGPU::SI_KILL_I1_TERMINATOR ||
587 Opcode == AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR ||
588 Opcode == AMDGPU::SI_DEMOTE_I1) {
589 KillInstrs.push_back(Elt: &MI);
590 BBI.NeedsLowering = true;
591 } else if (Opcode == AMDGPU::SI_INIT_EXEC ||
592 Opcode == AMDGPU::SI_INIT_EXEC_FROM_INPUT ||
593 Opcode == AMDGPU::SI_INIT_WHOLE_WAVE) {
594 InitExecInstrs.push_back(Elt: &MI);
595 } else if (WQMOutputs) {
596 // The function is in machine SSA form, which means that physical
597 // VGPRs correspond to shader inputs and outputs. Inputs are
598 // only used, outputs are only defined.
599 // FIXME: is this still valid?
600 for (const MachineOperand &MO : MI.defs()) {
601 Register Reg = MO.getReg();
602 if (Reg.isPhysical() &&
603 TRI->hasVectorRegisters(RC: TRI->getPhysRegBaseClass(Reg))) {
604 Flags = StateWQM;
605 break;
606 }
607 }
608 }
609
610 if (TII->hasUnwantedEffectsWhenEXECEmpty(MI)) {
611 for (auto &Op : MI.uses()) {
612 if (!Op.isReg())
613 continue;
614 if (!TRI->isVectorRegister(MRI: *MRI, Reg: Op.getReg()))
615 continue;
616
617 ExeczSideEffectInstrs.push_back(Elt: &MI);
618 break;
619 }
620 }
621
622 if (Flags) {
623 markInstruction(MI, Flag: Flags, Worklist);
624 GlobalFlags |= Flags;
625 }
626 }
627 }
628
629 // Mark sure that any SET_INACTIVE instructions are computed in WQM if WQM is
630 // ever used anywhere in the function. This implements the corresponding
631 // semantics of @llvm.amdgcn.set.inactive.
632 // Similarly for SOFT_WQM instructions, implementing @llvm.amdgcn.softwqm.
633 if (GlobalFlags & StateWQM) {
634 for (MachineInstr *MI : SetInactiveInstrs)
635 markInstruction(MI&: *MI, Flag: StateWQM, Worklist);
636 for (MachineInstr *MI : SoftWQMInstrs)
637 markInstruction(MI&: *MI, Flag: StateWQM, Worklist);
638 }
639
640 return GlobalFlags;
641}
642
643void SIWholeQuadMode::propagateInstruction(MachineInstr &MI,
644 std::vector<WorkItem>& Worklist) {
645 MachineBasicBlock *MBB = MI.getParent();
646 InstrInfo II = Instructions[&MI]; // take a copy to prevent dangling references
647 BlockInfo &BI = Blocks[MBB];
648
649 // Control flow-type instructions and stores to temporary memory that are
650 // followed by WQM computations must themselves be in WQM.
651 if ((II.OutNeeds & StateWQM) && !(II.Disabled & StateWQM) &&
652 (MI.isTerminator() || (TII->usesVM_CNT(MI) && MI.mayStore()))) {
653 Instructions[&MI].Needs = StateWQM;
654 II.Needs = StateWQM;
655 }
656
657 // Propagate to block level
658 if (II.Needs & StateWQM) {
659 BI.Needs |= StateWQM;
660 if (!(BI.InNeeds & StateWQM)) {
661 BI.InNeeds |= StateWQM;
662 Worklist.emplace_back(args&: MBB);
663 }
664 }
665
666 // Propagate backwards within block
667 if (MachineInstr *PrevMI = MI.getPrevNode()) {
668 char InNeeds = (II.Needs & ~StateStrict) | II.OutNeeds;
669 if (!PrevMI->isPHI()) {
670 InstrInfo &PrevII = Instructions[PrevMI];
671 if ((PrevII.OutNeeds | InNeeds) != PrevII.OutNeeds) {
672 PrevII.OutNeeds |= InNeeds;
673 Worklist.emplace_back(args&: PrevMI);
674 }
675 }
676 }
677
678 // Propagate WQM flag to instruction inputs
679 assert(!(II.Needs & StateExact));
680
681 if (II.Needs != 0)
682 markInstructionUses(MI, Flag: II.Needs, Worklist);
683
684 // Ensure we process a block containing StrictWWM/StrictWQM, even if it does
685 // not require any WQM transitions.
686 if (II.Needs & StateStrictWWM)
687 BI.Needs |= StateStrictWWM;
688 if (II.Needs & StateStrictWQM)
689 BI.Needs |= StateStrictWQM;
690}
691
692void SIWholeQuadMode::propagateBlock(MachineBasicBlock &MBB,
693 std::vector<WorkItem>& Worklist) {
694 BlockInfo BI = Blocks[&MBB]; // Make a copy to prevent dangling references.
695
696 // Propagate through instructions
697 if (!MBB.empty()) {
698 MachineInstr *LastMI = &*MBB.rbegin();
699 InstrInfo &LastII = Instructions[LastMI];
700 if ((LastII.OutNeeds | BI.OutNeeds) != LastII.OutNeeds) {
701 LastII.OutNeeds |= BI.OutNeeds;
702 Worklist.emplace_back(args&: LastMI);
703 }
704 }
705
706 // Predecessor blocks must provide for our WQM/Exact needs.
707 for (MachineBasicBlock *Pred : MBB.predecessors()) {
708 BlockInfo &PredBI = Blocks[Pred];
709 if ((PredBI.OutNeeds | BI.InNeeds) == PredBI.OutNeeds)
710 continue;
711
712 PredBI.OutNeeds |= BI.InNeeds;
713 PredBI.InNeeds |= BI.InNeeds;
714 Worklist.emplace_back(args&: Pred);
715 }
716
717 // All successors must be prepared to accept the same set of WQM/Exact data.
718 for (MachineBasicBlock *Succ : MBB.successors()) {
719 BlockInfo &SuccBI = Blocks[Succ];
720 if ((SuccBI.InNeeds | BI.OutNeeds) == SuccBI.InNeeds)
721 continue;
722
723 SuccBI.InNeeds |= BI.OutNeeds;
724 Worklist.emplace_back(args&: Succ);
725 }
726}
727
728char SIWholeQuadMode::analyzeFunction(MachineFunction &MF) {
729 std::vector<WorkItem> Worklist;
730 SmallVector<MachineInstr *> ExeczSideEffectInstrs;
731 char GlobalFlags = scanInstructions(MF, Worklist, ExeczSideEffectInstrs);
732
733 while (!Worklist.empty()) {
734 WorkItem WI = Worklist.back();
735 Worklist.pop_back();
736
737 if (WI.MI)
738 propagateInstruction(MI&: *WI.MI, Worklist);
739 else
740 propagateBlock(MBB&: *WI.MBB, Worklist);
741
742 if (Worklist.empty()) {
743 // Currently we let the instructions having sideeffect when execz to run
744 // under wqm, this avoids unwanted side-effect with exact mode if only
745 // helper lanes execute the parent block. At the same time, the wqm
746 // property should be back-propagated along the data-flow of their sources
747 // to ensure their sources have correct data for helper lanes.
748 for (auto *MI : ExeczSideEffectInstrs) {
749 InstrInfo II = Instructions[MI];
750 if (II.OutNeeds & StateWQM)
751 markInstructionUses(MI: *MI, Flag: StateWQM, Worklist);
752 }
753 // The side-effect backward propagation should not expand the wqm-region.
754 // So we only need to run the propagation once.
755 ExeczSideEffectInstrs.clear();
756 }
757 }
758
759 return GlobalFlags;
760}
761
762MachineBasicBlock::iterator
763SIWholeQuadMode::saveSCC(MachineBasicBlock &MBB,
764 MachineBasicBlock::iterator Before) {
765 Register SaveReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SReg_32_XM0RegClass);
766
767 MachineInstr *Save =
768 BuildMI(BB&: MBB, I: Before, MIMD: DebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: SaveReg)
769 .addReg(RegNo: AMDGPU::SCC);
770 MachineInstr *Restore =
771 BuildMI(BB&: MBB, I: Before, MIMD: DebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::SCC)
772 .addReg(RegNo: SaveReg);
773
774 LIS->InsertMachineInstrInMaps(MI&: *Save);
775 LIS->InsertMachineInstrInMaps(MI&: *Restore);
776 LIS->createAndComputeVirtRegInterval(Reg: SaveReg);
777
778 return Restore;
779}
780
781void SIWholeQuadMode::splitBlock(MachineInstr *TermMI) {
782 MachineBasicBlock *BB = TermMI->getParent();
783 LLVM_DEBUG(dbgs() << "Split block " << printMBBReference(*BB) << " @ "
784 << *TermMI << "\n");
785
786 MachineBasicBlock *SplitBB =
787 BB->splitAt(SplitInst&: *TermMI, /*UpdateLiveIns*/ true, LIS);
788
789 // Convert last instruction in block to a terminator.
790 // Note: this only covers the expected patterns
791 unsigned NewOpcode = 0;
792 switch (TermMI->getOpcode()) {
793 case AMDGPU::S_AND_B32:
794 NewOpcode = AMDGPU::S_AND_B32_term;
795 break;
796 case AMDGPU::S_AND_B64:
797 NewOpcode = AMDGPU::S_AND_B64_term;
798 break;
799 case AMDGPU::S_MOV_B32:
800 NewOpcode = AMDGPU::S_MOV_B32_term;
801 break;
802 case AMDGPU::S_MOV_B64:
803 NewOpcode = AMDGPU::S_MOV_B64_term;
804 break;
805 case AMDGPU::S_ANDN2_B32:
806 NewOpcode = AMDGPU::S_ANDN2_B32_term;
807 break;
808 case AMDGPU::S_ANDN2_B64:
809 NewOpcode = AMDGPU::S_ANDN2_B64_term;
810 break;
811 default:
812 llvm_unreachable("Unexpected instruction");
813 }
814
815 // These terminators fallthrough to the next block, no need to add an
816 // unconditional branch to the next block (SplitBB).
817 TermMI->setDesc(TII->get(Opcode: NewOpcode));
818
819 if (SplitBB != BB) {
820 // Update dominator trees
821 using DomTreeT = DomTreeBase<MachineBasicBlock>;
822 SmallVector<DomTreeT::UpdateType, 16> DTUpdates;
823 for (MachineBasicBlock *Succ : SplitBB->successors()) {
824 DTUpdates.push_back(Elt: {DomTreeT::Insert, SplitBB, Succ});
825 DTUpdates.push_back(Elt: {DomTreeT::Delete, BB, Succ});
826 }
827 DTUpdates.push_back(Elt: {DomTreeT::Insert, BB, SplitBB});
828 if (MDT)
829 MDT->applyUpdates(Updates: DTUpdates);
830 if (PDT)
831 PDT->applyUpdates(Updates: DTUpdates);
832 }
833}
834
835MachineInstr *SIWholeQuadMode::lowerKillF32(MachineInstr &MI) {
836 assert(LiveMaskReg.isVirtual());
837
838 const DebugLoc &DL = MI.getDebugLoc();
839 unsigned Opcode = 0;
840
841 assert(MI.getOperand(0).isReg());
842
843 // Comparison is for live lanes; however here we compute the inverse
844 // (killed lanes). This is because VCMP will always generate 0 bits
845 // for inactive lanes so a mask of live lanes would not be correct
846 // inside control flow.
847 // Invert the comparison by swapping the operands and adjusting
848 // the comparison codes.
849
850 switch (MI.getOperand(i: 2).getImm()) {
851 case ISD::SETUEQ:
852 Opcode = AMDGPU::V_CMP_LG_F32_e64;
853 break;
854 case ISD::SETUGT:
855 Opcode = AMDGPU::V_CMP_GE_F32_e64;
856 break;
857 case ISD::SETUGE:
858 Opcode = AMDGPU::V_CMP_GT_F32_e64;
859 break;
860 case ISD::SETULT:
861 Opcode = AMDGPU::V_CMP_LE_F32_e64;
862 break;
863 case ISD::SETULE:
864 Opcode = AMDGPU::V_CMP_LT_F32_e64;
865 break;
866 case ISD::SETUNE:
867 Opcode = AMDGPU::V_CMP_EQ_F32_e64;
868 break;
869 case ISD::SETO:
870 Opcode = AMDGPU::V_CMP_O_F32_e64;
871 break;
872 case ISD::SETUO:
873 Opcode = AMDGPU::V_CMP_U_F32_e64;
874 break;
875 case ISD::SETOEQ:
876 case ISD::SETEQ:
877 Opcode = AMDGPU::V_CMP_NEQ_F32_e64;
878 break;
879 case ISD::SETOGT:
880 case ISD::SETGT:
881 Opcode = AMDGPU::V_CMP_NLT_F32_e64;
882 break;
883 case ISD::SETOGE:
884 case ISD::SETGE:
885 Opcode = AMDGPU::V_CMP_NLE_F32_e64;
886 break;
887 case ISD::SETOLT:
888 case ISD::SETLT:
889 Opcode = AMDGPU::V_CMP_NGT_F32_e64;
890 break;
891 case ISD::SETOLE:
892 case ISD::SETLE:
893 Opcode = AMDGPU::V_CMP_NGE_F32_e64;
894 break;
895 case ISD::SETONE:
896 case ISD::SETNE:
897 Opcode = AMDGPU::V_CMP_NLG_F32_e64;
898 break;
899 default:
900 llvm_unreachable("invalid ISD:SET cond code");
901 }
902
903 MachineBasicBlock &MBB = *MI.getParent();
904
905 // Pick opcode based on comparison type.
906 MachineInstr *VcmpMI;
907 const MachineOperand &Op0 = MI.getOperand(i: 0);
908 const MachineOperand &Op1 = MI.getOperand(i: 1);
909
910 // VCC represents lanes killed.
911 if (TRI->isVGPR(MRI: *MRI, Reg: Op0.getReg())) {
912 Opcode = AMDGPU::getVOPe32(Opcode);
913 VcmpMI = BuildMI(BB&: MBB, I: &MI, MIMD: DL, MCID: TII->get(Opcode)).add(MO: Op1).add(MO: Op0);
914 } else {
915 VcmpMI = BuildMI(BB&: MBB, I: &MI, MIMD: DL, MCID: TII->get(Opcode))
916 .addReg(RegNo: LMC.VccReg, Flags: RegState::Define)
917 .addImm(Val: 0) // src0 modifiers
918 .add(MO: Op1)
919 .addImm(Val: 0) // src1 modifiers
920 .add(MO: Op0)
921 .addImm(Val: 0); // omod
922 }
923
924 MachineInstr *MaskUpdateMI =
925 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LMC.AndN2Opc), DestReg: LiveMaskReg)
926 .addReg(RegNo: LiveMaskReg)
927 .addReg(RegNo: LMC.VccReg);
928
929 // State of SCC represents whether any lanes are live in mask,
930 // if SCC is 0 then no lanes will be alive anymore.
931 MachineInstr *EarlyTermMI =
932 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::SI_EARLY_TERMINATE_SCC0));
933
934 MachineInstr *ExecMaskMI =
935 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LMC.AndN2Opc), DestReg: LMC.ExecReg)
936 .addReg(RegNo: LMC.ExecReg)
937 .addReg(RegNo: LMC.VccReg);
938
939 assert(MBB.succ_size() == 1);
940
941 // Update live intervals
942 LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *VcmpMI);
943 MBB.remove(I: &MI);
944
945 LIS->InsertMachineInstrInMaps(MI&: *MaskUpdateMI);
946 LIS->InsertMachineInstrInMaps(MI&: *EarlyTermMI);
947 LIS->InsertMachineInstrInMaps(MI&: *ExecMaskMI);
948
949 return ExecMaskMI;
950}
951
952MachineInstr *SIWholeQuadMode::lowerKillI1(MachineInstr &MI, bool IsWQM) {
953 assert(LiveMaskReg.isVirtual());
954
955 MachineBasicBlock &MBB = *MI.getParent();
956
957 const DebugLoc &DL = MI.getDebugLoc();
958 MachineInstr *MaskUpdateMI = nullptr;
959
960 const bool IsDemote = IsWQM && (MI.getOpcode() == AMDGPU::SI_DEMOTE_I1);
961 const MachineOperand &Op = MI.getOperand(i: 0);
962 int64_t KillVal = MI.getOperand(i: 1).getImm();
963 MachineInstr *ComputeKilledMaskMI = nullptr;
964 Register CndReg = !Op.isImm() ? Op.getReg() : Register();
965 Register TmpReg;
966
967 // Is this a static or dynamic kill?
968 if (Op.isImm()) {
969 if (Op.getImm() == KillVal) {
970 // Static: all active lanes are killed
971 MaskUpdateMI = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LMC.AndN2Opc), DestReg: LiveMaskReg)
972 .addReg(RegNo: LiveMaskReg)
973 .addReg(RegNo: LMC.ExecReg);
974 } else {
975 // Static: kill does nothing
976 bool IsLastTerminator = std::next(x: MI.getIterator()) == MBB.end();
977 if (!IsLastTerminator) {
978 LIS->RemoveMachineInstrFromMaps(MI);
979 } else {
980 assert(MBB.succ_size() == 1);
981 MachineInstr *NewTerm = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_BRANCH))
982 .addMBB(MBB: *MBB.succ_begin());
983 LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *NewTerm);
984 }
985 MBB.remove(I: &MI);
986 return nullptr;
987 }
988 } else {
989 if (!KillVal) {
990 // Op represents live lanes after kill,
991 // so exec mask needs to be factored in.
992 TmpReg = MRI->createVirtualRegister(RegClass: TRI->getBoolRC());
993 ComputeKilledMaskMI = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LMC.AndN2Opc), DestReg: TmpReg)
994 .addReg(RegNo: LMC.ExecReg)
995 .add(MO: Op);
996 MaskUpdateMI = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LMC.AndN2Opc), DestReg: LiveMaskReg)
997 .addReg(RegNo: LiveMaskReg)
998 .addReg(RegNo: TmpReg);
999 } else {
1000 // Op represents lanes to kill
1001 MaskUpdateMI = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LMC.AndN2Opc), DestReg: LiveMaskReg)
1002 .addReg(RegNo: LiveMaskReg)
1003 .add(MO: Op);
1004 }
1005 }
1006
1007 // State of SCC represents whether any lanes are live in mask,
1008 // if SCC is 0 then no lanes will be alive anymore.
1009 MachineInstr *EarlyTermMI =
1010 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::SI_EARLY_TERMINATE_SCC0));
1011
1012 // In the case we got this far some lanes are still live,
1013 // update EXEC to deactivate lanes as appropriate.
1014 MachineInstr *NewTerm;
1015 MachineInstr *WQMMaskMI = nullptr;
1016 Register LiveMaskWQM;
1017 if (IsDemote) {
1018 // Demote - deactivate quads with only helper lanes
1019 LiveMaskWQM = MRI->createVirtualRegister(RegClass: TRI->getBoolRC());
1020 WQMMaskMI = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LMC.WQMOpc), DestReg: LiveMaskWQM)
1021 .addReg(RegNo: LiveMaskReg);
1022 NewTerm = BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: LMC.AndOpc), DestReg: LMC.ExecReg)
1023 .addReg(RegNo: LMC.ExecReg)
1024 .addReg(RegNo: LiveMaskWQM);
1025 } else {
1026 // Kill - deactivate lanes no longer in live mask
1027 if (Op.isImm()) {
1028 NewTerm =
1029 BuildMI(BB&: MBB, I: &MI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg).addImm(Val: 0);
1030 } else if (!IsWQM) {
1031 NewTerm = BuildMI(BB&: MBB, I: &MI, MIMD: DL, MCID: TII->get(Opcode: LMC.AndOpc), DestReg: LMC.ExecReg)
1032 .addReg(RegNo: LMC.ExecReg)
1033 .addReg(RegNo: LiveMaskReg);
1034 } else {
1035 unsigned Opcode = KillVal ? LMC.AndN2Opc : LMC.AndOpc;
1036 NewTerm = BuildMI(BB&: MBB, I: &MI, MIMD: DL, MCID: TII->get(Opcode), DestReg: LMC.ExecReg)
1037 .addReg(RegNo: LMC.ExecReg)
1038 .add(MO: Op);
1039 }
1040 }
1041
1042 // Update live intervals
1043 LIS->RemoveMachineInstrFromMaps(MI);
1044 MBB.remove(I: &MI);
1045 assert(EarlyTermMI);
1046 assert(MaskUpdateMI);
1047 assert(NewTerm);
1048 if (ComputeKilledMaskMI)
1049 LIS->InsertMachineInstrInMaps(MI&: *ComputeKilledMaskMI);
1050 LIS->InsertMachineInstrInMaps(MI&: *MaskUpdateMI);
1051 LIS->InsertMachineInstrInMaps(MI&: *EarlyTermMI);
1052 if (WQMMaskMI)
1053 LIS->InsertMachineInstrInMaps(MI&: *WQMMaskMI);
1054 LIS->InsertMachineInstrInMaps(MI&: *NewTerm);
1055
1056 if (CndReg) {
1057 LIS->removeInterval(Reg: CndReg);
1058 LIS->createAndComputeVirtRegInterval(Reg: CndReg);
1059 }
1060 if (TmpReg)
1061 LIS->createAndComputeVirtRegInterval(Reg: TmpReg);
1062 if (LiveMaskWQM)
1063 LIS->createAndComputeVirtRegInterval(Reg: LiveMaskWQM);
1064
1065 return NewTerm;
1066}
1067
1068// Replace (or supplement) instructions accessing live mask.
1069// This can only happen once all the live mask registers have been created
1070// and the execute state (WQM/StrictWWM/Exact) of instructions is known.
1071void SIWholeQuadMode::lowerBlock(MachineBasicBlock &MBB, BlockInfo &BI) {
1072 if (!BI.NeedsLowering)
1073 return;
1074
1075 LLVM_DEBUG(dbgs() << "\nLowering block " << printMBBReference(MBB) << ":\n");
1076
1077 SmallVector<MachineInstr *, 4> SplitPoints;
1078 Register ActiveLanesReg = 0;
1079 char State = BI.InitialState;
1080
1081 for (MachineInstr &MI : llvm::make_early_inc_range(
1082 Range: llvm::make_range(x: MBB.getFirstNonPHI(), y: MBB.end()))) {
1083 auto MIState = StateTransition.find(Val: &MI);
1084 if (MIState != StateTransition.end())
1085 State = MIState->second;
1086
1087 MachineInstr *SplitPoint = nullptr;
1088 switch (MI.getOpcode()) {
1089 case AMDGPU::SI_DEMOTE_I1:
1090 case AMDGPU::SI_KILL_I1_TERMINATOR:
1091 SplitPoint = lowerKillI1(MI, IsWQM: State == StateWQM);
1092 break;
1093 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
1094 SplitPoint = lowerKillF32(MI);
1095 break;
1096 case AMDGPU::ENTER_STRICT_WWM:
1097 ActiveLanesReg = MI.getOperand(i: 0).getReg();
1098 break;
1099 case AMDGPU::EXIT_STRICT_WWM:
1100 ActiveLanesReg = 0;
1101 break;
1102 case AMDGPU::V_SET_INACTIVE_B32:
1103 if (ActiveLanesReg) {
1104 LiveInterval &LI = LIS->getInterval(Reg: MI.getOperand(i: 5).getReg());
1105 MRI->constrainRegClass(Reg: ActiveLanesReg, RC: TRI->getWaveMaskRegClass());
1106 MI.getOperand(i: 5).setReg(ActiveLanesReg);
1107 LIS->shrinkToUses(li: &LI);
1108 } else {
1109 assert(State == StateExact || State == StateWQM);
1110 }
1111 break;
1112 default:
1113 break;
1114 }
1115 if (SplitPoint)
1116 SplitPoints.push_back(Elt: SplitPoint);
1117 }
1118
1119 // Perform splitting after instruction scan to simplify iteration.
1120 for (MachineInstr *MI : SplitPoints)
1121 splitBlock(TermMI: MI);
1122}
1123
1124// Return an iterator in the (inclusive) range [First, Last] at which
1125// instructions can be safely inserted, keeping in mind that some of the
1126// instructions we want to add necessarily clobber SCC.
1127MachineBasicBlock::iterator SIWholeQuadMode::prepareInsertion(
1128 MachineBasicBlock &MBB, MachineBasicBlock::iterator First,
1129 MachineBasicBlock::iterator Last, bool PreferLast, bool SaveSCC) {
1130 if (!SaveSCC)
1131 return PreferLast ? Last : First;
1132
1133 LiveRange &LR =
1134 LIS->getRegUnit(Unit: *TRI->regunits(Reg: MCRegister::from(Val: AMDGPU::SCC)).begin());
1135 auto MBBE = MBB.end();
1136 // Skip debug instructions when getting slot indices, as they don't have
1137 // entries in the slot index map.
1138 auto FirstNonDbg = skipDebugInstructionsForward(It: First, End: MBBE);
1139 auto LastNonDbg = skipDebugInstructionsForward(It: Last, End: MBBE);
1140 SlotIndex FirstIdx = FirstNonDbg != MBBE
1141 ? LIS->getInstructionIndex(Instr: *FirstNonDbg)
1142 : LIS->getMBBEndIdx(mbb: &MBB);
1143 SlotIndex LastIdx = LastNonDbg != MBBE ? LIS->getInstructionIndex(Instr: *LastNonDbg)
1144 : LIS->getMBBEndIdx(mbb: &MBB);
1145 SlotIndex Idx = PreferLast ? LastIdx : FirstIdx;
1146 const LiveRange::Segment *S;
1147
1148 for (;;) {
1149 S = LR.getSegmentContaining(Idx);
1150 if (!S)
1151 break;
1152
1153 if (PreferLast) {
1154 SlotIndex Next = S->start.getBaseIndex();
1155 if (Next < FirstIdx)
1156 break;
1157 Idx = Next;
1158 } else {
1159 MachineInstr *EndMI = LIS->getInstructionFromIndex(index: S->end.getBaseIndex());
1160 assert(EndMI && "Segment does not end on valid instruction");
1161 auto NextI = next_nodbg(It: EndMI->getIterator(), End: MBB.instr_end());
1162 if (NextI == MBB.instr_end())
1163 break;
1164 SlotIndex Next = LIS->getInstructionIndex(Instr: *NextI);
1165 if (Next > LastIdx)
1166 break;
1167 Idx = Next;
1168 }
1169 }
1170
1171 MachineBasicBlock::iterator MBBI;
1172
1173 if (MachineInstr *MI = LIS->getInstructionFromIndex(index: Idx))
1174 MBBI = MI;
1175 else {
1176 assert(Idx == LIS->getMBBEndIdx(&MBB));
1177 MBBI = MBB.end();
1178 }
1179
1180 // Move insertion point past any operations modifying EXEC.
1181 // This assumes that the value of SCC defined by any of these operations
1182 // does not need to be preserved.
1183 while (MBBI != Last) {
1184 bool IsExecDef = false;
1185 for (const MachineOperand &MO : MBBI->all_defs()) {
1186 IsExecDef |=
1187 MO.getReg() == AMDGPU::EXEC_LO || MO.getReg() == AMDGPU::EXEC;
1188 }
1189 if (!IsExecDef)
1190 break;
1191 MBBI++;
1192 S = nullptr;
1193 }
1194
1195 if (S)
1196 MBBI = saveSCC(MBB, Before: MBBI);
1197
1198 return MBBI;
1199}
1200
1201void SIWholeQuadMode::toExact(MachineBasicBlock &MBB,
1202 MachineBasicBlock::iterator Before,
1203 Register SaveWQM) {
1204 assert(LiveMaskReg.isVirtual());
1205
1206 bool IsTerminator = Before == MBB.end();
1207 if (!IsTerminator) {
1208 auto FirstTerm = MBB.getFirstTerminator();
1209 if (FirstTerm != MBB.end()) {
1210 SlotIndex FirstTermIdx = LIS->getInstructionIndex(Instr: *FirstTerm);
1211 SlotIndex BeforeIdx = LIS->getInstructionIndex(Instr: *Before);
1212 IsTerminator = BeforeIdx > FirstTermIdx;
1213 }
1214 }
1215
1216 const DebugLoc &DL = MBB.findDebugLoc(MBBI: Before);
1217 MachineInstr *MI;
1218
1219 if (SaveWQM) {
1220 unsigned Opcode =
1221 IsTerminator ? LMC.AndSaveExecTermOpc : LMC.AndSaveExecOpc;
1222 MI =
1223 BuildMI(BB&: MBB, I: Before, MIMD: DL, MCID: TII->get(Opcode), DestReg: SaveWQM).addReg(RegNo: LiveMaskReg);
1224 } else {
1225 unsigned Opcode = IsTerminator ? LMC.AndTermOpc : LMC.AndOpc;
1226 MI = BuildMI(BB&: MBB, I: Before, MIMD: DL, MCID: TII->get(Opcode), DestReg: LMC.ExecReg)
1227 .addReg(RegNo: LMC.ExecReg)
1228 .addReg(RegNo: LiveMaskReg);
1229 }
1230
1231 LIS->InsertMachineInstrInMaps(MI&: *MI);
1232 LIS->removeAllRegUnitsForPhysReg(Reg: AMDGPU::EXEC);
1233 StateTransition[MI] = StateExact;
1234}
1235
1236void SIWholeQuadMode::toWQM(MachineBasicBlock &MBB,
1237 MachineBasicBlock::iterator Before,
1238 Register SavedWQM) {
1239 const DebugLoc &DL = MBB.findDebugLoc(MBBI: Before);
1240 MachineInstr *MI;
1241
1242 if (SavedWQM) {
1243 MI = BuildMI(BB&: MBB, I: Before, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: LMC.ExecReg)
1244 .addReg(RegNo: SavedWQM);
1245 } else {
1246 MI = BuildMI(BB&: MBB, I: Before, MIMD: DL, MCID: TII->get(Opcode: LMC.WQMOpc), DestReg: LMC.ExecReg)
1247 .addReg(RegNo: LMC.ExecReg);
1248 }
1249
1250 LIS->InsertMachineInstrInMaps(MI&: *MI);
1251 StateTransition[MI] = StateWQM;
1252}
1253
1254void SIWholeQuadMode::toStrictMode(MachineBasicBlock &MBB,
1255 MachineBasicBlock::iterator Before,
1256 Register SaveOrig, char StrictStateNeeded) {
1257 MachineInstr *MI;
1258 assert(SaveOrig);
1259 assert(StrictStateNeeded == StateStrictWWM ||
1260 StrictStateNeeded == StateStrictWQM);
1261
1262 const DebugLoc &DL = MBB.findDebugLoc(MBBI: Before);
1263
1264 if (StrictStateNeeded == StateStrictWWM) {
1265 MI = BuildMI(BB&: MBB, I: Before, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::ENTER_STRICT_WWM), DestReg: SaveOrig)
1266 .addImm(Val: -1);
1267 } else {
1268 MI = BuildMI(BB&: MBB, I: Before, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::ENTER_STRICT_WQM), DestReg: SaveOrig)
1269 .addImm(Val: -1);
1270 }
1271 LIS->InsertMachineInstrInMaps(MI&: *MI);
1272 StateTransition[MI] = StrictStateNeeded;
1273}
1274
1275void SIWholeQuadMode::fromStrictMode(MachineBasicBlock &MBB,
1276 MachineBasicBlock::iterator Before,
1277 Register SavedOrig, char NonStrictState,
1278 char CurrentStrictState) {
1279 MachineInstr *MI;
1280
1281 assert(SavedOrig);
1282 assert(CurrentStrictState == StateStrictWWM ||
1283 CurrentStrictState == StateStrictWQM);
1284
1285 const DebugLoc &DL = MBB.findDebugLoc(MBBI: Before);
1286
1287 if (CurrentStrictState == StateStrictWWM) {
1288 MI =
1289 BuildMI(BB&: MBB, I: Before, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::EXIT_STRICT_WWM), DestReg: LMC.ExecReg)
1290 .addReg(RegNo: SavedOrig);
1291 } else {
1292 MI =
1293 BuildMI(BB&: MBB, I: Before, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::EXIT_STRICT_WQM), DestReg: LMC.ExecReg)
1294 .addReg(RegNo: SavedOrig);
1295 }
1296 LIS->InsertMachineInstrInMaps(MI&: *MI);
1297 StateTransition[MI] = NonStrictState;
1298}
1299
1300void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, BlockInfo &BI,
1301 bool IsEntry) {
1302 // This is a non-entry block that is WQM throughout, so no need to do
1303 // anything.
1304 if (!IsEntry && BI.Needs == StateWQM && BI.OutNeeds != StateExact) {
1305 BI.InitialState = StateWQM;
1306 return;
1307 }
1308
1309 LLVM_DEBUG(dbgs() << "\nProcessing block " << printMBBReference(MBB)
1310 << ":\n");
1311
1312 Register SavedWQMReg;
1313 Register SavedNonStrictReg;
1314 bool WQMFromExec = IsEntry;
1315 char State = (IsEntry || !(BI.InNeeds & StateWQM)) ? StateExact : StateWQM;
1316 char NonStrictState = 0;
1317 const TargetRegisterClass *BoolRC = TRI->getBoolRC();
1318
1319 auto II = MBB.getFirstNonPHI(), IE = MBB.end();
1320 if (IsEntry) {
1321 // Skip the instruction that saves LiveMask
1322 if (II != IE && II->getOpcode() == AMDGPU::COPY &&
1323 II->getOperand(i: 1).getReg() == LMC.ExecReg)
1324 ++II;
1325 }
1326
1327 // This stores the first instruction where it's safe to switch from WQM to
1328 // Exact or vice versa.
1329 MachineBasicBlock::iterator FirstWQM = IE;
1330
1331 // This stores the first instruction where it's safe to switch from Strict
1332 // mode to Exact/WQM or to switch to Strict mode. It must always be the same
1333 // as, or after, FirstWQM since if it's safe to switch to/from Strict, it must
1334 // be safe to switch to/from WQM as well.
1335 MachineBasicBlock::iterator FirstStrict = IE;
1336
1337 // Record initial state is block information.
1338 BI.InitialState = State;
1339
1340 for (unsigned Idx = 0;; ++Idx) {
1341 MachineBasicBlock::iterator Next = II;
1342 char Needs = StateExact | StateWQM; // Strict mode is disabled by default.
1343 char OutNeeds = 0;
1344
1345 if (FirstWQM == IE)
1346 FirstWQM = II;
1347
1348 if (FirstStrict == IE)
1349 FirstStrict = II;
1350
1351 // Adjust needs if this is first instruction of WQM requiring shader.
1352 if (IsEntry && Idx == 0 && (BI.InNeeds & StateWQM))
1353 Needs = StateWQM;
1354
1355 // First, figure out the allowed states (Needs) based on the propagated
1356 // flags.
1357 if (II != IE) {
1358 MachineInstr &MI = *II;
1359
1360 if (MI.isTerminator() || TII->mayReadEXEC(MRI: *MRI, MI)) {
1361 auto III = Instructions.find(Val: &MI);
1362 if (III != Instructions.end()) {
1363 if (III->second.Needs & StateStrictWWM)
1364 Needs = StateStrictWWM;
1365 else if (III->second.Needs & StateStrictWQM)
1366 Needs = StateStrictWQM;
1367 else if (III->second.Needs & StateWQM)
1368 Needs = StateWQM;
1369 else
1370 Needs &= ~III->second.Disabled;
1371 OutNeeds = III->second.OutNeeds;
1372 }
1373 } else {
1374 // If the instruction doesn't actually need a correct EXEC, then we can
1375 // safely leave Strict mode enabled.
1376 Needs = StateExact | StateWQM | StateStrict;
1377 }
1378
1379 // Exact mode exit can occur in terminators, but must be before branches.
1380 if (MI.isBranch() && OutNeeds == StateExact)
1381 Needs = StateExact;
1382
1383 ++Next;
1384 } else {
1385 // End of basic block
1386 if (BI.OutNeeds & StateWQM)
1387 Needs = StateWQM;
1388 else if (BI.OutNeeds == StateExact)
1389 Needs = StateExact;
1390 else
1391 Needs = StateWQM | StateExact;
1392 }
1393
1394 // Now, transition if necessary.
1395 if (!(Needs & State)) {
1396 MachineBasicBlock::iterator First;
1397 if (State == StateStrictWWM || Needs == StateStrictWWM ||
1398 State == StateStrictWQM || Needs == StateStrictWQM) {
1399 // We must switch to or from Strict mode.
1400 First = FirstStrict;
1401 } else {
1402 // We only need to switch to/from WQM, so we can use FirstWQM.
1403 First = FirstWQM;
1404 }
1405
1406 // Whether we need to save SCC depends on start and end states.
1407 bool SaveSCC = false;
1408 switch (State) {
1409 case StateExact:
1410 case StateStrictWWM:
1411 case StateStrictWQM:
1412 // Exact/Strict -> Strict: save SCC
1413 // Exact/Strict -> WQM: save SCC if WQM mask is generated from exec
1414 // Exact/Strict -> Exact: no save
1415 SaveSCC = (Needs & StateStrict) || ((Needs & StateWQM) && WQMFromExec);
1416 break;
1417 case StateWQM:
1418 // WQM -> Exact/Strict: save SCC
1419 SaveSCC = !(Needs & StateWQM);
1420 break;
1421 default:
1422 llvm_unreachable("Unknown state");
1423 break;
1424 }
1425 char StartState = State & StateStrict ? NonStrictState : State;
1426 bool WQMToExact =
1427 StartState == StateWQM && (Needs & StateExact) && !(Needs & StateWQM);
1428 bool ExactToWQM = StartState == StateExact && (Needs & StateWQM) &&
1429 !(Needs & StateExact);
1430 bool PreferLast = Needs == StateWQM;
1431 // Exact regions in divergent control flow may run at EXEC=0, so try to
1432 // exclude instructions with unexpected effects from them.
1433 // FIXME: ideally we would branch over these when EXEC=0,
1434 // but this requires updating implicit values, live intervals and CFG.
1435 if ((WQMToExact && (OutNeeds & StateWQM)) || ExactToWQM) {
1436 for (MachineBasicBlock::iterator I = First; I != II; ++I) {
1437 if (TII->hasUnwantedEffectsWhenEXECEmpty(MI: *I)) {
1438 PreferLast = WQMToExact;
1439 break;
1440 }
1441 }
1442 }
1443 MachineBasicBlock::iterator Before =
1444 prepareInsertion(MBB, First, Last: II, PreferLast, SaveSCC);
1445
1446 if (State & StateStrict) {
1447 assert(State == StateStrictWWM || State == StateStrictWQM);
1448 assert(SavedNonStrictReg);
1449 fromStrictMode(MBB, Before, SavedOrig: SavedNonStrictReg, NonStrictState, CurrentStrictState: State);
1450
1451 LIS->createAndComputeVirtRegInterval(Reg: SavedNonStrictReg);
1452 SavedNonStrictReg = 0;
1453 State = NonStrictState;
1454 }
1455
1456 if (Needs & StateStrict) {
1457 NonStrictState = State;
1458 assert(Needs == StateStrictWWM || Needs == StateStrictWQM);
1459 assert(!SavedNonStrictReg);
1460 SavedNonStrictReg = MRI->createVirtualRegister(RegClass: BoolRC);
1461
1462 toStrictMode(MBB, Before, SaveOrig: SavedNonStrictReg, StrictStateNeeded: Needs);
1463 State = Needs;
1464 } else {
1465 if (WQMToExact) {
1466 if (!WQMFromExec && (OutNeeds & StateWQM)) {
1467 assert(!SavedWQMReg);
1468 SavedWQMReg = MRI->createVirtualRegister(RegClass: BoolRC);
1469 }
1470 Before = skipDebugInstructionsForward(It: Before, End: MBB.end());
1471 toExact(MBB, Before, SaveWQM: SavedWQMReg);
1472 State = StateExact;
1473 } else if (ExactToWQM) {
1474 assert(WQMFromExec == (SavedWQMReg == 0));
1475
1476 toWQM(MBB, Before, SavedWQM: SavedWQMReg);
1477
1478 if (SavedWQMReg) {
1479 LIS->createAndComputeVirtRegInterval(Reg: SavedWQMReg);
1480 SavedWQMReg = 0;
1481 }
1482 State = StateWQM;
1483 } else {
1484 // We can get here if we transitioned from StrictWWM to a
1485 // non-StrictWWM state that already matches our needs, but we
1486 // shouldn't need to do anything.
1487 assert(Needs & State);
1488 }
1489 }
1490 }
1491
1492 if (Needs != (StateExact | StateWQM | StateStrict)) {
1493 if (Needs != (StateExact | StateWQM))
1494 FirstWQM = IE;
1495 FirstStrict = IE;
1496 }
1497
1498 if (II == IE)
1499 break;
1500
1501 II = Next;
1502 }
1503 assert(!SavedWQMReg);
1504 assert(!SavedNonStrictReg);
1505}
1506
1507bool SIWholeQuadMode::lowerLiveMaskQueries() {
1508 for (MachineInstr *MI : LiveMaskQueries) {
1509 const DebugLoc &DL = MI->getDebugLoc();
1510 Register Dest = MI->getOperand(i: 0).getReg();
1511
1512 MachineInstr *Copy =
1513 BuildMI(BB&: *MI->getParent(), I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: Dest)
1514 .addReg(RegNo: LiveMaskReg);
1515
1516 LIS->ReplaceMachineInstrInMaps(MI&: *MI, NewMI&: *Copy);
1517 MI->eraseFromParent();
1518 }
1519 return !LiveMaskQueries.empty();
1520}
1521
1522bool SIWholeQuadMode::lowerCopyInstrs() {
1523 for (MachineInstr *MI : LowerToMovInstrs) {
1524 assert(MI->getNumExplicitOperands() == 2);
1525
1526 const Register Reg = MI->getOperand(i: 0).getReg();
1527
1528 const TargetRegisterClass *regClass =
1529 TRI->getRegClassForOperandReg(MRI: *MRI, MO: MI->getOperand(i: 0));
1530 if (TRI->isVGPRClass(RC: regClass)) {
1531 const unsigned MovOp = TII->getMovOpcode(DstRC: regClass);
1532 MI->setDesc(TII->get(Opcode: MovOp));
1533
1534 // Check that it already implicitly depends on exec (like all VALU movs
1535 // should do).
1536 assert(any_of(MI->implicit_operands(), [](const MachineOperand &MO) {
1537 return MO.isUse() && MO.getReg() == AMDGPU::EXEC;
1538 }));
1539 } else {
1540 // Remove early-clobber and exec dependency from simple SGPR copies.
1541 // This allows some to be eliminated during/post RA.
1542 LLVM_DEBUG(dbgs() << "simplify SGPR copy: " << *MI);
1543 if (MI->getOperand(i: 0).isEarlyClobber()) {
1544 LIS->removeInterval(Reg);
1545 MI->getOperand(i: 0).setIsEarlyClobber(false);
1546 LIS->createAndComputeVirtRegInterval(Reg);
1547 }
1548 int Index = MI->findRegisterUseOperandIdx(Reg: AMDGPU::EXEC, /*TRI=*/nullptr);
1549 while (Index >= 0) {
1550 MI->removeOperand(OpNo: Index);
1551 Index = MI->findRegisterUseOperandIdx(Reg: AMDGPU::EXEC, /*TRI=*/nullptr);
1552 }
1553 MI->setDesc(TII->get(Opcode: AMDGPU::COPY));
1554 LLVM_DEBUG(dbgs() << " -> " << *MI);
1555 }
1556 }
1557 for (MachineInstr *MI : LowerToCopyInstrs) {
1558 LLVM_DEBUG(dbgs() << "simplify: " << *MI);
1559
1560 if (MI->getOpcode() == AMDGPU::V_SET_INACTIVE_B32) {
1561 assert(MI->getNumExplicitOperands() == 6);
1562
1563 LiveInterval *RecomputeLI = nullptr;
1564 if (MI->getOperand(i: 4).isReg())
1565 RecomputeLI = &LIS->getInterval(Reg: MI->getOperand(i: 4).getReg());
1566
1567 MI->removeOperand(OpNo: 5);
1568 MI->removeOperand(OpNo: 4);
1569 MI->removeOperand(OpNo: 3);
1570 MI->removeOperand(OpNo: 1);
1571
1572 if (RecomputeLI)
1573 LIS->shrinkToUses(li: RecomputeLI);
1574 } else {
1575 assert(MI->getNumExplicitOperands() == 2);
1576 }
1577
1578 unsigned CopyOp = MI->getOperand(i: 1).isReg()
1579 ? (unsigned)AMDGPU::COPY
1580 : TII->getMovOpcode(DstRC: TRI->getRegClassForOperandReg(
1581 MRI: *MRI, MO: MI->getOperand(i: 0)));
1582 MI->setDesc(TII->get(Opcode: CopyOp));
1583 LLVM_DEBUG(dbgs() << " -> " << *MI);
1584 }
1585 return !LowerToCopyInstrs.empty() || !LowerToMovInstrs.empty();
1586}
1587
1588bool SIWholeQuadMode::lowerKillInstrs(bool IsWQM) {
1589 for (MachineInstr *MI : KillInstrs) {
1590 MachineInstr *SplitPoint = nullptr;
1591 switch (MI->getOpcode()) {
1592 case AMDGPU::SI_DEMOTE_I1:
1593 case AMDGPU::SI_KILL_I1_TERMINATOR:
1594 SplitPoint = lowerKillI1(MI&: *MI, IsWQM);
1595 break;
1596 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
1597 SplitPoint = lowerKillF32(MI&: *MI);
1598 break;
1599 }
1600 if (SplitPoint)
1601 splitBlock(TermMI: SplitPoint);
1602 }
1603 return !KillInstrs.empty();
1604}
1605
1606void SIWholeQuadMode::lowerInitExec(MachineInstr &MI) {
1607 MachineBasicBlock *MBB = MI.getParent();
1608
1609 if (MI.getOpcode() == AMDGPU::SI_INIT_WHOLE_WAVE) {
1610 assert(MBB == &MBB->getParent()->front() &&
1611 "init whole wave not in entry block");
1612 Register EntryExec = MRI->createVirtualRegister(RegClass: TRI->getBoolRC());
1613 MachineInstr *SaveExec = BuildMI(BB&: *MBB, I: MBB->begin(), MIMD: MI.getDebugLoc(),
1614 MCID: TII->get(Opcode: LMC.OrSaveExecOpc), DestReg: EntryExec)
1615 .addImm(Val: -1);
1616
1617 // Replace all uses of MI's destination reg with EntryExec.
1618 MRI->replaceRegWith(FromReg: MI.getOperand(i: 0).getReg(), ToReg: EntryExec);
1619
1620 if (LIS) {
1621 LIS->RemoveMachineInstrFromMaps(MI);
1622 }
1623
1624 MI.eraseFromParent();
1625
1626 if (LIS) {
1627 LIS->InsertMachineInstrInMaps(MI&: *SaveExec);
1628 LIS->createAndComputeVirtRegInterval(Reg: EntryExec);
1629 }
1630 return;
1631 }
1632
1633 if (MI.getOpcode() == AMDGPU::SI_INIT_EXEC) {
1634 // This should be before all vector instructions.
1635 MachineInstr *InitMI = BuildMI(BB&: *MBB, I: MBB->begin(), MIMD: MI.getDebugLoc(),
1636 MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
1637 .addImm(Val: MI.getOperand(i: 0).getImm());
1638 if (LIS) {
1639 LIS->RemoveMachineInstrFromMaps(MI);
1640 LIS->InsertMachineInstrInMaps(MI&: *InitMI);
1641 }
1642 MI.eraseFromParent();
1643 return;
1644 }
1645
1646 // Extract the thread count from an SGPR input and set EXEC accordingly.
1647 // Since BFM can't shift by 64, handle that case with CMP + CMOV.
1648 //
1649 // S_BFE_U32 count, input, {shift, 7}
1650 // S_BFM_B64 exec, count, 0
1651 // S_CMP_EQ_U32 count, 64
1652 // S_CMOV_B64 exec, -1
1653 Register InputReg = MI.getOperand(i: 0).getReg();
1654 MachineInstr *FirstMI = &*MBB->begin();
1655 if (InputReg.isVirtual()) {
1656 MachineInstr *DefInstr = MRI->getVRegDef(Reg: InputReg);
1657 assert(DefInstr && DefInstr->isCopy());
1658 if (DefInstr->getParent() == MBB) {
1659 if (DefInstr != FirstMI) {
1660 // If the `InputReg` is defined in current block, we also need to
1661 // move that instruction to the beginning of the block.
1662 DefInstr->removeFromParent();
1663 MBB->insert(I: FirstMI, MI: DefInstr);
1664 if (LIS)
1665 LIS->handleMove(MI&: *DefInstr);
1666 } else {
1667 // If first instruction is definition then move pointer after it.
1668 FirstMI = &*std::next(x: FirstMI->getIterator());
1669 }
1670 }
1671 }
1672
1673 // Insert instruction sequence at block beginning (before vector operations).
1674 const DebugLoc &DL = MI.getDebugLoc();
1675 const unsigned WavefrontSize = ST->getWavefrontSize();
1676 const unsigned Mask = (WavefrontSize << 1) - 1;
1677 Register CountReg = MRI->createVirtualRegister(RegClass: &AMDGPU::SGPR_32RegClass);
1678 auto BfeMI = BuildMI(BB&: *MBB, I: FirstMI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_BFE_U32), DestReg: CountReg)
1679 .addReg(RegNo: InputReg)
1680 .addImm(Val: (MI.getOperand(i: 1).getImm() & Mask) | 0x70000);
1681 auto BfmMI = BuildMI(BB&: *MBB, I: FirstMI, MIMD: DL, MCID: TII->get(Opcode: LMC.BfmOpc), DestReg: LMC.ExecReg)
1682 .addReg(RegNo: CountReg)
1683 .addImm(Val: 0);
1684 auto CmpMI = BuildMI(BB&: *MBB, I: FirstMI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMP_EQ_U32))
1685 .addReg(RegNo: CountReg, Flags: RegState::Kill)
1686 .addImm(Val: WavefrontSize);
1687 auto CmovMI =
1688 BuildMI(BB&: *MBB, I: FirstMI, MIMD: DL, MCID: TII->get(Opcode: LMC.CMovOpc), DestReg: LMC.ExecReg).addImm(Val: -1);
1689
1690 if (!LIS) {
1691 MI.eraseFromParent();
1692 return;
1693 }
1694
1695 LIS->RemoveMachineInstrFromMaps(MI);
1696 MI.eraseFromParent();
1697
1698 LIS->InsertMachineInstrInMaps(MI&: *BfeMI);
1699 LIS->InsertMachineInstrInMaps(MI&: *BfmMI);
1700 LIS->InsertMachineInstrInMaps(MI&: *CmpMI);
1701 LIS->InsertMachineInstrInMaps(MI&: *CmovMI);
1702
1703 LIS->removeInterval(Reg: InputReg);
1704 LIS->createAndComputeVirtRegInterval(Reg: InputReg);
1705 LIS->createAndComputeVirtRegInterval(Reg: CountReg);
1706}
1707
1708/// Lower INIT_EXEC instructions. Return a suitable insert point in \p Entry
1709/// for instructions that depend on EXEC.
1710MachineBasicBlock::iterator
1711SIWholeQuadMode::lowerInitExecInstrs(MachineBasicBlock &Entry, bool &Changed) {
1712 MachineBasicBlock::iterator InsertPt = Entry.getFirstNonPHI();
1713
1714 for (MachineInstr *MI : InitExecInstrs) {
1715 // Try to handle undefined cases gracefully:
1716 // - multiple INIT_EXEC instructions
1717 // - INIT_EXEC instructions not in the entry block
1718 if (MI->getParent() == &Entry)
1719 InsertPt = std::next(x: MI->getIterator());
1720
1721 lowerInitExec(MI&: *MI);
1722 Changed = true;
1723 }
1724
1725 return InsertPt;
1726}
1727
1728bool SIWholeQuadMode::run(MachineFunction &MF) {
1729 LLVM_DEBUG(dbgs() << "SI Whole Quad Mode on " << MF.getName()
1730 << " ------------- \n");
1731 LLVM_DEBUG(MF.dump(););
1732
1733 Instructions.clear();
1734 Blocks.clear();
1735 LiveMaskQueries.clear();
1736 LowerToCopyInstrs.clear();
1737 LowerToMovInstrs.clear();
1738 KillInstrs.clear();
1739 InitExecInstrs.clear();
1740 SetInactiveInstrs.clear();
1741 StateTransition.clear();
1742
1743 const char GlobalFlags = analyzeFunction(MF);
1744 bool Changed = false;
1745
1746 LiveMaskReg = LMC.ExecReg;
1747
1748 MachineBasicBlock &Entry = MF.front();
1749 MachineBasicBlock::iterator EntryMI = lowerInitExecInstrs(Entry, Changed);
1750
1751 // Store a copy of the original live mask when required
1752 const bool HasLiveMaskQueries = !LiveMaskQueries.empty();
1753 const bool HasWaveModes = GlobalFlags & ~StateExact;
1754 const bool HasKills = !KillInstrs.empty();
1755 const bool UsesWQM = GlobalFlags & StateWQM;
1756 if (HasKills || UsesWQM || (HasWaveModes && HasLiveMaskQueries)) {
1757 LiveMaskReg = MRI->createVirtualRegister(RegClass: TRI->getBoolRC());
1758 MachineInstr *MI =
1759 BuildMI(BB&: Entry, I: EntryMI, MIMD: DebugLoc(), MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: LiveMaskReg)
1760 .addReg(RegNo: LMC.ExecReg);
1761 LIS->InsertMachineInstrInMaps(MI&: *MI);
1762 Changed = true;
1763 }
1764
1765 // Check if V_SET_INACTIVE was touched by a strict state mode.
1766 // If so, promote to WWM; otherwise lower to COPY.
1767 for (MachineInstr *MI : SetInactiveInstrs) {
1768 if (LowerToCopyInstrs.contains(key: MI))
1769 continue;
1770 auto &Info = Instructions[MI];
1771 if (Info.MarkedStates & StateStrict) {
1772 Info.Needs |= StateStrictWWM;
1773 Info.Disabled &= ~StateStrictWWM;
1774 Blocks[MI->getParent()].Needs |= StateStrictWWM;
1775 } else {
1776 LLVM_DEBUG(dbgs() << "Has no WWM marking: " << *MI);
1777 LowerToCopyInstrs.insert(X: MI);
1778 }
1779 }
1780
1781 LLVM_DEBUG(printInfo());
1782
1783 Changed |= lowerLiveMaskQueries();
1784 Changed |= lowerCopyInstrs();
1785
1786 if (!HasWaveModes) {
1787 // No wave mode execution
1788 Changed |= lowerKillInstrs(IsWQM: false);
1789 } else if (GlobalFlags == StateWQM) {
1790 // Shader only needs WQM
1791 auto MI =
1792 BuildMI(BB&: Entry, I: EntryMI, MIMD: DebugLoc(), MCID: TII->get(Opcode: LMC.WQMOpc), DestReg: LMC.ExecReg)
1793 .addReg(RegNo: LMC.ExecReg);
1794 LIS->InsertMachineInstrInMaps(MI&: *MI);
1795 lowerKillInstrs(IsWQM: true);
1796 Changed = true;
1797 } else {
1798 // Mark entry for WQM if required.
1799 if (GlobalFlags & StateWQM)
1800 Blocks[&Entry].InNeeds |= StateWQM;
1801 // Wave mode switching requires full lowering pass.
1802 for (auto &BII : Blocks)
1803 processBlock(MBB&: *BII.first, BI&: BII.second, IsEntry: BII.first == &Entry);
1804 // Lowering blocks causes block splitting so perform as a second pass.
1805 for (auto &BII : Blocks)
1806 lowerBlock(MBB&: *BII.first, BI&: BII.second);
1807 Changed = true;
1808 }
1809
1810 // Compute live range for live mask
1811 if (LiveMaskReg != LMC.ExecReg)
1812 LIS->createAndComputeVirtRegInterval(Reg: LiveMaskReg);
1813
1814 // Physical registers like SCC aren't tracked by default anyway, so just
1815 // removing the ranges we computed is the simplest option for maintaining
1816 // the analysis results.
1817 LIS->removeAllRegUnitsForPhysReg(Reg: AMDGPU::SCC);
1818
1819 // If we performed any kills then recompute EXEC
1820 if (!KillInstrs.empty() || !InitExecInstrs.empty())
1821 LIS->removeAllRegUnitsForPhysReg(Reg: AMDGPU::EXEC);
1822
1823 return Changed;
1824}
1825
1826bool SIWholeQuadModeLegacy::runOnMachineFunction(MachineFunction &MF) {
1827 LiveIntervals *LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();
1828 auto *MDTWrapper = getAnalysisIfAvailable<MachineDominatorTreeWrapperPass>();
1829 MachineDominatorTree *MDT = MDTWrapper ? &MDTWrapper->getDomTree() : nullptr;
1830 auto *PDTWrapper =
1831 getAnalysisIfAvailable<MachinePostDominatorTreeWrapperPass>();
1832 MachinePostDominatorTree *PDT =
1833 PDTWrapper ? &PDTWrapper->getPostDomTree() : nullptr;
1834 SIWholeQuadMode Impl(MF, LIS, MDT, PDT);
1835 return Impl.run(MF);
1836}
1837
1838PreservedAnalyses
1839SIWholeQuadModePass::run(MachineFunction &MF,
1840 MachineFunctionAnalysisManager &MFAM) {
1841 MFPropsModifier _(*this, MF);
1842
1843 LiveIntervals *LIS = &MFAM.getResult<LiveIntervalsAnalysis>(IR&: MF);
1844 MachineDominatorTree *MDT =
1845 MFAM.getCachedResult<MachineDominatorTreeAnalysis>(IR&: MF);
1846 MachinePostDominatorTree *PDT =
1847 MFAM.getCachedResult<MachinePostDominatorTreeAnalysis>(IR&: MF);
1848 SIWholeQuadMode Impl(MF, LIS, MDT, PDT);
1849 bool Changed = Impl.run(MF);
1850 if (!Changed)
1851 return PreservedAnalyses::all();
1852
1853 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
1854 PA.preserve<SlotIndexesAnalysis>();
1855 PA.preserve<LiveIntervalsAnalysis>();
1856 PA.preserve<MachineDominatorTreeAnalysis>();
1857 PA.preserve<MachinePostDominatorTreeAnalysis>();
1858 return PA;
1859}
1860