1//===-- GCNPreRAAntiHints.cpp - MFMA register anti-hints ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Insert register allocation anti-hints.
11///
12//===----------------------------------------------------------------------===//
13
14#include "GCNPreRAAntiHints.h"
15#include "GCNSubtarget.h"
16#include "SIInstrInfo.h"
17#include "SIRegisterInfo.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/CodeGen/LiveIntervals.h"
20#include "llvm/CodeGen/MachineRegisterInfo.h"
21#include "llvm/CodeGen/SlotIndexes.h"
22#include "llvm/CodeGen/TargetSchedule.h"
23
24using namespace llvm;
25using namespace llvm::AMDGPU;
26
27#define DEBUG_TYPE "amdgpu-anti-hints"
28
29namespace HC = llvm::AMDGPU::HazardClass;
30
31enum class AntiHintRule {
32 None,
33 MFMAWAW,
34 MFMAWAR,
35 WMMAWARAB,
36 SWMMACWARIndex,
37 WMMAWAW,
38 TransWAR,
39 MemAddrWAR,
40 VAVdstWAR,
41 All,
42};
43
44static cl::list<AntiHintRule> AntiHintRuleSelection(
45 "amdgpu-anti-hints-rules", cl::Hidden, cl::CommaSeparated,
46 cl::desc("Anti-hints rules to select."),
47 cl::values(clEnumValN(AntiHintRule::None, "none", "Select no rules"),
48 clEnumValN(AntiHintRule::MFMAWAW, "mfma-waw",
49 "MFMA destination write-after-write"),
50 clEnumValN(AntiHintRule::MFMAWAR, "mfma-war",
51 "XDL MFMA src2 write-after-read"),
52 clEnumValN(AntiHintRule::WMMAWARAB, "wmma-war-ab",
53 "XDL WMMA A/B source write-after-read"),
54 clEnumValN(AntiHintRule::SWMMACWARIndex, "swmmac-war-index",
55 "XDL SWMMAC sparse index write-after-read"),
56 clEnumValN(AntiHintRule::WMMAWAW, "wmma-waw",
57 "XDL WMMA destination write-after-write"),
58 clEnumValN(AntiHintRule::TransWAR, "trans-war",
59 "TRANS source write-after-read"),
60 clEnumValN(AntiHintRule::MemAddrWAR, "mem-addr-war",
61 "Memory address/data write-after-read (s_wait_xcnt)"),
62 clEnumValN(AntiHintRule::VAVdstWAR, "va-vdst-war",
63 "VALU source write-after-read by a load "
64 "(s_wait_alu va_vdst)"),
65 clEnumValN(AntiHintRule::All, "all",
66 "Select all rules (default); per-rule "
67 "-amdgpu-anti-hints-for-* still apply")));
68
69static cl::opt<bool>
70 EnableAntiHintsForAddr("amdgpu-anti-hints-for-addr", cl::Hidden,
71 cl::desc("Enable Anti-Hints for memory address "
72 "operands and subsequent VGPR writes to "
73 "avoid wait xcnt."),
74 cl::init(Val: true));
75
76static cl::opt<bool>
77 EnableAntiHintsForVAVdst("amdgpu-anti-hints-for-va-vdst", cl::Hidden,
78 cl::desc("Enable Anti-Hints for VA-VDST."),
79 cl::init(Val: false));
80
81static cl::opt<unsigned>
82 VAVDSTLookbackWindow("amdgpu-va-vdst-lookback-window", cl::Hidden,
83 cl::desc("Lookback window for VA_VDST anti-hints."),
84 cl::init(Val: 32));
85
86static cl::opt<unsigned>
87 AddrAntiHintWindow("amdgpu-addr-anti-hint-window", cl::Hidden,
88 cl::desc("Number of later memory instructions an "
89 "address anti-hint stays open."),
90 cl::init(Val: 16));
91
92namespace {
93
94// Classify the MI into a HazardClassMask.
95HazardClassMask getInstHazardClass(const MachineInstr &MI,
96 const HazardContext &Ctx) {
97 const SIInstrInfo &TII = *Ctx.TII;
98 HazardClassMask Mask = HC::None;
99
100 if (TII.isLDSDMA(MI))
101 Mask = HC::VALU | HC::VMEM | HC::DS | HC::LDSDMA;
102 else if (TII.isWMMA(MI) || SIInstrInfo::isSWMMAC(MI))
103 Mask = HC::WMMA;
104 else if (TII.isMFMA(MI))
105 Mask = HC::MFMA;
106 else if (SIInstrInfo::isTRANS(MI))
107 Mask = HC::TRANS;
108 else if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
109 Mask = HC::VALU;
110 else if (TII.isDS(MI))
111 Mask = HC::DS;
112 else if (TII.isVMEM(MI))
113 Mask = HC::VMEM;
114 else if (TII.isSMRD(MI))
115 Mask = HC::SMEM;
116 else if (TII.isEXP(MI))
117 Mask = HC::EXP;
118 else if (SIInstrInfo::isSALU(MI))
119 Mask = HC::SALU;
120 else if (MI.isCopy() && MI.getOperand(i: 0).getReg().isVirtual() &&
121 Ctx.TRI->hasVGPRs(RC: Ctx.MRI->getRegClass(Reg: MI.getOperand(i: 0).getReg())))
122 Mask = HC::VALU;
123 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
124 Mask |= HC::RawVALU;
125 if (MI.mayLoad())
126 Mask |= HC::Load;
127 for (const MachineOperand &MO : MI.defs()) {
128 if (MO.isReg() && MO.getReg().isVirtual() &&
129 Ctx.TRI->hasVGPRs(RC: Ctx.MRI->getRegClass(Reg: MO.getReg()))) {
130 Mask |= HC::WritesVGPR;
131 break;
132 }
133 }
134
135 return Mask;
136}
137
138bool isVirtualVGPR(const HazardContext &Ctx, const MachineOperand &MO) {
139 if (!MO.isReg() || !MO.getReg().isVirtual())
140 return false;
141 return Ctx.TRI->hasVGPRs(RC: Ctx.MRI->getRegClass(Reg: MO.getReg()));
142}
143
144void collectOperandRegs(const MachineInstr &MI, HazardOperand Op,
145 const HazardContext &Ctx,
146 SmallVectorImpl<Register> &Out) {
147 const SIInstrInfo &TII = *Ctx.TII;
148 switch (Op) {
149 case HazardOperand::None:
150 break;
151 case HazardOperand::Def:
152 for (const MachineOperand &MO : MI.all_defs()) {
153 if (isVirtualVGPR(Ctx, MO))
154 Out.push_back(Elt: MO.getReg());
155 }
156 break;
157 case HazardOperand::Src0:
158 if (const MachineOperand *MO =
159 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src0)) {
160 if (isVirtualVGPR(Ctx, MO: *MO))
161 Out.push_back(Elt: MO->getReg());
162 }
163 break;
164 case HazardOperand::Src1:
165 if (const MachineOperand *MO =
166 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src1)) {
167 if (isVirtualVGPR(Ctx, MO: *MO))
168 Out.push_back(Elt: MO->getReg());
169 }
170 break;
171 case HazardOperand::Src2:
172 if (const MachineOperand *MO =
173 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src2)) {
174 if (isVirtualVGPR(Ctx, MO: *MO))
175 Out.push_back(Elt: MO->getReg());
176 }
177 break;
178 case HazardOperand::Src0Src1:
179 for (AMDGPU::OpName Name : {AMDGPU::OpName::src0, AMDGPU::OpName::src1}) {
180 if (const MachineOperand *MO = TII.getNamedOperand(MI, OperandName: Name)) {
181 if (isVirtualVGPR(Ctx, MO: *MO))
182 Out.push_back(Elt: MO->getReg());
183 }
184 }
185 break;
186 case HazardOperand::Idx:
187 if (const MachineOperand *MO =
188 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::idx)) {
189 if (isVirtualVGPR(Ctx, MO: *MO))
190 Out.push_back(Elt: MO->getReg());
191 }
192 break;
193 case HazardOperand::Vaddr:
194 if (const MachineOperand *MO =
195 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr)) {
196 if (isVirtualVGPR(Ctx, MO: *MO))
197 Out.push_back(Elt: MO->getReg());
198 }
199 break;
200 case HazardOperand::AnySrc:
201 for (AMDGPU::OpName Name :
202 {AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2}) {
203 if (const MachineOperand *MO = TII.getNamedOperand(MI, OperandName: Name)) {
204 if (isVirtualVGPR(Ctx, MO: *MO))
205 Out.push_back(Elt: MO->getReg());
206 }
207 }
208 break;
209 case HazardOperand::AnyUse:
210 for (const MachineOperand &MO : MI.all_uses()) {
211 if (isVirtualVGPR(Ctx, MO))
212 Out.push_back(Elt: MO.getReg());
213 }
214 break;
215 }
216}
217
218enum class MFMAHazardKind { RAW, WAW, WAR };
219
220// MFMA anti-hint wait-state window, mirroring GCNHazardRecognizer.cpp wait
221// states.
222unsigned mfmaWaitStates(const MachineInstr &MFMA, MFMAHazardKind Kind,
223 HazardClassMask ReaderClass, const HazardContext &Ctx) {
224 const SIInstrInfo &TII = *Ctx.TII;
225 const GCNSubtarget &ST = *Ctx.ST;
226 const int NumPasses = Ctx.SchedModel->computeInstrLatency(MI: &MFMA);
227 const bool IsDGEMM = SIInstrInfo::isDGEMM(Opcode: MFMA.getOpcode());
228 const bool Mem = ReaderClass & (HC::VMEM | HC::DS | HC::EXP);
229
230 auto GFX940NPass = [&]() -> unsigned {
231 return TII.isXDL(MI: MFMA)
232 ? NumPasses + 3 + (NumPasses != 2 && ST.hasGFX950Insts())
233 : NumPasses + 2;
234 };
235 auto SMFMANPass = [&]() -> unsigned {
236 switch (NumPasses) {
237 case 2:
238 return 5;
239 case 8:
240 return 11;
241 case 16:
242 return 19;
243 default:
244 return 0;
245 }
246 };
247
248 switch (Kind) {
249 case MFMAHazardKind::RAW:
250 if (IsDGEMM) {
251 switch (NumPasses) {
252 case 4:
253 return Mem ? 9 : 6;
254 case 8:
255 case 16:
256 return Mem ? 18 : (ST.hasGFX950Insts() ? 19 : 11);
257 default:
258 return 0;
259 }
260 }
261 return ST.hasGFX940Insts() ? GFX940NPass() : SMFMANPass();
262
263 case MFMAHazardKind::WAW:
264 if (IsDGEMM) {
265 switch (NumPasses) {
266 case 4:
267 return 6;
268 case 8:
269 case 16:
270 return 11;
271 default:
272 return 0;
273 }
274 }
275 return ST.hasGFX940Insts() ? GFX940NPass() : SMFMANPass();
276
277 case MFMAHazardKind::WAR:
278 switch (NumPasses) {
279 case 2:
280 return 1;
281 case 4:
282 return 3;
283 case 8:
284 return 7;
285 case 16:
286 return 15;
287 default:
288 return 15;
289 }
290 }
291 return 0;
292}
293
294unsigned mfmaWawWindow(const MachineInstr &P, HazardClassMask,
295 const HazardContext &Ctx) {
296 return mfmaWaitStates(MFMA: P, Kind: MFMAHazardKind::WAW, ReaderClass: HC::None, Ctx);
297}
298unsigned mfmaWarWindow(const MachineInstr &P, HazardClassMask,
299 const HazardContext &Ctx) {
300 return mfmaWaitStates(MFMA: P, Kind: MFMAHazardKind::WAR, ReaderClass: HC::None, Ctx);
301}
302
303constexpr unsigned NumWMMAHazardCategories = 7;
304unsigned wmmaHazardCategory(const MachineInstr &Producer,
305 const HazardContext &Ctx) {
306 const bool IsSWMMAC = SIInstrInfo::isSWMMAC(MI: Producer);
307 const bool LowestRate = Ctx.ST->hasGFX125xLowestRateWMMA();
308 switch (Ctx.SchedModel->computeInstrLatency(MI: &Producer)) {
309 case 4:
310 return 6;
311 case 8:
312 return IsSWMMAC ? 2 : 0;
313 case 16:
314 return LowestRate ? 4 : (IsSWMMAC ? 3 : 1);
315 case 32:
316 return 5;
317 default:
318 return NumWMMAHazardCategories;
319 }
320}
321
322// WMMA/SWMMAC co-exec window.
323unsigned wmmaCoexecWindow(const MachineInstr &Producer,
324 HazardClassMask ConsumerClass,
325 const HazardContext &Ctx) {
326 const unsigned Category = wmmaHazardCategory(Producer, Ctx);
327 if (Category >= NumWMMAHazardCategories)
328 return 0;
329
330 static constexpr unsigned WMMAWaitStates[] = {5, 9, 3, 5, 9, 17, 2};
331 static constexpr unsigned VALUWaitStates[] = {4, 8, 2, 4, 8, 16, 1};
332 return (ConsumerClass & HC::WMMA) ? WMMAWaitStates[Category]
333 : VALUWaitStates[Category];
334}
335
336unsigned mfmaReaderRawWindow(const MachineInstr &Producer,
337 HazardClassMask ReaderClass,
338 const HazardContext &Ctx) {
339 return mfmaWaitStates(MFMA: Producer, Kind: MFMAHazardKind::RAW, ReaderClass, Ctx);
340}
341
342bool hasMFMAHazard(const HazardContext &Ctx) {
343 return Ctx.ST->hasGFX90AInsts();
344}
345
346bool hasWMMACoexecHazard(const HazardContext &Ctx) {
347 return Ctx.ST->hasWMMACoexecutionHazards();
348}
349
350bool hasTransCoexecHazard(const HazardContext &Ctx) {
351 return Ctx.ST->hasTransCoexecutionHazard();
352}
353
354bool hasGFX1250Insts(const HazardContext &Ctx) {
355 return Ctx.ST->hasGFX1250Insts();
356}
357
358bool ruleSelected(AntiHintRule Rule) {
359 // No -amdgpu-anti-hints-rules on the command line selects every rule.
360 if (AntiHintRuleSelection.empty() ||
361 is_contained(Range&: AntiHintRuleSelection, Element: AntiHintRule::All))
362 return true;
363 return is_contained(Range&: AntiHintRuleSelection, Element: Rule);
364}
365
366bool isMFMAWAWRuleEnabled(const HazardContext &Ctx) {
367 return hasMFMAHazard(Ctx) && ruleSelected(Rule: AntiHintRule::MFMAWAW);
368}
369
370bool isMFMAWARRuleEnabled(const HazardContext &Ctx) {
371 return hasMFMAHazard(Ctx) && ruleSelected(Rule: AntiHintRule::MFMAWAR);
372}
373
374bool isWMMAWARABRuleEnabled(const HazardContext &Ctx) {
375 return hasWMMACoexecHazard(Ctx) && ruleSelected(Rule: AntiHintRule::WMMAWARAB);
376}
377
378bool isSWMMACWARIndexRuleEnabled(const HazardContext &Ctx) {
379 return hasWMMACoexecHazard(Ctx) && ruleSelected(Rule: AntiHintRule::SWMMACWARIndex);
380}
381
382bool isWMMAWAWRuleEnabled(const HazardContext &Ctx) {
383 return hasWMMACoexecHazard(Ctx) && ruleSelected(Rule: AntiHintRule::WMMAWAW);
384}
385
386bool isTransWARRuleEnabled(const HazardContext &Ctx) {
387 return hasTransCoexecHazard(Ctx) && ruleSelected(Rule: AntiHintRule::TransWAR);
388}
389
390bool isAddrWARRuleEnabled(const HazardContext &Ctx) {
391 return EnableAntiHintsForAddr && hasGFX1250Insts(Ctx) &&
392 ruleSelected(Rule: AntiHintRule::MemAddrWAR);
393}
394
395bool isVAVdstWARRuleEnabled(const HazardContext &Ctx) {
396 return EnableAntiHintsForVAVdst && hasGFX1250Insts(Ctx) &&
397 ruleSelected(Rule: AntiHintRule::VAVdstWAR);
398}
399
400bool isXDLMFMA(const MachineInstr &MI, const HazardContext &Ctx) {
401 return Ctx.TII->isXDL(MI);
402}
403
404bool isXDLWMMA(const MachineInstr &MI, const HazardContext &Ctx) {
405 return Ctx.TII->isXDLWMMA(MI);
406}
407
408bool isXDLSWMMAC(const MachineInstr &MI, const HazardContext &Ctx) {
409 return Ctx.TII->isXDLWMMA(MI) && SIInstrInfo::isSWMMAC(MI);
410}
411
412bool isDSorVMEMLoad(const MachineInstr &MI, const HazardContext &Ctx) {
413 return MI.mayLoad() && (Ctx.TII->isDS(MI) || Ctx.TII->isVMEM(MI));
414}
415
416unsigned resolveWindow(const ConsumerTarget &CT, const MachineInstr &MI,
417 const HazardContext &Ctx) {
418 // Explicity given window length overrides the computed one.
419 if (CT.Window.OptWindowLength)
420 return *CT.Window.OptWindowLength;
421 if (CT.Window.Fn)
422 return CT.Window.Fn(MI, CT.Side.Match.AnyOf, Ctx);
423 return CT.Window.WindowLength;
424}
425
426// Build the anti-hints rules.
427class HazardRuleSet {
428 SmallVector<HazardAntiHintRule, 0> Rules;
429
430public:
431 class RuleBuilder {
432
433 HazardRuleSet &S;
434 unsigned Idx;
435 HazardAntiHintRule &rule() const { return S.Rules[Idx]; }
436
437 public:
438 RuleBuilder(HazardRuleSet &S, unsigned Idx) : S(S), Idx(Idx) {}
439
440 RuleBuilder &producer(ClassMatch M, HazardOperand Op,
441 InstPredicate Predicate = nullptr) {
442 rule().Producer = {.Match: M, .Op: Op, .Predicate: Predicate};
443 return *this;
444 }
445
446 RuleBuilder &rawCredit(AdvanceForRawWindowFn AdvanceForRawWindow) {
447 rule().AdvanceForRawWindow = AdvanceForRawWindow;
448 return *this;
449 }
450
451 RuleBuilder &
452 consumer(ClassMatch M, HazardOperand Op, WindowSpec Window,
453 HazardClassMask CountMask = 0,
454 AntiHintDirection Direction = AntiHintDirection::OneDirectional,
455 InstPredicate Predicate = nullptr) {
456 rule().Consumers.push_back(
457 Elt: {.Side: {.Match: M, .Op: Op, .Predicate: Predicate}, .Window: Window, .CounterMask: CountMask, .Direction: Direction});
458 return *this;
459 }
460
461 RuleBuilder &enabledIf(RulePredicate Predicate) {
462 rule().Predicate = Predicate;
463 return *this;
464 }
465
466 RuleBuilder &lifetime(Lifetime Life) {
467 rule().Life = Life;
468 return *this;
469 }
470 };
471
472 RuleBuilder addRule() {
473 Rules.emplace_back();
474 return RuleBuilder(*this, Rules.size() - 1);
475 }
476
477 SmallVector<HazardAntiHintRule, 0> buildRules() { return std::move(Rules); }
478};
479
480// Here the anti-hints rules are inserted.
481SmallVector<HazardAntiHintRule, 0> buildAntiHintsRules() {
482 using HO = HazardOperand;
483 HazardRuleSet S;
484
485 // MFMA relevent windows and consumers
486 const WindowSpec MfmaWawWindow{.WindowLength: 0, .OptWindowLength: nullptr, .Fn: mfmaWawWindow};
487 const WindowSpec MfmaWarWindow{.WindowLength: 0, .OptWindowLength: nullptr, .Fn: mfmaWarWindow};
488 const ClassMatch MfmaConsumers = {.AnyOf: HC::DS | HC::VALU | HC::VMEM | HC::TRANS |
489 HC::EXP};
490
491 // WMMA relevent windows and consumers
492 const WindowSpec WmmaCoexecWindow{.WindowLength: 0, .OptWindowLength: nullptr, .Fn: wmmaCoexecWindow};
493 const ClassMatch WmmaCoexecConsumers = {/*AnyOf=*/HC::VALU | HC::TRANS,
494 /*AllOf=*/HC::None,
495 /*NoneOf=*/HC::LDSDMA};
496 const HazardClassMask CoexecCounters = HC::VALU | HC::TRANS | HC::WMMA;
497
498 // MFMA WAW rules
499 S.addRule()
500 .enabledIf(Predicate: isMFMAWAWRuleEnabled)
501 .producer(M: {.AnyOf: HC::MFMA}, Op: HO::Def)
502 .rawCredit(AdvanceForRawWindow: mfmaReaderRawWindow)
503 .consumer(M: MfmaConsumers, Op: HO::Def, Window: MfmaWawWindow, CountMask: HC::None,
504 Direction: AntiHintDirection::OneDirectional);
505
506 // MFMA WAR rules
507 S.addRule()
508 .enabledIf(Predicate: isMFMAWARRuleEnabled)
509 .producer(M: {.AnyOf: HC::MFMA}, Op: HO::Src2, Predicate: isXDLMFMA)
510 .rawCredit(AdvanceForRawWindow: mfmaReaderRawWindow)
511 .consumer(M: MfmaConsumers, Op: HO::Def, Window: MfmaWarWindow, CountMask: HC::None,
512 Direction: AntiHintDirection::OneDirectional);
513
514 // WMMA WAR.
515 S.addRule()
516 .enabledIf(Predicate: isWMMAWARABRuleEnabled)
517 .producer(M: {.AnyOf: HC::WMMA}, Op: HO::Src0Src1, Predicate: isXDLWMMA)
518 .consumer(M: WmmaCoexecConsumers, Op: HO::Def, Window: WmmaCoexecWindow, CountMask: CoexecCounters,
519 Direction: AntiHintDirection::OneDirectional)
520 .consumer(M: {.AnyOf: HC::VMEM | HC::DS}, Op: HO::Def, Window: WmmaCoexecWindow, CountMask: CoexecCounters,
521 Direction: AntiHintDirection::OneDirectional);
522
523 // SWMMAC WAR.
524 S.addRule()
525 .enabledIf(Predicate: isSWMMACWARIndexRuleEnabled)
526 .producer(M: {.AnyOf: HC::WMMA}, Op: HO::Src2, Predicate: isXDLSWMMAC)
527 .consumer(M: WmmaCoexecConsumers, Op: HO::Def, Window: WmmaCoexecWindow, CountMask: CoexecCounters,
528 Direction: AntiHintDirection::OneDirectional);
529
530 // WMMA WAW.
531 S.addRule()
532 .enabledIf(Predicate: isWMMAWAWRuleEnabled)
533 .producer(M: {.AnyOf: HC::WMMA}, Op: HO::Def, Predicate: isXDLWMMA)
534 .consumer(M: WmmaCoexecConsumers, Op: HO::Def, Window: WmmaCoexecWindow, CountMask: CoexecCounters,
535 Direction: AntiHintDirection::OneDirectional);
536
537 // TRANS WAR.
538 S.addRule()
539 .enabledIf(Predicate: isTransWARRuleEnabled)
540 .producer(M: {.AnyOf: HC::TRANS}, Op: HO::AnyUse)
541 .consumer(M: {.AnyOf: HC::VALU | HC::WMMA}, Op: HO::Def, Window: {/*WindowLength=*/1},
542 CountMask: CoexecCounters, Direction: AntiHintDirection::OneDirectional);
543
544 // Address WAR.
545 S.addRule()
546 .enabledIf(Predicate: isAddrWARRuleEnabled)
547 .producer(M: {.AnyOf: HC::VMEM}, Op: HO::AnyUse)
548 .consumer(M: {.AnyOf: HC::WritesVGPR}, Op: HO::Def, Window: {.WindowLength: 0, .OptWindowLength: &AddrAntiHintWindow}, CountMask: HC::VMEM,
549 Direction: AntiHintDirection::Symmetric);
550
551 // VA_VDST WAR.
552 S.addRule()
553 .enabledIf(Predicate: isVAVdstWARRuleEnabled)
554 .lifetime(Life: Lifetime::RegisterCount)
555 .producer(M: {.AnyOf: HC::RawVALU}, Op: HO::AnyUse)
556 .consumer(M: {.AnyOf: HC::Load}, Op: HO::Def, Window: {.WindowLength: 0, .OptWindowLength: &VAVDSTLookbackWindow}, CountMask: HC::None,
557 Direction: AntiHintDirection::Symmetric, Predicate: isDSorVMEMLoad);
558
559 return S.buildRules();
560}
561
562ArrayRef<HazardAntiHintRule> getAntiHintsRules() {
563 static const SmallVector<HazardAntiHintRule, 0> Rules = buildAntiHintsRules();
564 return Rules;
565}
566
567struct AntiHintWindow {
568 SmallVector<Register, 4> Regs;
569 const MachineInstr *Producer = nullptr;
570 unsigned Len = 0;
571 unsigned Elapsed = 0;
572};
573
574using ConsumerTracking = SmallVector<AntiHintWindow, 3>;
575// One per consumer target of a rule.
576using RuleTracking = SmallVector<ConsumerTracking, 3>;
577
578class AntiHintEngine {
579 const HazardContext &Ctx;
580 ArrayRef<HazardAntiHintRule> Rules;
581
582 SmallVector<bool, 8> RuleApplies;
583 bool AnyEnabled = false;
584
585public:
586 AntiHintEngine(const HazardContext &Ctx)
587 : Ctx(Ctx), Rules(getAntiHintsRules()), RuleApplies(Rules.size()) {
588 for (unsigned R = 0; R < Rules.size(); ++R) {
589 const HazardAntiHintRule &Rule = Rules[R];
590 RuleApplies[R] = !Rule.Predicate || Rule.Predicate(Ctx);
591 AnyEnabled |= RuleApplies[R];
592 }
593 }
594
595 void run(MachineFunction &MF) {
596 if (!AnyEnabled)
597 return;
598
599 SmallVector<RuleTracking, 8> Tracking(Rules.size());
600 for (unsigned R = 0; R < Rules.size(); ++R)
601 Tracking[R].resize(N: Rules[R].Consumers.size());
602
603 for (const MachineBasicBlock &MBB : MF) {
604 for (RuleTracking &RT : Tracking) {
605 for (ConsumerTracking &Track : RT)
606 Track.clear();
607 }
608 for (const MachineInstr &MI : MBB) {
609 if (MI.isMetaInstruction())
610 continue;
611 const HazardClassMask C = getInstHazardClass(MI, Ctx);
612 // Wait states this instruction contributes to an open window.
613 unsigned WaitStates = SIInstrInfo::getNumWaitStates(MI);
614 addAntiHintsAndExpire(MI, C, WaitStates, Tracking);
615 addWindows(MI, C, Tracking);
616 }
617 }
618 }
619
620private:
621 // Check if the right class and predicate matches.
622 bool sideMatches(const HazardSide &Side, const MachineInstr &MI,
623 HazardClassMask C) {
624 return Side.Match.matches(Mask: C) &&
625 (!Side.Predicate || Side.Predicate(MI, Ctx));
626 }
627
628 // Producer phase: open a window for each consumer target.
629 void addWindows(const MachineInstr &MI, HazardClassMask C,
630 MutableArrayRef<RuleTracking> Tracking) {
631 for (unsigned R = 0; R < Rules.size(); ++R) {
632 const HazardAntiHintRule &Rule = Rules[R];
633 if (!RuleApplies[R] || !sideMatches(Side: Rule.Producer, MI, C))
634 continue;
635
636 SmallVector<Register, 4> Regs;
637 // Collect the producer regs.
638 collectOperandRegs(MI, Op: Rule.Producer.Op, Ctx, Out&: Regs);
639 if (Regs.empty())
640 continue;
641 for (unsigned ConsumerIdx = 0, E = Rule.Consumers.size();
642 ConsumerIdx != E; ++ConsumerIdx)
643 seedWindow(Rule, CT: Rule.Consumers[ConsumerIdx], MI, Regs,
644 Track&: Tracking[R][ConsumerIdx]);
645 }
646 }
647
648 void seedWindow(const HazardAntiHintRule &Rule, const ConsumerTarget &CT,
649 const MachineInstr &MI, ArrayRef<Register> Regs,
650 ConsumerTracking &Track) {
651 unsigned Window = resolveWindow(CT, MI, Ctx);
652 if (!Window)
653 return;
654
655 if (Rule.Life == Lifetime::RegisterCount) {
656 // Window length caps the register count.
657 if (Track.empty())
658 Track.push_back(Elt: {.Regs: {}, .Producer: &MI, .Len: Window, .Elapsed: 0});
659 SmallVectorImpl<Register> &Recent = Track.front().Regs;
660 for (Register Reg : Regs) {
661 if (llvm::is_contained(Range&: Recent, Element: Reg))
662 continue;
663 Recent.push_back(Elt: Reg);
664 if (Recent.size() > Window)
665 Recent.erase(CI: Recent.begin());
666 }
667 return;
668 }
669
670 Track.push_back(Elt: {.Regs: SmallVector<Register, 4>(Regs), .Producer: &MI, .Len: Window, .Elapsed: 0});
671 }
672
673 void advanceByRawWindow(const MachineInstr &MI, HazardClassMask C,
674 const HazardAntiHintRule &Rule,
675 const ConsumerTarget &CT, ConsumerTracking &Track) {
676
677 if (!Rule.AdvanceForRawWindow || !CT.Side.Match.matches(Mask: C))
678 return;
679 for (AntiHintWindow &Window : Track) {
680 // Determine if def generated by producer is read by the MI consumer.
681 bool ReadsProducerDef = llvm::any_of(
682 Range: Window.Producer->all_defs(), P: [&](const MachineOperand &MO) {
683 return MO.getReg().isVirtual() &&
684 MI.readsVirtualRegister(Reg: MO.getReg());
685 });
686 // Advance by RAW window from the producer if that is larger than current
687 // Window.Elapsed.
688 if (ReadsProducerDef) {
689 Window.Elapsed = std::max(
690 a: Window.Elapsed, b: Rule.AdvanceForRawWindow(*Window.Producer, C, Ctx));
691 }
692 }
693 }
694
695 // Consumer phase: add anti-hints, then charge and expire open windows.
696 void addAntiHintsAndExpire(const MachineInstr &MI, HazardClassMask C,
697 unsigned WaitStates,
698 MutableArrayRef<RuleTracking> Tracking) {
699 for (unsigned R = 0; R < Rules.size(); ++R) {
700 const HazardAntiHintRule &Rule = Rules[R];
701 if (!RuleApplies[R])
702 continue;
703 for (unsigned ConsumerIdx = 0, E = Rule.Consumers.size();
704 ConsumerIdx != E; ++ConsumerIdx) {
705 const ConsumerTarget &CT = Rule.Consumers[ConsumerIdx];
706 ConsumerTracking &Track = Tracking[R][ConsumerIdx];
707 if (Track.empty())
708 continue;
709
710 // Before adding the anti-hints, see if advancing by RAW window will
711 // help remove the window.
712 const bool Counts = Rule.Life == Lifetime::WindowBudget;
713 if (Counts) {
714 advanceByRawWindow(MI, C, Rule, CT, Track);
715 llvm::erase_if(C&: Track, P: [](const AntiHintWindow &Window) {
716 return Window.Elapsed >= Window.Len;
717 });
718 }
719
720 // Add anti-hints if the consumer matches the instruction.
721 if (sideMatches(Side: CT.Side, MI, C))
722 addAntiHints(CT, MI, Track);
723
724 // This instruction's own wait states count toward the next one.
725 if (Counts && (!CT.CounterMask || (C & CT.CounterMask))) {
726 for (AntiHintWindow &Window : Track)
727 Window.Elapsed += WaitStates;
728 }
729 }
730 }
731 }
732
733 bool isCopyOf(Register Cand, Register HazardReg) const {
734 const MachineInstr *Def = Ctx.MRI->getUniqueVRegDef(Reg: Cand);
735 return Def && Def->isCopy() && Def->getOperand(i: 1).getReg() == HazardReg;
736 }
737
738 void addAntiHints(const ConsumerTarget &CT, const MachineInstr &MI,
739 const ConsumerTracking &Track) {
740 if (Track.empty())
741 return;
742 SmallVector<Register, 4> ConsumerRegs;
743 // Collect the consumer regs.
744 collectOperandRegs(MI, Op: CT.Side.Op, Ctx, Out&: ConsumerRegs);
745 if (ConsumerRegs.empty())
746 return;
747 SlotIndex Slot = Ctx.LIS->getInstructionIndex(Instr: MI).getRegSlot();
748 auto AntiHint = [&](Register ProducerReg) {
749 if (!Ctx.LIS->hasInterval(Reg: ProducerReg))
750 return;
751 const LiveInterval &ProducerLI = Ctx.LIS->getInterval(Reg: ProducerReg);
752 // Skip a live producer reg.
753 if (ProducerLI.liveAt(index: Slot))
754 return;
755 for (Register ConsumerReg : ConsumerRegs) {
756 if (ConsumerReg == ProducerReg || isCopyOf(Cand: ConsumerReg, HazardReg: ProducerReg))
757 continue;
758
759 Ctx.MRI->addRegAllocationAntiHints(VReg: ConsumerReg, AntiHintVRegs: ProducerReg);
760 if (CT.Direction == AntiHintDirection::Symmetric)
761 Ctx.MRI->addRegAllocationAntiHints(VReg: ProducerReg, AntiHintVRegs: ConsumerReg);
762 LLVM_DEBUG(
763 dbgs() << "anti-hint: keep " << printReg(ProducerReg, Ctx.TRI)
764 << (CT.Direction == AntiHintDirection::Symmetric ? " <-> "
765 : " <- ")
766 << printReg(ConsumerReg, Ctx.TRI) << " (consumer "
767 << Ctx.TII->getName(MI.getOpcode()) << ")\n");
768 }
769 };
770 for (const AntiHintWindow &Window : Track) {
771 for (Register ProducerReg : Window.Regs)
772 AntiHint(ProducerReg);
773 }
774 }
775};
776
777} // namespace
778
779void AMDGPU::applyAntiHintRules(MachineFunction &MF, const HazardContext &Ctx) {
780 AntiHintEngine(Ctx).run(MF);
781}
782