1//===-- GCNPreRAAntiHints.cpp - MFMA register anti-hints ------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Insert register allocation anti-hints.
11///
12//===----------------------------------------------------------------------===//
13
14#include "GCNPreRAAntiHints.h"
15#include "GCNSubtarget.h"
16#include "SIInstrInfo.h"
17#include "SIRegisterInfo.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/CodeGen/LiveIntervals.h"
20#include "llvm/CodeGen/MachineRegisterInfo.h"
21#include "llvm/CodeGen/SlotIndexes.h"
22#include "llvm/CodeGen/TargetSchedule.h"
23
24using namespace llvm;
25using namespace llvm::AMDGPU;
26
27#define DEBUG_TYPE "amdgpu-anti-hints"
28
29namespace HC = llvm::AMDGPU::HazardClass;
30
31namespace {
32
33// Classify the MI into a HazardClassMask.
34HazardClassMask getInstHazardClass(const MachineInstr &MI,
35 const HazardContext &Ctx) {
36 const SIInstrInfo &TII = *Ctx.TII;
37 HazardClassMask Mask = HC::None;
38
39 if (TII.isLDSDMA(MI))
40 Mask = HC::VALU | HC::VMEM | HC::DS | HC::LDSDMA;
41 else if (TII.isWMMA(MI) || SIInstrInfo::isSWMMAC(MI))
42 Mask = HC::WMMA;
43 else if (TII.isMFMA(MI))
44 Mask = HC::MFMA;
45 else if (SIInstrInfo::isTRANS(MI))
46 Mask = HC::TRANS;
47 else if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false))
48 Mask = HC::VALU;
49 else if (TII.isDS(MI))
50 Mask = HC::DS;
51 else if (TII.isVMEM(MI))
52 Mask = HC::VMEM;
53 else if (TII.isSMRD(MI))
54 Mask = HC::SMEM;
55 else if (TII.isEXP(MI))
56 Mask = HC::EXP;
57 else if (SIInstrInfo::isSALU(MI))
58 Mask = HC::SALU;
59 else if (MI.isCopy() && MI.getOperand(i: 0).getReg().isVirtual() &&
60 Ctx.TRI->hasVGPRs(RC: Ctx.MRI->getRegClass(Reg: MI.getOperand(i: 0).getReg())))
61 Mask = HC::VALU;
62 if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true))
63 Mask |= HC::RawVALU;
64 if (MI.mayLoad())
65 Mask |= HC::Load;
66 for (const MachineOperand &MO : MI.defs()) {
67 if (MO.isReg() && MO.getReg().isVirtual() &&
68 Ctx.TRI->hasVGPRs(RC: Ctx.MRI->getRegClass(Reg: MO.getReg()))) {
69 Mask |= HC::WritesVGPR;
70 break;
71 }
72 }
73
74 return Mask;
75}
76
77bool isVirtualVGPR(const HazardContext &Ctx, const MachineOperand &MO) {
78 if (!MO.isReg() || !MO.getReg().isVirtual())
79 return false;
80 return Ctx.TRI->hasVGPRs(RC: Ctx.MRI->getRegClass(Reg: MO.getReg()));
81}
82
83void collectOperandRegs(const MachineInstr &MI, HazardOperand Op,
84 const HazardContext &Ctx,
85 SmallVectorImpl<Register> &Out) {
86 const SIInstrInfo &TII = *Ctx.TII;
87 switch (Op) {
88 case HazardOperand::None:
89 break;
90 case HazardOperand::Def:
91 for (const MachineOperand &MO : MI.all_defs()) {
92 if (isVirtualVGPR(Ctx, MO))
93 Out.push_back(Elt: MO.getReg());
94 }
95 break;
96 case HazardOperand::Src0:
97 if (const MachineOperand *MO =
98 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src0)) {
99 if (isVirtualVGPR(Ctx, MO: *MO))
100 Out.push_back(Elt: MO->getReg());
101 }
102 break;
103 case HazardOperand::Src1:
104 if (const MachineOperand *MO =
105 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src1)) {
106 if (isVirtualVGPR(Ctx, MO: *MO))
107 Out.push_back(Elt: MO->getReg());
108 }
109 break;
110 case HazardOperand::Src2:
111 if (const MachineOperand *MO =
112 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src2)) {
113 if (isVirtualVGPR(Ctx, MO: *MO))
114 Out.push_back(Elt: MO->getReg());
115 }
116 break;
117 case HazardOperand::Src0Src1:
118 for (AMDGPU::OpName Name : {AMDGPU::OpName::src0, AMDGPU::OpName::src1}) {
119 if (const MachineOperand *MO = TII.getNamedOperand(MI, OperandName: Name)) {
120 if (isVirtualVGPR(Ctx, MO: *MO))
121 Out.push_back(Elt: MO->getReg());
122 }
123 }
124 break;
125 case HazardOperand::Idx:
126 if (const MachineOperand *MO =
127 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::idx)) {
128 if (isVirtualVGPR(Ctx, MO: *MO))
129 Out.push_back(Elt: MO->getReg());
130 }
131 break;
132 case HazardOperand::Vaddr:
133 if (const MachineOperand *MO =
134 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr)) {
135 if (isVirtualVGPR(Ctx, MO: *MO))
136 Out.push_back(Elt: MO->getReg());
137 }
138 break;
139 case HazardOperand::AnySrc:
140 for (AMDGPU::OpName Name :
141 {AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2}) {
142 if (const MachineOperand *MO = TII.getNamedOperand(MI, OperandName: Name)) {
143 if (isVirtualVGPR(Ctx, MO: *MO))
144 Out.push_back(Elt: MO->getReg());
145 }
146 }
147 break;
148 case HazardOperand::AnyUse:
149 for (const MachineOperand &MO : MI.all_uses()) {
150 if (isVirtualVGPR(Ctx, MO))
151 Out.push_back(Elt: MO.getReg());
152 }
153 break;
154 }
155}
156
157enum class MFMAHazardKind { RAW, WAW, WAR };
158
159// MFMA anti-hint wait-state window, mirroring GCNHazardRecognizer.cpp wait
160// states.
161unsigned mfmaWaitStates(const MachineInstr &MFMA, MFMAHazardKind Kind,
162 HazardClassMask ReaderClass, const HazardContext &Ctx) {
163 const SIInstrInfo &TII = *Ctx.TII;
164 const GCNSubtarget &ST = *Ctx.ST;
165 const int NumPasses = Ctx.SchedModel->computeInstrLatency(MI: &MFMA);
166 const bool IsDGEMM = SIInstrInfo::isDGEMM(Opcode: MFMA.getOpcode());
167 const bool Mem = ReaderClass & (HC::VMEM | HC::DS | HC::EXP);
168
169 auto GFX940NPass = [&]() -> unsigned {
170 return TII.isXDL(MI: MFMA)
171 ? NumPasses + 3 + (NumPasses != 2 && ST.hasGFX950Insts())
172 : NumPasses + 2;
173 };
174 auto SMFMANPass = [&]() -> unsigned {
175 switch (NumPasses) {
176 case 2:
177 return 5;
178 case 8:
179 return 11;
180 case 16:
181 return 19;
182 default:
183 return 0;
184 }
185 };
186
187 switch (Kind) {
188 case MFMAHazardKind::RAW:
189 if (IsDGEMM) {
190 switch (NumPasses) {
191 case 4:
192 return Mem ? 9 : 6;
193 case 8:
194 case 16:
195 return Mem ? 18 : (ST.hasGFX950Insts() ? 19 : 11);
196 default:
197 return 0;
198 }
199 }
200 return ST.hasGFX940Insts() ? GFX940NPass() : SMFMANPass();
201
202 case MFMAHazardKind::WAW:
203 if (IsDGEMM) {
204 switch (NumPasses) {
205 case 4:
206 return 6;
207 case 8:
208 case 16:
209 return 11;
210 default:
211 return 0;
212 }
213 }
214 return ST.hasGFX940Insts() ? GFX940NPass() : SMFMANPass();
215
216 case MFMAHazardKind::WAR:
217 switch (NumPasses) {
218 case 2:
219 return 1;
220 case 4:
221 return 3;
222 case 8:
223 return 7;
224 case 16:
225 return 15;
226 default:
227 return 15;
228 }
229 }
230 return 0;
231}
232
233unsigned mfmaWawWindow(const MachineInstr &P, HazardClassMask,
234 const HazardContext &Ctx) {
235 return mfmaWaitStates(MFMA: P, Kind: MFMAHazardKind::WAW, ReaderClass: HC::None, Ctx);
236}
237unsigned mfmaWarWindow(const MachineInstr &P, HazardClassMask,
238 const HazardContext &Ctx) {
239 return mfmaWaitStates(MFMA: P, Kind: MFMAHazardKind::WAR, ReaderClass: HC::None, Ctx);
240}
241unsigned addrAntiHintWindow(const MachineInstr &, HazardClassMask,
242 const HazardContext &Ctx) {
243 return Ctx.ST->getCLOpts().addr_anti_hint_window;
244}
245unsigned vaVdstLookbackWindow(const MachineInstr &, HazardClassMask,
246 const HazardContext &Ctx) {
247 return Ctx.ST->getCLOpts().va_vdst_lookback_window;
248}
249
250constexpr unsigned NumWMMAHazardCategories = 7;
251unsigned wmmaHazardCategory(const MachineInstr &Producer,
252 const HazardContext &Ctx) {
253 const bool IsSWMMAC = SIInstrInfo::isSWMMAC(MI: Producer);
254 const bool LowestRate = Ctx.ST->hasGFX125xLowestRateWMMA();
255 switch (Ctx.SchedModel->computeInstrLatency(MI: &Producer)) {
256 case 4:
257 return 6;
258 case 8:
259 return IsSWMMAC ? 2 : 0;
260 case 16:
261 return LowestRate ? 4 : (IsSWMMAC ? 3 : 1);
262 case 32:
263 return 5;
264 default:
265 return NumWMMAHazardCategories;
266 }
267}
268
269// WMMA/SWMMAC co-exec window.
270unsigned wmmaCoexecWindow(const MachineInstr &Producer,
271 HazardClassMask ConsumerClass,
272 const HazardContext &Ctx) {
273 const unsigned Category = wmmaHazardCategory(Producer, Ctx);
274 if (Category >= NumWMMAHazardCategories)
275 return 0;
276
277 static constexpr unsigned WMMAWaitStates[] = {5, 9, 3, 5, 9, 17, 2};
278 static constexpr unsigned VALUWaitStates[] = {4, 8, 2, 4, 8, 16, 1};
279 return (ConsumerClass & HC::WMMA) ? WMMAWaitStates[Category]
280 : VALUWaitStates[Category];
281}
282
283unsigned mfmaReaderRawWindow(const MachineInstr &Producer,
284 HazardClassMask ReaderClass,
285 const HazardContext &Ctx) {
286 return mfmaWaitStates(MFMA: Producer, Kind: MFMAHazardKind::RAW, ReaderClass, Ctx);
287}
288
289bool hasMFMAHazard(const HazardContext &Ctx) {
290 return Ctx.ST->hasGFX90AInsts();
291}
292
293bool hasWMMACoexecHazard(const HazardContext &Ctx) {
294 return Ctx.ST->hasWMMACoexecutionHazards();
295}
296
297bool hasTransCoexecHazard(const HazardContext &Ctx) {
298 return Ctx.ST->hasTransCoexecutionHazard();
299}
300
301bool hasGFX1250Insts(const HazardContext &Ctx) {
302 return Ctx.ST->hasGFX1250Insts();
303}
304
305bool ruleSelected(const HazardContext &Ctx, AntiHintRule Rule) {
306 ArrayRef<AntiHintRule> Selection = Ctx.ST->getCLOpts().anti_hints_rules;
307 // No -amdgpu-anti-hints-rules on the command line selects every rule.
308 if (Selection.empty() || is_contained(Range&: Selection, Element: AntiHintRule::All))
309 return true;
310 return is_contained(Range&: Selection, Element: Rule);
311}
312
313bool isMFMAWAWRuleEnabled(const HazardContext &Ctx) {
314 return hasMFMAHazard(Ctx) && ruleSelected(Ctx, Rule: AntiHintRule::MFMAWAW);
315}
316
317bool isMFMAWARRuleEnabled(const HazardContext &Ctx) {
318 return hasMFMAHazard(Ctx) && ruleSelected(Ctx, Rule: AntiHintRule::MFMAWAR);
319}
320
321bool isWMMAWARABRuleEnabled(const HazardContext &Ctx) {
322 return hasWMMACoexecHazard(Ctx) && ruleSelected(Ctx, Rule: AntiHintRule::WMMAWARAB);
323}
324
325bool isSWMMACWARIndexRuleEnabled(const HazardContext &Ctx) {
326 return hasWMMACoexecHazard(Ctx) &&
327 ruleSelected(Ctx, Rule: AntiHintRule::SWMMACWARIndex);
328}
329
330bool isWMMAWAWRuleEnabled(const HazardContext &Ctx) {
331 return hasWMMACoexecHazard(Ctx) && ruleSelected(Ctx, Rule: AntiHintRule::WMMAWAW);
332}
333
334bool isTransWARRuleEnabled(const HazardContext &Ctx) {
335 return hasTransCoexecHazard(Ctx) && ruleSelected(Ctx, Rule: AntiHintRule::TransWAR);
336}
337
338bool isAddrWARRuleEnabled(const HazardContext &Ctx) {
339 return Ctx.ST->getCLOpts().anti_hints_for_addr && hasGFX1250Insts(Ctx) &&
340 ruleSelected(Ctx, Rule: AntiHintRule::MemAddrWAR);
341}
342
343bool isVAVdstWARRuleEnabled(const HazardContext &Ctx) {
344 return Ctx.ST->getCLOpts().anti_hints_for_va_vdst && hasGFX1250Insts(Ctx) &&
345 ruleSelected(Ctx, Rule: AntiHintRule::VAVdstWAR);
346}
347
348bool isXDLMFMA(const MachineInstr &MI, const HazardContext &Ctx) {
349 return Ctx.TII->isXDL(MI);
350}
351
352bool isXDLWMMA(const MachineInstr &MI, const HazardContext &Ctx) {
353 return Ctx.TII->isXDLWMMA(MI);
354}
355
356bool isXDLSWMMAC(const MachineInstr &MI, const HazardContext &Ctx) {
357 return Ctx.TII->isXDLWMMA(MI) && SIInstrInfo::isSWMMAC(MI);
358}
359
360bool isDSorVMEMLoad(const MachineInstr &MI, const HazardContext &Ctx) {
361 return MI.mayLoad() && (Ctx.TII->isDS(MI) || Ctx.TII->isVMEM(MI));
362}
363
364unsigned resolveWindow(const ConsumerTarget &CT, const MachineInstr &MI,
365 const HazardContext &Ctx) {
366 if (CT.Window.Fn)
367 return CT.Window.Fn(MI, CT.Side.Match.AnyOf, Ctx);
368 return CT.Window.WindowLength;
369}
370
371// Build the anti-hints rules.
372class HazardRuleSet {
373 SmallVector<HazardAntiHintRule, 0> Rules;
374
375public:
376 class RuleBuilder {
377
378 HazardRuleSet &S;
379 unsigned Idx;
380 HazardAntiHintRule &rule() const { return S.Rules[Idx]; }
381
382 public:
383 RuleBuilder(HazardRuleSet &S, unsigned Idx) : S(S), Idx(Idx) {}
384
385 RuleBuilder &producer(ClassMatch M, HazardOperand Op,
386 InstPredicate Predicate = nullptr) {
387 rule().Producer = {.Match: M, .Op: Op, .Predicate: Predicate};
388 return *this;
389 }
390
391 RuleBuilder &rawCredit(AdvanceForRawWindowFn AdvanceForRawWindow) {
392 rule().AdvanceForRawWindow = AdvanceForRawWindow;
393 return *this;
394 }
395
396 RuleBuilder &
397 consumer(ClassMatch M, HazardOperand Op, WindowSpec Window,
398 HazardClassMask CountMask = 0,
399 AntiHintDirection Direction = AntiHintDirection::OneDirectional,
400 InstPredicate Predicate = nullptr) {
401 rule().Consumers.push_back(
402 Elt: {.Side: {.Match: M, .Op: Op, .Predicate: Predicate}, .Window: Window, .CounterMask: CountMask, .Direction: Direction});
403 return *this;
404 }
405
406 RuleBuilder &enabledIf(RulePredicate Predicate) {
407 rule().Predicate = Predicate;
408 return *this;
409 }
410
411 RuleBuilder &lifetime(Lifetime Life) {
412 rule().Life = Life;
413 return *this;
414 }
415 };
416
417 RuleBuilder addRule() {
418 Rules.emplace_back();
419 return RuleBuilder(*this, Rules.size() - 1);
420 }
421
422 SmallVector<HazardAntiHintRule, 0> buildRules() { return std::move(Rules); }
423};
424
425// Here the anti-hints rules are inserted.
426SmallVector<HazardAntiHintRule, 0> buildAntiHintsRules() {
427 using HO = HazardOperand;
428 HazardRuleSet S;
429
430 // MFMA relevent windows and consumers
431 const WindowSpec MfmaWawWindow{.WindowLength: 0, .Fn: mfmaWawWindow};
432 const WindowSpec MfmaWarWindow{.WindowLength: 0, .Fn: mfmaWarWindow};
433 const ClassMatch MfmaConsumers = {.AnyOf: HC::DS | HC::VALU | HC::VMEM | HC::TRANS |
434 HC::EXP};
435
436 // WMMA relevent windows and consumers
437 const WindowSpec WmmaCoexecWindow{.WindowLength: 0, .Fn: wmmaCoexecWindow};
438 const ClassMatch WmmaCoexecConsumers = {/*AnyOf=*/HC::VALU | HC::TRANS,
439 /*AllOf=*/HC::None,
440 /*NoneOf=*/HC::LDSDMA};
441 const HazardClassMask CoexecCounters = HC::VALU | HC::TRANS | HC::WMMA;
442
443 // MFMA WAW rules
444 S.addRule()
445 .enabledIf(Predicate: isMFMAWAWRuleEnabled)
446 .producer(M: {.AnyOf: HC::MFMA}, Op: HO::Def)
447 .rawCredit(AdvanceForRawWindow: mfmaReaderRawWindow)
448 .consumer(M: MfmaConsumers, Op: HO::Def, Window: MfmaWawWindow, CountMask: HC::None,
449 Direction: AntiHintDirection::OneDirectional);
450
451 // MFMA WAR rules
452 S.addRule()
453 .enabledIf(Predicate: isMFMAWARRuleEnabled)
454 .producer(M: {.AnyOf: HC::MFMA}, Op: HO::Src2, Predicate: isXDLMFMA)
455 .rawCredit(AdvanceForRawWindow: mfmaReaderRawWindow)
456 .consumer(M: MfmaConsumers, Op: HO::Def, Window: MfmaWarWindow, CountMask: HC::None,
457 Direction: AntiHintDirection::OneDirectional);
458
459 // WMMA WAR.
460 S.addRule()
461 .enabledIf(Predicate: isWMMAWARABRuleEnabled)
462 .producer(M: {.AnyOf: HC::WMMA}, Op: HO::Src0Src1, Predicate: isXDLWMMA)
463 .consumer(M: WmmaCoexecConsumers, Op: HO::Def, Window: WmmaCoexecWindow, CountMask: CoexecCounters,
464 Direction: AntiHintDirection::OneDirectional)
465 .consumer(M: {.AnyOf: HC::VMEM | HC::DS}, Op: HO::Def, Window: WmmaCoexecWindow, CountMask: CoexecCounters,
466 Direction: AntiHintDirection::OneDirectional);
467
468 // SWMMAC WAR.
469 S.addRule()
470 .enabledIf(Predicate: isSWMMACWARIndexRuleEnabled)
471 .producer(M: {.AnyOf: HC::WMMA}, Op: HO::Src2, Predicate: isXDLSWMMAC)
472 .consumer(M: WmmaCoexecConsumers, Op: HO::Def, Window: WmmaCoexecWindow, CountMask: CoexecCounters,
473 Direction: AntiHintDirection::OneDirectional);
474
475 // WMMA WAW.
476 S.addRule()
477 .enabledIf(Predicate: isWMMAWAWRuleEnabled)
478 .producer(M: {.AnyOf: HC::WMMA}, Op: HO::Def, Predicate: isXDLWMMA)
479 .consumer(M: WmmaCoexecConsumers, Op: HO::Def, Window: WmmaCoexecWindow, CountMask: CoexecCounters,
480 Direction: AntiHintDirection::OneDirectional);
481
482 // TRANS WAR.
483 S.addRule()
484 .enabledIf(Predicate: isTransWARRuleEnabled)
485 .producer(M: {.AnyOf: HC::TRANS}, Op: HO::AnyUse)
486 .consumer(M: {.AnyOf: HC::VALU | HC::WMMA}, Op: HO::Def, Window: {/*WindowLength=*/1},
487 CountMask: CoexecCounters, Direction: AntiHintDirection::OneDirectional);
488
489 // Address WAR.
490 S.addRule()
491 .enabledIf(Predicate: isAddrWARRuleEnabled)
492 .producer(M: {.AnyOf: HC::VMEM}, Op: HO::AnyUse)
493 .consumer(M: {.AnyOf: HC::WritesVGPR}, Op: HO::Def, Window: {.WindowLength: 0, .Fn: addrAntiHintWindow}, CountMask: HC::VMEM,
494 Direction: AntiHintDirection::Symmetric);
495
496 // VA_VDST WAR.
497 S.addRule()
498 .enabledIf(Predicate: isVAVdstWARRuleEnabled)
499 .lifetime(Life: Lifetime::RegisterCount)
500 .producer(M: {.AnyOf: HC::RawVALU}, Op: HO::AnyUse)
501 .consumer(M: {.AnyOf: HC::Load}, Op: HO::Def, Window: {.WindowLength: 0, .Fn: vaVdstLookbackWindow}, CountMask: HC::None,
502 Direction: AntiHintDirection::Symmetric, Predicate: isDSorVMEMLoad);
503
504 return S.buildRules();
505}
506
507ArrayRef<HazardAntiHintRule> getAntiHintsRules() {
508 static const SmallVector<HazardAntiHintRule, 0> Rules = buildAntiHintsRules();
509 return Rules;
510}
511
512struct AntiHintWindow {
513 SmallVector<Register, 4> Regs;
514 const MachineInstr *Producer = nullptr;
515 unsigned Len = 0;
516 unsigned Elapsed = 0;
517};
518
519using ConsumerTracking = SmallVector<AntiHintWindow, 3>;
520// One per consumer target of a rule.
521using RuleTracking = SmallVector<ConsumerTracking, 3>;
522
523class AntiHintEngine {
524 const HazardContext &Ctx;
525 ArrayRef<HazardAntiHintRule> Rules;
526
527 SmallVector<bool, 8> RuleApplies;
528 bool AnyEnabled = false;
529
530public:
531 AntiHintEngine(const HazardContext &Ctx)
532 : Ctx(Ctx), Rules(getAntiHintsRules()), RuleApplies(Rules.size()) {
533 for (unsigned R = 0; R < Rules.size(); ++R) {
534 const HazardAntiHintRule &Rule = Rules[R];
535 RuleApplies[R] = !Rule.Predicate || Rule.Predicate(Ctx);
536 AnyEnabled |= RuleApplies[R];
537 }
538 }
539
540 void run(MachineFunction &MF) {
541 if (!AnyEnabled)
542 return;
543
544 SmallVector<RuleTracking, 8> Tracking(Rules.size());
545 for (unsigned R = 0; R < Rules.size(); ++R)
546 Tracking[R].resize(N: Rules[R].Consumers.size());
547
548 for (const MachineBasicBlock &MBB : MF) {
549 for (RuleTracking &RT : Tracking) {
550 for (ConsumerTracking &Track : RT)
551 Track.clear();
552 }
553 for (const MachineInstr &MI : MBB) {
554 if (MI.isMetaInstruction())
555 continue;
556 const HazardClassMask C = getInstHazardClass(MI, Ctx);
557 // Wait states this instruction contributes to an open window.
558 unsigned WaitStates = SIInstrInfo::getNumWaitStates(MI);
559 addAntiHintsAndExpire(MI, C, WaitStates, Tracking);
560 addWindows(MI, C, Tracking);
561 }
562 }
563 }
564
565private:
566 // Check if the right class and predicate matches.
567 bool sideMatches(const HazardSide &Side, const MachineInstr &MI,
568 HazardClassMask C) {
569 return Side.Match.matches(Mask: C) &&
570 (!Side.Predicate || Side.Predicate(MI, Ctx));
571 }
572
573 // Producer phase: open a window for each consumer target.
574 void addWindows(const MachineInstr &MI, HazardClassMask C,
575 MutableArrayRef<RuleTracking> Tracking) {
576 for (unsigned R = 0; R < Rules.size(); ++R) {
577 const HazardAntiHintRule &Rule = Rules[R];
578 if (!RuleApplies[R] || !sideMatches(Side: Rule.Producer, MI, C))
579 continue;
580
581 SmallVector<Register, 4> Regs;
582 // Collect the producer regs.
583 collectOperandRegs(MI, Op: Rule.Producer.Op, Ctx, Out&: Regs);
584 if (Regs.empty())
585 continue;
586 for (unsigned ConsumerIdx = 0, E = Rule.Consumers.size();
587 ConsumerIdx != E; ++ConsumerIdx)
588 seedWindow(Rule, CT: Rule.Consumers[ConsumerIdx], MI, Regs,
589 Track&: Tracking[R][ConsumerIdx]);
590 }
591 }
592
593 void seedWindow(const HazardAntiHintRule &Rule, const ConsumerTarget &CT,
594 const MachineInstr &MI, ArrayRef<Register> Regs,
595 ConsumerTracking &Track) {
596 unsigned Window = resolveWindow(CT, MI, Ctx);
597 if (!Window)
598 return;
599
600 if (Rule.Life == Lifetime::RegisterCount) {
601 // Window length caps the register count.
602 if (Track.empty())
603 Track.push_back(Elt: {.Regs: {}, .Producer: &MI, .Len: Window, .Elapsed: 0});
604 SmallVectorImpl<Register> &Recent = Track.front().Regs;
605 for (Register Reg : Regs) {
606 if (llvm::is_contained(Range&: Recent, Element: Reg))
607 continue;
608 Recent.push_back(Elt: Reg);
609 if (Recent.size() > Window)
610 Recent.erase(CI: Recent.begin());
611 }
612 return;
613 }
614
615 Track.push_back(Elt: {.Regs: SmallVector<Register, 4>(Regs), .Producer: &MI, .Len: Window, .Elapsed: 0});
616 }
617
618 void advanceByRawWindow(const MachineInstr &MI, HazardClassMask C,
619 const HazardAntiHintRule &Rule,
620 const ConsumerTarget &CT, ConsumerTracking &Track) {
621
622 if (!Rule.AdvanceForRawWindow || !CT.Side.Match.matches(Mask: C))
623 return;
624 for (AntiHintWindow &Window : Track) {
625 // Determine if def generated by producer is read by the MI consumer.
626 bool ReadsProducerDef = llvm::any_of(
627 Range: Window.Producer->all_defs(), P: [&](const MachineOperand &MO) {
628 return MO.getReg().isVirtual() &&
629 MI.readsVirtualRegister(Reg: MO.getReg());
630 });
631 // Advance by RAW window from the producer if that is larger than current
632 // Window.Elapsed.
633 if (ReadsProducerDef) {
634 Window.Elapsed = std::max(
635 a: Window.Elapsed, b: Rule.AdvanceForRawWindow(*Window.Producer, C, Ctx));
636 }
637 }
638 }
639
640 // Consumer phase: add anti-hints, then charge and expire open windows.
641 void addAntiHintsAndExpire(const MachineInstr &MI, HazardClassMask C,
642 unsigned WaitStates,
643 MutableArrayRef<RuleTracking> Tracking) {
644 for (unsigned R = 0; R < Rules.size(); ++R) {
645 const HazardAntiHintRule &Rule = Rules[R];
646 if (!RuleApplies[R])
647 continue;
648 for (unsigned ConsumerIdx = 0, E = Rule.Consumers.size();
649 ConsumerIdx != E; ++ConsumerIdx) {
650 const ConsumerTarget &CT = Rule.Consumers[ConsumerIdx];
651 ConsumerTracking &Track = Tracking[R][ConsumerIdx];
652 if (Track.empty())
653 continue;
654
655 // Before adding the anti-hints, see if advancing by RAW window will
656 // help remove the window.
657 const bool Counts = Rule.Life == Lifetime::WindowBudget;
658 if (Counts) {
659 advanceByRawWindow(MI, C, Rule, CT, Track);
660 llvm::erase_if(C&: Track, P: [](const AntiHintWindow &Window) {
661 return Window.Elapsed >= Window.Len;
662 });
663 }
664
665 // Add anti-hints if the consumer matches the instruction.
666 if (sideMatches(Side: CT.Side, MI, C))
667 addAntiHints(CT, MI, Track);
668
669 // This instruction's own wait states count toward the next one.
670 if (Counts && (!CT.CounterMask || (C & CT.CounterMask))) {
671 for (AntiHintWindow &Window : Track)
672 Window.Elapsed += WaitStates;
673 }
674 }
675 }
676 }
677
678 bool isCopyOf(Register Cand, Register HazardReg) const {
679 const MachineInstr *Def = Ctx.MRI->getUniqueVRegDef(Reg: Cand);
680 return Def && Def->isCopy() && Def->getOperand(i: 1).getReg() == HazardReg;
681 }
682
683 void addAntiHints(const ConsumerTarget &CT, const MachineInstr &MI,
684 const ConsumerTracking &Track) {
685 if (Track.empty())
686 return;
687 SmallVector<Register, 4> ConsumerRegs;
688 // Collect the consumer regs.
689 collectOperandRegs(MI, Op: CT.Side.Op, Ctx, Out&: ConsumerRegs);
690 if (ConsumerRegs.empty())
691 return;
692 SlotIndex Slot = Ctx.LIS->getInstructionIndex(Instr: MI).getRegSlot();
693 auto AntiHint = [&](Register ProducerReg) {
694 if (!Ctx.LIS->hasInterval(Reg: ProducerReg))
695 return;
696 const LiveInterval &ProducerLI = Ctx.LIS->getInterval(Reg: ProducerReg);
697 // Skip a live producer reg.
698 if (ProducerLI.liveAt(index: Slot))
699 return;
700 for (Register ConsumerReg : ConsumerRegs) {
701 if (ConsumerReg == ProducerReg || isCopyOf(Cand: ConsumerReg, HazardReg: ProducerReg))
702 continue;
703
704 Ctx.MRI->addRegAllocationAntiHints(VReg: ConsumerReg, AntiHintVRegs: ProducerReg);
705 if (CT.Direction == AntiHintDirection::Symmetric)
706 Ctx.MRI->addRegAllocationAntiHints(VReg: ProducerReg, AntiHintVRegs: ConsumerReg);
707 LLVM_DEBUG(
708 dbgs() << "anti-hint: keep " << printReg(ProducerReg, Ctx.TRI)
709 << (CT.Direction == AntiHintDirection::Symmetric ? " <-> "
710 : " <- ")
711 << printReg(ConsumerReg, Ctx.TRI) << " (consumer "
712 << Ctx.TII->getName(MI.getOpcode()) << ")\n");
713 }
714 };
715 for (const AntiHintWindow &Window : Track) {
716 for (Register ProducerReg : Window.Regs)
717 AntiHint(ProducerReg);
718 }
719 }
720};
721
722} // namespace
723
724void AMDGPU::applyAntiHintRules(MachineFunction &MF, const HazardContext &Ctx) {
725 AntiHintEngine(Ctx).run(MF);
726}
727