| 1 | //===-- GCNPreRAAntiHints.cpp - MFMA register anti-hints ------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | /// \file |
| 10 | /// Insert register allocation anti-hints. |
| 11 | /// |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "GCNPreRAAntiHints.h" |
| 15 | #include "GCNSubtarget.h" |
| 16 | #include "SIInstrInfo.h" |
| 17 | #include "SIRegisterInfo.h" |
| 18 | #include "llvm/ADT/STLExtras.h" |
| 19 | #include "llvm/CodeGen/LiveIntervals.h" |
| 20 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
| 21 | #include "llvm/CodeGen/SlotIndexes.h" |
| 22 | #include "llvm/CodeGen/TargetSchedule.h" |
| 23 | |
| 24 | using namespace llvm; |
| 25 | using namespace llvm::AMDGPU; |
| 26 | |
| 27 | #define DEBUG_TYPE "amdgpu-anti-hints" |
| 28 | |
| 29 | namespace HC = llvm::AMDGPU::HazardClass; |
| 30 | |
| 31 | namespace { |
| 32 | |
| 33 | // Classify the MI into a HazardClassMask. |
| 34 | HazardClassMask getInstHazardClass(const MachineInstr &MI, |
| 35 | const HazardContext &Ctx) { |
| 36 | const SIInstrInfo &TII = *Ctx.TII; |
| 37 | HazardClassMask Mask = HC::None; |
| 38 | |
| 39 | if (TII.isLDSDMA(MI)) |
| 40 | Mask = HC::VALU | HC::VMEM | HC::DS | HC::LDSDMA; |
| 41 | else if (TII.isWMMA(MI) || SIInstrInfo::isSWMMAC(MI)) |
| 42 | Mask = HC::WMMA; |
| 43 | else if (TII.isMFMA(MI)) |
| 44 | Mask = HC::MFMA; |
| 45 | else if (SIInstrInfo::isTRANS(MI)) |
| 46 | Mask = HC::TRANS; |
| 47 | else if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/false)) |
| 48 | Mask = HC::VALU; |
| 49 | else if (TII.isDS(MI)) |
| 50 | Mask = HC::DS; |
| 51 | else if (TII.isVMEM(MI)) |
| 52 | Mask = HC::VMEM; |
| 53 | else if (TII.isSMRD(MI)) |
| 54 | Mask = HC::SMEM; |
| 55 | else if (TII.isEXP(MI)) |
| 56 | Mask = HC::EXP; |
| 57 | else if (SIInstrInfo::isSALU(MI)) |
| 58 | Mask = HC::SALU; |
| 59 | else if (MI.isCopy() && MI.getOperand(i: 0).getReg().isVirtual() && |
| 60 | Ctx.TRI->hasVGPRs(RC: Ctx.MRI->getRegClass(Reg: MI.getOperand(i: 0).getReg()))) |
| 61 | Mask = HC::VALU; |
| 62 | if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true)) |
| 63 | Mask |= HC::RawVALU; |
| 64 | if (MI.mayLoad()) |
| 65 | Mask |= HC::Load; |
| 66 | for (const MachineOperand &MO : MI.defs()) { |
| 67 | if (MO.isReg() && MO.getReg().isVirtual() && |
| 68 | Ctx.TRI->hasVGPRs(RC: Ctx.MRI->getRegClass(Reg: MO.getReg()))) { |
| 69 | Mask |= HC::WritesVGPR; |
| 70 | break; |
| 71 | } |
| 72 | } |
| 73 | |
| 74 | return Mask; |
| 75 | } |
| 76 | |
| 77 | bool isVirtualVGPR(const HazardContext &Ctx, const MachineOperand &MO) { |
| 78 | if (!MO.isReg() || !MO.getReg().isVirtual()) |
| 79 | return false; |
| 80 | return Ctx.TRI->hasVGPRs(RC: Ctx.MRI->getRegClass(Reg: MO.getReg())); |
| 81 | } |
| 82 | |
| 83 | void collectOperandRegs(const MachineInstr &MI, HazardOperand Op, |
| 84 | const HazardContext &Ctx, |
| 85 | SmallVectorImpl<Register> &Out) { |
| 86 | const SIInstrInfo &TII = *Ctx.TII; |
| 87 | switch (Op) { |
| 88 | case HazardOperand::None: |
| 89 | break; |
| 90 | case HazardOperand::Def: |
| 91 | for (const MachineOperand &MO : MI.all_defs()) { |
| 92 | if (isVirtualVGPR(Ctx, MO)) |
| 93 | Out.push_back(Elt: MO.getReg()); |
| 94 | } |
| 95 | break; |
| 96 | case HazardOperand::Src0: |
| 97 | if (const MachineOperand *MO = |
| 98 | TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src0)) { |
| 99 | if (isVirtualVGPR(Ctx, MO: *MO)) |
| 100 | Out.push_back(Elt: MO->getReg()); |
| 101 | } |
| 102 | break; |
| 103 | case HazardOperand::Src1: |
| 104 | if (const MachineOperand *MO = |
| 105 | TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src1)) { |
| 106 | if (isVirtualVGPR(Ctx, MO: *MO)) |
| 107 | Out.push_back(Elt: MO->getReg()); |
| 108 | } |
| 109 | break; |
| 110 | case HazardOperand::Src2: |
| 111 | if (const MachineOperand *MO = |
| 112 | TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::src2)) { |
| 113 | if (isVirtualVGPR(Ctx, MO: *MO)) |
| 114 | Out.push_back(Elt: MO->getReg()); |
| 115 | } |
| 116 | break; |
| 117 | case HazardOperand::Src0Src1: |
| 118 | for (AMDGPU::OpName Name : {AMDGPU::OpName::src0, AMDGPU::OpName::src1}) { |
| 119 | if (const MachineOperand *MO = TII.getNamedOperand(MI, OperandName: Name)) { |
| 120 | if (isVirtualVGPR(Ctx, MO: *MO)) |
| 121 | Out.push_back(Elt: MO->getReg()); |
| 122 | } |
| 123 | } |
| 124 | break; |
| 125 | case HazardOperand::Idx: |
| 126 | if (const MachineOperand *MO = |
| 127 | TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::idx)) { |
| 128 | if (isVirtualVGPR(Ctx, MO: *MO)) |
| 129 | Out.push_back(Elt: MO->getReg()); |
| 130 | } |
| 131 | break; |
| 132 | case HazardOperand::Vaddr: |
| 133 | if (const MachineOperand *MO = |
| 134 | TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::vaddr)) { |
| 135 | if (isVirtualVGPR(Ctx, MO: *MO)) |
| 136 | Out.push_back(Elt: MO->getReg()); |
| 137 | } |
| 138 | break; |
| 139 | case HazardOperand::AnySrc: |
| 140 | for (AMDGPU::OpName Name : |
| 141 | {AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2}) { |
| 142 | if (const MachineOperand *MO = TII.getNamedOperand(MI, OperandName: Name)) { |
| 143 | if (isVirtualVGPR(Ctx, MO: *MO)) |
| 144 | Out.push_back(Elt: MO->getReg()); |
| 145 | } |
| 146 | } |
| 147 | break; |
| 148 | case HazardOperand::AnyUse: |
| 149 | for (const MachineOperand &MO : MI.all_uses()) { |
| 150 | if (isVirtualVGPR(Ctx, MO)) |
| 151 | Out.push_back(Elt: MO.getReg()); |
| 152 | } |
| 153 | break; |
| 154 | } |
| 155 | } |
| 156 | |
| 157 | enum class MFMAHazardKind { RAW, WAW, WAR }; |
| 158 | |
| 159 | // MFMA anti-hint wait-state window, mirroring GCNHazardRecognizer.cpp wait |
| 160 | // states. |
| 161 | unsigned mfmaWaitStates(const MachineInstr &MFMA, MFMAHazardKind Kind, |
| 162 | HazardClassMask ReaderClass, const HazardContext &Ctx) { |
| 163 | const SIInstrInfo &TII = *Ctx.TII; |
| 164 | const GCNSubtarget &ST = *Ctx.ST; |
| 165 | const int NumPasses = Ctx.SchedModel->computeInstrLatency(MI: &MFMA); |
| 166 | const bool IsDGEMM = SIInstrInfo::isDGEMM(Opcode: MFMA.getOpcode()); |
| 167 | const bool Mem = ReaderClass & (HC::VMEM | HC::DS | HC::EXP); |
| 168 | |
| 169 | auto GFX940NPass = [&]() -> unsigned { |
| 170 | return TII.isXDL(MI: MFMA) |
| 171 | ? NumPasses + 3 + (NumPasses != 2 && ST.hasGFX950Insts()) |
| 172 | : NumPasses + 2; |
| 173 | }; |
| 174 | auto SMFMANPass = [&]() -> unsigned { |
| 175 | switch (NumPasses) { |
| 176 | case 2: |
| 177 | return 5; |
| 178 | case 8: |
| 179 | return 11; |
| 180 | case 16: |
| 181 | return 19; |
| 182 | default: |
| 183 | return 0; |
| 184 | } |
| 185 | }; |
| 186 | |
| 187 | switch (Kind) { |
| 188 | case MFMAHazardKind::RAW: |
| 189 | if (IsDGEMM) { |
| 190 | switch (NumPasses) { |
| 191 | case 4: |
| 192 | return Mem ? 9 : 6; |
| 193 | case 8: |
| 194 | case 16: |
| 195 | return Mem ? 18 : (ST.hasGFX950Insts() ? 19 : 11); |
| 196 | default: |
| 197 | return 0; |
| 198 | } |
| 199 | } |
| 200 | return ST.hasGFX940Insts() ? GFX940NPass() : SMFMANPass(); |
| 201 | |
| 202 | case MFMAHazardKind::WAW: |
| 203 | if (IsDGEMM) { |
| 204 | switch (NumPasses) { |
| 205 | case 4: |
| 206 | return 6; |
| 207 | case 8: |
| 208 | case 16: |
| 209 | return 11; |
| 210 | default: |
| 211 | return 0; |
| 212 | } |
| 213 | } |
| 214 | return ST.hasGFX940Insts() ? GFX940NPass() : SMFMANPass(); |
| 215 | |
| 216 | case MFMAHazardKind::WAR: |
| 217 | switch (NumPasses) { |
| 218 | case 2: |
| 219 | return 1; |
| 220 | case 4: |
| 221 | return 3; |
| 222 | case 8: |
| 223 | return 7; |
| 224 | case 16: |
| 225 | return 15; |
| 226 | default: |
| 227 | return 15; |
| 228 | } |
| 229 | } |
| 230 | return 0; |
| 231 | } |
| 232 | |
| 233 | unsigned mfmaWawWindow(const MachineInstr &P, HazardClassMask, |
| 234 | const HazardContext &Ctx) { |
| 235 | return mfmaWaitStates(MFMA: P, Kind: MFMAHazardKind::WAW, ReaderClass: HC::None, Ctx); |
| 236 | } |
| 237 | unsigned mfmaWarWindow(const MachineInstr &P, HazardClassMask, |
| 238 | const HazardContext &Ctx) { |
| 239 | return mfmaWaitStates(MFMA: P, Kind: MFMAHazardKind::WAR, ReaderClass: HC::None, Ctx); |
| 240 | } |
| 241 | unsigned addrAntiHintWindow(const MachineInstr &, HazardClassMask, |
| 242 | const HazardContext &Ctx) { |
| 243 | return Ctx.ST->getCLOpts().addr_anti_hint_window; |
| 244 | } |
| 245 | unsigned vaVdstLookbackWindow(const MachineInstr &, HazardClassMask, |
| 246 | const HazardContext &Ctx) { |
| 247 | return Ctx.ST->getCLOpts().va_vdst_lookback_window; |
| 248 | } |
| 249 | |
| 250 | constexpr unsigned NumWMMAHazardCategories = 7; |
| 251 | unsigned wmmaHazardCategory(const MachineInstr &Producer, |
| 252 | const HazardContext &Ctx) { |
| 253 | const bool IsSWMMAC = SIInstrInfo::isSWMMAC(MI: Producer); |
| 254 | const bool LowestRate = Ctx.ST->hasGFX125xLowestRateWMMA(); |
| 255 | switch (Ctx.SchedModel->computeInstrLatency(MI: &Producer)) { |
| 256 | case 4: |
| 257 | return 6; |
| 258 | case 8: |
| 259 | return IsSWMMAC ? 2 : 0; |
| 260 | case 16: |
| 261 | return LowestRate ? 4 : (IsSWMMAC ? 3 : 1); |
| 262 | case 32: |
| 263 | return 5; |
| 264 | default: |
| 265 | return NumWMMAHazardCategories; |
| 266 | } |
| 267 | } |
| 268 | |
| 269 | // WMMA/SWMMAC co-exec window. |
| 270 | unsigned wmmaCoexecWindow(const MachineInstr &Producer, |
| 271 | HazardClassMask ConsumerClass, |
| 272 | const HazardContext &Ctx) { |
| 273 | const unsigned Category = wmmaHazardCategory(Producer, Ctx); |
| 274 | if (Category >= NumWMMAHazardCategories) |
| 275 | return 0; |
| 276 | |
| 277 | static constexpr unsigned WMMAWaitStates[] = {5, 9, 3, 5, 9, 17, 2}; |
| 278 | static constexpr unsigned VALUWaitStates[] = {4, 8, 2, 4, 8, 16, 1}; |
| 279 | return (ConsumerClass & HC::WMMA) ? WMMAWaitStates[Category] |
| 280 | : VALUWaitStates[Category]; |
| 281 | } |
| 282 | |
| 283 | unsigned mfmaReaderRawWindow(const MachineInstr &Producer, |
| 284 | HazardClassMask ReaderClass, |
| 285 | const HazardContext &Ctx) { |
| 286 | return mfmaWaitStates(MFMA: Producer, Kind: MFMAHazardKind::RAW, ReaderClass, Ctx); |
| 287 | } |
| 288 | |
| 289 | bool hasMFMAHazard(const HazardContext &Ctx) { |
| 290 | return Ctx.ST->hasGFX90AInsts(); |
| 291 | } |
| 292 | |
| 293 | bool hasWMMACoexecHazard(const HazardContext &Ctx) { |
| 294 | return Ctx.ST->hasWMMACoexecutionHazards(); |
| 295 | } |
| 296 | |
| 297 | bool hasTransCoexecHazard(const HazardContext &Ctx) { |
| 298 | return Ctx.ST->hasTransCoexecutionHazard(); |
| 299 | } |
| 300 | |
| 301 | bool hasGFX1250Insts(const HazardContext &Ctx) { |
| 302 | return Ctx.ST->hasGFX1250Insts(); |
| 303 | } |
| 304 | |
| 305 | bool ruleSelected(const HazardContext &Ctx, AntiHintRule Rule) { |
| 306 | ArrayRef<AntiHintRule> Selection = Ctx.ST->getCLOpts().anti_hints_rules; |
| 307 | // No -amdgpu-anti-hints-rules on the command line selects every rule. |
| 308 | if (Selection.empty() || is_contained(Range&: Selection, Element: AntiHintRule::All)) |
| 309 | return true; |
| 310 | return is_contained(Range&: Selection, Element: Rule); |
| 311 | } |
| 312 | |
| 313 | bool isMFMAWAWRuleEnabled(const HazardContext &Ctx) { |
| 314 | return hasMFMAHazard(Ctx) && ruleSelected(Ctx, Rule: AntiHintRule::MFMAWAW); |
| 315 | } |
| 316 | |
| 317 | bool isMFMAWARRuleEnabled(const HazardContext &Ctx) { |
| 318 | return hasMFMAHazard(Ctx) && ruleSelected(Ctx, Rule: AntiHintRule::MFMAWAR); |
| 319 | } |
| 320 | |
| 321 | bool isWMMAWARABRuleEnabled(const HazardContext &Ctx) { |
| 322 | return hasWMMACoexecHazard(Ctx) && ruleSelected(Ctx, Rule: AntiHintRule::WMMAWARAB); |
| 323 | } |
| 324 | |
| 325 | bool isSWMMACWARIndexRuleEnabled(const HazardContext &Ctx) { |
| 326 | return hasWMMACoexecHazard(Ctx) && |
| 327 | ruleSelected(Ctx, Rule: AntiHintRule::SWMMACWARIndex); |
| 328 | } |
| 329 | |
| 330 | bool isWMMAWAWRuleEnabled(const HazardContext &Ctx) { |
| 331 | return hasWMMACoexecHazard(Ctx) && ruleSelected(Ctx, Rule: AntiHintRule::WMMAWAW); |
| 332 | } |
| 333 | |
| 334 | bool isTransWARRuleEnabled(const HazardContext &Ctx) { |
| 335 | return hasTransCoexecHazard(Ctx) && ruleSelected(Ctx, Rule: AntiHintRule::TransWAR); |
| 336 | } |
| 337 | |
| 338 | bool isAddrWARRuleEnabled(const HazardContext &Ctx) { |
| 339 | return Ctx.ST->getCLOpts().anti_hints_for_addr && hasGFX1250Insts(Ctx) && |
| 340 | ruleSelected(Ctx, Rule: AntiHintRule::MemAddrWAR); |
| 341 | } |
| 342 | |
| 343 | bool isVAVdstWARRuleEnabled(const HazardContext &Ctx) { |
| 344 | return Ctx.ST->getCLOpts().anti_hints_for_va_vdst && hasGFX1250Insts(Ctx) && |
| 345 | ruleSelected(Ctx, Rule: AntiHintRule::VAVdstWAR); |
| 346 | } |
| 347 | |
| 348 | bool isXDLMFMA(const MachineInstr &MI, const HazardContext &Ctx) { |
| 349 | return Ctx.TII->isXDL(MI); |
| 350 | } |
| 351 | |
| 352 | bool isXDLWMMA(const MachineInstr &MI, const HazardContext &Ctx) { |
| 353 | return Ctx.TII->isXDLWMMA(MI); |
| 354 | } |
| 355 | |
| 356 | bool isXDLSWMMAC(const MachineInstr &MI, const HazardContext &Ctx) { |
| 357 | return Ctx.TII->isXDLWMMA(MI) && SIInstrInfo::isSWMMAC(MI); |
| 358 | } |
| 359 | |
| 360 | bool isDSorVMEMLoad(const MachineInstr &MI, const HazardContext &Ctx) { |
| 361 | return MI.mayLoad() && (Ctx.TII->isDS(MI) || Ctx.TII->isVMEM(MI)); |
| 362 | } |
| 363 | |
| 364 | unsigned resolveWindow(const ConsumerTarget &CT, const MachineInstr &MI, |
| 365 | const HazardContext &Ctx) { |
| 366 | if (CT.Window.Fn) |
| 367 | return CT.Window.Fn(MI, CT.Side.Match.AnyOf, Ctx); |
| 368 | return CT.Window.WindowLength; |
| 369 | } |
| 370 | |
| 371 | // Build the anti-hints rules. |
| 372 | class HazardRuleSet { |
| 373 | SmallVector<HazardAntiHintRule, 0> Rules; |
| 374 | |
| 375 | public: |
| 376 | class RuleBuilder { |
| 377 | |
| 378 | HazardRuleSet &S; |
| 379 | unsigned Idx; |
| 380 | HazardAntiHintRule &rule() const { return S.Rules[Idx]; } |
| 381 | |
| 382 | public: |
| 383 | RuleBuilder(HazardRuleSet &S, unsigned Idx) : S(S), Idx(Idx) {} |
| 384 | |
| 385 | RuleBuilder &producer(ClassMatch M, HazardOperand Op, |
| 386 | InstPredicate Predicate = nullptr) { |
| 387 | rule().Producer = {.Match: M, .Op: Op, .Predicate: Predicate}; |
| 388 | return *this; |
| 389 | } |
| 390 | |
| 391 | RuleBuilder &rawCredit(AdvanceForRawWindowFn AdvanceForRawWindow) { |
| 392 | rule().AdvanceForRawWindow = AdvanceForRawWindow; |
| 393 | return *this; |
| 394 | } |
| 395 | |
| 396 | RuleBuilder & |
| 397 | consumer(ClassMatch M, HazardOperand Op, WindowSpec Window, |
| 398 | HazardClassMask CountMask = 0, |
| 399 | AntiHintDirection Direction = AntiHintDirection::OneDirectional, |
| 400 | InstPredicate Predicate = nullptr) { |
| 401 | rule().Consumers.push_back( |
| 402 | Elt: {.Side: {.Match: M, .Op: Op, .Predicate: Predicate}, .Window: Window, .CounterMask: CountMask, .Direction: Direction}); |
| 403 | return *this; |
| 404 | } |
| 405 | |
| 406 | RuleBuilder &enabledIf(RulePredicate Predicate) { |
| 407 | rule().Predicate = Predicate; |
| 408 | return *this; |
| 409 | } |
| 410 | |
| 411 | RuleBuilder &lifetime(Lifetime Life) { |
| 412 | rule().Life = Life; |
| 413 | return *this; |
| 414 | } |
| 415 | }; |
| 416 | |
| 417 | RuleBuilder addRule() { |
| 418 | Rules.emplace_back(); |
| 419 | return RuleBuilder(*this, Rules.size() - 1); |
| 420 | } |
| 421 | |
| 422 | SmallVector<HazardAntiHintRule, 0> buildRules() { return std::move(Rules); } |
| 423 | }; |
| 424 | |
| 425 | // Here the anti-hints rules are inserted. |
| 426 | SmallVector<HazardAntiHintRule, 0> buildAntiHintsRules() { |
| 427 | using HO = HazardOperand; |
| 428 | HazardRuleSet S; |
| 429 | |
| 430 | // MFMA relevent windows and consumers |
| 431 | const WindowSpec MfmaWawWindow{.WindowLength: 0, .Fn: mfmaWawWindow}; |
| 432 | const WindowSpec MfmaWarWindow{.WindowLength: 0, .Fn: mfmaWarWindow}; |
| 433 | const ClassMatch MfmaConsumers = {.AnyOf: HC::DS | HC::VALU | HC::VMEM | HC::TRANS | |
| 434 | HC::EXP}; |
| 435 | |
| 436 | // WMMA relevent windows and consumers |
| 437 | const WindowSpec WmmaCoexecWindow{.WindowLength: 0, .Fn: wmmaCoexecWindow}; |
| 438 | const ClassMatch WmmaCoexecConsumers = {/*AnyOf=*/HC::VALU | HC::TRANS, |
| 439 | /*AllOf=*/HC::None, |
| 440 | /*NoneOf=*/HC::LDSDMA}; |
| 441 | const HazardClassMask CoexecCounters = HC::VALU | HC::TRANS | HC::WMMA; |
| 442 | |
| 443 | // MFMA WAW rules |
| 444 | S.addRule() |
| 445 | .enabledIf(Predicate: isMFMAWAWRuleEnabled) |
| 446 | .producer(M: {.AnyOf: HC::MFMA}, Op: HO::Def) |
| 447 | .rawCredit(AdvanceForRawWindow: mfmaReaderRawWindow) |
| 448 | .consumer(M: MfmaConsumers, Op: HO::Def, Window: MfmaWawWindow, CountMask: HC::None, |
| 449 | Direction: AntiHintDirection::OneDirectional); |
| 450 | |
| 451 | // MFMA WAR rules |
| 452 | S.addRule() |
| 453 | .enabledIf(Predicate: isMFMAWARRuleEnabled) |
| 454 | .producer(M: {.AnyOf: HC::MFMA}, Op: HO::Src2, Predicate: isXDLMFMA) |
| 455 | .rawCredit(AdvanceForRawWindow: mfmaReaderRawWindow) |
| 456 | .consumer(M: MfmaConsumers, Op: HO::Def, Window: MfmaWarWindow, CountMask: HC::None, |
| 457 | Direction: AntiHintDirection::OneDirectional); |
| 458 | |
| 459 | // WMMA WAR. |
| 460 | S.addRule() |
| 461 | .enabledIf(Predicate: isWMMAWARABRuleEnabled) |
| 462 | .producer(M: {.AnyOf: HC::WMMA}, Op: HO::Src0Src1, Predicate: isXDLWMMA) |
| 463 | .consumer(M: WmmaCoexecConsumers, Op: HO::Def, Window: WmmaCoexecWindow, CountMask: CoexecCounters, |
| 464 | Direction: AntiHintDirection::OneDirectional) |
| 465 | .consumer(M: {.AnyOf: HC::VMEM | HC::DS}, Op: HO::Def, Window: WmmaCoexecWindow, CountMask: CoexecCounters, |
| 466 | Direction: AntiHintDirection::OneDirectional); |
| 467 | |
| 468 | // SWMMAC WAR. |
| 469 | S.addRule() |
| 470 | .enabledIf(Predicate: isSWMMACWARIndexRuleEnabled) |
| 471 | .producer(M: {.AnyOf: HC::WMMA}, Op: HO::Src2, Predicate: isXDLSWMMAC) |
| 472 | .consumer(M: WmmaCoexecConsumers, Op: HO::Def, Window: WmmaCoexecWindow, CountMask: CoexecCounters, |
| 473 | Direction: AntiHintDirection::OneDirectional); |
| 474 | |
| 475 | // WMMA WAW. |
| 476 | S.addRule() |
| 477 | .enabledIf(Predicate: isWMMAWAWRuleEnabled) |
| 478 | .producer(M: {.AnyOf: HC::WMMA}, Op: HO::Def, Predicate: isXDLWMMA) |
| 479 | .consumer(M: WmmaCoexecConsumers, Op: HO::Def, Window: WmmaCoexecWindow, CountMask: CoexecCounters, |
| 480 | Direction: AntiHintDirection::OneDirectional); |
| 481 | |
| 482 | // TRANS WAR. |
| 483 | S.addRule() |
| 484 | .enabledIf(Predicate: isTransWARRuleEnabled) |
| 485 | .producer(M: {.AnyOf: HC::TRANS}, Op: HO::AnyUse) |
| 486 | .consumer(M: {.AnyOf: HC::VALU | HC::WMMA}, Op: HO::Def, Window: {/*WindowLength=*/1}, |
| 487 | CountMask: CoexecCounters, Direction: AntiHintDirection::OneDirectional); |
| 488 | |
| 489 | // Address WAR. |
| 490 | S.addRule() |
| 491 | .enabledIf(Predicate: isAddrWARRuleEnabled) |
| 492 | .producer(M: {.AnyOf: HC::VMEM}, Op: HO::AnyUse) |
| 493 | .consumer(M: {.AnyOf: HC::WritesVGPR}, Op: HO::Def, Window: {.WindowLength: 0, .Fn: addrAntiHintWindow}, CountMask: HC::VMEM, |
| 494 | Direction: AntiHintDirection::Symmetric); |
| 495 | |
| 496 | // VA_VDST WAR. |
| 497 | S.addRule() |
| 498 | .enabledIf(Predicate: isVAVdstWARRuleEnabled) |
| 499 | .lifetime(Life: Lifetime::RegisterCount) |
| 500 | .producer(M: {.AnyOf: HC::RawVALU}, Op: HO::AnyUse) |
| 501 | .consumer(M: {.AnyOf: HC::Load}, Op: HO::Def, Window: {.WindowLength: 0, .Fn: vaVdstLookbackWindow}, CountMask: HC::None, |
| 502 | Direction: AntiHintDirection::Symmetric, Predicate: isDSorVMEMLoad); |
| 503 | |
| 504 | return S.buildRules(); |
| 505 | } |
| 506 | |
| 507 | ArrayRef<HazardAntiHintRule> getAntiHintsRules() { |
| 508 | static const SmallVector<HazardAntiHintRule, 0> Rules = buildAntiHintsRules(); |
| 509 | return Rules; |
| 510 | } |
| 511 | |
| 512 | struct AntiHintWindow { |
| 513 | SmallVector<Register, 4> Regs; |
| 514 | const MachineInstr *Producer = nullptr; |
| 515 | unsigned Len = 0; |
| 516 | unsigned Elapsed = 0; |
| 517 | }; |
| 518 | |
| 519 | using ConsumerTracking = SmallVector<AntiHintWindow, 3>; |
| 520 | // One per consumer target of a rule. |
| 521 | using RuleTracking = SmallVector<ConsumerTracking, 3>; |
| 522 | |
| 523 | class AntiHintEngine { |
| 524 | const HazardContext &Ctx; |
| 525 | ArrayRef<HazardAntiHintRule> Rules; |
| 526 | |
| 527 | SmallVector<bool, 8> RuleApplies; |
| 528 | bool AnyEnabled = false; |
| 529 | |
| 530 | public: |
| 531 | AntiHintEngine(const HazardContext &Ctx) |
| 532 | : Ctx(Ctx), Rules(getAntiHintsRules()), RuleApplies(Rules.size()) { |
| 533 | for (unsigned R = 0; R < Rules.size(); ++R) { |
| 534 | const HazardAntiHintRule &Rule = Rules[R]; |
| 535 | RuleApplies[R] = !Rule.Predicate || Rule.Predicate(Ctx); |
| 536 | AnyEnabled |= RuleApplies[R]; |
| 537 | } |
| 538 | } |
| 539 | |
| 540 | void run(MachineFunction &MF) { |
| 541 | if (!AnyEnabled) |
| 542 | return; |
| 543 | |
| 544 | SmallVector<RuleTracking, 8> Tracking(Rules.size()); |
| 545 | for (unsigned R = 0; R < Rules.size(); ++R) |
| 546 | Tracking[R].resize(N: Rules[R].Consumers.size()); |
| 547 | |
| 548 | for (const MachineBasicBlock &MBB : MF) { |
| 549 | for (RuleTracking &RT : Tracking) { |
| 550 | for (ConsumerTracking &Track : RT) |
| 551 | Track.clear(); |
| 552 | } |
| 553 | for (const MachineInstr &MI : MBB) { |
| 554 | if (MI.isMetaInstruction()) |
| 555 | continue; |
| 556 | const HazardClassMask C = getInstHazardClass(MI, Ctx); |
| 557 | // Wait states this instruction contributes to an open window. |
| 558 | unsigned WaitStates = SIInstrInfo::getNumWaitStates(MI); |
| 559 | addAntiHintsAndExpire(MI, C, WaitStates, Tracking); |
| 560 | addWindows(MI, C, Tracking); |
| 561 | } |
| 562 | } |
| 563 | } |
| 564 | |
| 565 | private: |
| 566 | // Check if the right class and predicate matches. |
| 567 | bool sideMatches(const HazardSide &Side, const MachineInstr &MI, |
| 568 | HazardClassMask C) { |
| 569 | return Side.Match.matches(Mask: C) && |
| 570 | (!Side.Predicate || Side.Predicate(MI, Ctx)); |
| 571 | } |
| 572 | |
| 573 | // Producer phase: open a window for each consumer target. |
| 574 | void addWindows(const MachineInstr &MI, HazardClassMask C, |
| 575 | MutableArrayRef<RuleTracking> Tracking) { |
| 576 | for (unsigned R = 0; R < Rules.size(); ++R) { |
| 577 | const HazardAntiHintRule &Rule = Rules[R]; |
| 578 | if (!RuleApplies[R] || !sideMatches(Side: Rule.Producer, MI, C)) |
| 579 | continue; |
| 580 | |
| 581 | SmallVector<Register, 4> Regs; |
| 582 | // Collect the producer regs. |
| 583 | collectOperandRegs(MI, Op: Rule.Producer.Op, Ctx, Out&: Regs); |
| 584 | if (Regs.empty()) |
| 585 | continue; |
| 586 | for (unsigned ConsumerIdx = 0, E = Rule.Consumers.size(); |
| 587 | ConsumerIdx != E; ++ConsumerIdx) |
| 588 | seedWindow(Rule, CT: Rule.Consumers[ConsumerIdx], MI, Regs, |
| 589 | Track&: Tracking[R][ConsumerIdx]); |
| 590 | } |
| 591 | } |
| 592 | |
| 593 | void seedWindow(const HazardAntiHintRule &Rule, const ConsumerTarget &CT, |
| 594 | const MachineInstr &MI, ArrayRef<Register> Regs, |
| 595 | ConsumerTracking &Track) { |
| 596 | unsigned Window = resolveWindow(CT, MI, Ctx); |
| 597 | if (!Window) |
| 598 | return; |
| 599 | |
| 600 | if (Rule.Life == Lifetime::RegisterCount) { |
| 601 | // Window length caps the register count. |
| 602 | if (Track.empty()) |
| 603 | Track.push_back(Elt: {.Regs: {}, .Producer: &MI, .Len: Window, .Elapsed: 0}); |
| 604 | SmallVectorImpl<Register> &Recent = Track.front().Regs; |
| 605 | for (Register Reg : Regs) { |
| 606 | if (llvm::is_contained(Range&: Recent, Element: Reg)) |
| 607 | continue; |
| 608 | Recent.push_back(Elt: Reg); |
| 609 | if (Recent.size() > Window) |
| 610 | Recent.erase(CI: Recent.begin()); |
| 611 | } |
| 612 | return; |
| 613 | } |
| 614 | |
| 615 | Track.push_back(Elt: {.Regs: SmallVector<Register, 4>(Regs), .Producer: &MI, .Len: Window, .Elapsed: 0}); |
| 616 | } |
| 617 | |
| 618 | void advanceByRawWindow(const MachineInstr &MI, HazardClassMask C, |
| 619 | const HazardAntiHintRule &Rule, |
| 620 | const ConsumerTarget &CT, ConsumerTracking &Track) { |
| 621 | |
| 622 | if (!Rule.AdvanceForRawWindow || !CT.Side.Match.matches(Mask: C)) |
| 623 | return; |
| 624 | for (AntiHintWindow &Window : Track) { |
| 625 | // Determine if def generated by producer is read by the MI consumer. |
| 626 | bool ReadsProducerDef = llvm::any_of( |
| 627 | Range: Window.Producer->all_defs(), P: [&](const MachineOperand &MO) { |
| 628 | return MO.getReg().isVirtual() && |
| 629 | MI.readsVirtualRegister(Reg: MO.getReg()); |
| 630 | }); |
| 631 | // Advance by RAW window from the producer if that is larger than current |
| 632 | // Window.Elapsed. |
| 633 | if (ReadsProducerDef) { |
| 634 | Window.Elapsed = std::max( |
| 635 | a: Window.Elapsed, b: Rule.AdvanceForRawWindow(*Window.Producer, C, Ctx)); |
| 636 | } |
| 637 | } |
| 638 | } |
| 639 | |
| 640 | // Consumer phase: add anti-hints, then charge and expire open windows. |
| 641 | void addAntiHintsAndExpire(const MachineInstr &MI, HazardClassMask C, |
| 642 | unsigned WaitStates, |
| 643 | MutableArrayRef<RuleTracking> Tracking) { |
| 644 | for (unsigned R = 0; R < Rules.size(); ++R) { |
| 645 | const HazardAntiHintRule &Rule = Rules[R]; |
| 646 | if (!RuleApplies[R]) |
| 647 | continue; |
| 648 | for (unsigned ConsumerIdx = 0, E = Rule.Consumers.size(); |
| 649 | ConsumerIdx != E; ++ConsumerIdx) { |
| 650 | const ConsumerTarget &CT = Rule.Consumers[ConsumerIdx]; |
| 651 | ConsumerTracking &Track = Tracking[R][ConsumerIdx]; |
| 652 | if (Track.empty()) |
| 653 | continue; |
| 654 | |
| 655 | // Before adding the anti-hints, see if advancing by RAW window will |
| 656 | // help remove the window. |
| 657 | const bool Counts = Rule.Life == Lifetime::WindowBudget; |
| 658 | if (Counts) { |
| 659 | advanceByRawWindow(MI, C, Rule, CT, Track); |
| 660 | llvm::erase_if(C&: Track, P: [](const AntiHintWindow &Window) { |
| 661 | return Window.Elapsed >= Window.Len; |
| 662 | }); |
| 663 | } |
| 664 | |
| 665 | // Add anti-hints if the consumer matches the instruction. |
| 666 | if (sideMatches(Side: CT.Side, MI, C)) |
| 667 | addAntiHints(CT, MI, Track); |
| 668 | |
| 669 | // This instruction's own wait states count toward the next one. |
| 670 | if (Counts && (!CT.CounterMask || (C & CT.CounterMask))) { |
| 671 | for (AntiHintWindow &Window : Track) |
| 672 | Window.Elapsed += WaitStates; |
| 673 | } |
| 674 | } |
| 675 | } |
| 676 | } |
| 677 | |
| 678 | bool isCopyOf(Register Cand, Register HazardReg) const { |
| 679 | const MachineInstr *Def = Ctx.MRI->getUniqueVRegDef(Reg: Cand); |
| 680 | return Def && Def->isCopy() && Def->getOperand(i: 1).getReg() == HazardReg; |
| 681 | } |
| 682 | |
| 683 | void addAntiHints(const ConsumerTarget &CT, const MachineInstr &MI, |
| 684 | const ConsumerTracking &Track) { |
| 685 | if (Track.empty()) |
| 686 | return; |
| 687 | SmallVector<Register, 4> ConsumerRegs; |
| 688 | // Collect the consumer regs. |
| 689 | collectOperandRegs(MI, Op: CT.Side.Op, Ctx, Out&: ConsumerRegs); |
| 690 | if (ConsumerRegs.empty()) |
| 691 | return; |
| 692 | SlotIndex Slot = Ctx.LIS->getInstructionIndex(Instr: MI).getRegSlot(); |
| 693 | auto AntiHint = [&](Register ProducerReg) { |
| 694 | if (!Ctx.LIS->hasInterval(Reg: ProducerReg)) |
| 695 | return; |
| 696 | const LiveInterval &ProducerLI = Ctx.LIS->getInterval(Reg: ProducerReg); |
| 697 | // Skip a live producer reg. |
| 698 | if (ProducerLI.liveAt(index: Slot)) |
| 699 | return; |
| 700 | for (Register ConsumerReg : ConsumerRegs) { |
| 701 | if (ConsumerReg == ProducerReg || isCopyOf(Cand: ConsumerReg, HazardReg: ProducerReg)) |
| 702 | continue; |
| 703 | |
| 704 | Ctx.MRI->addRegAllocationAntiHints(VReg: ConsumerReg, AntiHintVRegs: ProducerReg); |
| 705 | if (CT.Direction == AntiHintDirection::Symmetric) |
| 706 | Ctx.MRI->addRegAllocationAntiHints(VReg: ProducerReg, AntiHintVRegs: ConsumerReg); |
| 707 | LLVM_DEBUG( |
| 708 | dbgs() << "anti-hint: keep " << printReg(ProducerReg, Ctx.TRI) |
| 709 | << (CT.Direction == AntiHintDirection::Symmetric ? " <-> " |
| 710 | : " <- " ) |
| 711 | << printReg(ConsumerReg, Ctx.TRI) << " (consumer " |
| 712 | << Ctx.TII->getName(MI.getOpcode()) << ")\n" ); |
| 713 | } |
| 714 | }; |
| 715 | for (const AntiHintWindow &Window : Track) { |
| 716 | for (Register ProducerReg : Window.Regs) |
| 717 | AntiHint(ProducerReg); |
| 718 | } |
| 719 | } |
| 720 | }; |
| 721 | |
| 722 | } // namespace |
| 723 | |
| 724 | void AMDGPU::applyAntiHintRules(MachineFunction &MF, const HazardContext &Ctx) { |
| 725 | AntiHintEngine(Ctx).run(MF); |
| 726 | } |
| 727 | |