| 1 | //===- GCNPreRAAntiHints.h --------------------------------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #ifndef LLVM_LIB_TARGET_AMDGPU_GCNPRERAANTIHINTS_H |
| 10 | #define LLVM_LIB_TARGET_AMDGPU_GCNPRERAANTIHINTS_H |
| 11 | |
| 12 | #include "llvm/ADT/SmallVector.h" |
| 13 | |
| 14 | namespace llvm { |
| 15 | |
| 16 | class MachineFunction; |
| 17 | class MachineInstr; |
| 18 | class MachineRegisterInfo; |
| 19 | class LiveIntervals; |
| 20 | class SIRegisterInfo; |
| 21 | class SIInstrInfo; |
| 22 | class GCNSubtarget; |
| 23 | class TargetSchedModel; |
| 24 | |
| 25 | namespace AMDGPU { |
| 26 | |
| 27 | using HazardClassMask = uint32_t; |
| 28 | namespace HazardClass { |
| 29 | |
| 30 | // Coherent with CoExec instruction types |
| 31 | enum : HazardClassMask { |
| 32 | None = 0, |
| 33 | CTRL = 1u << 0, |
| 34 | VALU = 1u << 1, |
| 35 | TRANS = 1u << 2, |
| 36 | SALU = 1u << 3, |
| 37 | DS = 1u << 4, |
| 38 | VMEM = 1u << 5, |
| 39 | SMEM = 1u << 6, |
| 40 | WMMA = 1u << 7, |
| 41 | MFMA = 1u << 8, |
| 42 | EXP = 1u << 9, |
| 43 | |
| 44 | Load = 1u << 10, |
| 45 | WritesVGPR = 1u << 11, |
| 46 | RawVALU = 1u << 12, |
| 47 | LDSDMA = 1u << 13, |
| 48 | }; |
| 49 | } // namespace HazardClass |
| 50 | |
| 51 | enum class HazardOperand : uint8_t { |
| 52 | None, |
| 53 | Def, |
| 54 | Src0, |
| 55 | Src1, |
| 56 | Src2, |
| 57 | Src0Src1, |
| 58 | Idx, |
| 59 | Vaddr, |
| 60 | AnySrc, |
| 61 | AnyUse, |
| 62 | }; |
| 63 | |
| 64 | enum class AntiHintDirection : uint8_t { |
| 65 | OneDirectional, |
| 66 | Symmetric, |
| 67 | }; |
| 68 | |
| 69 | enum class Lifetime : uint8_t { |
| 70 | // Wait-state budget. |
| 71 | WindowBudget, |
| 72 | // Most recent producer regs count cap. |
| 73 | RegisterCount, |
| 74 | }; |
| 75 | struct HazardContext { |
| 76 | const SIInstrInfo *TII; |
| 77 | const SIRegisterInfo *TRI; |
| 78 | MachineRegisterInfo *MRI; |
| 79 | LiveIntervals *LIS; |
| 80 | const GCNSubtarget *ST; |
| 81 | const TargetSchedModel *SchedModel; |
| 82 | }; |
| 83 | |
| 84 | struct ClassMatch { |
| 85 | HazardClassMask AnyOf = 0; |
| 86 | HazardClassMask AllOf = 0; |
| 87 | HazardClassMask NoneOf = 0; |
| 88 | bool matches(HazardClassMask Mask) const { |
| 89 | return (!AnyOf || (Mask & AnyOf)) && ((Mask & AllOf) == AllOf) && |
| 90 | !(Mask & NoneOf); |
| 91 | } |
| 92 | }; |
| 93 | |
| 94 | // To enable or disable the whole rule. |
| 95 | using RulePredicate = bool (*)(const HazardContext &); |
| 96 | |
| 97 | // To gate the rule for producer or consumer. |
| 98 | using InstPredicate = bool (*)(const MachineInstr &, const HazardContext &); |
| 99 | |
| 100 | // This helps advancing a window for the WAW/WAR case where a RAW sits between |
| 101 | // the producer and the consumer. |
| 102 | using AdvanceForRawWindowFn = unsigned (*)(const MachineInstr &Producer, |
| 103 | HazardClassMask ReaderClass, |
| 104 | const HazardContext &Ctx); |
| 105 | |
| 106 | struct HazardSide { |
| 107 | ClassMatch Match; |
| 108 | HazardOperand Op = HazardOperand::None; |
| 109 | InstPredicate Predicate = nullptr; |
| 110 | }; |
| 111 | |
| 112 | // Length of a hazard window in wait states. |
| 113 | // Precedence: Fn --> WindowLength. |
| 114 | struct WindowSpec { |
| 115 | unsigned WindowLength = 0; |
| 116 | unsigned (*Fn)(const MachineInstr &Producer, HazardClassMask ConsumerClass, |
| 117 | const HazardContext &Ctx) = nullptr; |
| 118 | }; |
| 119 | |
| 120 | struct ConsumerTarget { |
| 121 | HazardSide Side; |
| 122 | WindowSpec Window; |
| 123 | HazardClassMask CounterMask = HazardClass::None; |
| 124 | AntiHintDirection Direction = AntiHintDirection::OneDirectional; |
| 125 | }; |
| 126 | |
| 127 | struct HazardAntiHintRule { |
| 128 | RulePredicate Predicate = nullptr; |
| 129 | Lifetime Life = Lifetime::WindowBudget; |
| 130 | HazardSide Producer; |
| 131 | SmallVector<ConsumerTarget, 3> Consumers; |
| 132 | AdvanceForRawWindowFn AdvanceForRawWindow = nullptr; |
| 133 | }; |
| 134 | |
| 135 | void applyAntiHintRules(MachineFunction &MF, const HazardContext &Ctx); |
| 136 | |
| 137 | } // namespace AMDGPU |
| 138 | } // namespace llvm |
| 139 | |
| 140 | #endif // LLVM_LIB_TARGET_AMDGPU_GCNPRERAANTIHINTS_H |
| 141 | |