1//===- AMDGPUCoExecSchedStrategy.h - CoExec Scheduling Strategy -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Coexecution-focused scheduling strategy for AMDGPU.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUCOEXECSCHEDSTRATEGY_H
15#define LLVM_LIB_TARGET_AMDGPU_AMDGPUCOEXECSCHEDSTRATEGY_H
16
17#include "AMDGPUCoExecInfo.h"
18#include "GCNSchedStrategy.h"
19#include "llvm/CodeGen/MachineScheduler.h"
20
21namespace llvm {
22
23namespace AMDGPU {
24namespace DefaultBufferSizes {
25constexpr unsigned DS = 16;
26} // namespace DefaultBufferSizes
27
28/// AMDGPU-specific scheduling decision reasons. These provide more granularity
29/// than the generic CandReason enum for debugging purposes.
30enum class AMDGPUSchedReason : uint8_t {
31 None,
32 CritResourceBalance, // tryCriticalResource chose based on resource pressure
33 CritResourceDep, // tryCriticalResourceDependency chose based on enabling
34 NUM_REASONS
35};
36
37inline StringRef getReasonName(AMDGPUSchedReason R) {
38 switch (R) {
39 case AMDGPUSchedReason::None:
40 return "None";
41 case AMDGPUSchedReason::CritResourceBalance:
42 return "CritResource";
43 case AMDGPUSchedReason::CritResourceDep:
44 return "CritResourceDep";
45 case AMDGPUSchedReason::NUM_REASONS:
46 llvm_unreachable("Unknown AMDGPUSchedReason");
47 }
48 llvm_unreachable("Unknown AMDGPUSchedReason");
49}
50
51} // End namespace AMDGPU
52
53//===----------------------------------------------------------------------===//
54// Hardware Unit Information
55//===----------------------------------------------------------------------===//
56
57/// HardwareUnitInfo is a wrapper class which maps to some real hardware
58/// resource. This is used to model hardware resource pressure per region, and
59/// guide scheduling heuristics.
60class HardwareUnitInfo {
61private:
62 /// PrioritySUs maintains a list of the SUs we want to prioritize scheduling
63 /// for this HardwareUnit. This is used for agreement between
64 /// tryCriticalResourceDependency and tryCriticalResource: we schedule the
65 /// dependencies for a SU on critical resource, then schedule that same SU on
66 /// the critical resource. This agreement results in shorter live ranges and
67 /// more regular HardwareUnit access patterns. SUs are prioritized based on
68 /// depth for top-down scheduling.
69 SmallSetVector<SUnit *, 16> PrioritySUs;
70 /// All the SUs in the region that consume this resource.
71 SmallSetVector<SUnit *, 16> AllSUs;
72 /// All the SUs for this HardwareUnit that have already been scheduled.
73 SmallVector<SUnit *, 16> ScheduledSUs;
74 /// The total number of busy cycles for this HardwareUnit for a given region.
75 unsigned TotalCycles = 0;
76 /// InstructionFlavor mapping.
77 AMDGPU::InstructionFlavor Type;
78 /// Whether or not instructions on this HardwareUnit may produce a window in
79 /// which instructions in other HardwareUnits can coexecute. For example, WMMA
80 /// / MFMA instructions may take multiple cycles, which may be overlapped with
81 /// instructions on other HardwareUnits.
82 bool ProducesCoexecWindow = false;
83 /// How many instructions can be held simultaneously for this HardwareUnit.
84 /// A value of 0 means there is no limit. A value of 1 models an unbuffered
85 /// resource with a single in-flight instruction.
86 ///
87 /// This may approximate the hardware. For example, for LDS instructions
88 /// it is a well-known phenomena that oversubscribing the LDS unit results in
89 /// longer latency for the LDS instructions. While it is true that there is a
90 /// hard limit to the amount of simulatenous in-flight LDS instructions, good
91 /// scheduling would also cool off the LDS to avoid other forms of hardware
92 /// contention and increasing LDS latency. Thus, we limit the amount of LDS
93 /// instructions we are willing to schedule close together, though this does
94 /// not correspond 1:1 with a hardware mechanism.
95 unsigned BufferSize = 0;
96 /// How many cycles it takes for an instruction to clear the buffer.
97 ///
98 /// Again, this may be an apprxoimation. For example, for memory FIFOs, the
99 /// actual amount of cycles it will take to clear it is dependent on how
100 /// quickly prior instructions evacuate the FIFO, which is based on runtime
101 /// behavior which is not modelled in the compiler.
102 unsigned BufferCycles = 0;
103
104public:
105 HardwareUnitInfo() {}
106
107 unsigned size() { return AllSUs.size(); }
108
109 unsigned getTotalCycles() { return TotalCycles; }
110
111 void setType(unsigned TheType) {
112 assert(TheType < (unsigned)AMDGPU::InstructionFlavor::NUM_FLAVORS);
113 Type = (AMDGPU::InstructionFlavor)(TheType);
114 }
115
116 AMDGPU::InstructionFlavor getType() const { return Type; }
117
118 bool producesCoexecWindow() const { return ProducesCoexecWindow; }
119
120 void setProducesCoexecWindow(bool Val) { ProducesCoexecWindow = Val; }
121
122 bool contains(SUnit *SU) const { return AllSUs.contains(key: SU); }
123
124 void setBufferSize(unsigned Size) { BufferSize = Size; }
125
126 unsigned getBufferSize() { return BufferSize; }
127
128 /// \returns the next cycle where there is space in the buffer.
129 unsigned getBufferAvailableCycle(unsigned CurrCycle) {
130 // An unlimited buffer is always available.
131 if (BufferSize == 0)
132 return CurrCycle;
133
134 // Buffer is available now.
135 if (ScheduledSUs.size() < BufferSize)
136 return CurrCycle;
137
138 return BufferCycles +
139 ScheduledSUs[ScheduledSUs.size() - BufferSize]->TopReadyCycle;
140 }
141
142 /// \returns the SUnit with higher priority or nullptr if they are the same.
143 /// This method looks through the PrioritySUs to determine if one SU is more
144 /// prioritized than the other. If neither are in the PrioritySUs list, then
145 /// neither have priority over each other.
146 SUnit *getHigherPriority(SUnit *SU, SUnit *Other) const {
147 for (SUnit *SUOrder : PrioritySUs) {
148 if (SUOrder == SU)
149 return SU;
150
151 if (SUOrder == Other)
152 return Other;
153 }
154 return nullptr;
155 }
156
157 void reset() {
158 AllSUs.clear();
159 PrioritySUs.clear();
160 ScheduledSUs.clear();
161 TotalCycles = 0;
162 Type = AMDGPU::InstructionFlavor::Other;
163 ProducesCoexecWindow = false;
164 BufferSize = 0;
165 BufferCycles = 0;
166 }
167
168 /// \returns the next SU in PrioritySUs that is not ready. If \p LookDeep is
169 /// set, we will look beyond the PrioritySUs (if all the PrioritySUs are
170 /// ready) to AllSUs to attempt to find a target SU. When looking through
171 /// AllSUs we sort pick the target SU by minimal depth for top-down
172 /// scheduling. getNextTargetSU is useful for determining which SU on this
173 /// HardwareUnit we are trying to schedule - this info helps us determine
174 /// which dependencies to schedule. LookDeep is useful if the dependencies are
175 /// long latency (e.g. memory instructions). If we have many long latency
176 /// dependencies, it is beneficial to enable SUs multiple levels ahead.
177 SUnit *getNextTargetSU(bool LookDeep = false) const;
178 /// Insert the \p SU into AllSUs and account its \p BlockingCycles into
179 /// the TotalCycles. This maintains the list of PrioritySUs.
180 void insert(SUnit *SU, unsigned BlockingCycles);
181 /// Update the state for \p SU being scheduled by removing it from the AllSUs
182 /// and reducing its \p BlockingCycles from the TotalCycles. This maintains
183 /// the list of PrioritySUs.
184 void markScheduled(SUnit *SU, unsigned BlockingCycles);
185 /// After we've collected all the region pressure for this HWUI, correct for
186 /// any specifics of the behavior of this resource. For example, if the
187 /// HardwareUnit can hold N instructions simultaneously, then there is no
188 /// penalty for scheduling N instructions back to back.
189 void finalizeCycles();
190};
191
192//===----------------------------------------------------------------------===//
193// Candidate Heuristics
194//===----------------------------------------------------------------------===//
195
196/// CandidateHeuristics contains state and implementations to facilitate making
197/// per instruction scheduling decisions; it contains methods used in
198/// tryCandidate to decide which instruction to schedule next.
199class CandidateHeuristics {
200protected:
201 ScheduleDAGMI *DAG;
202 const SIInstrInfo *SII;
203 const SIRegisterInfo *SRI;
204 const TargetSchedModel *SchedModel;
205 SmallVector<HardwareUnitInfo, 8> HWUInfo;
206
207 /// Walk over the region and collect total usage per HardwareUnit.
208 void collectHWUIPressure();
209
210 /// Compute the blocking cycles for the appropriate HardwareUnit given an \p
211 /// SU.
212 unsigned getHWUICyclesForInst(SUnit *SU);
213
214public:
215 CandidateHeuristics() = default;
216
217 void initialize(ScheduleDAGMI *DAG, const TargetSchedModel *SchedModel,
218 const TargetRegisterInfo *TRI);
219
220 /// Update the state to reflect that \p SU is going to be scheduled.
221 void updateForScheduling(SUnit *SU);
222
223 /// Given a \p Flavor , find the corresponding HardwareUnit. \returns the
224 /// mapped HardwareUnit.
225 HardwareUnitInfo *getHWUIFromFlavor(AMDGPU::InstructionFlavor Flavor);
226
227 /// Sort the HardwarUnitInfo vector. After sorting, the HWUI that are highest
228 /// priority are first. Priority is determined by maximizing coexecution and
229 /// keeping the critical HardwareUnit busy.
230 void sortHWUIResources();
231
232 /// Check for critical resource consumption. Prefer the candidate that uses
233 /// the most prioritized HardwareUnit. If both candidates use the same
234 /// HarwareUnit, prefer the candidate with higher priority on that
235 /// HardwareUnit.
236 bool tryCriticalResource(GenericSchedulerBase::SchedCandidate &TryCand,
237 GenericSchedulerBase::SchedCandidate &Cand,
238 SchedBoundary *Zone) const;
239
240 /// Check for dependencies of instructions that use prioritized HardwareUnits.
241 /// Prefer the candidate that is a dependency of an instruction that uses the
242 /// most prioritized HardwareUnit. If both candidates enable the same
243 /// HardwareUnit, prefer the candidate that enables the higher priority
244 /// instruction on that HardwareUnit.
245 bool
246 tryCriticalResourceDependency(GenericSchedulerBase::SchedCandidate &TryCand,
247 GenericSchedulerBase::SchedCandidate &Cand,
248 SchedBoundary *Zone) const;
249
250 void dumpRegionSummary();
251};
252
253class AMDGPUCoExecSchedStrategy final : public GCNSchedStrategy {
254protected:
255 bool tryEffectiveStall(SchedCandidate &Cand, SchedCandidate &TryCand,
256 SchedBoundary &Zone);
257 AMDGPU::AMDGPUSchedReason LastAMDGPUReason = AMDGPU::AMDGPUSchedReason::None;
258 CandidateHeuristics Heurs;
259
260#ifndef NDEBUG
261 void dumpPickSummary(SUnit *SU, bool IsTopNode, SchedCandidate &Cand);
262#endif
263
264 bool tryCandidateCoexec(SchedCandidate &Cand, SchedCandidate &TryCand,
265 SchedBoundary *Zone);
266 void pickNodeFromQueue(SchedBoundary &Zone, const CandPolicy &ZonePolicy,
267 const RegPressureTracker &RPTracker,
268 SchedCandidate &Cand, bool &PickedPending,
269 bool IsBottomUp);
270
271public:
272 AMDGPUCoExecSchedStrategy(const MachineSchedContext *C);
273
274 void initPolicy(MachineBasicBlock::iterator Begin,
275 MachineBasicBlock::iterator End,
276 unsigned NumRegionInstrs) override;
277 void initialize(ScheduleDAGMI *DAG) override;
278 SUnit *pickNode(bool &IsTopNode) override;
279 void schedNode(SUnit *SU, bool IsTopNode) override;
280};
281
282ScheduleDAGInstrs *createGCNCoExecMachineScheduler(MachineSchedContext *C);
283ScheduleDAGInstrs *createGCNNoopPostMachineScheduler(MachineSchedContext *C);
284
285} // End namespace llvm
286
287#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUCOEXECSCHEDSTRATEGY_H
288