1//===- AMDGPUCoExecSchedStrategy.cpp - CoExec Scheduling Strategy ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Coexecution-focused scheduling strategy for AMDGPU.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUCoExecSchedStrategy.h"
15#include "AMDGPUIGroupLP.h"
16#include "GCNHazardRecognizer.h"
17#include "llvm/Support/Debug.h"
18
19using namespace llvm;
20using namespace llvm::AMDGPU;
21
22#define DEBUG_TYPE "machine-scheduler"
23namespace {
24enum class CarriedLatency { Off, Fence, All };
25} // namespace
26
27static cl::opt<CarriedLatency> BlockCarriedLatency(
28 "amdgpu-block-carried-latency", cl::Hidden, cl::init(Val: CarriedLatency::Off),
29 cl::desc("Estimate block-carried latency and include it in the effective "
30 "candidate stall cost."),
31 cl::values(
32 clEnumValN(CarriedLatency::Off, "off",
33 "Disabled - do not pad latency."),
34 clEnumValN(CarriedLatency::Fence, "fence",
35 "Only pad latency for memory fence (e.g. those surrounding "
36 "barrier_signal/wait)."),
37 clEnumValN(
38 CarriedLatency::All, "all",
39 "Pad latency for any SU with an incoming ds_load dependency.")));
40
41namespace {
42
43// Used to disable post-RA scheduling with function level granularity.
44class GCNNoopPostScheduleDAG final : public ScheduleDAGInstrs {
45public:
46 explicit GCNNoopPostScheduleDAG(MachineSchedContext *C)
47 : ScheduleDAGInstrs(*C->MF, C->MLI, /*RemoveKillFlags=*/true) {}
48
49 // Do nothing.
50 void schedule() override {}
51};
52
53} // namespace
54
55static SUnit *pickOnlyChoice(SchedBoundary &Zone) {
56 // pickOnlyChoice() releases pending instructions and checks for new hazards.
57 SUnit *OnlyChoice = Zone.pickOnlyChoice();
58 if (!Zone.Pending.empty())
59 return nullptr;
60
61 return OnlyChoice;
62}
63
64/// Apply \p ExtraBits to every slot in \p Info starting with \p StartIndex
65/// Used by MFMA co-exec rules, because MFMA co-exec slots are incremental, i.e.
66/// for every slot N it supports all instructions which were supported by the
67/// previous slot N-1 and may support something extra.
68static void allowCoExec(llvm::AMDGPU::CoExecInfo &Info,
69 llvm::AMDGPU::CoExecMaskT ExtraBits,
70 unsigned StartIndex) {
71 for (unsigned Index = StartIndex; Index < Info.TotalWindow; ++Index)
72 Info.Slots[Index].Mask |= ExtraBits;
73}
74
75/// Get co-execution info for a gfx950 MFMA instruction.
76/// The occupancy (cycles until the next MFMA may issue) is expressed as the
77/// first stage carrying the WMMA bit.
78llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
79 using namespace llvm;
80 using namespace llvm::AMDGPU;
81 CoExecInfo Res;
82 for (unsigned I = 0; I < MaxCoExecStages; ++I)
83 Res.Slots[I].Mask = CoExecMask::None;
84
85 // TODO: Implement proper patterns support (for debugging purposes).
86 // Existing pattern letters are WMMA-specific and will probably be confusing
87 // if used as-is for MFMA. Inventing new MFMA-specific letters is an option,
88 // but perhaps the pattern should be instead dynamically reconstructed when
89 // needed by printing specific slots in full instead of a key for them.
90 Res.Pattern = "undefinedundefinedundefinedundefined";
91
92 switch (Opcode) {
93 // 4-cycle occupancy, 8-cycle window.
94 case V_MFMA_F32_16X16X128_F8F6F4_f4_f4_e64:
95 case V_MFMA_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64:
96 case V_MFMA_F32_16X16X128_F8F6F4_f4_f4_gfx940_acd:
97 case V_MFMA_F32_16X16X128_F8F6F4_f4_f4_gfx940_vcd:
98 case V_MFMA_F32_16X16X128_F8F6F4_f4_f6_e64:
99 case V_MFMA_F32_16X16X128_F8F6F4_f4_f6_vgprcd_e64:
100 case V_MFMA_F32_16X16X128_F8F6F4_f4_f6_gfx940_acd:
101 case V_MFMA_F32_16X16X128_F8F6F4_f4_f6_gfx940_vcd:
102 case V_MFMA_F32_16X16X128_F8F6F4_f6_f4_e64:
103 case V_MFMA_F32_16X16X128_F8F6F4_f6_f4_vgprcd_e64:
104 case V_MFMA_F32_16X16X128_F8F6F4_f6_f4_gfx940_acd:
105 case V_MFMA_F32_16X16X128_F8F6F4_f6_f4_gfx940_vcd:
106 case V_MFMA_F32_16X16X128_F8F6F4_f6_f6_e64:
107 case V_MFMA_F32_16X16X128_F8F6F4_f6_f6_vgprcd_e64:
108 case V_MFMA_F32_16X16X128_F8F6F4_f6_f6_gfx940_acd:
109 case V_MFMA_F32_16X16X128_F8F6F4_f6_f6_gfx940_vcd:
110 case V_MFMA_F32_16X16X32_BF16_e64:
111 case V_MFMA_F32_16X16X32_BF16_vgprcd_e64:
112 case V_MFMA_F32_16X16X32_BF16_gfx940_acd:
113 case V_MFMA_F32_16X16X32_BF16_gfx940_vcd:
114 case V_MFMA_I32_16X16X64_I8_e64:
115 case V_MFMA_I32_16X16X64_I8_vgprcd_e64:
116 case V_MFMA_I32_16X16X64_I8_gfx940_acd:
117 case V_MFMA_I32_16X16X64_I8_gfx940_vcd:
118 case V_MFMA_F32_16X16X32_F16_e64:
119 case V_MFMA_F32_16X16X32_F16_vgprcd_e64:
120 case V_MFMA_F32_16X16X32_F16_gfx940_acd:
121 case V_MFMA_F32_16X16X32_F16_gfx940_vcd:
122 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64:
123 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64:
124 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_gfx940_acd:
125 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_gfx940_vcd:
126 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6_e64:
127 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6_vgprcd_e64:
128 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6_gfx940_acd:
129 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6_gfx940_vcd:
130 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4_e64:
131 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4_vgprcd_e64:
132 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4_gfx940_acd:
133 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4_gfx940_vcd:
134 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_e64:
135 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_vgprcd_e64:
136 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_gfx940_acd:
137 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_gfx940_vcd:
138 // GFX9 Shader Programming Guide lists those SMFMAC separately, but for
139 // intended purposes here all those instructions are the same. This comment
140 // is to simplify reverse mapping to the SPG.
141 case V_SMFMAC_F32_16X16X64_BF16_e64:
142 case V_SMFMAC_F32_16X16X64_BF16_gfx940:
143 case V_SMFMAC_I32_16X16X128_I8_e64:
144 case V_SMFMAC_I32_16X16X128_I8_gfx940:
145 case V_SMFMAC_F32_16X16X128_BF8_BF8_e64:
146 case V_SMFMAC_F32_16X16X128_BF8_BF8_gfx940:
147 case V_SMFMAC_F32_16X16X128_BF8_FP8_e64:
148 case V_SMFMAC_F32_16X16X128_BF8_FP8_gfx940:
149 case V_SMFMAC_F32_16X16X128_FP8_BF8_e64:
150 case V_SMFMAC_F32_16X16X128_FP8_BF8_gfx940:
151 case V_SMFMAC_F32_16X16X128_FP8_FP8_e64:
152 case V_SMFMAC_F32_16X16X128_FP8_FP8_gfx940:
153 case V_SMFMAC_F32_16X16X64_F16_e64:
154 case V_SMFMAC_F32_16X16X64_F16_gfx940:
155 Res.TotalWindow = 8;
156 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
157 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, StartIndex: 2);
158 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 4);
159 return Res;
160
161 // 8-cycle occupancy, 12-cycle window.
162 case V_MFMA_F32_16X16X128_F8F6F4_f4_f8_e64:
163 case V_MFMA_F32_16X16X128_F8F6F4_f4_f8_vgprcd_e64:
164 case V_MFMA_F32_16X16X128_F8F6F4_f4_f8_gfx940_acd:
165 case V_MFMA_F32_16X16X128_F8F6F4_f4_f8_gfx940_vcd:
166 case V_MFMA_F32_16X16X128_F8F6F4_f6_f8_e64:
167 case V_MFMA_F32_16X16X128_F8F6F4_f6_f8_vgprcd_e64:
168 case V_MFMA_F32_16X16X128_F8F6F4_f6_f8_gfx940_acd:
169 case V_MFMA_F32_16X16X128_F8F6F4_f6_f8_gfx940_vcd:
170 case V_MFMA_F32_16X16X128_F8F6F4_f8_f4_e64:
171 case V_MFMA_F32_16X16X128_F8F6F4_f8_f4_vgprcd_e64:
172 case V_MFMA_F32_16X16X128_F8F6F4_f8_f4_gfx940_acd:
173 case V_MFMA_F32_16X16X128_F8F6F4_f8_f4_gfx940_vcd:
174 case V_MFMA_F32_16X16X128_F8F6F4_f8_f6_e64:
175 case V_MFMA_F32_16X16X128_F8F6F4_f8_f6_vgprcd_e64:
176 case V_MFMA_F32_16X16X128_F8F6F4_f8_f6_gfx940_acd:
177 case V_MFMA_F32_16X16X128_F8F6F4_f8_f6_gfx940_vcd:
178 case V_MFMA_F32_16X16X128_F8F6F4_f8_f8_e64:
179 case V_MFMA_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64:
180 case V_MFMA_F32_16X16X128_F8F6F4_f8_f8_gfx940_acd:
181 case V_MFMA_F32_16X16X128_F8F6F4_f8_f8_gfx940_vcd:
182 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f8_e64:
183 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f8_vgprcd_e64:
184 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f8_gfx940_acd:
185 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f8_gfx940_vcd:
186 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f8_e64:
187 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f8_vgprcd_e64:
188 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f8_gfx940_acd:
189 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f8_gfx940_vcd:
190 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4_e64:
191 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4_vgprcd_e64:
192 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4_gfx940_acd:
193 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4_gfx940_vcd:
194 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6_e64:
195 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6_vgprcd_e64:
196 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6_gfx940_acd:
197 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6_gfx940_vcd:
198 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_e64:
199 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64:
200 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_gfx940_acd:
201 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_gfx940_vcd:
202 Res.TotalWindow = 12;
203 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
204 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VMEM, StartIndex: 2);
205 allowCoExec(Info&: Res, ExtraBits: CoExecMask::VALU, StartIndex: 3);
206 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 8);
207 return Res;
208
209 // 4-cycle occupancy, 8-cycle window.
210 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_e64:
211 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_mac_e64:
212 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_mac_vgprcd_e64:
213 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_vgprcd_e64:
214 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_gfx940_acd:
215 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_gfx940_vcd:
216 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_e64:
217 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_mac_e64:
218 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_mac_vgprcd_e64:
219 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_vgprcd_e64:
220 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_gfx940_acd:
221 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_gfx940_vcd:
222 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_e64:
223 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_mac_e64:
224 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_mac_vgprcd_e64:
225 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_vgprcd_e64:
226 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_gfx940_acd:
227 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_gfx940_vcd:
228 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_e64:
229 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_mac_e64:
230 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_mac_vgprcd_e64:
231 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_vgprcd_e64:
232 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_gfx940_acd:
233 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_gfx940_vcd:
234 case V_MFMA_F32_32X32X16_BF16_e64:
235 case V_MFMA_F32_32X32X16_BF16_mac_e64:
236 case V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64:
237 case V_MFMA_F32_32X32X16_BF16_vgprcd_e64:
238 case V_MFMA_F32_32X32X16_BF16_gfx940_acd:
239 case V_MFMA_F32_32X32X16_BF16_gfx940_vcd:
240 case V_MFMA_I32_32X32X32_I8_e64:
241 case V_MFMA_I32_32X32X32_I8_mac_e64:
242 case V_MFMA_I32_32X32X32_I8_mac_vgprcd_e64:
243 case V_MFMA_I32_32X32X32_I8_vgprcd_e64:
244 case V_MFMA_I32_32X32X32_I8_gfx940_acd:
245 case V_MFMA_I32_32X32X32_I8_gfx940_vcd:
246 case V_MFMA_F32_32X32X16_F16_e64:
247 case V_MFMA_F32_32X32X16_F16_mac_e64:
248 case V_MFMA_F32_32X32X16_F16_mac_vgprcd_e64:
249 case V_MFMA_F32_32X32X16_F16_vgprcd_e64:
250 case V_MFMA_F32_32X32X16_F16_gfx940_acd:
251 case V_MFMA_F32_32X32X16_F16_gfx940_vcd:
252 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_gfx940_acd:
253 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_gfx940_vcd:
254 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_gfx940_acd:
255 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_gfx940_vcd:
256 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_gfx940_acd:
257 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_gfx940_vcd:
258 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_gfx940_acd:
259 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_gfx940_vcd:
260 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_e64:
261 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_e64:
262 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_e64:
263 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_e64:
264 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_vgprcd_e64:
265 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_vgprcd_e64:
266 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_vgprcd_e64:
267 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_vgprcd_e64:
268 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_mac_e64:
269 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_mac_e64:
270 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_mac_e64:
271 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_mac_e64:
272 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_mac_vgprcd_e64:
273 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_mac_vgprcd_e64:
274 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_mac_vgprcd_e64:
275 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_mac_vgprcd_e64:
276 Res.TotalWindow = 8;
277 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
278 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, StartIndex: 2);
279 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 4);
280 return Res;
281
282 // 16-cycle occupancy, 20-cycle window.
283 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_e64:
284 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_mac_e64:
285 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_mac_vgprcd_e64:
286 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_vgprcd_e64:
287 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_gfx940_acd:
288 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_gfx940_vcd:
289 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_e64:
290 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_mac_e64:
291 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_mac_vgprcd_e64:
292 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_vgprcd_e64:
293 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_gfx940_acd:
294 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_gfx940_vcd:
295 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_e64:
296 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_mac_e64:
297 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_mac_vgprcd_e64:
298 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_vgprcd_e64:
299 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_gfx940_acd:
300 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_gfx940_vcd:
301 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_e64:
302 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_mac_e64:
303 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_mac_vgprcd_e64:
304 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_vgprcd_e64:
305 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_gfx940_acd:
306 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_gfx940_vcd:
307 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_e64:
308 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_e64:
309 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64:
310 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_vgprcd_e64:
311 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_gfx940_acd:
312 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_gfx940_vcd:
313 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_e64:
314 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_e64:
315 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_e64:
316 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_e64:
317 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_e64:
318 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_vgprcd_e64:
319 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_vgprcd_e64:
320 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_vgprcd_e64:
321 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_vgprcd_e64:
322 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_vgprcd_e64:
323 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_mac_e64:
324 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_mac_e64:
325 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_mac_e64:
326 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_mac_e64:
327 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_mac_e64:
328 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_mac_vgprcd_e64:
329 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_mac_vgprcd_e64:
330 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_mac_vgprcd_e64:
331 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_mac_vgprcd_e64:
332 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64:
333 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_gfx940_acd:
334 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_gfx940_vcd:
335 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_gfx940_acd:
336 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_gfx940_vcd:
337 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_gfx940_acd:
338 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_gfx940_vcd:
339 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_gfx940_acd:
340 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_gfx940_vcd:
341 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_gfx940_acd:
342 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_gfx940_vcd:
343 Res.TotalWindow = 20;
344 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
345 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VMEM, StartIndex: 2);
346 allowCoExec(Info&: Res, ExtraBits: CoExecMask::VALU, StartIndex: 3);
347 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 16);
348 return Res;
349
350 // 9-cycle occupancy, 12-cycle window.
351 case V_SMFMAC_F32_32X32X32_BF16_e64:
352 case V_SMFMAC_F32_32X32X32_BF16_gfx940:
353 case V_SMFMAC_I32_32X32X64_I8_e64:
354 case V_SMFMAC_I32_32X32X64_I8_gfx940:
355 case V_SMFMAC_F32_32X32X64_BF8_BF8_e64:
356 case V_SMFMAC_F32_32X32X64_BF8_BF8_gfx940:
357 case V_SMFMAC_F32_32X32X64_BF8_FP8_e64:
358 case V_SMFMAC_F32_32X32X64_BF8_FP8_gfx940:
359 case V_SMFMAC_F32_32X32X64_FP8_BF8_e64:
360 case V_SMFMAC_F32_32X32X64_FP8_BF8_gfx940:
361 case V_SMFMAC_F32_32X32X64_FP8_FP8_e64:
362 case V_SMFMAC_F32_32X32X64_FP8_FP8_gfx940:
363 case V_SMFMAC_F32_32X32X32_F16_e64:
364 case V_SMFMAC_F32_32X32X32_F16_gfx940:
365 Res.TotalWindow = 12;
366 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
367 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, StartIndex: 4);
368 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 9);
369 return Res;
370
371 // 18-cycle occupancy, 19-cycle window.
372 case V_MFMA_F64_16X16X4F64_e64:
373 case V_MFMA_F64_16X16X4F64_mac_e64:
374 case V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
375 case V_MFMA_F64_16X16X4F64_vgprcd_e64:
376 Res.TotalWindow = 19;
377 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::SALU | CoExecMask::VMEM, StartIndex: 0);
378 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA | CoExecMask::VALU, StartIndex: 18);
379 return Res;
380
381 default:
382 // Default fallback: permissive 8-cycle pattern
383 return CoExecInfo::build(UnitOccupancy: 0, TotalWindow: 9, Pattern: "AAAAAAAAA");
384 }
385}
386
387InstructionFlavor llvm::AMDGPU::classifyFlavor(const MachineInstr &MI,
388 const SIInstrInfo &SII) {
389 if (MI.isDebugInstr())
390 return InstructionFlavor::Other;
391
392 unsigned Opc = MI.getOpcode();
393
394 // Check for specific opcodes first.
395 if (Opc == AMDGPU::ATOMIC_FENCE || Opc == AMDGPU::S_WAIT_ASYNCCNT ||
396 Opc == AMDGPU::S_WAIT_TENSORCNT || Opc == AMDGPU::S_BARRIER_WAIT ||
397 Opc == AMDGPU::S_BARRIER_SIGNAL_IMM)
398 return InstructionFlavor::Fence;
399
400 if (SII.isLDSDMA(MI))
401 return InstructionFlavor::DMA;
402
403 if (SII.isMFMA(MI)) {
404 // TODO: Consider further sub-classifying this (XDL, XDL2x, S/DGEMM).
405 // GFX9 SPG sub-classifies MFMA into XDL, XDL2x and S/DGEMM, because only
406 // certain sub-classes can be co-executed in certain slots. For now, we
407 // simply treat them all as one to simplify the change and leave the rest
408 // to a follow-up fine-tuning.
409 return InstructionFlavor::WMMA;
410 }
411
412 if (SII.isWMMA(MI) || SII.isSWMMAC(MI))
413 return InstructionFlavor::WMMA;
414
415 if (SII.isTRANS(MI))
416 return InstructionFlavor::TRANS;
417
418 if (SII.isVALU(MI, /*AllowLDSDMA=*/false)) {
419 if (SII.getBlockingCycles(MI) > 1)
420 return InstructionFlavor::MultiCycleVALU;
421
422 return InstructionFlavor::SingleCycleVALU;
423 }
424
425 if (SII.isSMRD(MI))
426 return InstructionFlavor::SMEM;
427
428 if (SII.isDS(MI))
429 return InstructionFlavor::DS;
430
431 if (SII.isVMEM(MI))
432 return InstructionFlavor::VMEM;
433
434 if (SII.isSALU(MI))
435 return InstructionFlavor::SALU;
436
437 return InstructionFlavor::Other;
438}
439
440SUnit *HardwareUnitInfo::getNextTargetSU(bool LookDeep) const {
441 for (SUnit *PrioritySU : PrioritySUs) {
442 if (!PrioritySU->isTopReady())
443 return PrioritySU;
444 }
445
446 if (!LookDeep)
447 return nullptr;
448
449 unsigned MinDepth = std::numeric_limits<unsigned int>::max();
450 SUnit *TargetSU = nullptr;
451 for (auto *SU : AllSUs) {
452 if (SU->isScheduled)
453 continue;
454
455 if (SU->isTopReady())
456 continue;
457
458 if (SU->getDepth() < MinDepth) {
459 MinDepth = SU->getDepth();
460 TargetSU = SU;
461 }
462 }
463 return TargetSU;
464}
465
466void HardwareUnitInfo::insert(SUnit *SU, unsigned BlockingCycles) {
467 if (!AllSUs.insert(X: SU))
468 llvm_unreachable("HardwareUnit already contains SU!");
469
470 TotalCycles += BlockingCycles;
471
472 if (PrioritySUs.empty()) {
473 PrioritySUs.insert(X: SU);
474 return;
475 }
476 unsigned SUDepth = SU->getDepth();
477 unsigned CurrDepth = (*PrioritySUs.begin())->getDepth();
478 if (SUDepth > CurrDepth)
479 return;
480
481 if (SUDepth == CurrDepth) {
482 PrioritySUs.insert(X: SU);
483 return;
484 }
485
486 // SU is lower depth and should be prioritized.
487 PrioritySUs.clear();
488 PrioritySUs.insert(X: SU);
489}
490
491void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles) {
492 // We may want to ignore some HWUIs (e.g. InstructionFlavor::Other). To do so,
493 // we just clear the HWUI. However, we still have instructions which map to
494 // this HWUI. Don't bother managing the state for these HWUI.
495 if (TotalCycles == 0)
496 return;
497
498 ScheduledSUs.push_back(Elt: SU);
499 AllSUs.remove(X: SU);
500 PrioritySUs.remove(X: SU);
501
502 // BufferSize 0 is unlimited, while size 1 has no parallel buffering. In
503 // either case, each SU uses the HardwareUnit for BlockingCycles.
504 if (BufferSize <= 1 || (ScheduledSUs.size() % BufferSize == 0))
505 TotalCycles -= std::min(a: TotalCycles, b: BlockingCycles);
506
507 if (AllSUs.empty())
508 return;
509 if (PrioritySUs.empty()) {
510 for (auto SU : AllSUs) {
511 if (PrioritySUs.empty()) {
512 PrioritySUs.insert(X: SU);
513 continue;
514 }
515 unsigned SUDepth = SU->getDepth();
516 unsigned CurrDepth = (*PrioritySUs.begin())->getDepth();
517 if (SUDepth > CurrDepth)
518 continue;
519
520 if (SUDepth == CurrDepth) {
521 PrioritySUs.insert(X: SU);
522 continue;
523 }
524
525 // SU is lower depth and should be prioritized.
526 PrioritySUs.clear();
527 PrioritySUs.insert(X: SU);
528 }
529 }
530}
531
532void HardwareUnitInfo::finalizeCycles() {
533 if (BufferSize == 0 || AllSUs.empty())
534 return;
535
536 // We estimate the amount of cycles it takes to free up a slot in the buffer
537 // as the average cycles per SU.
538 BufferCycles = TotalCycles / AllSUs.size();
539 // A single-entry buffer does not reduce TotalCycles.
540 if (BufferSize == 1)
541 return;
542
543 // The TotalCycles is normalized against the BufferSize.
544 // This provides an estimate of the TotalCycles which is not always accurate
545 // -- particularly in cases where we have fewer instructions than the
546 // BufferSize. For example, if we have 2 instructions which each take 50
547 // cycles and a BufferSize of 16, then a TotalCycles of 51 cycles would be
548 // somewhat accurate. This normalization calculates TotalCycles as 6. However,
549 // if we have 64 of these instructions, our normalized estimate of 200 is more
550 // reasonable, given the more accurate measure is 264. Having a completely
551 // accurate measure is not very important, since this metric is mainly used to
552 // compare the relative demand per HardwareUnit across the region. The simpler
553 // estimate makes managing the metric incrementally during scheduling much
554 // simpler.
555 TotalCycles /= BufferSize;
556}
557
558HardwareUnitInfo *
559CandidateHeuristics::getHWUIFromFlavor(InstructionFlavor Flavor) {
560 for (HardwareUnitInfo &HWUICand : HWUInfo) {
561 if (HWUICand.getType() == Flavor) {
562 return &HWUICand;
563 }
564 }
565 return nullptr;
566}
567
568unsigned CandidateHeuristics::getMaxBlockingCycles(const MCSchedClassDesc *SC,
569 const MachineInstr *MI) {
570 // Loads and stores are not pipelined.
571 if (MI->mayLoadOrStore())
572 return SchedModel->computeInstrLatency(MI, UseDefaultDefLatency: false);
573
574 unsigned ReleaseAtCycle = 0;
575 for (TargetSchedModel::ProcResIter PI = SchedModel->getWriteProcResBegin(SC),
576 PE = SchedModel->getWriteProcResEnd(SC);
577 PI != PE; ++PI) {
578 ReleaseAtCycle =
579 std::max(a: ReleaseAtCycle, b: static_cast<unsigned>(PI->ReleaseAtCycle));
580 }
581 ReleaseAtCycle = std::max(a: ReleaseAtCycle, b: SII->getBlockingCycles(MI: *MI));
582 return ReleaseAtCycle;
583}
584
585unsigned CandidateHeuristics::getHWUICyclesForSU(SUnit *SU) {
586 assert(SchedModel && SchedModel->hasInstrSchedModel());
587 MachineInstr *MI = SU->getInstr();
588 if (SII->isDS(MI: *MI))
589 return SchedModel->computeInstrLatency(MI);
590 return getMaxBlockingCycles(SC: DAG->getSchedClass(SU), MI);
591}
592
593unsigned CandidateHeuristics::getHWUICyclesForMI(MachineInstr *MI) {
594 assert(SchedModel && SchedModel->hasInstrSchedModel());
595 return getMaxBlockingCycles(SC: SchedModel->resolveSchedClass(MI), MI);
596}
597
598void CandidateHeuristics::updateForScheduling(SUnit *SU) {
599 HardwareUnitInfo *HWUI =
600 getHWUIFromFlavor(Flavor: classifyFlavor(MI: *SU->getInstr(), SII: *SII));
601 assert(HWUI);
602 HWUI->markScheduled(SU, BlockingCycles: getHWUICyclesForSU(SU));
603}
604
605void CandidateHeuristics::initialize(ScheduleDAGMI *SchedDAG,
606 const TargetSchedModel *TargetSchedModel,
607 const TargetRegisterInfo *TRI) {
608 DAG = SchedDAG;
609 SchedModel = TargetSchedModel;
610 assert(SchedModel && SchedModel->hasInstrSchedModel());
611
612 SRI = static_cast<const SIRegisterInfo *>(TRI);
613 SII = static_cast<const SIInstrInfo *>(DAG->TII);
614
615 HWUInfo.resize(N: static_cast<int>(InstructionFlavor::NUM_FLAVORS));
616
617 for (unsigned I = 0; I < HWUInfo.size(); I++) {
618 HWUInfo[I].reset();
619 HWUInfo[I].setType(I);
620 }
621
622 HWUInfo[static_cast<int>(InstructionFlavor::WMMA)].setProducesCoexecWindow(
623 true);
624 HWUInfo[static_cast<int>(InstructionFlavor::MultiCycleVALU)]
625 .setProducesCoexecWindow(true);
626 HWUInfo[static_cast<int>(InstructionFlavor::TRANS)].setProducesCoexecWindow(
627 true);
628 HWUInfo[static_cast<int>(InstructionFlavor::DS)].setBufferSize(
629 DefaultBufferSizes::DS);
630
631 collectRegionSummary();
632}
633
634unsigned CandidateHeuristics::getCarriedLatency(SUnit *SU) {
635 if (BlockCarriedLatency == CarriedLatency::Off)
636 return 0;
637
638 MachineInstr *MI = SU->getInstr();
639 unsigned CarriedLatency = 0;
640 const InstructionFlavor Flavor = classifyFlavor(MI: *MI, SII: *SII);
641 if (Flavor == InstructionFlavor::Fence) {
642 MachineBasicBlock *MBB = MI->getParent();
643 // Scan each direct predecessor back to its nearest Fence or block start for
644 // DS instructions.
645 for (auto PredMBB : MBB->predecessors()) {
646 auto I = PredMBB->rbegin();
647 auto E = PredMBB->rend();
648 for (; I != E; I++) {
649 const InstructionFlavor ItFlavor = classifyFlavor(MI: *I, SII: *SII);
650 if (ItFlavor == InstructionFlavor::Fence)
651 break;
652
653 // Found carried latency.
654 if (ItFlavor == InstructionFlavor::DS)
655 CarriedLatency = std::max(a: CarriedLatency, b: getHWUICyclesForMI(MI: &*I));
656 }
657 }
658 }
659
660 if (BlockCarriedLatency == CarriedLatency::Fence)
661 return CarriedLatency;
662
663 for (MachineOperand &Op : MI->all_uses()) {
664 auto Reg = Op.getReg();
665 if (!Reg.isVirtual())
666 continue;
667
668 for (MachineInstr &Def : DAG->MRI.def_instructions(Reg)) {
669 // We don't have the proper modelling to accurately measure all carried
670 // latency. Just try to measure carried latency for long latency loads to
671 // avoid long stalls.
672 if (!Def.mayLoad())
673 continue;
674
675 unsigned Latency = getHWUICyclesForMI(MI: &Def);
676
677 // Load is carried across block.
678 if (Def.getParent() != MI->getParent()) {
679 bool FoundUseInDefBlock = false;
680 for (MachineInstr &Use : DAG->MRI.use_nodbg_instructions(Reg)) {
681 if (Use.getParent() != Def.getParent())
682 continue;
683
684 SlotIndex DefIdx = DAG->getLIS()->getInstructionIndex(Instr: Def);
685 SlotIndex UseIdx = DAG->getLIS()->getInstructionIndex(Instr: Use);
686 // We have a use of this load in the def block that occurs after the
687 // load. In this case we must wait for the load in the def block, and
688 // we do not have any carried latency from this load.
689 if (SlotIndex::isEarlierInstr(A: DefIdx, B: UseIdx)) {
690 FoundUseInDefBlock = true;
691 break;
692 }
693 }
694 if (!FoundUseInDefBlock)
695 CarriedLatency = std::max(a: Latency, b: CarriedLatency);
696
697 continue;
698 }
699
700 assert(Def.getParent() == MI->getParent());
701 // Load is in the same block.
702 SlotIndex LoadIdx = DAG->getLIS()->getInstructionIndex(Instr: Def);
703 SlotIndex UseIdx = DAG->getLIS()->getInstructionIndex(Instr: *MI);
704 // The load occurs after this use -- the latency is carried across loop
705 // backedge.
706 if (SlotIndex::isEarlierInstr(A: UseIdx, B: LoadIdx))
707 CarriedLatency = std::max(a: Latency, b: CarriedLatency);
708 }
709 }
710 return CarriedLatency;
711}
712
713void CandidateHeuristics::collectRegionSummary() {
714 CarriedLatencies.clear();
715 if (!SchedModel || !SchedModel->hasInstrSchedModel())
716 return;
717
718 for (auto &SU : DAG->SUnits) {
719 MachineInstr *MI = SU.getInstr();
720 const InstructionFlavor Flavor = classifyFlavor(MI: *MI, SII: *SII);
721 HWUInfo[static_cast<int>(Flavor)].insert(SU: &SU, BlockingCycles: getHWUICyclesForSU(SU: &SU));
722 unsigned CarriedLatency = getCarriedLatency(SU: &SU);
723 if (CarriedLatency)
724 CarriedLatencies[MI] = CarriedLatency;
725 }
726
727 for (auto &HWUI : HWUInfo)
728 HWUI.finalizeCycles();
729
730 LLVM_DEBUG(dumpRegionSummary());
731}
732
733void CandidateHeuristics::dumpRegionSummary() {
734 MachineBasicBlock *BB = DAG->begin()->getParent();
735 dbgs() << "\n=== Region: " << DAG->MF.getName() << " BB" << BB->getNumber()
736 << " (" << DAG->SUnits.size() << " SUs) ===\n";
737
738 dbgs() << "\nHWUI Resource Pressure:\n";
739 for (auto &HWUI : HWUInfo) {
740 if (HWUI.getTotalCycles() == 0)
741 continue;
742
743 StringRef Name = getFlavorName(F: HWUI.getType());
744 dbgs() << " " << Name << ": " << HWUI.getTotalCycles() << " cycles, "
745 << HWUI.size() << " instrs\n";
746 }
747 dbgs() << "\n";
748}
749
750void CandidateHeuristics::sortHWUIResources() {
751 // Highest priority should be first.
752 llvm::sort(C&: HWUInfo, Comp: [](HardwareUnitInfo &A, HardwareUnitInfo &B) {
753 // Prefer CoexecWindow producers
754 if (A.producesCoexecWindow() != B.producesCoexecWindow())
755 return A.producesCoexecWindow();
756
757 // Prefer more demanded resources
758 if (A.getTotalCycles() != B.getTotalCycles())
759 return A.getTotalCycles() > B.getTotalCycles();
760
761 // In ties -- prefer the resource with more instructions
762 if (A.size() != B.size())
763 return A.size() < B.size();
764
765 // Default to Flavor order
766 return static_cast<unsigned>(A.getType()) <
767 static_cast<unsigned>(B.getType());
768 });
769}
770
771unsigned CandidateHeuristics::getStructuralStallCycles(SchedBoundary &Zone,
772 SUnit *SU) {
773 // Only implemented for top-down scheduling currently.
774 if (!Zone.isTop() || !SU)
775 return 0;
776
777 MachineInstr *MI = SU->getInstr();
778 unsigned CurrCycle = Zone.getCurrCycle();
779 unsigned Stall = 0;
780
781 // Query SchedModel for resource stalls (unbuffered resources).
782 if (SchedModel->hasInstrSchedModel() && SU->hasReservedResource) {
783 const MCSchedClassDesc *SC = DAG->getSchedClass(SU);
784 for (const MCWriteProcResEntry &PE :
785 make_range(x: SchedModel->getWriteProcResBegin(SC),
786 y: SchedModel->getWriteProcResEnd(SC))) {
787 unsigned NextAvail =
788 Zone.getNextResourceCycle(SC, PIdx: PE.ProcResourceIdx, ReleaseAtCycle: PE.ReleaseAtCycle,
789 AcquireAtCycle: PE.AcquireAtCycle)
790 .first;
791 if (NextAvail > CurrCycle)
792 Stall = std::max(a: Stall, b: NextAvail - CurrCycle);
793 }
794 }
795
796 // Query HazardRecognizer for sequence-dependent hazard penalties.
797 if (Zone.HazardRec && Zone.HazardRec->isEnabled()) {
798 auto *HR = static_cast<GCNHazardRecognizer *>(Zone.HazardRec.get());
799 Stall = std::max(a: Stall, b: HR->getHazardWaitStates(MI));
800 }
801
802 return Stall;
803}
804
805bool CandidateHeuristics::tryEffectiveStall(
806 GenericSchedulerBase::SchedCandidate &Cand,
807 GenericSchedulerBase::SchedCandidate &TryCand, SchedBoundary &Zone) {
808
809 // Treat structural and latency stalls as a single scheduling cost for the
810 // current cycle.
811 struct StallCosts {
812 unsigned Ready = 0;
813 unsigned Structural = 0;
814 unsigned Latency = 0;
815 unsigned Effective = 0;
816 unsigned Carried = 0;
817 unsigned Buffer = 0;
818 };
819
820 auto getBufferFullStalls = [this, &Zone](SUnit *SU) -> unsigned {
821 InstructionFlavor Flavor = classifyFlavor(
822 MI: *SU->getInstr(), SII: *static_cast<const SIInstrInfo *>(DAG->TII));
823 HardwareUnitInfo *HWUI = getHWUIFromFlavor(Flavor);
824
825 // A BufferSize of 0 means "unlimited" buffer, thus we will never fill it.
826 if (HWUI->getBufferSize() == 0)
827 return 0;
828
829 // getBufferAvailableCycle assumes top-down scheduling.
830 assert(Zone.isTop());
831 unsigned CurrCycle = Zone.getCurrCycle();
832 unsigned BufferReadyCycle = HWUI->getBufferAvailableCycle(CurrCycle);
833 if (BufferReadyCycle <= CurrCycle)
834 return 0;
835
836 return BufferReadyCycle - CurrCycle;
837 };
838
839 unsigned CurrCycle = Zone.getCurrCycle();
840 auto GetStallCosts = [&](SUnit *SU) {
841 unsigned ReadyCycle = Zone.isTop() ? SU->TopReadyCycle : SU->BotReadyCycle;
842 StallCosts Costs;
843 Costs.Ready = ReadyCycle > CurrCycle ? ReadyCycle - CurrCycle : 0;
844 Costs.Structural = getStructuralStallCycles(Zone, SU);
845 Costs.Latency = Zone.getLatencyStallCycles(SU);
846 unsigned CarriedLatency = CarriedLatencies.lookup_or(Val: SU->getInstr(), Default: 0);
847 Costs.Carried = CarriedLatency > CurrCycle ? CarriedLatency - CurrCycle : 0;
848 Costs.Buffer = getBufferFullStalls(SU);
849
850 Costs.Effective = std::max(l: {Costs.Ready, Costs.Structural, Costs.Latency,
851 Costs.Carried, Costs.Buffer});
852 return Costs;
853 };
854
855 StallCosts TryCosts = GetStallCosts(TryCand.SU);
856 StallCosts CandCosts = GetStallCosts(Cand.SU);
857
858 LLVM_DEBUG(if (TryCosts.Effective || CandCosts.Effective) {
859 dbgs() << "Effective stalls: try=" << TryCosts.Effective
860 << " (ready=" << TryCosts.Ready << ", struct=" << TryCosts.Structural
861 << ", lat=" << TryCosts.Latency << ", carried=" << TryCosts.Carried
862 << ", buffer=" << TryCosts.Buffer << ") cand=" << CandCosts.Effective
863 << " (ready=" << CandCosts.Ready
864 << ", struct=" << CandCosts.Structural
865 << ", lat=" << CandCosts.Latency << ", carried=" << CandCosts.Carried
866 << ", buffer=" << CandCosts.Buffer << ")\n";
867 });
868
869 return tryLess(TryVal: TryCosts.Effective, CandVal: CandCosts.Effective, TryCand, Cand,
870 Reason: AMDGPUCoExecSchedStrategy::Stall);
871}
872
873bool CandidateHeuristics::tryCriticalResourceDependency(
874 GenericSchedulerBase::SchedCandidate &TryCand,
875 GenericSchedulerBase::SchedCandidate &Cand, SchedBoundary *Zone) const {
876
877 auto HasPrioritySU = [this, &Cand, &TryCand](unsigned ResourceIdx) {
878 const HardwareUnitInfo &HWUI = HWUInfo[ResourceIdx];
879
880 auto CandFlavor = classifyFlavor(MI: *Cand.SU->getInstr(), SII: *SII);
881 auto TryCandFlavor = classifyFlavor(MI: *TryCand.SU->getInstr(), SII: *SII);
882 bool LookDeep = (CandFlavor == InstructionFlavor::DS ||
883 TryCandFlavor == InstructionFlavor::DS) &&
884 HWUI.getType() == InstructionFlavor::WMMA;
885 auto *TargetSU = HWUI.getNextTargetSU(LookDeep);
886
887 // If we do not have a TargetSU for this resource, then it is not critical.
888 if (!TargetSU)
889 return false;
890
891 return true;
892 };
893
894 auto TryEnablesResource = [&Cand, &TryCand, this](unsigned ResourceIdx) {
895 const HardwareUnitInfo &HWUI = HWUInfo[ResourceIdx];
896 auto CandFlavor = classifyFlavor(MI: *Cand.SU->getInstr(), SII: *SII);
897
898 // We want to ensure our DS order matches WMMA order.
899 bool LookDeep = CandFlavor == InstructionFlavor::DS &&
900 HWUI.getType() == InstructionFlavor::WMMA;
901 auto *TargetSU = HWUI.getNextTargetSU(LookDeep);
902
903 bool CandEnables =
904 TargetSU != Cand.SU && DAG->IsReachable(SU: TargetSU, TargetSU: Cand.SU);
905 bool TryCandEnables =
906 TargetSU != TryCand.SU && DAG->IsReachable(SU: TargetSU, TargetSU: TryCand.SU);
907
908 if (!CandEnables && !TryCandEnables)
909 return false;
910
911 if (CandEnables && !TryCandEnables) {
912 if (Cand.Reason > GenericSchedulerBase::RegCritical)
913 Cand.Reason = GenericSchedulerBase::RegCritical;
914
915 return true;
916 }
917
918 if (!CandEnables && TryCandEnables) {
919 TryCand.Reason = GenericSchedulerBase::RegCritical;
920 return true;
921 }
922
923 // Both enable, prefer the critical path.
924 unsigned CandHeight = Cand.SU->getHeight();
925 unsigned TryCandHeight = TryCand.SU->getHeight();
926
927 if (CandHeight > TryCandHeight) {
928 if (Cand.Reason > GenericSchedulerBase::RegCritical)
929 Cand.Reason = GenericSchedulerBase::RegCritical;
930
931 return true;
932 }
933
934 if (CandHeight < TryCandHeight) {
935 TryCand.Reason = GenericSchedulerBase::RegCritical;
936 return true;
937 }
938
939 // Same critical path, just prefer original candidate.
940 if (Cand.Reason > GenericSchedulerBase::RegCritical)
941 Cand.Reason = GenericSchedulerBase::RegCritical;
942
943 return true;
944 };
945
946 for (unsigned I = 0; I < HWUInfo.size(); I++) {
947 // If we have encountered a resource that is not critical, then neither
948 // candidate enables a critical resource
949 if (!HasPrioritySU(I))
950 continue;
951
952 bool Enabled = TryEnablesResource(I);
953 // If neither has enabled the resource, continue to the next resource
954 if (Enabled)
955 return true;
956 }
957 return false;
958}
959
960bool CandidateHeuristics::tryCriticalResource(
961 GenericSchedulerBase::SchedCandidate &TryCand,
962 GenericSchedulerBase::SchedCandidate &Cand, SchedBoundary *Zone) const {
963 for (unsigned I = 0; I < HWUInfo.size(); I++) {
964 const HardwareUnitInfo &HWUI = HWUInfo[I];
965
966 bool CandUsesCrit = HWUI.contains(SU: Cand.SU);
967 bool TryCandUsesCrit = HWUI.contains(SU: TryCand.SU);
968
969 if (!CandUsesCrit && !TryCandUsesCrit)
970 continue;
971
972 if (CandUsesCrit != TryCandUsesCrit) {
973 if (CandUsesCrit) {
974 if (Cand.Reason > GenericSchedulerBase::RegCritical)
975 Cand.Reason = GenericSchedulerBase::RegCritical;
976 return true;
977 }
978 TryCand.Reason = GenericSchedulerBase::RegCritical;
979 return true;
980 }
981
982 // Otherwise, both use the critical resource
983 // For longer latency InstructionFlavors, we should prioritize first by
984 // their enablement of critical resources
985 if (HWUI.getType() == InstructionFlavor::DS) {
986 if (tryCriticalResourceDependency(TryCand, Cand, Zone))
987 return true;
988 }
989
990 // Prioritize based on HWUI priorities.
991 SUnit *Match = HWUI.getHigherPriority(SU: Cand.SU, Other: TryCand.SU);
992 if (Match) {
993 if (Match == Cand.SU) {
994 if (Cand.Reason > GenericSchedulerBase::RegCritical)
995 Cand.Reason = GenericSchedulerBase::RegCritical;
996 return true;
997 }
998 TryCand.Reason = GenericSchedulerBase::RegCritical;
999 return true;
1000 }
1001 }
1002
1003 return false;
1004}
1005
1006AMDGPUCoExecSchedStrategy::AMDGPUCoExecSchedStrategy(
1007 const MachineSchedContext *C)
1008 : GCNSchedStrategy(C) {
1009 SchedStages.push_back(Elt: GCNSchedStageID::ILPInitialSchedule);
1010 SchedStages.push_back(Elt: GCNSchedStageID::RewriteMFMAForm);
1011 SchedStages.push_back(Elt: GCNSchedStageID::PreRARematerialize);
1012 // Use more accurate GCN pressure trackers.
1013 UseGCNTrackers = true;
1014}
1015
1016void AMDGPUCoExecSchedStrategy::initPolicy(MachineBasicBlock::iterator Begin,
1017 MachineBasicBlock::iterator End,
1018 unsigned NumRegionInstrs) {
1019 GCNSchedStrategy::initPolicy(Begin, End, NumRegionInstrs);
1020 assert((PreRADirection == MISched::Unspecified ||
1021 PreRADirection == MISched::TopDown) &&
1022 "coexec scheduler only supports top-down scheduling");
1023 RegionPolicy.OnlyTopDown = true;
1024 RegionPolicy.OnlyBottomUp = false;
1025 RegionPolicy.ShouldTrackLaneMasks = true;
1026}
1027
1028void AMDGPUCoExecSchedStrategy::initialize(ScheduleDAGMI *DAG) {
1029 // Coexecution scheduling strategy is only done top-down to support new
1030 // resource balancing heuristics.
1031 RegionPolicy.OnlyTopDown = true;
1032 RegionPolicy.OnlyBottomUp = false;
1033
1034 GCNSchedStrategy::initialize(DAG);
1035 Heurs.initialize(SchedDAG: DAG, TargetSchedModel: SchedModel, TRI);
1036
1037 // Replace the default hazard recognizer with our PreRA one so that pre-RA
1038 // scheduling accounts for WMMA co-execution slot constraints. This must
1039 // happen after GCNSchedStrategy::initialize() because
1040 // GenericScheduler::initialize() calls SchedBoundary::reset(), which deletes
1041 // and recreates the hazard recognizer each region.
1042 Top.HazardRec = std::make_unique<GCNHazardRecognizer>(
1043 args&: DAG->MF, args: GCNHazardRecognizer::OperatingMode::PreRA);
1044}
1045
1046void AMDGPUCoExecSchedStrategy::schedNode(SUnit *SU, bool IsTopNode) {
1047 Heurs.updateForScheduling(SU);
1048 GCNSchedStrategy::schedNode(SU, IsTopNode);
1049}
1050
1051SUnit *AMDGPUCoExecSchedStrategy::pickNode(bool &IsTopNode) {
1052 assert(RegionPolicy.OnlyTopDown && !RegionPolicy.OnlyBottomUp &&
1053 "coexec scheduler only supports top-down scheduling");
1054
1055 if (DAG->top() == DAG->bottom()) {
1056 assert(Top.Available.empty() && Top.Pending.empty() &&
1057 Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage");
1058 return nullptr;
1059 }
1060
1061 bool PickedPending = false;
1062 SUnit *SU = nullptr;
1063#ifndef NDEBUG
1064 SchedCandidate *PickedCand = nullptr;
1065#endif
1066 do {
1067 PickedPending = false;
1068 SU = pickOnlyChoice(Zone&: Top);
1069 if (!SU) {
1070 CandPolicy NoPolicy;
1071 TopCand.reset(NewPolicy: NoPolicy);
1072 pickNodeFromQueue(Zone&: Top, ZonePolicy: NoPolicy, RPTracker: DAG->getTopRPTracker(), Cand&: TopCand,
1073 PickedPending, /*IsBottomUp=*/false);
1074 assert(TopCand.Reason != NoCand && "failed to find a candidate");
1075 SU = TopCand.SU;
1076#ifndef NDEBUG
1077 PickedCand = &TopCand;
1078#endif
1079 }
1080 IsTopNode = true;
1081 } while (SU->isScheduled);
1082
1083 LLVM_DEBUG(if (PickedCand) dumpPickSummary(SU, IsTopNode, *PickedCand));
1084
1085 if (PickedPending) {
1086 unsigned ReadyCycle = SU->TopReadyCycle;
1087 unsigned CurrentCycle = Top.getCurrCycle();
1088 if (ReadyCycle > CurrentCycle)
1089 Top.bumpCycle(NextCycle: ReadyCycle);
1090
1091 // checkHazard() does not expose the exact cycle where the hazard clears.
1092 while (Top.checkHazard(SU))
1093 Top.bumpCycle(NextCycle: Top.getCurrCycle() + 1);
1094
1095 Top.releasePending();
1096 }
1097
1098 if (SU->isTopReady())
1099 Top.removeReady(SU);
1100 if (SU->isBottomReady())
1101 Bot.removeReady(SU);
1102
1103 LLVM_DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") "
1104 << *SU->getInstr());
1105
1106 assert(IsTopNode && "coexec scheduler must only schedule from top boundary");
1107 return SU;
1108}
1109
1110void AMDGPUCoExecSchedStrategy::pickNodeFromQueue(
1111 SchedBoundary &Zone, const CandPolicy &ZonePolicy,
1112 const RegPressureTracker &RPTracker, SchedCandidate &Cand,
1113 bool &PickedPending, bool IsBottomUp) {
1114 assert(Zone.isTop() && "coexec scheduler only supports top boundary");
1115 assert(!IsBottomUp && "coexec scheduler only supports top-down scheduling");
1116
1117 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo *>(TRI);
1118 ArrayRef<unsigned> Pressure = RPTracker.getRegSetPressureAtPos();
1119 unsigned SGPRPressure = 0;
1120 unsigned VGPRPressure = 0;
1121 unsigned AGPRPressure = 0;
1122 PickedPending = false;
1123 if (DAG->isTrackingPressure()) {
1124 if (!useGCNTrackers()) {
1125 SGPRPressure = Pressure[AMDGPU::RegisterPressureSets::SReg_32];
1126 VGPRPressure = Pressure[AMDGPU::RegisterPressureSets::VGPR_32];
1127 AGPRPressure = Pressure[AMDGPU::RegisterPressureSets::AGPR_32];
1128 } else {
1129 SGPRPressure = DownwardTracker.getPressure().getSGPRNum();
1130 VGPRPressure = DownwardTracker.getPressure().getArchVGPRNum();
1131 AGPRPressure = DownwardTracker.getPressure().getAGPRNum();
1132 }
1133 }
1134
1135 auto EvaluateQueue = [&](ReadyQueue &Q, bool FromPending) {
1136 for (SUnit *SU : Q) {
1137 SchedCandidate TryCand(ZonePolicy);
1138 initCandidate(Cand&: TryCand, SU, AtTop: Zone.isTop(), RPTracker, SRI, SGPRPressure,
1139 VGPRPressure, AGPRPressure, IsBottomUp);
1140 SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
1141 tryCandidateCoexec(Cand, TryCand, Zone: ZoneArg);
1142 if (TryCand.Reason != NoCand) {
1143 if (TryCand.ResDelta == SchedResourceDelta())
1144 TryCand.initResourceDelta(DAG: Zone.DAG, SchedModel);
1145 LLVM_DEBUG(printCandidateDecision(Cand, TryCand));
1146 PickedPending = FromPending;
1147 Cand.setBest(TryCand);
1148 } else {
1149 LLVM_DEBUG(printCandidateDecision(TryCand, Cand));
1150 }
1151 }
1152 };
1153
1154 LLVM_DEBUG(dbgs() << "Available Q:\n");
1155 EvaluateQueue(Zone.Available, /*FromPending=*/false);
1156
1157 LLVM_DEBUG(dbgs() << "Pending Q:\n");
1158 EvaluateQueue(Zone.Pending, /*FromPending=*/true);
1159}
1160
1161#ifndef NDEBUG
1162void AMDGPUCoExecSchedStrategy::dumpPickSummary(SUnit *SU, bool IsTopNode,
1163 SchedCandidate &Cand) {
1164 const SIInstrInfo *SII = static_cast<const SIInstrInfo *>(DAG->TII);
1165 unsigned Cycle = IsTopNode ? Top.getCurrCycle() : Bot.getCurrCycle();
1166
1167 dbgs() << "=== Pick @ Cycle " << Cycle << " ===\n";
1168
1169 const InstructionFlavor Flavor = classifyFlavor(*SU->getInstr(), *SII);
1170 dbgs() << "Picked: SU(" << SU->NodeNum << ") ";
1171 SU->getInstr()->print(dbgs(), /*IsStandalone=*/true, /*SkipOpers=*/false,
1172 /*SkipDebugLoc=*/true);
1173 dbgs() << " [" << getFlavorName(Flavor) << "]\n";
1174
1175 dbgs() << " Reason: ";
1176 if (LastAMDGPUReason != AMDGPUSchedReason::None)
1177 dbgs() << getReasonName(LastAMDGPUReason);
1178 else if (Cand.Reason != NoCand)
1179 dbgs() << GenericSchedulerBase::getReasonStr(Cand.Reason);
1180 else
1181 dbgs() << "Unknown";
1182 dbgs() << "\n\n";
1183
1184 LastAMDGPUReason = AMDGPUSchedReason::None;
1185}
1186#endif
1187
1188bool AMDGPUCoExecSchedStrategy::tryCandidateCoexec(SchedCandidate &Cand,
1189 SchedCandidate &TryCand,
1190 SchedBoundary *Zone) {
1191 // Initialize the candidate if needed.
1192 if (!Cand.isValid()) {
1193 TryCand.Reason = FirstValid;
1194 return true;
1195 }
1196
1197 // Bias PhysReg Defs and copies to their uses and defined respectively.
1198 if (tryGreater(TryVal: biasPhysReg(SU: TryCand.SU, isTop: TryCand.AtTop),
1199 CandVal: biasPhysReg(SU: Cand.SU, isTop: Cand.AtTop), TryCand, Cand, Reason: PhysReg))
1200 return TryCand.Reason != NoCand;
1201
1202 // Avoid exceeding the target's limit.
1203 if (DAG->isTrackingPressure() &&
1204 tryPressure(TryP: TryCand.RPDelta.Excess, CandP: Cand.RPDelta.Excess, TryCand, Cand,
1205 Reason: RegExcess, TRI, MF: DAG->MF))
1206 return TryCand.Reason != NoCand;
1207
1208 // We only compare a subset of features when comparing nodes between
1209 // Top and Bottom boundary. Some properties are simply incomparable, in many
1210 // other instances we should only override the other boundary if something
1211 // is a clear good pick on one boundary. Skip heuristics that are more
1212 // "tie-breaking" in nature.
1213 bool SameBoundary = Zone != nullptr;
1214 if (SameBoundary) {
1215 // Compare candidates by the stall they would introduce if
1216 // scheduled in the current cycle.
1217 if (Heurs.tryEffectiveStall(Cand, TryCand, Zone&: *Zone))
1218 return TryCand.Reason != NoCand;
1219
1220 Heurs.sortHWUIResources();
1221 if (Heurs.tryCriticalResource(TryCand, Cand, Zone)) {
1222 LastAMDGPUReason = AMDGPUSchedReason::CritResourceBalance;
1223 return TryCand.Reason != NoCand;
1224 }
1225
1226 if (Heurs.tryCriticalResourceDependency(TryCand, Cand, Zone)) {
1227 LastAMDGPUReason = AMDGPUSchedReason::CritResourceDep;
1228 return TryCand.Reason != NoCand;
1229 }
1230 }
1231
1232 // Keep clustered nodes together to encourage downstream peephole
1233 // optimizations which may reduce resource requirements.
1234 //
1235 // This is a best effort to set things up for a post-RA pass. Optimizations
1236 // like generating loads of multiple registers should ideally be done within
1237 // the scheduler pass by combining the loads during DAG postprocessing.
1238 unsigned CandZoneCluster = Cand.AtTop ? TopClusterID : BotClusterID;
1239 unsigned TryCandZoneCluster = TryCand.AtTop ? TopClusterID : BotClusterID;
1240 bool CandIsClusterSucc =
1241 isTheSameCluster(A: CandZoneCluster, B: Cand.SU->ParentClusterIdx);
1242 bool TryCandIsClusterSucc =
1243 isTheSameCluster(A: TryCandZoneCluster, B: TryCand.SU->ParentClusterIdx);
1244
1245 if (tryGreater(TryVal: TryCandIsClusterSucc, CandVal: CandIsClusterSucc, TryCand, Cand,
1246 Reason: Cluster))
1247 return TryCand.Reason != NoCand;
1248
1249 if (SameBoundary) {
1250 // Weak edges are for clustering and other constraints.
1251 if (tryLess(TryVal: getWeakLeft(SU: TryCand.SU, isTop: TryCand.AtTop),
1252 CandVal: getWeakLeft(SU: Cand.SU, isTop: Cand.AtTop), TryCand, Cand, Reason: Weak))
1253 return TryCand.Reason != NoCand;
1254 }
1255
1256 // Avoid increasing the max pressure of the entire region.
1257 if (DAG->isTrackingPressure() &&
1258 tryPressure(TryP: TryCand.RPDelta.CurrentMax, CandP: Cand.RPDelta.CurrentMax, TryCand,
1259 Cand, Reason: RegMax, TRI, MF: DAG->MF))
1260 return TryCand.Reason != NoCand;
1261
1262 if (SameBoundary) {
1263 // Avoid serializing long latency dependence chains.
1264 // For acyclic path limited loops, latency was already checked above.
1265 if (!RegionPolicy.DisableLatencyHeuristic && TryCand.Policy.ReduceLatency &&
1266 !Rem.IsAcyclicLatencyLimited && tryLatency(TryCand, Cand, Zone&: *Zone))
1267 return TryCand.Reason != NoCand;
1268
1269 // Fall through to original instruction order.
1270 if ((Zone->isTop() && TryCand.SU->NodeNum < Cand.SU->NodeNum) ||
1271 (!Zone->isTop() && TryCand.SU->NodeNum > Cand.SU->NodeNum)) {
1272 TryCand.Reason = NodeOrder;
1273 return true;
1274 }
1275 }
1276
1277 return false;
1278}
1279
1280ScheduleDAGInstrs *
1281llvm::createGCNCoExecMachineScheduler(MachineSchedContext *C) {
1282 LLVM_DEBUG(dbgs() << "AMDGPU coexec preRA scheduler selected for "
1283 << C->MF->getName() << '\n');
1284 ScheduleDAGMILive *DAG = new GCNScheduleDAGMILive(
1285 C, std::make_unique<AMDGPUCoExecSchedStrategy>(args&: C));
1286 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
1287 return DAG;
1288}
1289
1290ScheduleDAGInstrs *
1291llvm::createGCNNoopPostMachineScheduler(MachineSchedContext *C) {
1292 LLVM_DEBUG(dbgs() << "AMDGPU nop postRA scheduler selected for "
1293 << C->MF->getName() << '\n');
1294 return new GCNNoopPostScheduleDAG(C);
1295}
1296