1//===- AMDGPUCoExecSchedStrategy.cpp - CoExec Scheduling Strategy ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Coexecution-focused scheduling strategy for AMDGPU.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUCoExecSchedStrategy.h"
15#include "AMDGPUBarrierLatency.h"
16#include "AMDGPUIGroupLP.h"
17#include "GCNHazardRecognizer.h"
18#include "llvm/Support/Debug.h"
19
20using namespace llvm;
21using namespace llvm::AMDGPU;
22
23#define DEBUG_TYPE "machine-scheduler"
24namespace {
25enum class CarriedLatency { Off, Fence, All };
26} // namespace
27
28static cl::opt<CarriedLatency> BlockCarriedLatency(
29 "amdgpu-block-carried-latency", cl::Hidden, cl::init(Val: CarriedLatency::Off),
30 cl::desc("Estimate block-carried latency and include it in the effective "
31 "candidate stall cost."),
32 cl::values(
33 clEnumValN(CarriedLatency::Off, "off",
34 "Disabled - do not pad latency."),
35 clEnumValN(CarriedLatency::Fence, "fence",
36 "Only pad latency for memory fence (e.g. those surrounding "
37 "barrier_signal/wait)."),
38 clEnumValN(
39 CarriedLatency::All, "all",
40 "Pad latency for any SU with an incoming ds_load dependency.")));
41
42// Default VGPR threshold percent for coexec scheduler.
43static constexpr unsigned DefaultCoExecVGPRThresholdPercent = 100;
44
45namespace {
46
47// Used to disable post-RA scheduling with function level granularity.
48class GCNNoopPostScheduleDAG final : public ScheduleDAGInstrs {
49public:
50 explicit GCNNoopPostScheduleDAG(MachineSchedContext *C)
51 : ScheduleDAGInstrs(*C->MF, C->MLI, /*RemoveKillFlags=*/true) {}
52
53 // Do nothing.
54 void schedule() override {}
55};
56
57} // namespace
58
59static SUnit *pickOnlyChoice(SchedBoundary &Zone) {
60 // pickOnlyChoice() releases pending instructions and checks for new hazards.
61 SUnit *OnlyChoice = Zone.pickOnlyChoice();
62 if (!Zone.Pending.empty())
63 return nullptr;
64
65 return OnlyChoice;
66}
67
68/// Apply \p ExtraBits to every slot in \p Info starting with \p StartIndex
69/// Used by MFMA co-exec rules, because MFMA co-exec slots are incremental, i.e.
70/// for every slot N it supports all instructions which were supported by the
71/// previous slot N-1 and may support something extra.
72static void allowCoExec(llvm::AMDGPU::CoExecInfo &Info,
73 llvm::AMDGPU::CoExecMaskT ExtraBits,
74 unsigned StartIndex) {
75 for (unsigned Index = StartIndex; Index < Info.TotalWindow; ++Index)
76 Info.Slots[Index].Mask |= ExtraBits;
77}
78
79/// Get co-execution info for a gfx950 MFMA instruction.
80/// The occupancy (cycles until the next MFMA may issue) is expressed as the
81/// first stage carrying the WMMA bit.
82llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
83 using namespace llvm;
84 using namespace llvm::AMDGPU;
85 CoExecInfo Res;
86 for (unsigned I = 0; I < MaxCoExecStages; ++I)
87 Res.Slots[I].Mask = CoExecMask::None;
88
89 // TODO: Implement proper patterns support (for debugging purposes).
90 // Existing pattern letters are WMMA-specific and will probably be confusing
91 // if used as-is for MFMA. Inventing new MFMA-specific letters is an option,
92 // but perhaps the pattern should be instead dynamically reconstructed when
93 // needed by printing specific slots in full instead of a key for them.
94 Res.Pattern = "undefinedundefinedundefinedundefined";
95
96 switch (Opcode) {
97 // 4-cycle occupancy, 8-cycle window.
98 case V_MFMA_F32_16X16X128_F8F6F4_f4_f4_e64:
99 case V_MFMA_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64:
100 case V_MFMA_F32_16X16X128_F8F6F4_f4_f4_gfx940_acd:
101 case V_MFMA_F32_16X16X128_F8F6F4_f4_f4_gfx940_vcd:
102 case V_MFMA_F32_16X16X128_F8F6F4_f4_f6_e64:
103 case V_MFMA_F32_16X16X128_F8F6F4_f4_f6_vgprcd_e64:
104 case V_MFMA_F32_16X16X128_F8F6F4_f4_f6_gfx940_acd:
105 case V_MFMA_F32_16X16X128_F8F6F4_f4_f6_gfx940_vcd:
106 case V_MFMA_F32_16X16X128_F8F6F4_f6_f4_e64:
107 case V_MFMA_F32_16X16X128_F8F6F4_f6_f4_vgprcd_e64:
108 case V_MFMA_F32_16X16X128_F8F6F4_f6_f4_gfx940_acd:
109 case V_MFMA_F32_16X16X128_F8F6F4_f6_f4_gfx940_vcd:
110 case V_MFMA_F32_16X16X128_F8F6F4_f6_f6_e64:
111 case V_MFMA_F32_16X16X128_F8F6F4_f6_f6_vgprcd_e64:
112 case V_MFMA_F32_16X16X128_F8F6F4_f6_f6_gfx940_acd:
113 case V_MFMA_F32_16X16X128_F8F6F4_f6_f6_gfx940_vcd:
114 case V_MFMA_F32_16X16X32_BF16_e64:
115 case V_MFMA_F32_16X16X32_BF16_vgprcd_e64:
116 case V_MFMA_F32_16X16X32_BF16_gfx940_acd:
117 case V_MFMA_F32_16X16X32_BF16_gfx940_vcd:
118 case V_MFMA_I32_16X16X64_I8_e64:
119 case V_MFMA_I32_16X16X64_I8_vgprcd_e64:
120 case V_MFMA_I32_16X16X64_I8_gfx940_acd:
121 case V_MFMA_I32_16X16X64_I8_gfx940_vcd:
122 case V_MFMA_F32_16X16X32_F16_e64:
123 case V_MFMA_F32_16X16X32_F16_vgprcd_e64:
124 case V_MFMA_F32_16X16X32_F16_gfx940_acd:
125 case V_MFMA_F32_16X16X32_F16_gfx940_vcd:
126 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64:
127 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64:
128 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_gfx940_acd:
129 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_gfx940_vcd:
130 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6_e64:
131 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6_vgprcd_e64:
132 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6_gfx940_acd:
133 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6_gfx940_vcd:
134 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4_e64:
135 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4_vgprcd_e64:
136 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4_gfx940_acd:
137 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4_gfx940_vcd:
138 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_e64:
139 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_vgprcd_e64:
140 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_gfx940_acd:
141 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_gfx940_vcd:
142 // GFX9 Shader Programming Guide lists those SMFMAC separately, but for
143 // intended purposes here all those instructions are the same. This comment
144 // is to simplify reverse mapping to the SPG.
145 case V_SMFMAC_F32_16X16X64_BF16_e64:
146 case V_SMFMAC_F32_16X16X64_BF16_gfx940:
147 case V_SMFMAC_I32_16X16X128_I8_e64:
148 case V_SMFMAC_I32_16X16X128_I8_gfx940:
149 case V_SMFMAC_F32_16X16X128_BF8_BF8_e64:
150 case V_SMFMAC_F32_16X16X128_BF8_BF8_gfx940:
151 case V_SMFMAC_F32_16X16X128_BF8_FP8_e64:
152 case V_SMFMAC_F32_16X16X128_BF8_FP8_gfx940:
153 case V_SMFMAC_F32_16X16X128_FP8_BF8_e64:
154 case V_SMFMAC_F32_16X16X128_FP8_BF8_gfx940:
155 case V_SMFMAC_F32_16X16X128_FP8_FP8_e64:
156 case V_SMFMAC_F32_16X16X128_FP8_FP8_gfx940:
157 case V_SMFMAC_F32_16X16X64_F16_e64:
158 case V_SMFMAC_F32_16X16X64_F16_gfx940:
159 Res.TotalWindow = 8;
160 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
161 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, StartIndex: 2);
162 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 4);
163 return Res;
164
165 // 8-cycle occupancy, 12-cycle window.
166 case V_MFMA_F32_16X16X128_F8F6F4_f4_f8_e64:
167 case V_MFMA_F32_16X16X128_F8F6F4_f4_f8_vgprcd_e64:
168 case V_MFMA_F32_16X16X128_F8F6F4_f4_f8_gfx940_acd:
169 case V_MFMA_F32_16X16X128_F8F6F4_f4_f8_gfx940_vcd:
170 case V_MFMA_F32_16X16X128_F8F6F4_f6_f8_e64:
171 case V_MFMA_F32_16X16X128_F8F6F4_f6_f8_vgprcd_e64:
172 case V_MFMA_F32_16X16X128_F8F6F4_f6_f8_gfx940_acd:
173 case V_MFMA_F32_16X16X128_F8F6F4_f6_f8_gfx940_vcd:
174 case V_MFMA_F32_16X16X128_F8F6F4_f8_f4_e64:
175 case V_MFMA_F32_16X16X128_F8F6F4_f8_f4_vgprcd_e64:
176 case V_MFMA_F32_16X16X128_F8F6F4_f8_f4_gfx940_acd:
177 case V_MFMA_F32_16X16X128_F8F6F4_f8_f4_gfx940_vcd:
178 case V_MFMA_F32_16X16X128_F8F6F4_f8_f6_e64:
179 case V_MFMA_F32_16X16X128_F8F6F4_f8_f6_vgprcd_e64:
180 case V_MFMA_F32_16X16X128_F8F6F4_f8_f6_gfx940_acd:
181 case V_MFMA_F32_16X16X128_F8F6F4_f8_f6_gfx940_vcd:
182 case V_MFMA_F32_16X16X128_F8F6F4_f8_f8_e64:
183 case V_MFMA_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64:
184 case V_MFMA_F32_16X16X128_F8F6F4_f8_f8_gfx940_acd:
185 case V_MFMA_F32_16X16X128_F8F6F4_f8_f8_gfx940_vcd:
186 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f8_e64:
187 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f8_vgprcd_e64:
188 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f8_gfx940_acd:
189 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f8_gfx940_vcd:
190 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f8_e64:
191 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f8_vgprcd_e64:
192 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f8_gfx940_acd:
193 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f8_gfx940_vcd:
194 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4_e64:
195 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4_vgprcd_e64:
196 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4_gfx940_acd:
197 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4_gfx940_vcd:
198 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6_e64:
199 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6_vgprcd_e64:
200 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6_gfx940_acd:
201 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6_gfx940_vcd:
202 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_e64:
203 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64:
204 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_gfx940_acd:
205 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_gfx940_vcd:
206 Res.TotalWindow = 12;
207 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
208 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VMEM, StartIndex: 2);
209 allowCoExec(Info&: Res, ExtraBits: CoExecMask::VALU, StartIndex: 3);
210 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 8);
211 return Res;
212
213 // 4-cycle occupancy, 8-cycle window.
214 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_e64:
215 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_mac_e64:
216 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_mac_vgprcd_e64:
217 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_vgprcd_e64:
218 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_gfx940_acd:
219 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_gfx940_vcd:
220 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_e64:
221 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_mac_e64:
222 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_mac_vgprcd_e64:
223 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_vgprcd_e64:
224 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_gfx940_acd:
225 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_gfx940_vcd:
226 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_e64:
227 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_mac_e64:
228 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_mac_vgprcd_e64:
229 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_vgprcd_e64:
230 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_gfx940_acd:
231 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_gfx940_vcd:
232 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_e64:
233 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_mac_e64:
234 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_mac_vgprcd_e64:
235 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_vgprcd_e64:
236 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_gfx940_acd:
237 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_gfx940_vcd:
238 case V_MFMA_F32_32X32X16_BF16_e64:
239 case V_MFMA_F32_32X32X16_BF16_mac_e64:
240 case V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64:
241 case V_MFMA_F32_32X32X16_BF16_vgprcd_e64:
242 case V_MFMA_F32_32X32X16_BF16_gfx940_acd:
243 case V_MFMA_F32_32X32X16_BF16_gfx940_vcd:
244 case V_MFMA_I32_32X32X32_I8_e64:
245 case V_MFMA_I32_32X32X32_I8_mac_e64:
246 case V_MFMA_I32_32X32X32_I8_mac_vgprcd_e64:
247 case V_MFMA_I32_32X32X32_I8_vgprcd_e64:
248 case V_MFMA_I32_32X32X32_I8_gfx940_acd:
249 case V_MFMA_I32_32X32X32_I8_gfx940_vcd:
250 case V_MFMA_F32_32X32X16_F16_e64:
251 case V_MFMA_F32_32X32X16_F16_mac_e64:
252 case V_MFMA_F32_32X32X16_F16_mac_vgprcd_e64:
253 case V_MFMA_F32_32X32X16_F16_vgprcd_e64:
254 case V_MFMA_F32_32X32X16_F16_gfx940_acd:
255 case V_MFMA_F32_32X32X16_F16_gfx940_vcd:
256 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_gfx940_acd:
257 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_gfx940_vcd:
258 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_gfx940_acd:
259 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_gfx940_vcd:
260 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_gfx940_acd:
261 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_gfx940_vcd:
262 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_gfx940_acd:
263 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_gfx940_vcd:
264 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_e64:
265 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_e64:
266 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_e64:
267 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_e64:
268 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_vgprcd_e64:
269 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_vgprcd_e64:
270 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_vgprcd_e64:
271 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_vgprcd_e64:
272 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_mac_e64:
273 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_mac_e64:
274 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_mac_e64:
275 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_mac_e64:
276 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_mac_vgprcd_e64:
277 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_mac_vgprcd_e64:
278 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_mac_vgprcd_e64:
279 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_mac_vgprcd_e64:
280 Res.TotalWindow = 8;
281 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
282 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, StartIndex: 2);
283 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 4);
284 return Res;
285
286 // 16-cycle occupancy, 20-cycle window.
287 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_e64:
288 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_mac_e64:
289 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_mac_vgprcd_e64:
290 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_vgprcd_e64:
291 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_gfx940_acd:
292 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_gfx940_vcd:
293 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_e64:
294 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_mac_e64:
295 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_mac_vgprcd_e64:
296 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_vgprcd_e64:
297 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_gfx940_acd:
298 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_gfx940_vcd:
299 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_e64:
300 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_mac_e64:
301 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_mac_vgprcd_e64:
302 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_vgprcd_e64:
303 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_gfx940_acd:
304 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_gfx940_vcd:
305 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_e64:
306 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_mac_e64:
307 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_mac_vgprcd_e64:
308 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_vgprcd_e64:
309 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_gfx940_acd:
310 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_gfx940_vcd:
311 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_e64:
312 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_e64:
313 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64:
314 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_vgprcd_e64:
315 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_gfx940_acd:
316 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_gfx940_vcd:
317 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_e64:
318 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_e64:
319 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_e64:
320 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_e64:
321 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_e64:
322 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_vgprcd_e64:
323 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_vgprcd_e64:
324 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_vgprcd_e64:
325 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_vgprcd_e64:
326 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_vgprcd_e64:
327 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_mac_e64:
328 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_mac_e64:
329 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_mac_e64:
330 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_mac_e64:
331 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_mac_e64:
332 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_mac_vgprcd_e64:
333 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_mac_vgprcd_e64:
334 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_mac_vgprcd_e64:
335 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_mac_vgprcd_e64:
336 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64:
337 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_gfx940_acd:
338 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_gfx940_vcd:
339 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_gfx940_acd:
340 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_gfx940_vcd:
341 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_gfx940_acd:
342 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_gfx940_vcd:
343 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_gfx940_acd:
344 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_gfx940_vcd:
345 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_gfx940_acd:
346 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_gfx940_vcd:
347 Res.TotalWindow = 20;
348 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
349 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VMEM, StartIndex: 2);
350 allowCoExec(Info&: Res, ExtraBits: CoExecMask::VALU, StartIndex: 3);
351 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 16);
352 return Res;
353
354 // 9-cycle occupancy, 12-cycle window.
355 case V_SMFMAC_F32_32X32X32_BF16_e64:
356 case V_SMFMAC_F32_32X32X32_BF16_gfx940:
357 case V_SMFMAC_I32_32X32X64_I8_e64:
358 case V_SMFMAC_I32_32X32X64_I8_gfx940:
359 case V_SMFMAC_F32_32X32X64_BF8_BF8_e64:
360 case V_SMFMAC_F32_32X32X64_BF8_BF8_gfx940:
361 case V_SMFMAC_F32_32X32X64_BF8_FP8_e64:
362 case V_SMFMAC_F32_32X32X64_BF8_FP8_gfx940:
363 case V_SMFMAC_F32_32X32X64_FP8_BF8_e64:
364 case V_SMFMAC_F32_32X32X64_FP8_BF8_gfx940:
365 case V_SMFMAC_F32_32X32X64_FP8_FP8_e64:
366 case V_SMFMAC_F32_32X32X64_FP8_FP8_gfx940:
367 case V_SMFMAC_F32_32X32X32_F16_e64:
368 case V_SMFMAC_F32_32X32X32_F16_gfx940:
369 Res.TotalWindow = 12;
370 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
371 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, StartIndex: 4);
372 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 9);
373 return Res;
374
375 // 18-cycle occupancy, 19-cycle window.
376 case V_MFMA_F64_16X16X4F64_e64:
377 case V_MFMA_F64_16X16X4F64_mac_e64:
378 case V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
379 case V_MFMA_F64_16X16X4F64_vgprcd_e64:
380 Res.TotalWindow = 19;
381 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::SALU | CoExecMask::VMEM, StartIndex: 0);
382 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA | CoExecMask::VALU, StartIndex: 18);
383 return Res;
384
385 default:
386 // Default fallback: permissive 8-cycle pattern
387 return CoExecInfo::build(UnitOccupancy: 0, TotalWindow: 9, Pattern: "AAAAAAAAA");
388 }
389}
390
391InstructionFlavor llvm::AMDGPU::classifyFlavor(const MachineInstr &MI,
392 const SIInstrInfo &SII) {
393 if (MI.isDebugInstr())
394 return InstructionFlavor::Other;
395
396 unsigned Opc = MI.getOpcode();
397
398 // Check for specific opcodes first.
399 if (Opc == AMDGPU::ATOMIC_FENCE || Opc == AMDGPU::S_BARRIER_WAIT ||
400 Opc == AMDGPU::S_BARRIER_SIGNAL_IMM || SII.isWaitcnt(Opcode: Opc))
401 return InstructionFlavor::Fence;
402
403 if (SII.isLDSDMA(MI))
404 return InstructionFlavor::DMA;
405
406 if (SII.isMFMA(MI)) {
407 // TODO: Consider further sub-classifying this (XDL, XDL2x, S/DGEMM).
408 // GFX9 SPG sub-classifies MFMA into XDL, XDL2x and S/DGEMM, because only
409 // certain sub-classes can be co-executed in certain slots. For now, we
410 // simply treat them all as one to simplify the change and leave the rest
411 // to a follow-up fine-tuning.
412 return InstructionFlavor::WMMA;
413 }
414
415 if (SII.isWMMA(MI) || SII.isSWMMAC(MI))
416 return InstructionFlavor::WMMA;
417
418 if (SII.isTRANS(MI))
419 return InstructionFlavor::TRANS;
420
421 if (SII.isVALU(MI, /*AllowLDSDMA=*/false)) {
422 if (SII.getBlockingCycles(MI) > 1)
423 return InstructionFlavor::MultiCycleVALU;
424
425 return InstructionFlavor::SingleCycleVALU;
426 }
427
428 if (SII.isSMRD(MI))
429 return InstructionFlavor::SMEM;
430
431 if (SII.isDS(MI))
432 return InstructionFlavor::DS;
433
434 if (SII.isVMEM(MI))
435 return InstructionFlavor::VMEM;
436
437 if (SII.isSALU(MI))
438 return InstructionFlavor::SALU;
439
440 return InstructionFlavor::Other;
441}
442
443SUnit *HardwareUnitInfo::getNextTargetSU(bool LookDeep) const {
444 for (SUnit *PrioritySU : PrioritySUs) {
445 if (!PrioritySU->isTopReady())
446 return PrioritySU;
447 }
448
449 if (!LookDeep)
450 return nullptr;
451
452 unsigned MinDepth = std::numeric_limits<unsigned int>::max();
453 SUnit *TargetSU = nullptr;
454 for (auto *SU : AllSUs) {
455 if (SU->isScheduled)
456 continue;
457
458 if (SU->isTopReady())
459 continue;
460
461 if (SU->getDepth() < MinDepth) {
462 MinDepth = SU->getDepth();
463 TargetSU = SU;
464 }
465 }
466 return TargetSU;
467}
468
469void HardwareUnitInfo::insert(SUnit *SU, unsigned BlockingCycles) {
470 if (!AllSUs.insert(X: SU))
471 llvm_unreachable("HardwareUnit already contains SU!");
472
473 TotalCycles += BlockingCycles;
474
475 if (PrioritySUs.empty()) {
476 PrioritySUs.insert(X: SU);
477 return;
478 }
479 unsigned SUDepth = SU->getDepth();
480 unsigned CurrDepth = (*PrioritySUs.begin())->getDepth();
481 if (SUDepth > CurrDepth)
482 return;
483
484 if (SUDepth == CurrDepth) {
485 PrioritySUs.insert(X: SU);
486 return;
487 }
488
489 // SU is lower depth and should be prioritized.
490 PrioritySUs.clear();
491 PrioritySUs.insert(X: SU);
492}
493
494void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles) {
495 // We may want to ignore some HWUIs (e.g. InstructionFlavor::Other). To do so,
496 // we just clear the HWUI. However, we still have instructions which map to
497 // this HWUI. Don't bother managing the state for these HWUI.
498 if (TotalCycles == 0)
499 return;
500
501 ScheduledSUs.push_back(Elt: SU);
502 AllSUs.remove(X: SU);
503 PrioritySUs.remove(X: SU);
504
505 // BufferSize 0 is unlimited, while size 1 has no parallel buffering. In
506 // either case, each SU uses the HardwareUnit for BlockingCycles.
507 if (BufferSize <= 1 || (ScheduledSUs.size() % BufferSize == 0))
508 TotalCycles -= std::min(a: TotalCycles, b: BlockingCycles);
509
510 if (AllSUs.empty())
511 return;
512 if (PrioritySUs.empty()) {
513 for (auto SU : AllSUs) {
514 if (PrioritySUs.empty()) {
515 PrioritySUs.insert(X: SU);
516 continue;
517 }
518 unsigned SUDepth = SU->getDepth();
519 unsigned CurrDepth = (*PrioritySUs.begin())->getDepth();
520 if (SUDepth > CurrDepth)
521 continue;
522
523 if (SUDepth == CurrDepth) {
524 PrioritySUs.insert(X: SU);
525 continue;
526 }
527
528 // SU is lower depth and should be prioritized.
529 PrioritySUs.clear();
530 PrioritySUs.insert(X: SU);
531 }
532 }
533}
534
535void HardwareUnitInfo::finalizeCycles() {
536 if (BufferSize == 0 || AllSUs.empty())
537 return;
538
539 // We estimate the amount of cycles it takes to free up a slot in the buffer
540 // as the average cycles per SU.
541 BufferCycles = TotalCycles / AllSUs.size();
542 // A single-entry buffer does not reduce TotalCycles.
543 if (BufferSize == 1)
544 return;
545
546 // The TotalCycles is normalized against the BufferSize.
547 // This provides an estimate of the TotalCycles which is not always accurate
548 // -- particularly in cases where we have fewer instructions than the
549 // BufferSize. For example, if we have 2 instructions which each take 50
550 // cycles and a BufferSize of 16, then a TotalCycles of 51 cycles would be
551 // somewhat accurate. This normalization calculates TotalCycles as 6. However,
552 // if we have 64 of these instructions, our normalized estimate of 200 is more
553 // reasonable, given the more accurate measure is 264. Having a completely
554 // accurate measure is not very important, since this metric is mainly used to
555 // compare the relative demand per HardwareUnit across the region. The simpler
556 // estimate makes managing the metric incrementally during scheduling much
557 // simpler.
558 TotalCycles /= BufferSize;
559}
560
561HardwareUnitInfo *
562CandidateHeuristics::getHWUIFromFlavor(InstructionFlavor Flavor) {
563 for (HardwareUnitInfo &HWUICand : HWUInfo) {
564 if (HWUICand.getType() == Flavor) {
565 return &HWUICand;
566 }
567 }
568 return nullptr;
569}
570
571unsigned CandidateHeuristics::getMaxBlockingCycles(const MCSchedClassDesc *SC,
572 const MachineInstr *MI) {
573 unsigned ReleaseAtCycle = 0;
574 for (TargetSchedModel::ProcResIter PI = SchedModel->getWriteProcResBegin(SC),
575 PE = SchedModel->getWriteProcResEnd(SC);
576 PI != PE; ++PI) {
577 ReleaseAtCycle =
578 std::max(a: ReleaseAtCycle, b: static_cast<unsigned>(PI->ReleaseAtCycle));
579 }
580 ReleaseAtCycle = std::max(a: ReleaseAtCycle, b: SII->getBlockingCycles(MI: *MI));
581 return ReleaseAtCycle;
582}
583
584unsigned CandidateHeuristics::getHWUICyclesForMI(MachineInstr *MI) {
585 assert(SchedModel && SchedModel->hasInstrSchedModel());
586 if (MI->mayLoadOrStore())
587 return SchedModel->computeInstrLatency(MI, UseDefaultDefLatency: false);
588 return getMaxBlockingCycles(SC: SchedModel->resolveSchedClass(MI), MI);
589}
590
591void CandidateHeuristics::updateForScheduling(SUnit *SU) {
592 MachineInstr *MI = SU->getInstr();
593 HardwareUnitInfo *HWUI = getHWUIFromFlavor(Flavor: classifyFlavor(MI: *MI, SII: *SII));
594 assert(HWUI);
595 HWUI->markScheduled(SU, BlockingCycles: getHWUICyclesForMI(MI));
596}
597
598void CandidateHeuristics::initialize(ScheduleDAGMI *SchedDAG,
599 const TargetSchedModel *TargetSchedModel,
600 const TargetRegisterInfo *TRI) {
601 DAG = SchedDAG;
602 SchedModel = TargetSchedModel;
603 assert(SchedModel && SchedModel->hasInstrSchedModel());
604
605 SRI = static_cast<const SIRegisterInfo *>(TRI);
606 SII = static_cast<const SIInstrInfo *>(DAG->TII);
607
608 HWUInfo.resize(N: static_cast<int>(InstructionFlavor::NUM_FLAVORS));
609
610 for (unsigned I = 0; I < HWUInfo.size(); I++) {
611 HWUInfo[I].reset();
612 HWUInfo[I].setType(I);
613 }
614
615 HWUInfo[static_cast<int>(InstructionFlavor::WMMA)].setProducesCoexecWindow(
616 true);
617 HWUInfo[static_cast<int>(InstructionFlavor::MultiCycleVALU)]
618 .setProducesCoexecWindow(true);
619 HWUInfo[static_cast<int>(InstructionFlavor::TRANS)].setProducesCoexecWindow(
620 true);
621 HWUInfo[static_cast<int>(InstructionFlavor::DS)].setBufferSize(
622 DefaultBufferSizes::DS);
623
624 collectRegionSummary();
625}
626
627unsigned CandidateHeuristics::getCarriedLatency(SUnit *SU) {
628 if (BlockCarriedLatency == CarriedLatency::Off)
629 return 0;
630
631 MachineInstr *MI = SU->getInstr();
632 unsigned CarriedLatency = 0;
633 const InstructionFlavor Flavor = classifyFlavor(MI: *MI, SII: *SII);
634 if (Flavor == InstructionFlavor::Fence) {
635 MachineBasicBlock *MBB = MI->getParent();
636 // Scan each direct predecessor back to its nearest Fence or block start for
637 // DS instructions.
638 for (auto PredMBB : MBB->predecessors()) {
639 auto I = PredMBB->rbegin();
640 auto E = PredMBB->rend();
641 for (; I != E; I++) {
642 const InstructionFlavor ItFlavor = classifyFlavor(MI: *I, SII: *SII);
643 if (ItFlavor == InstructionFlavor::Fence)
644 break;
645
646 // Found carried latency.
647 if (ItFlavor == InstructionFlavor::DS)
648 CarriedLatency = std::max(a: CarriedLatency, b: getHWUICyclesForMI(MI: &*I));
649 }
650 }
651 }
652
653 if (BlockCarriedLatency == CarriedLatency::Fence)
654 return CarriedLatency;
655
656 for (MachineOperand &Op : MI->all_uses()) {
657 auto Reg = Op.getReg();
658 if (!Reg.isVirtual())
659 continue;
660
661 for (MachineInstr &Def : DAG->MRI.def_instructions(Reg)) {
662 // We don't have the proper modelling to accurately measure all carried
663 // latency. Just try to measure carried latency for long latency loads to
664 // avoid long stalls.
665 if (!Def.mayLoad())
666 continue;
667
668 unsigned Latency = getHWUICyclesForMI(MI: &Def);
669
670 // Load is carried across block.
671 if (Def.getParent() != MI->getParent()) {
672 bool FoundUseInDefBlock = false;
673 for (MachineInstr &Use : DAG->MRI.use_nodbg_instructions(Reg)) {
674 if (Use.getParent() != Def.getParent())
675 continue;
676
677 SlotIndex DefIdx = DAG->getLIS()->getInstructionIndex(Instr: Def);
678 SlotIndex UseIdx = DAG->getLIS()->getInstructionIndex(Instr: Use);
679 // We have a use of this load in the def block that occurs after the
680 // load. In this case we must wait for the load in the def block, and
681 // we do not have any carried latency from this load.
682 if (SlotIndex::isEarlierInstr(A: DefIdx, B: UseIdx)) {
683 FoundUseInDefBlock = true;
684 break;
685 }
686 }
687 if (!FoundUseInDefBlock)
688 CarriedLatency = std::max(a: Latency, b: CarriedLatency);
689
690 continue;
691 }
692
693 assert(Def.getParent() == MI->getParent());
694 // Load is in the same block.
695 SlotIndex LoadIdx = DAG->getLIS()->getInstructionIndex(Instr: Def);
696 SlotIndex UseIdx = DAG->getLIS()->getInstructionIndex(Instr: *MI);
697 // The load occurs after this use -- the latency is carried across loop
698 // backedge.
699 if (SlotIndex::isEarlierInstr(A: UseIdx, B: LoadIdx))
700 CarriedLatency = std::max(a: Latency, b: CarriedLatency);
701 }
702 }
703 return CarriedLatency;
704}
705
706void CandidateHeuristics::collectRegionSummary() {
707 CarriedLatencies.clear();
708 if (!SchedModel || !SchedModel->hasInstrSchedModel())
709 return;
710
711 for (auto &SU : DAG->SUnits) {
712 MachineInstr *MI = SU.getInstr();
713 const InstructionFlavor Flavor = classifyFlavor(MI: *MI, SII: *SII);
714 HWUInfo[static_cast<int>(Flavor)].insert(SU: &SU, BlockingCycles: getHWUICyclesForMI(MI));
715 unsigned CarriedLatency = getCarriedLatency(SU: &SU);
716 if (CarriedLatency)
717 CarriedLatencies[MI] = CarriedLatency;
718 }
719
720 for (auto &HWUI : HWUInfo)
721 HWUI.finalizeCycles();
722
723 LLVM_DEBUG(dumpRegionSummary());
724}
725
726void CandidateHeuristics::dumpRegionSummary() {
727 MachineBasicBlock *BB = DAG->begin()->getParent();
728 dbgs() << "\n=== Region: " << DAG->MF.getName() << " BB" << BB->getNumber()
729 << " (" << DAG->SUnits.size() << " SUs) ===\n";
730
731 dbgs() << "\nHWUI Resource Pressure:\n";
732 for (auto &HWUI : HWUInfo) {
733 if (HWUI.getTotalCycles() == 0)
734 continue;
735
736 StringRef Name = getFlavorName(F: HWUI.getType());
737 dbgs() << " " << Name << ": " << HWUI.getTotalCycles() << " cycles, "
738 << HWUI.size() << " instrs\n";
739 }
740 dbgs() << "\n";
741}
742
743void CandidateHeuristics::sortHWUIResources() {
744 // Highest priority should be first.
745 llvm::sort(C&: HWUInfo, Comp: [](HardwareUnitInfo &A, HardwareUnitInfo &B) {
746 // Prefer CoexecWindow producers
747 if (A.producesCoexecWindow() != B.producesCoexecWindow())
748 return A.producesCoexecWindow();
749
750 // Prefer more demanded resources
751 if (A.getTotalCycles() != B.getTotalCycles())
752 return A.getTotalCycles() > B.getTotalCycles();
753
754 // In ties -- prefer the resource with more instructions
755 if (A.size() != B.size())
756 return A.size() < B.size();
757
758 // Default to Flavor order
759 return static_cast<unsigned>(A.getType()) <
760 static_cast<unsigned>(B.getType());
761 });
762}
763
764unsigned CandidateHeuristics::getStructuralStallCycles(SchedBoundary &Zone,
765 SUnit *SU) {
766 assert(Zone.isTop() && "effective stall comparison requires top boundary");
767 if (!SU)
768 return 0;
769
770 MachineInstr *MI = SU->getInstr();
771 unsigned CurrCycle = Zone.getCurrCycle();
772 unsigned Stall = 0;
773
774 // Query SchedModel for resource stalls (unbuffered resources).
775 if (SchedModel->hasInstrSchedModel() && SU->hasReservedResource) {
776 const MCSchedClassDesc *SC = DAG->getSchedClass(SU);
777 for (const MCWriteProcResEntry &PE :
778 make_range(x: SchedModel->getWriteProcResBegin(SC),
779 y: SchedModel->getWriteProcResEnd(SC))) {
780 unsigned NextAvail =
781 Zone.getNextResourceCycle(SC, PIdx: PE.ProcResourceIdx, ReleaseAtCycle: PE.ReleaseAtCycle,
782 AcquireAtCycle: PE.AcquireAtCycle)
783 .first;
784 if (NextAvail > CurrCycle)
785 Stall = std::max(a: Stall, b: NextAvail - CurrCycle);
786 }
787 }
788
789 // Query HazardRecognizer for sequence-dependent hazard penalties.
790 if (Zone.HazardRec && Zone.HazardRec->isEnabled()) {
791 auto *HR = static_cast<GCNHazardRecognizer *>(Zone.HazardRec.get());
792 Stall = std::max(a: Stall, b: HR->getHazardWaitStates(MI));
793 }
794
795 return Stall;
796}
797
798CandidateHeuristics::StallCosts
799CandidateHeuristics::getStallCosts(SUnit *SU, SchedBoundary &Zone) {
800 if (!Zone.isTop())
801 return {};
802
803 auto getBufferFullStalls = [this, &Zone](SUnit *SU) -> unsigned {
804 InstructionFlavor Flavor = classifyFlavor(
805 MI: *SU->getInstr(), SII: *static_cast<const SIInstrInfo *>(DAG->TII));
806 HardwareUnitInfo *HWUI = getHWUIFromFlavor(Flavor);
807
808 // A BufferSize of 0 means "unlimited" buffer, thus we will never fill it.
809 if (HWUI->getBufferSize() == 0)
810 return 0;
811
812 unsigned CurrCycle = Zone.getCurrCycle();
813 unsigned BufferReadyCycle = HWUI->getBufferAvailableCycle(CurrCycle);
814 if (BufferReadyCycle <= CurrCycle)
815 return 0;
816
817 return BufferReadyCycle - CurrCycle;
818 };
819
820 unsigned CurrCycle = Zone.getCurrCycle();
821
822 auto getFenceStalls = [this, &CurrCycle](SUnit *SU) -> unsigned {
823 InstructionFlavor Flavor = classifyFlavor(
824 MI: *SU->getInstr(), SII: *static_cast<const SIInstrInfo *>(DAG->TII));
825
826 if (Flavor != InstructionFlavor::Fence)
827 return 0;
828
829 HardwareUnitInfo *ConsumerHWUI = getHWUIFromFlavor(Flavor);
830 HardwareUnitInfo *ProducerHWUI = getHWUIFromFlavor(Flavor: InstructionFlavor::DS);
831
832 SUnit *LastProducer = ProducerHWUI->getLastScheduledSU();
833 if (!LastProducer)
834 return 0;
835
836 SUnit *LastConsumer = ConsumerHWUI->getLastScheduledSU();
837 unsigned LastConsumerCycle = LastConsumer ? LastConsumer->TopReadyCycle : 0;
838 unsigned LastProducerCycle = LastProducer->TopReadyCycle;
839
840 if (LastProducerCycle < LastConsumerCycle)
841 return 0;
842
843 unsigned FenceStallFinish =
844 LastProducerCycle + getHWUICyclesForMI(MI: LastProducer->getInstr());
845 return FenceStallFinish <= CurrCycle ? 0 : FenceStallFinish - CurrCycle;
846 };
847
848 unsigned ReadyCycle = SU->TopReadyCycle;
849 StallCosts Costs;
850 Costs.Ready = ReadyCycle > CurrCycle ? ReadyCycle - CurrCycle : 0;
851 Costs.Structural = getStructuralStallCycles(Zone, SU);
852 Costs.Latency = Zone.getLatencyStallCycles(SU);
853 unsigned CarriedLatency = CarriedLatencies.lookup_or(Val: SU->getInstr(), Default: 0);
854 Costs.Carried = CarriedLatency > CurrCycle ? CarriedLatency - CurrCycle : 0;
855 Costs.Buffer = getBufferFullStalls(SU);
856 Costs.Fence = getFenceStalls(SU);
857 Costs.Effective = std::max(l: {Costs.Ready, Costs.Structural, Costs.Latency,
858 Costs.Carried, Costs.Buffer, Costs.Fence});
859 return Costs;
860}
861
862bool CandidateHeuristics::tryEffectiveStall(
863 GenericSchedulerBase::SchedCandidate &TryCand,
864 GenericSchedulerBase::SchedCandidate &Cand, SchedBoundary &Zone) {
865 // Only implemented for top-down scheduling
866 if (!Zone.isTop())
867 return 0;
868
869 StallCosts TryCosts = getStallCosts(SU: TryCand.SU, Zone);
870 StallCosts CandCosts = getStallCosts(SU: Cand.SU, Zone);
871
872 LLVM_DEBUG(if (TryCosts.Effective || CandCosts.Effective) {
873 dbgs() << "Effective stalls: try=" << TryCosts.Effective
874 << " (ready=" << TryCosts.Ready << ", struct=" << TryCosts.Structural
875 << ", lat=" << TryCosts.Latency << ", carried=" << TryCosts.Carried
876 << ", buffer=" << TryCosts.Buffer << ", fence=" << TryCosts.Fence
877 << ") cand=" << CandCosts.Effective << " (ready=" << CandCosts.Ready
878 << ", struct=" << CandCosts.Structural
879 << ", lat=" << CandCosts.Latency << ", carried=" << CandCosts.Carried
880 << ", buffer=" << CandCosts.Buffer << ", fence=" << CandCosts.Fence
881 << ")\n";
882 });
883
884 return tryLess(TryVal: TryCosts.Effective, CandVal: CandCosts.Effective, TryCand, Cand,
885 Reason: AMDGPUCoExecSchedStrategy::Stall);
886}
887
888bool CandidateHeuristics::tryMemoryPipeline(
889 GenericSchedulerBase::SchedCandidate &TryCand,
890 GenericSchedulerBase::SchedCandidate &Cand, SchedBoundary &Zone) {
891
892 InstructionFlavor TryFlavor = classifyFlavor(MI: *TryCand.SU->getInstr(), SII: *SII);
893
894 InstructionFlavor CandFlavor = classifyFlavor(MI: *Cand.SU->getInstr(), SII: *SII);
895
896 bool TryIsMemoryPipeline = TryFlavor == InstructionFlavor::DMA ||
897 TryFlavor == InstructionFlavor::Fence;
898 bool CandIsMemoryPipeline = CandFlavor == InstructionFlavor::DMA ||
899 CandFlavor == InstructionFlavor::Fence;
900
901 if (!(TryIsMemoryPipeline || CandIsMemoryPipeline))
902 return false;
903
904 if (TryIsMemoryPipeline)
905 TryIsMemoryPipeline &= getStallCosts(SU: TryCand.SU, Zone).Effective == 0;
906
907 if (CandIsMemoryPipeline)
908 CandIsMemoryPipeline &= getStallCosts(SU: Cand.SU, Zone).Effective == 0;
909
910 if (TryIsMemoryPipeline == CandIsMemoryPipeline)
911 return false;
912
913 if (CandIsMemoryPipeline) {
914 if (Cand.Reason > GenericSchedulerBase::RegCritical)
915 Cand.Reason = GenericSchedulerBase::RegCritical;
916
917 return true;
918 }
919
920 TryCand.Reason = GenericSchedulerBase::RegCritical;
921 return true;
922}
923
924bool CandidateHeuristics::tryCriticalResourceDependency(
925 GenericSchedulerBase::SchedCandidate &TryCand,
926 GenericSchedulerBase::SchedCandidate &Cand, SchedBoundary *Zone) const {
927
928 auto HasPrioritySU = [this, &Cand, &TryCand](unsigned ResourceIdx) {
929 const HardwareUnitInfo &HWUI = HWUInfo[ResourceIdx];
930
931 auto CandFlavor = classifyFlavor(MI: *Cand.SU->getInstr(), SII: *SII);
932 auto TryCandFlavor = classifyFlavor(MI: *TryCand.SU->getInstr(), SII: *SII);
933 bool LookDeep = (CandFlavor == InstructionFlavor::DS ||
934 TryCandFlavor == InstructionFlavor::DS) &&
935 HWUI.getType() == InstructionFlavor::WMMA;
936 auto *TargetSU = HWUI.getNextTargetSU(LookDeep);
937
938 // If we do not have a TargetSU for this resource, then it is not critical.
939 if (!TargetSU)
940 return false;
941
942 return true;
943 };
944
945 auto TryEnablesResource = [&Cand, &TryCand, this](unsigned ResourceIdx) {
946 const HardwareUnitInfo &HWUI = HWUInfo[ResourceIdx];
947 auto CandFlavor = classifyFlavor(MI: *Cand.SU->getInstr(), SII: *SII);
948
949 // We want to ensure our DS order matches WMMA order.
950 bool LookDeep = CandFlavor == InstructionFlavor::DS &&
951 HWUI.getType() == InstructionFlavor::WMMA;
952 auto *TargetSU = HWUI.getNextTargetSU(LookDeep);
953
954 bool CandEnables =
955 TargetSU != Cand.SU && DAG->IsReachable(SU: TargetSU, TargetSU: Cand.SU);
956 bool TryCandEnables =
957 TargetSU != TryCand.SU && DAG->IsReachable(SU: TargetSU, TargetSU: TryCand.SU);
958
959 if (!CandEnables && !TryCandEnables)
960 return false;
961
962 if (CandEnables && !TryCandEnables) {
963 if (Cand.Reason > GenericSchedulerBase::RegCritical)
964 Cand.Reason = GenericSchedulerBase::RegCritical;
965
966 return true;
967 }
968
969 if (!CandEnables && TryCandEnables) {
970 TryCand.Reason = GenericSchedulerBase::RegCritical;
971 return true;
972 }
973
974 // Both enable, prefer the critical path.
975 unsigned CandHeight = Cand.SU->getHeight();
976 unsigned TryCandHeight = TryCand.SU->getHeight();
977
978 if (CandHeight > TryCandHeight) {
979 if (Cand.Reason > GenericSchedulerBase::RegCritical)
980 Cand.Reason = GenericSchedulerBase::RegCritical;
981
982 return true;
983 }
984
985 if (CandHeight < TryCandHeight) {
986 TryCand.Reason = GenericSchedulerBase::RegCritical;
987 return true;
988 }
989
990 // Same critical path, just prefer original candidate.
991 if (Cand.Reason > GenericSchedulerBase::RegCritical)
992 Cand.Reason = GenericSchedulerBase::RegCritical;
993
994 return true;
995 };
996
997 for (unsigned I = 0; I < HWUInfo.size(); I++) {
998 // If we have encountered a resource that is not critical, then neither
999 // candidate enables a critical resource
1000 if (!HasPrioritySU(I))
1001 continue;
1002
1003 bool Enabled = TryEnablesResource(I);
1004 // If neither has enabled the resource, continue to the next resource
1005 if (Enabled)
1006 return true;
1007 }
1008 return false;
1009}
1010
1011bool CandidateHeuristics::tryCriticalResource(
1012 GenericSchedulerBase::SchedCandidate &TryCand,
1013 GenericSchedulerBase::SchedCandidate &Cand, SchedBoundary *Zone) const {
1014 for (unsigned I = 0; I < HWUInfo.size(); I++) {
1015 const HardwareUnitInfo &HWUI = HWUInfo[I];
1016
1017 bool CandUsesCrit = HWUI.contains(SU: Cand.SU);
1018 bool TryCandUsesCrit = HWUI.contains(SU: TryCand.SU);
1019
1020 if (!CandUsesCrit && !TryCandUsesCrit)
1021 continue;
1022
1023 if (CandUsesCrit != TryCandUsesCrit) {
1024 if (CandUsesCrit) {
1025 if (Cand.Reason > GenericSchedulerBase::RegCritical)
1026 Cand.Reason = GenericSchedulerBase::RegCritical;
1027 return true;
1028 }
1029 TryCand.Reason = GenericSchedulerBase::RegCritical;
1030 return true;
1031 }
1032
1033 // Otherwise, both use the critical resource
1034 // For longer latency InstructionFlavors, we should prioritize first by
1035 // their enablement of critical resources
1036 if (HWUI.getType() == InstructionFlavor::DS) {
1037 if (tryCriticalResourceDependency(TryCand, Cand, Zone))
1038 return true;
1039 }
1040
1041 // Prioritize based on HWUI priorities.
1042 SUnit *Match = HWUI.getHigherPriority(SU: Cand.SU, Other: TryCand.SU);
1043 if (Match) {
1044 if (Match == Cand.SU) {
1045 if (Cand.Reason > GenericSchedulerBase::RegCritical)
1046 Cand.Reason = GenericSchedulerBase::RegCritical;
1047 return true;
1048 }
1049 TryCand.Reason = GenericSchedulerBase::RegCritical;
1050 return true;
1051 }
1052 }
1053
1054 return false;
1055}
1056
1057AMDGPUCoExecSchedStrategy::AMDGPUCoExecSchedStrategy(
1058 const MachineSchedContext *C)
1059 : GCNSchedStrategy(C) {
1060 SchedStages.push_back(Elt: GCNSchedStageID::ILPInitialSchedule);
1061 SchedStages.push_back(Elt: GCNSchedStageID::RewriteMFMAForm);
1062 SchedStages.push_back(Elt: GCNSchedStageID::LiveIntervalRPReschedule);
1063 SchedStages.push_back(Elt: GCNSchedStageID::PreRARematerialize);
1064 // Use more accurate GCN pressure trackers.
1065 UseGCNTrackers = true;
1066
1067 if (!VGPRThresholdPercentOpt.getNumOccurrences())
1068 VGPRThresholdPercent = DefaultCoExecVGPRThresholdPercent;
1069}
1070
1071void AMDGPUCoExecSchedStrategy::initPolicy(MachineBasicBlock::iterator Begin,
1072 MachineBasicBlock::iterator End,
1073 unsigned NumRegionInstrs) {
1074 GCNSchedStrategy::initPolicy(Begin, End, NumRegionInstrs);
1075 if (getPreRADirection() == MISched::BottomUp ||
1076 getPreRADirection() == MISched::Bidirectional)
1077 report_fatal_error(reason: "CoExecSchedStrategy only support TopDown scheduling.");
1078 RegionPolicy.OnlyTopDown = true;
1079 RegionPolicy.OnlyBottomUp = false;
1080 RegionPolicy.ShouldTrackLaneMasks = true;
1081}
1082
1083void AMDGPUCoExecSchedStrategy::initialize(ScheduleDAGMI *DAG) {
1084 // Coexecution scheduling strategy is only done top-down to support new
1085 // resource balancing heuristics.
1086 RegionPolicy.OnlyTopDown = true;
1087 RegionPolicy.OnlyBottomUp = false;
1088
1089 GCNSchedStrategy::initialize(DAG);
1090 Heurs.initialize(SchedDAG: DAG, TargetSchedModel: SchedModel, TRI);
1091
1092 // Replace the default hazard recognizer with our PreRA one so that pre-RA
1093 // scheduling accounts for WMMA co-execution slot constraints. This must
1094 // happen after GCNSchedStrategy::initialize() because
1095 // GenericScheduler::initialize() calls SchedBoundary::reset(), which deletes
1096 // and recreates the hazard recognizer each region.
1097 Top.HazardRec = std::make_unique<GCNHazardRecognizer>(
1098 args&: DAG->MF, args: GCNHazardRecognizer::OperatingMode::PreRA);
1099}
1100
1101void AMDGPUCoExecSchedStrategy::schedNode(SUnit *SU, bool IsTopNode) {
1102 Heurs.updateForScheduling(SU);
1103 GCNSchedStrategy::schedNode(SU, IsTopNode);
1104}
1105
1106SUnit *AMDGPUCoExecSchedStrategy::pickNode(bool &IsTopNode) {
1107 assert(RegionPolicy.OnlyTopDown && !RegionPolicy.OnlyBottomUp &&
1108 "coexec scheduler only supports top-down scheduling");
1109
1110 if (DAG->top() == DAG->bottom()) {
1111 assert(Top.Available.empty() && Top.Pending.empty() &&
1112 Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage");
1113 return nullptr;
1114 }
1115
1116 bool PickedPending = false;
1117 SUnit *SU = nullptr;
1118#ifndef NDEBUG
1119 SchedCandidate *PickedCand = nullptr;
1120#endif
1121 do {
1122 PickedPending = false;
1123 SU = pickOnlyChoice(Zone&: Top);
1124 if (!SU) {
1125 CandPolicy NoPolicy;
1126 TopCand.reset(NewPolicy: NoPolicy);
1127 pickNodeFromQueue(Zone&: Top, ZonePolicy: NoPolicy, RPTracker: DAG->getTopRPTracker(), Cand&: TopCand,
1128 PickedPending, /*IsBottomUp=*/false);
1129 assert(TopCand.Reason != NoCand && "failed to find a candidate");
1130 SU = TopCand.SU;
1131#ifndef NDEBUG
1132 PickedCand = &TopCand;
1133#endif
1134 }
1135 IsTopNode = true;
1136 } while (SU->isScheduled);
1137
1138 LLVM_DEBUG(if (PickedCand) dumpPickSummary(SU, IsTopNode, *PickedCand));
1139
1140 if (PickedPending) {
1141 unsigned ReadyCycle = SU->TopReadyCycle;
1142 unsigned CurrentCycle = Top.getCurrCycle();
1143 if (ReadyCycle > CurrentCycle)
1144 Top.bumpCycle(NextCycle: ReadyCycle);
1145
1146 // checkHazard() does not expose the exact cycle where the hazard clears.
1147 while (Top.checkHazard(SU))
1148 Top.bumpCycle(NextCycle: Top.getCurrCycle() + 1);
1149
1150 Top.releasePending();
1151 }
1152
1153 if (SU->isTopReady())
1154 Top.removeReady(SU);
1155 if (SU->isBottomReady())
1156 Bot.removeReady(SU);
1157
1158 LLVM_DEBUG(dbgs() << "Scheduling " << *SU << " " << *SU->getInstr());
1159
1160 assert(IsTopNode && "coexec scheduler must only schedule from top boundary");
1161 return SU;
1162}
1163
1164void AMDGPUCoExecSchedStrategy::pickNodeFromQueue(
1165 SchedBoundary &Zone, const CandPolicy &ZonePolicy,
1166 const RegPressureTracker &RPTracker, SchedCandidate &Cand,
1167 bool &PickedPending, bool IsBottomUp) {
1168 assert(Zone.isTop() && "coexec scheduler only supports top boundary");
1169 assert(!IsBottomUp && "coexec scheduler only supports top-down scheduling");
1170
1171 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo *>(TRI);
1172 ArrayRef<unsigned> Pressure = RPTracker.getRegSetPressureAtPos();
1173 unsigned SGPRPressure = 0;
1174 unsigned VGPRPressure = 0;
1175 unsigned AGPRPressure = 0;
1176 PickedPending = false;
1177 if (DAG->isTrackingPressure()) {
1178 if (!useGCNTrackers()) {
1179 SGPRPressure = Pressure[AMDGPU::RegisterPressureSets::SReg_32];
1180 VGPRPressure = Pressure[AMDGPU::RegisterPressureSets::VGPR_32];
1181 AGPRPressure = Pressure[AMDGPU::RegisterPressureSets::AGPR_32];
1182 } else {
1183 SGPRPressure = DownwardTracker.getPressure().getSGPRNum();
1184 VGPRPressure = DownwardTracker.getPressure().getArchVGPRNum();
1185 AGPRPressure = DownwardTracker.getPressure().getAGPRNum();
1186 }
1187 }
1188
1189 auto EvaluateQueue = [&](ReadyQueue &Q, bool FromPending) {
1190 for (SUnit *SU : Q) {
1191 SchedCandidate TryCand(ZonePolicy);
1192 initCandidate(Cand&: TryCand, SU, AtTop: Zone.isTop(), RPTracker, SRI, SGPRPressure,
1193 VGPRPressure, AGPRPressure, IsBottomUp);
1194 SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
1195 tryCandidateCoexec(Cand, TryCand, Zone: ZoneArg);
1196 if (TryCand.Reason != NoCand) {
1197 if (TryCand.ResDelta == SchedResourceDelta())
1198 TryCand.initResourceDelta(DAG: Zone.DAG, SchedModel);
1199 LLVM_DEBUG(printCandidateDecision(Cand, TryCand));
1200 PickedPending = FromPending;
1201 Cand.setBest(TryCand);
1202 } else {
1203 LLVM_DEBUG(printCandidateDecision(TryCand, Cand));
1204 }
1205 }
1206 };
1207
1208 LLVM_DEBUG(dbgs() << "Available Q:\n");
1209 EvaluateQueue(Zone.Available, /*FromPending=*/false);
1210
1211 LLVM_DEBUG(dbgs() << "Pending Q:\n");
1212 EvaluateQueue(Zone.Pending, /*FromPending=*/true);
1213}
1214
1215#ifndef NDEBUG
1216void AMDGPUCoExecSchedStrategy::dumpPickSummary(SUnit *SU, bool IsTopNode,
1217 SchedCandidate &Cand) {
1218 const SIInstrInfo *SII = static_cast<const SIInstrInfo *>(DAG->TII);
1219 unsigned Cycle = IsTopNode ? Top.getCurrCycle() : Bot.getCurrCycle();
1220
1221 dbgs() << "=== Pick @ Cycle " << Cycle << " ===\n";
1222
1223 const InstructionFlavor Flavor = classifyFlavor(*SU->getInstr(), *SII);
1224 dbgs() << "Picked: " << *SU << " ";
1225 SU->getInstr()->print(dbgs(), /*IsStandalone=*/true, /*SkipOpers=*/false,
1226 /*SkipDebugLoc=*/true);
1227 dbgs() << " [" << getFlavorName(Flavor) << "]\n";
1228
1229 dbgs() << " Reason: ";
1230 if (LastAMDGPUReason != AMDGPUSchedReason::None)
1231 dbgs() << getReasonName(LastAMDGPUReason);
1232 else if (Cand.Reason != NoCand)
1233 dbgs() << GenericSchedulerBase::getReasonStr(Cand.Reason);
1234 else
1235 dbgs() << "Unknown";
1236 dbgs() << "\n\n";
1237
1238 LastAMDGPUReason = AMDGPUSchedReason::None;
1239}
1240#endif
1241
1242bool AMDGPUCoExecSchedStrategy::tryCandidateCoexec(SchedCandidate &Cand,
1243 SchedCandidate &TryCand,
1244 SchedBoundary *Zone) {
1245 // Initialize the candidate if needed.
1246 if (!Cand.isValid()) {
1247 TryCand.Reason = FirstValid;
1248 return true;
1249 }
1250
1251 // Bias PhysReg Defs and copies to their uses and defined respectively.
1252 if (tryGreater(TryVal: biasPhysReg(SU: TryCand.SU, isTop: TryCand.AtTop),
1253 CandVal: biasPhysReg(SU: Cand.SU, isTop: Cand.AtTop), TryCand, Cand, Reason: PhysReg))
1254 return TryCand.Reason != NoCand;
1255
1256 // Avoid exceeding the target's limit.
1257 if (DAG->isTrackingPressure() &&
1258 tryPressure(TryP: TryCand.RPDelta.Excess, CandP: Cand.RPDelta.Excess, TryCand, Cand,
1259 Reason: RegExcess, TRI, MF: DAG->MF))
1260 return TryCand.Reason != NoCand;
1261
1262 // We only compare a subset of features when comparing nodes between
1263 // Top and Bottom boundary. Some properties are simply incomparable, in many
1264 // other instances we should only override the other boundary if something
1265 // is a clear good pick on one boundary. Skip heuristics that are more
1266 // "tie-breaking" in nature.
1267 bool SameBoundary = Zone != nullptr;
1268 if (SameBoundary) {
1269 // Compare candidates by the stall they would introduce if
1270 // scheduled in the current cycle.
1271 if (Heurs.tryEffectiveStall(TryCand, Cand, Zone&: *Zone)) {
1272 LastAMDGPUReason = AMDGPUSchedReason::Stall;
1273 return TryCand.Reason != NoCand;
1274 }
1275
1276 if (Heurs.tryMemoryPipeline(TryCand, Cand, Zone&: *Zone)) {
1277 LastAMDGPUReason = AMDGPUSchedReason::MemoryPipeline;
1278 return TryCand.Reason != NoCand;
1279 }
1280
1281 Heurs.sortHWUIResources();
1282 if (Heurs.tryCriticalResource(TryCand, Cand, Zone)) {
1283 LastAMDGPUReason = AMDGPUSchedReason::CritResourceBalance;
1284 return TryCand.Reason != NoCand;
1285 }
1286
1287 if (Heurs.tryCriticalResourceDependency(TryCand, Cand, Zone)) {
1288 LastAMDGPUReason = AMDGPUSchedReason::CritResourceDep;
1289 return TryCand.Reason != NoCand;
1290 }
1291 }
1292
1293 // Keep clustered nodes together to encourage downstream peephole
1294 // optimizations which may reduce resource requirements.
1295 //
1296 // This is a best effort to set things up for a post-RA pass. Optimizations
1297 // like generating loads of multiple registers should ideally be done within
1298 // the scheduler pass by combining the loads during DAG postprocessing.
1299 unsigned CandZoneCluster = Cand.AtTop ? TopClusterID : BotClusterID;
1300 unsigned TryCandZoneCluster = TryCand.AtTop ? TopClusterID : BotClusterID;
1301 bool CandIsClusterSucc =
1302 isTheSameCluster(A: CandZoneCluster, B: Cand.SU->ParentClusterIdx);
1303 bool TryCandIsClusterSucc =
1304 isTheSameCluster(A: TryCandZoneCluster, B: TryCand.SU->ParentClusterIdx);
1305
1306 if (tryGreater(TryVal: TryCandIsClusterSucc, CandVal: CandIsClusterSucc, TryCand, Cand,
1307 Reason: Cluster))
1308 return TryCand.Reason != NoCand;
1309
1310 if (SameBoundary) {
1311 // Weak edges are for clustering and other constraints.
1312 if (tryLess(TryVal: getWeakLeft(SU: TryCand.SU, isTop: TryCand.AtTop),
1313 CandVal: getWeakLeft(SU: Cand.SU, isTop: Cand.AtTop), TryCand, Cand, Reason: Weak))
1314 return TryCand.Reason != NoCand;
1315 }
1316
1317 // Avoid increasing the max pressure of the entire region.
1318 if (DAG->isTrackingPressure() &&
1319 tryPressure(TryP: TryCand.RPDelta.CurrentMax, CandP: Cand.RPDelta.CurrentMax, TryCand,
1320 Cand, Reason: RegMax, TRI, MF: DAG->MF))
1321 return TryCand.Reason != NoCand;
1322
1323 if (SameBoundary) {
1324 // Avoid serializing long latency dependence chains.
1325 // For acyclic path limited loops, latency was already checked above.
1326 if (!RegionPolicy.DisableLatencyHeuristic && TryCand.Policy.ReduceLatency &&
1327 !Rem.IsAcyclicLatencyLimited && tryLatency(TryCand, Cand, Zone&: *Zone))
1328 return TryCand.Reason != NoCand;
1329
1330 // Fall through to original instruction order.
1331 if ((Zone->isTop() && TryCand.SU->NodeNum < Cand.SU->NodeNum) ||
1332 (!Zone->isTop() && TryCand.SU->NodeNum > Cand.SU->NodeNum)) {
1333 TryCand.Reason = NodeOrder;
1334 return true;
1335 }
1336 }
1337
1338 return false;
1339}
1340
1341ScheduleDAGInstrs *
1342llvm::createGCNCoExecMachineScheduler(MachineSchedContext *C) {
1343 LLVM_DEBUG(dbgs() << "AMDGPU coexec preRA scheduler selected for "
1344 << C->MF->getName() << '\n');
1345 ScheduleDAGMILive *DAG = new GCNScheduleDAGMILive(
1346 C, std::make_unique<AMDGPUCoExecSchedStrategy>(args&: C));
1347 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
1348 DAG->addMutation(Mutation: createAMDGPUBarrierLatencyDAGMutation(MF: C->MF));
1349 return DAG;
1350}
1351
1352ScheduleDAGInstrs *
1353llvm::createGCNNoopPostMachineScheduler(MachineSchedContext *C) {
1354 LLVM_DEBUG(dbgs() << "AMDGPU nop postRA scheduler selected for "
1355 << C->MF->getName() << '\n');
1356 return new GCNNoopPostScheduleDAG(C);
1357}
1358