1//===- AMDGPUCoExecSchedStrategy.cpp - CoExec Scheduling Strategy ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Coexecution-focused scheduling strategy for AMDGPU.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUCoExecSchedStrategy.h"
15#include "AMDGPUBarrierLatency.h"
16#include "AMDGPUIGroupLP.h"
17#include "GCNHazardRecognizer.h"
18#include "llvm/Support/Debug.h"
19
20using namespace llvm;
21using namespace llvm::AMDGPU;
22
23#define DEBUG_TYPE "machine-scheduler"
24
25static cl::opt<CarriedLatency> BlockCarriedLatency(
26 "amdgpu-block-carried-latency", cl::Hidden, cl::init(Val: CarriedLatency::Off),
27 cl::desc("Estimate block-carried latency and include it in the effective "
28 "candidate stall cost."),
29 cl::values(
30 clEnumValN(CarriedLatency::Off, "off",
31 "Disabled - do not pad latency."),
32 clEnumValN(CarriedLatency::Fence, "fence",
33 "Only pad latency for memory fence (e.g. those surrounding "
34 "barrier_signal/wait)."),
35 clEnumValN(
36 CarriedLatency::All, "all",
37 "Pad latency for any SU with an incoming ds_load dependency.")));
38
39// Default VGPR threshold percent for coexec scheduler.
40static constexpr unsigned DefaultCoExecVGPRThresholdPercent = 100;
41
42namespace {
43
44// Used to disable post-RA scheduling with function level granularity.
45class GCNNoopPostScheduleDAG final : public ScheduleDAGInstrs {
46public:
47 explicit GCNNoopPostScheduleDAG(MachineSchedContext *C)
48 : ScheduleDAGInstrs(*C->MF, C->MLI, /*RemoveKillFlags=*/true) {}
49
50 // Do nothing.
51 void schedule() override {}
52};
53
54} // namespace
55
56static SUnit *pickOnlyChoice(SchedBoundary &Zone) {
57 // pickOnlyChoice() releases pending instructions and checks for new hazards.
58 SUnit *OnlyChoice = Zone.pickOnlyChoice();
59 if (!Zone.Pending.empty())
60 return nullptr;
61
62 return OnlyChoice;
63}
64
65/// Apply \p ExtraBits to every slot in \p Info starting with \p StartIndex
66/// Used by MFMA co-exec rules, because MFMA co-exec slots are incremental, i.e.
67/// for every slot N it supports all instructions which were supported by the
68/// previous slot N-1 and may support something extra.
69static void allowCoExec(llvm::AMDGPU::CoExecInfo &Info,
70 llvm::AMDGPU::CoExecMaskT ExtraBits,
71 unsigned StartIndex) {
72 for (unsigned Index = StartIndex; Index < Info.TotalWindow; ++Index)
73 Info.Slots[Index].Mask |= ExtraBits;
74}
75
76/// Get co-execution info for a gfx950 MFMA instruction.
77/// The occupancy (cycles until the next MFMA may issue) is expressed as the
78/// first stage carrying the WMMA bit.
79llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
80 using namespace llvm;
81 using namespace llvm::AMDGPU;
82 CoExecInfo Res;
83 for (unsigned I = 0; I < MaxCoExecStages; ++I)
84 Res.Slots[I].Mask = CoExecMask::None;
85
86 // TODO: Implement proper patterns support (for debugging purposes).
87 // Existing pattern letters are WMMA-specific and will probably be confusing
88 // if used as-is for MFMA. Inventing new MFMA-specific letters is an option,
89 // but perhaps the pattern should be instead dynamically reconstructed when
90 // needed by printing specific slots in full instead of a key for them.
91 Res.Pattern = "undefinedundefinedundefinedundefined";
92
93 switch (Opcode) {
94 // 4-cycle occupancy, 8-cycle window.
95 case V_MFMA_F32_16X16X128_F8F6F4_f4_f4_e64:
96 case V_MFMA_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64:
97 case V_MFMA_F32_16X16X128_F8F6F4_f4_f4_gfx940_acd:
98 case V_MFMA_F32_16X16X128_F8F6F4_f4_f4_gfx940_vcd:
99 case V_MFMA_F32_16X16X128_F8F6F4_f4_f6_e64:
100 case V_MFMA_F32_16X16X128_F8F6F4_f4_f6_vgprcd_e64:
101 case V_MFMA_F32_16X16X128_F8F6F4_f4_f6_gfx940_acd:
102 case V_MFMA_F32_16X16X128_F8F6F4_f4_f6_gfx940_vcd:
103 case V_MFMA_F32_16X16X128_F8F6F4_f6_f4_e64:
104 case V_MFMA_F32_16X16X128_F8F6F4_f6_f4_vgprcd_e64:
105 case V_MFMA_F32_16X16X128_F8F6F4_f6_f4_gfx940_acd:
106 case V_MFMA_F32_16X16X128_F8F6F4_f6_f4_gfx940_vcd:
107 case V_MFMA_F32_16X16X128_F8F6F4_f6_f6_e64:
108 case V_MFMA_F32_16X16X128_F8F6F4_f6_f6_vgprcd_e64:
109 case V_MFMA_F32_16X16X128_F8F6F4_f6_f6_gfx940_acd:
110 case V_MFMA_F32_16X16X128_F8F6F4_f6_f6_gfx940_vcd:
111 case V_MFMA_F32_16X16X32_BF16_e64:
112 case V_MFMA_F32_16X16X32_BF16_vgprcd_e64:
113 case V_MFMA_F32_16X16X32_BF16_gfx940_acd:
114 case V_MFMA_F32_16X16X32_BF16_gfx940_vcd:
115 case V_MFMA_I32_16X16X64_I8_e64:
116 case V_MFMA_I32_16X16X64_I8_vgprcd_e64:
117 case V_MFMA_I32_16X16X64_I8_gfx940_acd:
118 case V_MFMA_I32_16X16X64_I8_gfx940_vcd:
119 case V_MFMA_F32_16X16X32_F16_e64:
120 case V_MFMA_F32_16X16X32_F16_vgprcd_e64:
121 case V_MFMA_F32_16X16X32_F16_gfx940_acd:
122 case V_MFMA_F32_16X16X32_F16_gfx940_vcd:
123 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64:
124 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64:
125 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_gfx940_acd:
126 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_gfx940_vcd:
127 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6_e64:
128 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6_vgprcd_e64:
129 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6_gfx940_acd:
130 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6_gfx940_vcd:
131 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4_e64:
132 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4_vgprcd_e64:
133 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4_gfx940_acd:
134 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4_gfx940_vcd:
135 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_e64:
136 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_vgprcd_e64:
137 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_gfx940_acd:
138 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_gfx940_vcd:
139 // GFX9 Shader Programming Guide lists those SMFMAC separately, but for
140 // intended purposes here all those instructions are the same. This comment
141 // is to simplify reverse mapping to the SPG.
142 case V_SMFMAC_F32_16X16X64_BF16_e64:
143 case V_SMFMAC_F32_16X16X64_BF16_gfx940:
144 case V_SMFMAC_I32_16X16X128_I8_e64:
145 case V_SMFMAC_I32_16X16X128_I8_gfx940:
146 case V_SMFMAC_F32_16X16X128_BF8_BF8_e64:
147 case V_SMFMAC_F32_16X16X128_BF8_BF8_gfx940:
148 case V_SMFMAC_F32_16X16X128_BF8_FP8_e64:
149 case V_SMFMAC_F32_16X16X128_BF8_FP8_gfx940:
150 case V_SMFMAC_F32_16X16X128_FP8_BF8_e64:
151 case V_SMFMAC_F32_16X16X128_FP8_BF8_gfx940:
152 case V_SMFMAC_F32_16X16X128_FP8_FP8_e64:
153 case V_SMFMAC_F32_16X16X128_FP8_FP8_gfx940:
154 case V_SMFMAC_F32_16X16X64_F16_e64:
155 case V_SMFMAC_F32_16X16X64_F16_gfx940:
156 Res.TotalWindow = 8;
157 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
158 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, StartIndex: 2);
159 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 4);
160 return Res;
161
162 // 8-cycle occupancy, 12-cycle window.
163 case V_MFMA_F32_16X16X128_F8F6F4_f4_f8_e64:
164 case V_MFMA_F32_16X16X128_F8F6F4_f4_f8_vgprcd_e64:
165 case V_MFMA_F32_16X16X128_F8F6F4_f4_f8_gfx940_acd:
166 case V_MFMA_F32_16X16X128_F8F6F4_f4_f8_gfx940_vcd:
167 case V_MFMA_F32_16X16X128_F8F6F4_f6_f8_e64:
168 case V_MFMA_F32_16X16X128_F8F6F4_f6_f8_vgprcd_e64:
169 case V_MFMA_F32_16X16X128_F8F6F4_f6_f8_gfx940_acd:
170 case V_MFMA_F32_16X16X128_F8F6F4_f6_f8_gfx940_vcd:
171 case V_MFMA_F32_16X16X128_F8F6F4_f8_f4_e64:
172 case V_MFMA_F32_16X16X128_F8F6F4_f8_f4_vgprcd_e64:
173 case V_MFMA_F32_16X16X128_F8F6F4_f8_f4_gfx940_acd:
174 case V_MFMA_F32_16X16X128_F8F6F4_f8_f4_gfx940_vcd:
175 case V_MFMA_F32_16X16X128_F8F6F4_f8_f6_e64:
176 case V_MFMA_F32_16X16X128_F8F6F4_f8_f6_vgprcd_e64:
177 case V_MFMA_F32_16X16X128_F8F6F4_f8_f6_gfx940_acd:
178 case V_MFMA_F32_16X16X128_F8F6F4_f8_f6_gfx940_vcd:
179 case V_MFMA_F32_16X16X128_F8F6F4_f8_f8_e64:
180 case V_MFMA_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64:
181 case V_MFMA_F32_16X16X128_F8F6F4_f8_f8_gfx940_acd:
182 case V_MFMA_F32_16X16X128_F8F6F4_f8_f8_gfx940_vcd:
183 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f8_e64:
184 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f8_vgprcd_e64:
185 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f8_gfx940_acd:
186 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f8_gfx940_vcd:
187 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f8_e64:
188 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f8_vgprcd_e64:
189 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f8_gfx940_acd:
190 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f8_gfx940_vcd:
191 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4_e64:
192 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4_vgprcd_e64:
193 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4_gfx940_acd:
194 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4_gfx940_vcd:
195 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6_e64:
196 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6_vgprcd_e64:
197 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6_gfx940_acd:
198 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6_gfx940_vcd:
199 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_e64:
200 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64:
201 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_gfx940_acd:
202 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_gfx940_vcd:
203 Res.TotalWindow = 12;
204 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
205 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VMEM, StartIndex: 2);
206 allowCoExec(Info&: Res, ExtraBits: CoExecMask::VALU, StartIndex: 3);
207 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 8);
208 return Res;
209
210 // 4-cycle occupancy, 8-cycle window.
211 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_e64:
212 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_mac_e64:
213 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_mac_vgprcd_e64:
214 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_vgprcd_e64:
215 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_gfx940_acd:
216 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_gfx940_vcd:
217 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_e64:
218 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_mac_e64:
219 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_mac_vgprcd_e64:
220 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_vgprcd_e64:
221 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_gfx940_acd:
222 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_gfx940_vcd:
223 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_e64:
224 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_mac_e64:
225 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_mac_vgprcd_e64:
226 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_vgprcd_e64:
227 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_gfx940_acd:
228 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_gfx940_vcd:
229 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_e64:
230 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_mac_e64:
231 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_mac_vgprcd_e64:
232 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_vgprcd_e64:
233 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_gfx940_acd:
234 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_gfx940_vcd:
235 case V_MFMA_F32_32X32X16_BF16_e64:
236 case V_MFMA_F32_32X32X16_BF16_mac_e64:
237 case V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64:
238 case V_MFMA_F32_32X32X16_BF16_vgprcd_e64:
239 case V_MFMA_F32_32X32X16_BF16_gfx940_acd:
240 case V_MFMA_F32_32X32X16_BF16_gfx940_vcd:
241 case V_MFMA_I32_32X32X32_I8_e64:
242 case V_MFMA_I32_32X32X32_I8_mac_e64:
243 case V_MFMA_I32_32X32X32_I8_mac_vgprcd_e64:
244 case V_MFMA_I32_32X32X32_I8_vgprcd_e64:
245 case V_MFMA_I32_32X32X32_I8_gfx940_acd:
246 case V_MFMA_I32_32X32X32_I8_gfx940_vcd:
247 case V_MFMA_F32_32X32X16_F16_e64:
248 case V_MFMA_F32_32X32X16_F16_mac_e64:
249 case V_MFMA_F32_32X32X16_F16_mac_vgprcd_e64:
250 case V_MFMA_F32_32X32X16_F16_vgprcd_e64:
251 case V_MFMA_F32_32X32X16_F16_gfx940_acd:
252 case V_MFMA_F32_32X32X16_F16_gfx940_vcd:
253 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_gfx940_acd:
254 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_gfx940_vcd:
255 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_gfx940_acd:
256 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_gfx940_vcd:
257 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_gfx940_acd:
258 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_gfx940_vcd:
259 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_gfx940_acd:
260 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_gfx940_vcd:
261 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_e64:
262 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_e64:
263 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_e64:
264 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_e64:
265 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_vgprcd_e64:
266 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_vgprcd_e64:
267 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_vgprcd_e64:
268 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_vgprcd_e64:
269 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_mac_e64:
270 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_mac_e64:
271 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_mac_e64:
272 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_mac_e64:
273 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_mac_vgprcd_e64:
274 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_mac_vgprcd_e64:
275 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_mac_vgprcd_e64:
276 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_mac_vgprcd_e64:
277 Res.TotalWindow = 8;
278 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
279 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, StartIndex: 2);
280 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 4);
281 return Res;
282
283 // 16-cycle occupancy, 20-cycle window.
284 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_e64:
285 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_mac_e64:
286 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_mac_vgprcd_e64:
287 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_vgprcd_e64:
288 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_gfx940_acd:
289 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_gfx940_vcd:
290 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_e64:
291 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_mac_e64:
292 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_mac_vgprcd_e64:
293 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_vgprcd_e64:
294 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_gfx940_acd:
295 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_gfx940_vcd:
296 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_e64:
297 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_mac_e64:
298 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_mac_vgprcd_e64:
299 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_vgprcd_e64:
300 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_gfx940_acd:
301 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_gfx940_vcd:
302 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_e64:
303 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_mac_e64:
304 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_mac_vgprcd_e64:
305 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_vgprcd_e64:
306 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_gfx940_acd:
307 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_gfx940_vcd:
308 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_e64:
309 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_e64:
310 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64:
311 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_vgprcd_e64:
312 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_gfx940_acd:
313 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_gfx940_vcd:
314 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_e64:
315 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_e64:
316 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_e64:
317 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_e64:
318 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_e64:
319 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_vgprcd_e64:
320 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_vgprcd_e64:
321 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_vgprcd_e64:
322 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_vgprcd_e64:
323 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_vgprcd_e64:
324 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_mac_e64:
325 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_mac_e64:
326 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_mac_e64:
327 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_mac_e64:
328 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_mac_e64:
329 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_mac_vgprcd_e64:
330 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_mac_vgprcd_e64:
331 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_mac_vgprcd_e64:
332 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_mac_vgprcd_e64:
333 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64:
334 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_gfx940_acd:
335 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_gfx940_vcd:
336 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_gfx940_acd:
337 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_gfx940_vcd:
338 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_gfx940_acd:
339 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_gfx940_vcd:
340 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_gfx940_acd:
341 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_gfx940_vcd:
342 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_gfx940_acd:
343 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_gfx940_vcd:
344 Res.TotalWindow = 20;
345 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
346 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VMEM, StartIndex: 2);
347 allowCoExec(Info&: Res, ExtraBits: CoExecMask::VALU, StartIndex: 3);
348 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 16);
349 return Res;
350
351 // 9-cycle occupancy, 12-cycle window.
352 case V_SMFMAC_F32_32X32X32_BF16_e64:
353 case V_SMFMAC_F32_32X32X32_BF16_gfx940:
354 case V_SMFMAC_I32_32X32X64_I8_e64:
355 case V_SMFMAC_I32_32X32X64_I8_gfx940:
356 case V_SMFMAC_F32_32X32X64_BF8_BF8_e64:
357 case V_SMFMAC_F32_32X32X64_BF8_BF8_gfx940:
358 case V_SMFMAC_F32_32X32X64_BF8_FP8_e64:
359 case V_SMFMAC_F32_32X32X64_BF8_FP8_gfx940:
360 case V_SMFMAC_F32_32X32X64_FP8_BF8_e64:
361 case V_SMFMAC_F32_32X32X64_FP8_BF8_gfx940:
362 case V_SMFMAC_F32_32X32X64_FP8_FP8_e64:
363 case V_SMFMAC_F32_32X32X64_FP8_FP8_gfx940:
364 case V_SMFMAC_F32_32X32X32_F16_e64:
365 case V_SMFMAC_F32_32X32X32_F16_gfx940:
366 Res.TotalWindow = 12;
367 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
368 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, StartIndex: 4);
369 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 9);
370 return Res;
371
372 // 18-cycle occupancy, 19-cycle window.
373 case V_MFMA_F64_16X16X4F64_e64:
374 case V_MFMA_F64_16X16X4F64_mac_e64:
375 case V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
376 case V_MFMA_F64_16X16X4F64_vgprcd_e64:
377 Res.TotalWindow = 19;
378 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::SALU | CoExecMask::VMEM, StartIndex: 0);
379 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA | CoExecMask::VALU, StartIndex: 18);
380 return Res;
381
382 default:
383 // Default fallback: permissive 8-cycle pattern
384 return CoExecInfo::build(UnitOccupancy: 0, TotalWindow: 9, Pattern: "AAAAAAAAA");
385 }
386}
387
388InstructionFlavor llvm::AMDGPU::classifyFlavor(const MachineInstr &MI,
389 const SIInstrInfo &SII) {
390 if (MI.isDebugInstr())
391 return InstructionFlavor::Other;
392
393 unsigned Opc = MI.getOpcode();
394
395 // Check for specific opcodes first.
396 if (Opc == AMDGPU::ATOMIC_FENCE || Opc == AMDGPU::S_BARRIER_WAIT ||
397 Opc == AMDGPU::S_BARRIER_SIGNAL_IMM || SII.isWaitcnt(Opcode: Opc))
398 return InstructionFlavor::Fence;
399
400 if (SII.isLDSDMA(MI))
401 return InstructionFlavor::DMA;
402
403 if (SII.isMFMA(MI)) {
404 // TODO: Consider further sub-classifying this (XDL, XDL2x, S/DGEMM).
405 // GFX9 SPG sub-classifies MFMA into XDL, XDL2x and S/DGEMM, because only
406 // certain sub-classes can be co-executed in certain slots. For now, we
407 // simply treat them all as one to simplify the change and leave the rest
408 // to a follow-up fine-tuning.
409 return InstructionFlavor::WMMA;
410 }
411
412 if (SII.isWMMA(MI) || SII.isSWMMAC(MI))
413 return InstructionFlavor::WMMA;
414
415 if (SII.isTRANS(MI))
416 return InstructionFlavor::TRANS;
417
418 if (SII.isVALU(MI, /*AllowLDSDMA=*/false)) {
419 if (SII.getBlockingCycles(MI) > 1)
420 return InstructionFlavor::MultiCycleVALU;
421
422 return InstructionFlavor::SingleCycleVALU;
423 }
424
425 if (SII.isSMRD(MI))
426 return InstructionFlavor::SMEM;
427
428 if (SII.isDS(MI))
429 return InstructionFlavor::DS;
430
431 if (SII.isVMEM(MI))
432 return InstructionFlavor::VMEM;
433
434 if (SII.isSALU(MI))
435 return InstructionFlavor::SALU;
436
437 return InstructionFlavor::Other;
438}
439
440SUnit *HardwareUnitInfo::getNextTargetSU(bool LookDeep) const {
441 for (SUnit *PrioritySU : PrioritySUs) {
442 if (!PrioritySU->isTopReady())
443 return PrioritySU;
444 }
445
446 if (!LookDeep)
447 return nullptr;
448
449 unsigned MinDepth = std::numeric_limits<unsigned int>::max();
450 SUnit *TargetSU = nullptr;
451 for (auto *SU : AllSUs) {
452 if (SU->isScheduled)
453 continue;
454
455 if (SU->isTopReady())
456 continue;
457
458 if (SU->getDepth() < MinDepth) {
459 MinDepth = SU->getDepth();
460 TargetSU = SU;
461 }
462 }
463 return TargetSU;
464}
465
466void HardwareUnitInfo::insert(SUnit *SU, unsigned BlockingCycles) {
467 if (!AllSUs.insert(X: SU))
468 llvm_unreachable("HardwareUnit already contains SU!");
469
470 TotalCycles += BlockingCycles;
471
472 if (PrioritySUs.empty()) {
473 PrioritySUs.insert(X: SU);
474 return;
475 }
476 unsigned SUDepth = SU->getDepth();
477 unsigned CurrDepth = (*PrioritySUs.begin())->getDepth();
478 if (SUDepth > CurrDepth)
479 return;
480
481 if (SUDepth == CurrDepth) {
482 PrioritySUs.insert(X: SU);
483 return;
484 }
485
486 // SU is lower depth and should be prioritized.
487 PrioritySUs.clear();
488 PrioritySUs.insert(X: SU);
489}
490
491void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles) {
492 // We may want to ignore some HWUIs (e.g. InstructionFlavor::Other). To do so,
493 // we just clear the HWUI. However, we still have instructions which map to
494 // this HWUI. Don't bother managing the state for these HWUI.
495 if (TotalCycles == 0)
496 return;
497
498 ScheduledSUs.push_back(Elt: SU);
499 AllSUs.remove(X: SU);
500 PrioritySUs.remove(X: SU);
501
502 // BufferSize 0 is unlimited, while size 1 has no parallel buffering. In
503 // either case, each SU uses the HardwareUnit for BlockingCycles.
504 if (BufferSize <= 1 || (ScheduledSUs.size() % BufferSize == 0))
505 TotalCycles -= std::min(a: TotalCycles, b: BlockingCycles);
506
507 if (AllSUs.empty())
508 return;
509 if (PrioritySUs.empty()) {
510 for (auto SU : AllSUs) {
511 if (PrioritySUs.empty()) {
512 PrioritySUs.insert(X: SU);
513 continue;
514 }
515 unsigned SUDepth = SU->getDepth();
516 unsigned CurrDepth = (*PrioritySUs.begin())->getDepth();
517 if (SUDepth > CurrDepth)
518 continue;
519
520 if (SUDepth == CurrDepth) {
521 PrioritySUs.insert(X: SU);
522 continue;
523 }
524
525 // SU is lower depth and should be prioritized.
526 PrioritySUs.clear();
527 PrioritySUs.insert(X: SU);
528 }
529 }
530}
531
532void HardwareUnitInfo::finalizeCycles() {
533 if (BufferSize == 0 || AllSUs.empty())
534 return;
535
536 // We estimate the amount of cycles it takes to free up a slot in the buffer
537 // as the average cycles per SU.
538 BufferCycles = TotalCycles / AllSUs.size();
539 // A single-entry buffer does not reduce TotalCycles.
540 if (BufferSize == 1)
541 return;
542
543 // The TotalCycles is normalized against the BufferSize.
544 // This provides an estimate of the TotalCycles which is not always accurate
545 // -- particularly in cases where we have fewer instructions than the
546 // BufferSize. For example, if we have 2 instructions which each take 50
547 // cycles and a BufferSize of 16, then a TotalCycles of 51 cycles would be
548 // somewhat accurate. This normalization calculates TotalCycles as 6. However,
549 // if we have 64 of these instructions, our normalized estimate of 200 is more
550 // reasonable, given the more accurate measure is 264. Having a completely
551 // accurate measure is not very important, since this metric is mainly used to
552 // compare the relative demand per HardwareUnit across the region. The simpler
553 // estimate makes managing the metric incrementally during scheduling much
554 // simpler.
555 TotalCycles /= BufferSize;
556}
557
558HardwareUnitInfo *
559CandidateHeuristics::getHWUIFromFlavor(InstructionFlavor Flavor) {
560 for (HardwareUnitInfo &HWUICand : HWUInfo) {
561 if (HWUICand.getType() == Flavor) {
562 return &HWUICand;
563 }
564 }
565 return nullptr;
566}
567
568unsigned CandidateHeuristics::getMaxBlockingCycles(const MCSchedClassDesc *SC,
569 const MachineInstr *MI) {
570 unsigned ReleaseAtCycle = 0;
571 for (TargetSchedModel::ProcResIter PI = SchedModel->getWriteProcResBegin(SC),
572 PE = SchedModel->getWriteProcResEnd(SC);
573 PI != PE; ++PI) {
574 ReleaseAtCycle =
575 std::max(a: ReleaseAtCycle, b: static_cast<unsigned>(PI->ReleaseAtCycle));
576 }
577 ReleaseAtCycle = std::max(a: ReleaseAtCycle, b: SII->getBlockingCycles(MI: *MI));
578 return ReleaseAtCycle;
579}
580
581unsigned CandidateHeuristics::getHWUICyclesForMI(MachineInstr *MI) {
582 assert(SchedModel && SchedModel->hasInstrSchedModel());
583 if (MI->mayLoadOrStore())
584 return SchedModel->computeInstrLatency(MI, UseDefaultDefLatency: false);
585 return getMaxBlockingCycles(SC: SchedModel->resolveSchedClass(MI), MI);
586}
587
588void CandidateHeuristics::updateForScheduling(SUnit *SU) {
589 MachineInstr *MI = SU->getInstr();
590 HardwareUnitInfo *HWUI = getHWUIFromFlavor(Flavor: classifyFlavor(MI: *MI, SII: *SII));
591 assert(HWUI);
592 HWUI->markScheduled(SU, BlockingCycles: getHWUICyclesForMI(MI));
593}
594
595void CandidateHeuristics::initialize(ScheduleDAGMI *SchedDAG,
596 const TargetSchedModel *TargetSchedModel,
597 const TargetRegisterInfo *TRI,
598 const MachineLoopInfo *LoopInfo) {
599 DAG = SchedDAG;
600 SchedModel = TargetSchedModel;
601 MLI = LoopInfo;
602 assert(SchedModel && SchedModel->hasInstrSchedModel());
603
604 SRI = static_cast<const SIRegisterInfo *>(TRI);
605 SII = static_cast<const SIInstrInfo *>(DAG->TII);
606
607 HWUInfo.resize(N: static_cast<int>(InstructionFlavor::NUM_FLAVORS));
608
609 for (unsigned I = 0; I < HWUInfo.size(); I++) {
610 HWUInfo[I].reset();
611 HWUInfo[I].setType(I);
612 }
613
614 HWUInfo[static_cast<int>(InstructionFlavor::WMMA)].setProducesCoexecWindow(
615 true);
616 HWUInfo[static_cast<int>(InstructionFlavor::MultiCycleVALU)]
617 .setProducesCoexecWindow(true);
618 HWUInfo[static_cast<int>(InstructionFlavor::TRANS)].setProducesCoexecWindow(
619 true);
620 HWUInfo[static_cast<int>(InstructionFlavor::DS)].setBufferSize(
621 DefaultBufferSizes::DS);
622
623 collectRegionSummary();
624}
625
626unsigned CandidateHeuristics::getCarriedLatency(SUnit *SU) {
627 if (RegionCarriedLatency == CarriedLatency::Off)
628 return 0;
629
630 MachineInstr *MI = SU->getInstr();
631 unsigned CarriedLatency = 0;
632 const InstructionFlavor Flavor = classifyFlavor(MI: *MI, SII: *SII);
633 if (Flavor == InstructionFlavor::Fence) {
634 MachineBasicBlock *MBB = MI->getParent();
635 // Scan each direct predecessor back to its nearest Fence or block start for
636 // DS instructions.
637 for (auto PredMBB : MBB->predecessors()) {
638 auto I = PredMBB->rbegin();
639 auto E = PredMBB->rend();
640 for (; I != E; I++) {
641 const InstructionFlavor ItFlavor = classifyFlavor(MI: *I, SII: *SII);
642 if (ItFlavor == InstructionFlavor::Fence)
643 break;
644
645 // Found carried latency.
646 if (ItFlavor == InstructionFlavor::DS)
647 CarriedLatency = std::max(a: CarriedLatency, b: getHWUICyclesForMI(MI: &*I));
648 }
649 }
650 }
651
652 if (RegionCarriedLatency == CarriedLatency::Fence)
653 return CarriedLatency;
654
655 for (MachineOperand &Op : MI->all_uses()) {
656 auto Reg = Op.getReg();
657 if (!Reg.isVirtual())
658 continue;
659
660 for (MachineInstr &Def : DAG->MRI.def_instructions(Reg)) {
661 // We don't have the proper modelling to accurately measure all carried
662 // latency. Just try to measure carried latency for long latency loads to
663 // avoid long stalls.
664 if (!Def.mayLoad())
665 continue;
666
667 unsigned Latency = getHWUICyclesForMI(MI: &Def);
668
669 // Load is carried across block.
670 if (Def.getParent() != MI->getParent()) {
671 bool FoundUseInDefBlock = false;
672 for (MachineInstr &Use : DAG->MRI.use_nodbg_instructions(Reg)) {
673 if (Use.getParent() != Def.getParent())
674 continue;
675
676 SlotIndex DefIdx = DAG->getLIS()->getInstructionIndex(Instr: Def);
677 SlotIndex UseIdx = DAG->getLIS()->getInstructionIndex(Instr: Use);
678 // We have a use of this load in the def block that occurs after the
679 // load. In this case we must wait for the load in the def block, and
680 // we do not have any carried latency from this load.
681 if (SlotIndex::isEarlierInstr(A: DefIdx, B: UseIdx)) {
682 FoundUseInDefBlock = true;
683 break;
684 }
685 }
686 if (!FoundUseInDefBlock)
687 CarriedLatency = std::max(a: Latency, b: CarriedLatency);
688
689 continue;
690 }
691
692 assert(Def.getParent() == MI->getParent());
693 // Load is in the same block.
694 SlotIndex LoadIdx = DAG->getLIS()->getInstructionIndex(Instr: Def);
695 SlotIndex UseIdx = DAG->getLIS()->getInstructionIndex(Instr: *MI);
696 // The load occurs after this use -- the latency is carried across loop
697 // backedge.
698 if (SlotIndex::isEarlierInstr(A: UseIdx, B: LoadIdx))
699 CarriedLatency = std::max(a: Latency, b: CarriedLatency);
700 }
701 }
702 return CarriedLatency;
703}
704
705void CandidateHeuristics::collectRegionSummary() {
706 CarriedLatencies.clear();
707 RegionCarriedLatency = CarriedLatency::Off;
708
709 if (!SchedModel || !SchedModel->hasInstrSchedModel())
710 return;
711
712 // In some cases, we may end up with loads at the end of a predecssor block.
713 // In these cases, it is preferable to defer scheduling a fence which
714 // corresponds with a waitcnt for those loads until the latency of the load
715 // has cleared. Unfortunately, there is no reliable way to determine whether
716 // or not a predecessor block will end up scheduling loads at the end. Here,
717 // we inspect the dependency structure to define a rough heuristic: if we
718 // must schedule ds_loads after wmma, then we carry the latency of ds_loads
719 // to successor block fences. Currently this only checks for the loop-carried
720 // case, as this will generally have the largest impact on performance.
721 // TODO: 1. extend to different memory instructions, 2. teach carried
722 // latencies about fence legalization.
723 auto mustHaveDSAfter = [this]() {
724 if (!DAG->SUnits.size())
725 return false;
726
727 MachineBasicBlock *MBB = DAG->begin()->getParent();
728 MachineLoop *Loop = MLI->getLoopFor(BB: MBB);
729 if (!Loop || Loop->getNumBlocks() != 1)
730 return false;
731
732 SmallVector<SUnit *, 16> RegionWMMAs;
733
734 for (auto &SU : DAG->SUnits) {
735 if (!SU.getInstr())
736 continue;
737 MachineInstr *MI = SU.getInstr();
738 const InstructionFlavor Flavor = classifyFlavor(MI: *MI, SII: *SII);
739 if (Flavor == InstructionFlavor::WMMA)
740 RegionWMMAs.push_back(Elt: &SU);
741 }
742
743 bool MustHaveDSAfter = RegionWMMAs.size();
744
745 for (SUnit *SU : RegionWMMAs) {
746 bool HasDSSucc = false;
747 for (auto &Succ : SU->Succs) {
748 if (!Succ.getSUnit()->getInstr())
749 continue;
750 if (classifyFlavor(MI: *Succ.getSUnit()->getInstr(), SII: *SII) ==
751 InstructionFlavor::DS) {
752 HasDSSucc = true;
753 break;
754 }
755 }
756
757 if (!HasDSSucc) {
758 MustHaveDSAfter = false;
759 break;
760 }
761 }
762
763 return MustHaveDSAfter;
764 };
765
766 RegionCarriedLatency = BlockCarriedLatency.getNumOccurrences()
767 ? BlockCarriedLatency
768 : mustHaveDSAfter() ? CarriedLatency::Fence
769 : CarriedLatency::Off;
770
771 for (auto &SU : DAG->SUnits) {
772 MachineInstr *MI = SU.getInstr();
773 const InstructionFlavor Flavor = classifyFlavor(MI: *MI, SII: *SII);
774 HWUInfo[static_cast<int>(Flavor)].insert(SU: &SU, BlockingCycles: getHWUICyclesForMI(MI));
775 unsigned CarriedLatency = getCarriedLatency(SU: &SU);
776 if (CarriedLatency)
777 CarriedLatencies[MI] = CarriedLatency;
778 }
779
780 for (auto &HWUI : HWUInfo)
781 HWUI.finalizeCycles();
782
783 LLVM_DEBUG(dumpRegionSummary());
784}
785
786void CandidateHeuristics::dumpRegionSummary() {
787 MachineBasicBlock *BB = DAG->begin()->getParent();
788 dbgs() << "\n=== Region: " << DAG->MF.getName() << " BB" << BB->getNumber()
789 << " (" << DAG->SUnits.size() << " SUs) ===\n";
790
791 dbgs() << "\nHWUI Resource Pressure:\n";
792 for (auto &HWUI : HWUInfo) {
793 if (HWUI.getTotalCycles() == 0)
794 continue;
795
796 StringRef Name = getFlavorName(F: HWUI.getType());
797 dbgs() << " " << Name << ": " << HWUI.getTotalCycles() << " cycles, "
798 << HWUI.size() << " instrs\n";
799 }
800 dbgs() << "\n";
801}
802
803void CandidateHeuristics::sortHWUIResources() {
804 // Highest priority should be first.
805 llvm::sort(C&: HWUInfo, Comp: [](HardwareUnitInfo &A, HardwareUnitInfo &B) {
806 // Prefer CoexecWindow producers
807 if (A.producesCoexecWindow() != B.producesCoexecWindow())
808 return A.producesCoexecWindow();
809
810 // Prefer more demanded resources
811 if (A.getTotalCycles() != B.getTotalCycles())
812 return A.getTotalCycles() > B.getTotalCycles();
813
814 // In ties -- prefer the resource with more instructions
815 if (A.size() != B.size())
816 return A.size() < B.size();
817
818 // Default to Flavor order
819 return static_cast<unsigned>(A.getType()) <
820 static_cast<unsigned>(B.getType());
821 });
822}
823
824unsigned CandidateHeuristics::getStructuralStallCycles(SchedBoundary &Zone,
825 SUnit *SU) {
826 assert(Zone.isTop() && "effective stall comparison requires top boundary");
827 if (!SU)
828 return 0;
829
830 MachineInstr *MI = SU->getInstr();
831 unsigned CurrCycle = Zone.getCurrCycle();
832 unsigned Stall = 0;
833
834 // Query SchedModel for resource stalls (unbuffered resources).
835 if (SchedModel->hasInstrSchedModel() && SU->hasReservedResource) {
836 const MCSchedClassDesc *SC = DAG->getSchedClass(SU);
837 for (const MCWriteProcResEntry &PE :
838 make_range(x: SchedModel->getWriteProcResBegin(SC),
839 y: SchedModel->getWriteProcResEnd(SC))) {
840 unsigned NextAvail =
841 Zone.getNextResourceCycle(SC, PIdx: PE.ProcResourceIdx, ReleaseAtCycle: PE.ReleaseAtCycle,
842 AcquireAtCycle: PE.AcquireAtCycle)
843 .first;
844 if (NextAvail > CurrCycle)
845 Stall = std::max(a: Stall, b: NextAvail - CurrCycle);
846 }
847 }
848
849 // Query HazardRecognizer for sequence-dependent hazard penalties.
850 if (Zone.HazardRec && Zone.HazardRec->isEnabled()) {
851 auto *HR = static_cast<GCNHazardRecognizer *>(Zone.HazardRec.get());
852 Stall = std::max(a: Stall, b: HR->getHazardWaitStates(MI));
853 }
854
855 return Stall;
856}
857
858CandidateHeuristics::StallCosts
859CandidateHeuristics::getStallCosts(SUnit *SU, SchedBoundary &Zone) {
860 if (!Zone.isTop())
861 return {};
862
863 auto getBufferFullStalls = [this, &Zone](SUnit *SU) -> unsigned {
864 InstructionFlavor Flavor = classifyFlavor(
865 MI: *SU->getInstr(), SII: *static_cast<const SIInstrInfo *>(DAG->TII));
866 HardwareUnitInfo *HWUI = getHWUIFromFlavor(Flavor);
867
868 // A BufferSize of 0 means "unlimited" buffer, thus we will never fill it.
869 if (HWUI->getBufferSize() == 0)
870 return 0;
871
872 unsigned CurrCycle = Zone.getCurrCycle();
873 unsigned BufferReadyCycle = HWUI->getBufferAvailableCycle(CurrCycle);
874 if (BufferReadyCycle <= CurrCycle)
875 return 0;
876
877 return BufferReadyCycle - CurrCycle;
878 };
879
880 unsigned CurrCycle = Zone.getCurrCycle();
881
882 auto getFenceStalls = [this, &CurrCycle](SUnit *SU) -> unsigned {
883 InstructionFlavor Flavor = classifyFlavor(
884 MI: *SU->getInstr(), SII: *static_cast<const SIInstrInfo *>(DAG->TII));
885
886 if (Flavor != InstructionFlavor::Fence)
887 return 0;
888
889 HardwareUnitInfo *ConsumerHWUI = getHWUIFromFlavor(Flavor);
890 HardwareUnitInfo *ProducerHWUI = getHWUIFromFlavor(Flavor: InstructionFlavor::DS);
891
892 SUnit *LastProducer = ProducerHWUI->getLastScheduledSU();
893 if (!LastProducer)
894 return 0;
895
896 SUnit *LastConsumer = ConsumerHWUI->getLastScheduledSU();
897 unsigned LastConsumerCycle = LastConsumer ? LastConsumer->TopReadyCycle : 0;
898 unsigned LastProducerCycle = LastProducer->TopReadyCycle;
899
900 if (LastProducerCycle < LastConsumerCycle)
901 return 0;
902
903 unsigned FenceStallFinish =
904 LastProducerCycle + getHWUICyclesForMI(MI: LastProducer->getInstr());
905 return FenceStallFinish <= CurrCycle ? 0 : FenceStallFinish - CurrCycle;
906 };
907
908 unsigned ReadyCycle = SU->TopReadyCycle;
909 StallCosts Costs;
910 Costs.Ready = ReadyCycle > CurrCycle ? ReadyCycle - CurrCycle : 0;
911 Costs.Structural = getStructuralStallCycles(Zone, SU);
912 Costs.Latency = Zone.getLatencyStallCycles(SU);
913 unsigned CarriedLatency = CarriedLatencies.lookup_or(Val: SU->getInstr(), Default: 0);
914 Costs.Carried = CarriedLatency > CurrCycle ? CarriedLatency - CurrCycle : 0;
915 Costs.Buffer = getBufferFullStalls(SU);
916 Costs.Fence = getFenceStalls(SU);
917 Costs.Effective = std::max(l: {Costs.Ready, Costs.Structural, Costs.Latency,
918 Costs.Carried, Costs.Buffer, Costs.Fence});
919 return Costs;
920}
921
922bool CandidateHeuristics::tryEffectiveStall(
923 GenericSchedulerBase::SchedCandidate &TryCand,
924 GenericSchedulerBase::SchedCandidate &Cand, SchedBoundary &Zone) {
925 // Only implemented for top-down scheduling
926 if (!Zone.isTop())
927 return 0;
928
929 StallCosts TryCosts = getStallCosts(SU: TryCand.SU, Zone);
930 StallCosts CandCosts = getStallCosts(SU: Cand.SU, Zone);
931
932 LLVM_DEBUG(if (TryCosts.Effective || CandCosts.Effective) {
933 dbgs() << "Effective stalls: try=" << TryCosts.Effective
934 << " (ready=" << TryCosts.Ready << ", struct=" << TryCosts.Structural
935 << ", lat=" << TryCosts.Latency << ", carried=" << TryCosts.Carried
936 << ", buffer=" << TryCosts.Buffer << ", fence=" << TryCosts.Fence
937 << ") cand=" << CandCosts.Effective << " (ready=" << CandCosts.Ready
938 << ", struct=" << CandCosts.Structural
939 << ", lat=" << CandCosts.Latency << ", carried=" << CandCosts.Carried
940 << ", buffer=" << CandCosts.Buffer << ", fence=" << CandCosts.Fence
941 << ")\n";
942 });
943
944 return tryLess(TryVal: TryCosts.Effective, CandVal: CandCosts.Effective, TryCand, Cand,
945 Reason: AMDGPUCoExecSchedStrategy::Stall);
946}
947
948bool CandidateHeuristics::tryMemoryPipeline(
949 GenericSchedulerBase::SchedCandidate &TryCand,
950 GenericSchedulerBase::SchedCandidate &Cand, SchedBoundary &Zone) {
951
952 InstructionFlavor TryFlavor = classifyFlavor(MI: *TryCand.SU->getInstr(), SII: *SII);
953
954 InstructionFlavor CandFlavor = classifyFlavor(MI: *Cand.SU->getInstr(), SII: *SII);
955
956 bool TryIsMemoryPipeline = TryFlavor == InstructionFlavor::DMA ||
957 TryFlavor == InstructionFlavor::Fence;
958 bool CandIsMemoryPipeline = CandFlavor == InstructionFlavor::DMA ||
959 CandFlavor == InstructionFlavor::Fence;
960
961 if (!(TryIsMemoryPipeline || CandIsMemoryPipeline))
962 return false;
963
964 if (TryIsMemoryPipeline)
965 TryIsMemoryPipeline &= getStallCosts(SU: TryCand.SU, Zone).Effective == 0;
966
967 if (CandIsMemoryPipeline)
968 CandIsMemoryPipeline &= getStallCosts(SU: Cand.SU, Zone).Effective == 0;
969
970 if (TryIsMemoryPipeline == CandIsMemoryPipeline)
971 return false;
972
973 if (CandIsMemoryPipeline) {
974 if (Cand.Reason > GenericSchedulerBase::RegCritical)
975 Cand.Reason = GenericSchedulerBase::RegCritical;
976
977 return true;
978 }
979
980 TryCand.Reason = GenericSchedulerBase::RegCritical;
981 return true;
982}
983
984bool CandidateHeuristics::tryCriticalResourceDependency(
985 GenericSchedulerBase::SchedCandidate &TryCand,
986 GenericSchedulerBase::SchedCandidate &Cand, SchedBoundary *Zone) const {
987
988 auto HasPrioritySU = [this, &Cand, &TryCand](unsigned ResourceIdx) {
989 const HardwareUnitInfo &HWUI = HWUInfo[ResourceIdx];
990
991 auto CandFlavor = classifyFlavor(MI: *Cand.SU->getInstr(), SII: *SII);
992 auto TryCandFlavor = classifyFlavor(MI: *TryCand.SU->getInstr(), SII: *SII);
993 bool LookDeep = (CandFlavor == InstructionFlavor::DS ||
994 TryCandFlavor == InstructionFlavor::DS) &&
995 HWUI.getType() == InstructionFlavor::WMMA;
996 auto *TargetSU = HWUI.getNextTargetSU(LookDeep);
997
998 // If we do not have a TargetSU for this resource, then it is not critical.
999 if (!TargetSU)
1000 return false;
1001
1002 return true;
1003 };
1004
1005 auto TryEnablesResource = [&Cand, &TryCand, this](unsigned ResourceIdx) {
1006 const HardwareUnitInfo &HWUI = HWUInfo[ResourceIdx];
1007 auto CandFlavor = classifyFlavor(MI: *Cand.SU->getInstr(), SII: *SII);
1008
1009 // We want to ensure our DS order matches WMMA order.
1010 bool LookDeep = CandFlavor == InstructionFlavor::DS &&
1011 HWUI.getType() == InstructionFlavor::WMMA;
1012 auto *TargetSU = HWUI.getNextTargetSU(LookDeep);
1013
1014 bool CandEnables =
1015 TargetSU != Cand.SU && DAG->IsReachable(SU: TargetSU, TargetSU: Cand.SU);
1016 bool TryCandEnables =
1017 TargetSU != TryCand.SU && DAG->IsReachable(SU: TargetSU, TargetSU: TryCand.SU);
1018
1019 if (!CandEnables && !TryCandEnables)
1020 return false;
1021
1022 if (CandEnables && !TryCandEnables) {
1023 if (Cand.Reason > GenericSchedulerBase::RegCritical)
1024 Cand.Reason = GenericSchedulerBase::RegCritical;
1025
1026 return true;
1027 }
1028
1029 if (!CandEnables && TryCandEnables) {
1030 TryCand.Reason = GenericSchedulerBase::RegCritical;
1031 return true;
1032 }
1033
1034 // Both enable, prefer the critical path.
1035 unsigned CandHeight = Cand.SU->getHeight();
1036 unsigned TryCandHeight = TryCand.SU->getHeight();
1037
1038 if (CandHeight > TryCandHeight) {
1039 if (Cand.Reason > GenericSchedulerBase::RegCritical)
1040 Cand.Reason = GenericSchedulerBase::RegCritical;
1041
1042 return true;
1043 }
1044
1045 if (CandHeight < TryCandHeight) {
1046 TryCand.Reason = GenericSchedulerBase::RegCritical;
1047 return true;
1048 }
1049
1050 // Same critical path, just prefer original candidate.
1051 if (Cand.Reason > GenericSchedulerBase::RegCritical)
1052 Cand.Reason = GenericSchedulerBase::RegCritical;
1053
1054 return true;
1055 };
1056
1057 for (unsigned I = 0; I < HWUInfo.size(); I++) {
1058 // If we have encountered a resource that is not critical, then neither
1059 // candidate enables a critical resource
1060 if (!HasPrioritySU(I))
1061 continue;
1062
1063 bool Enabled = TryEnablesResource(I);
1064 // If neither has enabled the resource, continue to the next resource
1065 if (Enabled)
1066 return true;
1067 }
1068 return false;
1069}
1070
1071bool CandidateHeuristics::tryCriticalResource(
1072 GenericSchedulerBase::SchedCandidate &TryCand,
1073 GenericSchedulerBase::SchedCandidate &Cand, SchedBoundary *Zone) const {
1074 for (unsigned I = 0; I < HWUInfo.size(); I++) {
1075 const HardwareUnitInfo &HWUI = HWUInfo[I];
1076
1077 bool CandUsesCrit = HWUI.contains(SU: Cand.SU);
1078 bool TryCandUsesCrit = HWUI.contains(SU: TryCand.SU);
1079
1080 if (!CandUsesCrit && !TryCandUsesCrit)
1081 continue;
1082
1083 if (CandUsesCrit != TryCandUsesCrit) {
1084 if (CandUsesCrit) {
1085 if (Cand.Reason > GenericSchedulerBase::RegCritical)
1086 Cand.Reason = GenericSchedulerBase::RegCritical;
1087 return true;
1088 }
1089 TryCand.Reason = GenericSchedulerBase::RegCritical;
1090 return true;
1091 }
1092
1093 // Otherwise, both use the critical resource
1094 // For longer latency InstructionFlavors, we should prioritize first by
1095 // their enablement of critical resources
1096 if (HWUI.getType() == InstructionFlavor::DS) {
1097 if (tryCriticalResourceDependency(TryCand, Cand, Zone))
1098 return true;
1099 }
1100
1101 // Prioritize based on HWUI priorities.
1102 SUnit *Match = HWUI.getHigherPriority(SU: Cand.SU, Other: TryCand.SU);
1103 if (Match) {
1104 if (Match == Cand.SU) {
1105 if (Cand.Reason > GenericSchedulerBase::RegCritical)
1106 Cand.Reason = GenericSchedulerBase::RegCritical;
1107 return true;
1108 }
1109 TryCand.Reason = GenericSchedulerBase::RegCritical;
1110 return true;
1111 }
1112 }
1113
1114 return false;
1115}
1116
1117AMDGPUCoExecSchedStrategy::AMDGPUCoExecSchedStrategy(
1118 const MachineSchedContext *C)
1119 : GCNSchedStrategy(C) {
1120 SchedStages.push_back(Elt: GCNSchedStageID::ILPInitialSchedule);
1121 SchedStages.push_back(Elt: GCNSchedStageID::RewriteMFMAForm);
1122 SchedStages.push_back(Elt: GCNSchedStageID::LiveIntervalRPReschedule);
1123 SchedStages.push_back(Elt: GCNSchedStageID::PreRARematerialize);
1124 // Use more accurate GCN pressure trackers.
1125 UseGCNTrackers = true;
1126
1127 if (!VGPRThresholdPercentOpt.getNumOccurrences())
1128 VGPRThresholdPercent = DefaultCoExecVGPRThresholdPercent;
1129}
1130
1131void AMDGPUCoExecSchedStrategy::initPolicy(MachineBasicBlock::iterator Begin,
1132 MachineBasicBlock::iterator End,
1133 unsigned NumRegionInstrs) {
1134 GCNSchedStrategy::initPolicy(Begin, End, NumRegionInstrs);
1135 if (getPreRADirection() == MISched::BottomUp ||
1136 getPreRADirection() == MISched::Bidirectional)
1137 report_fatal_error(reason: "CoExecSchedStrategy only support TopDown scheduling.");
1138 RegionPolicy.OnlyTopDown = true;
1139 RegionPolicy.OnlyBottomUp = false;
1140 RegionPolicy.ShouldTrackLaneMasks = true;
1141}
1142
1143void AMDGPUCoExecSchedStrategy::initialize(ScheduleDAGMI *DAG) {
1144 // Coexecution scheduling strategy is only done top-down to support new
1145 // resource balancing heuristics.
1146 RegionPolicy.OnlyTopDown = true;
1147 RegionPolicy.OnlyBottomUp = false;
1148
1149 GCNSchedStrategy::initialize(DAG);
1150 Heurs.initialize(SchedDAG: DAG, TargetSchedModel: SchedModel, TRI, LoopInfo: Context->MLI);
1151
1152 // Replace the default hazard recognizer with our PreRA one so that pre-RA
1153 // scheduling accounts for WMMA co-execution slot constraints. This must
1154 // happen after GCNSchedStrategy::initialize() because
1155 // GenericScheduler::initialize() calls SchedBoundary::reset(), which deletes
1156 // and recreates the hazard recognizer each region.
1157 Top.HazardRec = std::make_unique<GCNHazardRecognizer>(
1158 args&: DAG->MF, args: GCNHazardRecognizer::OperatingMode::PreRA);
1159}
1160
1161void AMDGPUCoExecSchedStrategy::schedNode(SUnit *SU, bool IsTopNode) {
1162 Heurs.updateForScheduling(SU);
1163 GCNSchedStrategy::schedNode(SU, IsTopNode);
1164}
1165
1166SUnit *AMDGPUCoExecSchedStrategy::pickNode(bool &IsTopNode) {
1167 assert(RegionPolicy.OnlyTopDown && !RegionPolicy.OnlyBottomUp &&
1168 "coexec scheduler only supports top-down scheduling");
1169
1170 if (DAG->top() == DAG->bottom()) {
1171 assert(Top.Available.empty() && Top.Pending.empty() &&
1172 Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage");
1173 return nullptr;
1174 }
1175
1176 bool PickedPending = false;
1177 SUnit *SU = nullptr;
1178#ifndef NDEBUG
1179 SchedCandidate *PickedCand = nullptr;
1180#endif
1181 do {
1182 PickedPending = false;
1183 SU = pickOnlyChoice(Zone&: Top);
1184 if (!SU) {
1185 CandPolicy NoPolicy;
1186 TopCand.reset(NewPolicy: NoPolicy);
1187 pickNodeFromQueue(Zone&: Top, ZonePolicy: NoPolicy, RPTracker: DAG->getTopRPTracker(), Cand&: TopCand,
1188 PickedPending, /*IsBottomUp=*/false);
1189 assert(TopCand.Reason != NoCand && "failed to find a candidate");
1190 SU = TopCand.SU;
1191#ifndef NDEBUG
1192 PickedCand = &TopCand;
1193#endif
1194 }
1195 IsTopNode = true;
1196 } while (SU->isScheduled);
1197
1198 LLVM_DEBUG(if (PickedCand) dumpPickSummary(SU, IsTopNode, *PickedCand));
1199
1200 if (PickedPending) {
1201 unsigned ReadyCycle = SU->TopReadyCycle;
1202 unsigned CurrentCycle = Top.getCurrCycle();
1203 if (ReadyCycle > CurrentCycle)
1204 Top.bumpCycle(NextCycle: ReadyCycle);
1205
1206 // checkHazard() does not expose the exact cycle where the hazard clears.
1207 while (Top.checkHazard(SU))
1208 Top.bumpCycle(NextCycle: Top.getCurrCycle() + 1);
1209
1210 Top.releasePending();
1211 }
1212
1213 if (SU->isTopReady())
1214 Top.removeReady(SU);
1215 if (SU->isBottomReady())
1216 Bot.removeReady(SU);
1217
1218 LLVM_DEBUG(dbgs() << "Scheduling " << *SU << " " << *SU->getInstr());
1219
1220 assert(IsTopNode && "coexec scheduler must only schedule from top boundary");
1221 return SU;
1222}
1223
1224void AMDGPUCoExecSchedStrategy::pickNodeFromQueue(
1225 SchedBoundary &Zone, const CandPolicy &ZonePolicy,
1226 const RegPressureTracker &RPTracker, SchedCandidate &Cand,
1227 bool &PickedPending, bool IsBottomUp) {
1228 assert(Zone.isTop() && "coexec scheduler only supports top boundary");
1229 assert(!IsBottomUp && "coexec scheduler only supports top-down scheduling");
1230
1231 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo *>(TRI);
1232 ArrayRef<unsigned> Pressure = RPTracker.getRegSetPressureAtPos();
1233 unsigned SGPRPressure = 0;
1234 unsigned VGPRPressure = 0;
1235 unsigned AGPRPressure = 0;
1236 PickedPending = false;
1237 if (DAG->isTrackingPressure()) {
1238 if (!useGCNTrackers()) {
1239 SGPRPressure = Pressure[AMDGPU::RegisterPressureSets::SReg_32];
1240 VGPRPressure = Pressure[AMDGPU::RegisterPressureSets::VGPR_32];
1241 AGPRPressure = Pressure[AMDGPU::RegisterPressureSets::AGPR_32];
1242 } else {
1243 SGPRPressure = DownwardTracker.getPressure().getSGPRNum();
1244 VGPRPressure = DownwardTracker.getPressure().getArchVGPRNum();
1245 AGPRPressure = DownwardTracker.getPressure().getAGPRNum();
1246 }
1247 }
1248
1249 auto EvaluateQueue = [&](ReadyQueue &Q, bool FromPending) {
1250 for (SUnit *SU : Q) {
1251 SchedCandidate TryCand(ZonePolicy);
1252 initCandidate(Cand&: TryCand, SU, AtTop: Zone.isTop(), RPTracker, SRI, SGPRPressure,
1253 VGPRPressure, AGPRPressure, IsBottomUp);
1254 SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
1255 tryCandidateCoexec(Cand, TryCand, Zone: ZoneArg);
1256 if (TryCand.Reason != NoCand) {
1257 if (TryCand.ResDelta == SchedResourceDelta())
1258 TryCand.initResourceDelta(DAG: Zone.DAG, SchedModel);
1259 LLVM_DEBUG(printCandidateDecision(Cand, TryCand));
1260 PickedPending = FromPending;
1261 Cand.setBest(TryCand);
1262 } else {
1263 LLVM_DEBUG(printCandidateDecision(TryCand, Cand));
1264 }
1265 }
1266 };
1267
1268 LLVM_DEBUG(dbgs() << "Available Q:\n");
1269 EvaluateQueue(Zone.Available, /*FromPending=*/false);
1270
1271 LLVM_DEBUG(dbgs() << "Pending Q:\n");
1272 EvaluateQueue(Zone.Pending, /*FromPending=*/true);
1273}
1274
1275#ifndef NDEBUG
1276void AMDGPUCoExecSchedStrategy::dumpPickSummary(SUnit *SU, bool IsTopNode,
1277 SchedCandidate &Cand) {
1278 const SIInstrInfo *SII = static_cast<const SIInstrInfo *>(DAG->TII);
1279 unsigned Cycle = IsTopNode ? Top.getCurrCycle() : Bot.getCurrCycle();
1280
1281 dbgs() << "=== Pick @ Cycle " << Cycle << " ===\n";
1282
1283 const InstructionFlavor Flavor = classifyFlavor(*SU->getInstr(), *SII);
1284 dbgs() << "Picked: " << *SU << " ";
1285 SU->getInstr()->print(dbgs(), /*IsStandalone=*/true, /*SkipOpers=*/false,
1286 /*SkipDebugLoc=*/true);
1287 dbgs() << " [" << getFlavorName(Flavor) << "]\n";
1288
1289 dbgs() << " Reason: ";
1290 if (LastAMDGPUReason != AMDGPUSchedReason::None)
1291 dbgs() << getReasonName(LastAMDGPUReason);
1292 else if (Cand.Reason != NoCand)
1293 dbgs() << GenericSchedulerBase::getReasonStr(Cand.Reason);
1294 else
1295 dbgs() << "Unknown";
1296 dbgs() << "\n\n";
1297
1298 LastAMDGPUReason = AMDGPUSchedReason::None;
1299}
1300#endif
1301
1302bool AMDGPUCoExecSchedStrategy::tryCandidateCoexec(SchedCandidate &Cand,
1303 SchedCandidate &TryCand,
1304 SchedBoundary *Zone) {
1305 // Initialize the candidate if needed.
1306 if (!Cand.isValid()) {
1307 TryCand.Reason = FirstValid;
1308 return true;
1309 }
1310
1311 // Bias PhysReg Defs and copies to their uses and defined respectively.
1312 if (tryGreater(TryVal: biasPhysReg(SU: TryCand.SU, isTop: TryCand.AtTop),
1313 CandVal: biasPhysReg(SU: Cand.SU, isTop: Cand.AtTop), TryCand, Cand, Reason: PhysReg))
1314 return TryCand.Reason != NoCand;
1315
1316 // Avoid exceeding the target's limit.
1317 if (DAG->isTrackingPressure() &&
1318 tryPressure(TryP: TryCand.RPDelta.Excess, CandP: Cand.RPDelta.Excess, TryCand, Cand,
1319 Reason: RegExcess, TRI, MF: DAG->MF))
1320 return TryCand.Reason != NoCand;
1321
1322 // We only compare a subset of features when comparing nodes between
1323 // Top and Bottom boundary. Some properties are simply incomparable, in many
1324 // other instances we should only override the other boundary if something
1325 // is a clear good pick on one boundary. Skip heuristics that are more
1326 // "tie-breaking" in nature.
1327 bool SameBoundary = Zone != nullptr;
1328 if (SameBoundary) {
1329 // Compare candidates by the stall they would introduce if
1330 // scheduled in the current cycle.
1331 if (Heurs.tryEffectiveStall(TryCand, Cand, Zone&: *Zone)) {
1332 LastAMDGPUReason = AMDGPUSchedReason::Stall;
1333 return TryCand.Reason != NoCand;
1334 }
1335
1336 if (Heurs.tryMemoryPipeline(TryCand, Cand, Zone&: *Zone)) {
1337 LastAMDGPUReason = AMDGPUSchedReason::MemoryPipeline;
1338 return TryCand.Reason != NoCand;
1339 }
1340
1341 Heurs.sortHWUIResources();
1342 if (Heurs.tryCriticalResource(TryCand, Cand, Zone)) {
1343 LastAMDGPUReason = AMDGPUSchedReason::CritResourceBalance;
1344 return TryCand.Reason != NoCand;
1345 }
1346
1347 if (Heurs.tryCriticalResourceDependency(TryCand, Cand, Zone)) {
1348 LastAMDGPUReason = AMDGPUSchedReason::CritResourceDep;
1349 return TryCand.Reason != NoCand;
1350 }
1351 }
1352
1353 // Keep clustered nodes together to encourage downstream peephole
1354 // optimizations which may reduce resource requirements.
1355 //
1356 // This is a best effort to set things up for a post-RA pass. Optimizations
1357 // like generating loads of multiple registers should ideally be done within
1358 // the scheduler pass by combining the loads during DAG postprocessing.
1359 unsigned CandZoneCluster = Cand.AtTop ? TopClusterID : BotClusterID;
1360 unsigned TryCandZoneCluster = TryCand.AtTop ? TopClusterID : BotClusterID;
1361 bool CandIsClusterSucc =
1362 isTheSameCluster(A: CandZoneCluster, B: Cand.SU->ParentClusterIdx);
1363 bool TryCandIsClusterSucc =
1364 isTheSameCluster(A: TryCandZoneCluster, B: TryCand.SU->ParentClusterIdx);
1365
1366 if (tryGreater(TryVal: TryCandIsClusterSucc, CandVal: CandIsClusterSucc, TryCand, Cand,
1367 Reason: Cluster))
1368 return TryCand.Reason != NoCand;
1369
1370 if (SameBoundary) {
1371 // Weak edges are for clustering and other constraints.
1372 if (tryLess(TryVal: getWeakLeft(SU: TryCand.SU, isTop: TryCand.AtTop),
1373 CandVal: getWeakLeft(SU: Cand.SU, isTop: Cand.AtTop), TryCand, Cand, Reason: Weak))
1374 return TryCand.Reason != NoCand;
1375 }
1376
1377 // Avoid increasing the max pressure of the entire region.
1378 if (DAG->isTrackingPressure() &&
1379 tryPressure(TryP: TryCand.RPDelta.CurrentMax, CandP: Cand.RPDelta.CurrentMax, TryCand,
1380 Cand, Reason: RegMax, TRI, MF: DAG->MF))
1381 return TryCand.Reason != NoCand;
1382
1383 if (SameBoundary) {
1384 // Avoid serializing long latency dependence chains.
1385 // For acyclic path limited loops, latency was already checked above.
1386 if (!RegionPolicy.DisableLatencyHeuristic && TryCand.Policy.ReduceLatency &&
1387 !Rem.IsAcyclicLatencyLimited && tryLatency(TryCand, Cand, Zone&: *Zone))
1388 return TryCand.Reason != NoCand;
1389
1390 // Fall through to original instruction order.
1391 if ((Zone->isTop() && TryCand.SU->NodeNum < Cand.SU->NodeNum) ||
1392 (!Zone->isTop() && TryCand.SU->NodeNum > Cand.SU->NodeNum)) {
1393 TryCand.Reason = NodeOrder;
1394 return true;
1395 }
1396 }
1397
1398 return false;
1399}
1400
1401ScheduleDAGInstrs *
1402llvm::createGCNCoExecMachineScheduler(MachineSchedContext *C) {
1403 LLVM_DEBUG(dbgs() << "AMDGPU coexec preRA scheduler selected for "
1404 << C->MF->getName() << '\n');
1405 ScheduleDAGMILive *DAG = new GCNScheduleDAGMILive(
1406 C, std::make_unique<AMDGPUCoExecSchedStrategy>(args&: C));
1407 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
1408 DAG->addMutation(Mutation: createAMDGPUBarrierLatencyDAGMutation(MF: C->MF));
1409 return DAG;
1410}
1411
1412ScheduleDAGInstrs *
1413llvm::createGCNNoopPostMachineScheduler(MachineSchedContext *C) {
1414 LLVM_DEBUG(dbgs() << "AMDGPU nop postRA scheduler selected for "
1415 << C->MF->getName() << '\n');
1416 return new GCNNoopPostScheduleDAG(C);
1417}
1418