1//===- AMDGPUCoExecSchedStrategy.cpp - CoExec Scheduling Strategy ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Coexecution-focused scheduling strategy for AMDGPU.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUCoExecSchedStrategy.h"
15#include "AMDGPUBarrierLatency.h"
16#include "AMDGPUIGroupLP.h"
17#include "GCNHazardRecognizer.h"
18#include "llvm/Support/Debug.h"
19
20using namespace llvm;
21using namespace llvm::AMDGPU;
22
23#define DEBUG_TYPE "machine-scheduler"
24
25// Default VGPR threshold percent for coexec scheduler.
26static constexpr unsigned DefaultCoExecVGPRThresholdPercent = 100;
27
28namespace {
29
30// Used to disable post-RA scheduling with function level granularity.
31class GCNNoopPostScheduleDAG final : public ScheduleDAGInstrs {
32public:
33 explicit GCNNoopPostScheduleDAG(MachineSchedContext *C)
34 : ScheduleDAGInstrs(*C->MF, C->MLI, /*RemoveKillFlags=*/true) {}
35
36 // Do nothing.
37 void schedule() override {}
38};
39
40} // namespace
41
42static SUnit *pickOnlyChoice(SchedBoundary &Zone) {
43 // pickOnlyChoice() releases pending instructions and checks for new hazards.
44 SUnit *OnlyChoice = Zone.pickOnlyChoice();
45 if (!Zone.Pending.empty())
46 return nullptr;
47
48 return OnlyChoice;
49}
50
51/// Apply \p ExtraBits to every slot in \p Info starting with \p StartIndex
52/// Used by MFMA co-exec rules, because MFMA co-exec slots are incremental, i.e.
53/// for every slot N it supports all instructions which were supported by the
54/// previous slot N-1 and may support something extra.
55static void allowCoExec(llvm::AMDGPU::CoExecInfo &Info,
56 llvm::AMDGPU::CoExecMaskT ExtraBits,
57 unsigned StartIndex) {
58 for (unsigned Index = StartIndex; Index < Info.TotalWindow; ++Index)
59 Info.Slots[Index].Mask |= ExtraBits;
60}
61
62/// Get co-execution info for a gfx950 MFMA instruction.
63/// The occupancy (cycles until the next MFMA may issue) is expressed as the
64/// first stage carrying the WMMA bit.
65llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) {
66 using namespace llvm;
67 using namespace llvm::AMDGPU;
68 CoExecInfo Res;
69 for (unsigned I = 0; I < MaxCoExecStages; ++I)
70 Res.Slots[I].Mask = CoExecMask::None;
71
72 // TODO: Implement proper patterns support (for debugging purposes).
73 // Existing pattern letters are WMMA-specific and will probably be confusing
74 // if used as-is for MFMA. Inventing new MFMA-specific letters is an option,
75 // but perhaps the pattern should be instead dynamically reconstructed when
76 // needed by printing specific slots in full instead of a key for them.
77 Res.Pattern = "undefinedundefinedundefinedundefined";
78
79 switch (Opcode) {
80 // 4-cycle occupancy, 8-cycle window.
81 case V_MFMA_F32_16X16X128_F8F6F4_f4_f4_e64:
82 case V_MFMA_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64:
83 case V_MFMA_F32_16X16X128_F8F6F4_f4_f4_gfx940_acd:
84 case V_MFMA_F32_16X16X128_F8F6F4_f4_f4_gfx940_vcd:
85 case V_MFMA_F32_16X16X128_F8F6F4_f4_f6_e64:
86 case V_MFMA_F32_16X16X128_F8F6F4_f4_f6_vgprcd_e64:
87 case V_MFMA_F32_16X16X128_F8F6F4_f4_f6_gfx940_acd:
88 case V_MFMA_F32_16X16X128_F8F6F4_f4_f6_gfx940_vcd:
89 case V_MFMA_F32_16X16X128_F8F6F4_f6_f4_e64:
90 case V_MFMA_F32_16X16X128_F8F6F4_f6_f4_vgprcd_e64:
91 case V_MFMA_F32_16X16X128_F8F6F4_f6_f4_gfx940_acd:
92 case V_MFMA_F32_16X16X128_F8F6F4_f6_f4_gfx940_vcd:
93 case V_MFMA_F32_16X16X128_F8F6F4_f6_f6_e64:
94 case V_MFMA_F32_16X16X128_F8F6F4_f6_f6_vgprcd_e64:
95 case V_MFMA_F32_16X16X128_F8F6F4_f6_f6_gfx940_acd:
96 case V_MFMA_F32_16X16X128_F8F6F4_f6_f6_gfx940_vcd:
97 case V_MFMA_F32_16X16X32_BF16_e64:
98 case V_MFMA_F32_16X16X32_BF16_vgprcd_e64:
99 case V_MFMA_F32_16X16X32_BF16_gfx940_acd:
100 case V_MFMA_F32_16X16X32_BF16_gfx940_vcd:
101 case V_MFMA_I32_16X16X64_I8_e64:
102 case V_MFMA_I32_16X16X64_I8_vgprcd_e64:
103 case V_MFMA_I32_16X16X64_I8_gfx940_acd:
104 case V_MFMA_I32_16X16X64_I8_gfx940_vcd:
105 case V_MFMA_F32_16X16X32_F16_e64:
106 case V_MFMA_F32_16X16X32_F16_vgprcd_e64:
107 case V_MFMA_F32_16X16X32_F16_gfx940_acd:
108 case V_MFMA_F32_16X16X32_F16_gfx940_vcd:
109 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_e64:
110 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_vgprcd_e64:
111 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_gfx940_acd:
112 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f4_gfx940_vcd:
113 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6_e64:
114 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6_vgprcd_e64:
115 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6_gfx940_acd:
116 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6_gfx940_vcd:
117 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4_e64:
118 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4_vgprcd_e64:
119 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4_gfx940_acd:
120 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4_gfx940_vcd:
121 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_e64:
122 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_vgprcd_e64:
123 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_gfx940_acd:
124 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_gfx940_vcd:
125 // GFX9 Shader Programming Guide lists those SMFMAC separately, but for
126 // intended purposes here all those instructions are the same. This comment
127 // is to simplify reverse mapping to the SPG.
128 case V_SMFMAC_F32_16X16X64_BF16_e64:
129 case V_SMFMAC_F32_16X16X64_BF16_gfx940:
130 case V_SMFMAC_I32_16X16X128_I8_e64:
131 case V_SMFMAC_I32_16X16X128_I8_gfx940:
132 case V_SMFMAC_F32_16X16X128_BF8_BF8_e64:
133 case V_SMFMAC_F32_16X16X128_BF8_BF8_gfx940:
134 case V_SMFMAC_F32_16X16X128_BF8_FP8_e64:
135 case V_SMFMAC_F32_16X16X128_BF8_FP8_gfx940:
136 case V_SMFMAC_F32_16X16X128_FP8_BF8_e64:
137 case V_SMFMAC_F32_16X16X128_FP8_BF8_gfx940:
138 case V_SMFMAC_F32_16X16X128_FP8_FP8_e64:
139 case V_SMFMAC_F32_16X16X128_FP8_FP8_gfx940:
140 case V_SMFMAC_F32_16X16X64_F16_e64:
141 case V_SMFMAC_F32_16X16X64_F16_gfx940:
142 Res.TotalWindow = 8;
143 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
144 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, StartIndex: 2);
145 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 4);
146 return Res;
147
148 // 8-cycle occupancy, 12-cycle window.
149 case V_MFMA_F32_16X16X128_F8F6F4_f4_f8_e64:
150 case V_MFMA_F32_16X16X128_F8F6F4_f4_f8_vgprcd_e64:
151 case V_MFMA_F32_16X16X128_F8F6F4_f4_f8_gfx940_acd:
152 case V_MFMA_F32_16X16X128_F8F6F4_f4_f8_gfx940_vcd:
153 case V_MFMA_F32_16X16X128_F8F6F4_f6_f8_e64:
154 case V_MFMA_F32_16X16X128_F8F6F4_f6_f8_vgprcd_e64:
155 case V_MFMA_F32_16X16X128_F8F6F4_f6_f8_gfx940_acd:
156 case V_MFMA_F32_16X16X128_F8F6F4_f6_f8_gfx940_vcd:
157 case V_MFMA_F32_16X16X128_F8F6F4_f8_f4_e64:
158 case V_MFMA_F32_16X16X128_F8F6F4_f8_f4_vgprcd_e64:
159 case V_MFMA_F32_16X16X128_F8F6F4_f8_f4_gfx940_acd:
160 case V_MFMA_F32_16X16X128_F8F6F4_f8_f4_gfx940_vcd:
161 case V_MFMA_F32_16X16X128_F8F6F4_f8_f6_e64:
162 case V_MFMA_F32_16X16X128_F8F6F4_f8_f6_vgprcd_e64:
163 case V_MFMA_F32_16X16X128_F8F6F4_f8_f6_gfx940_acd:
164 case V_MFMA_F32_16X16X128_F8F6F4_f8_f6_gfx940_vcd:
165 case V_MFMA_F32_16X16X128_F8F6F4_f8_f8_e64:
166 case V_MFMA_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64:
167 case V_MFMA_F32_16X16X128_F8F6F4_f8_f8_gfx940_acd:
168 case V_MFMA_F32_16X16X128_F8F6F4_f8_f8_gfx940_vcd:
169 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f8_e64:
170 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f8_vgprcd_e64:
171 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f8_gfx940_acd:
172 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f8_gfx940_vcd:
173 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f8_e64:
174 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f8_vgprcd_e64:
175 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f8_gfx940_acd:
176 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f8_gfx940_vcd:
177 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4_e64:
178 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4_vgprcd_e64:
179 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4_gfx940_acd:
180 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4_gfx940_vcd:
181 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6_e64:
182 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6_vgprcd_e64:
183 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6_gfx940_acd:
184 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6_gfx940_vcd:
185 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_e64:
186 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_vgprcd_e64:
187 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_gfx940_acd:
188 case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8_gfx940_vcd:
189 Res.TotalWindow = 12;
190 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
191 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VMEM, StartIndex: 2);
192 allowCoExec(Info&: Res, ExtraBits: CoExecMask::VALU, StartIndex: 3);
193 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 8);
194 return Res;
195
196 // 4-cycle occupancy, 8-cycle window.
197 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_e64:
198 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_mac_e64:
199 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_mac_vgprcd_e64:
200 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_vgprcd_e64:
201 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_gfx940_acd:
202 case V_MFMA_F32_32X32X64_F8F6F4_f4_f4_gfx940_vcd:
203 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_e64:
204 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_mac_e64:
205 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_mac_vgprcd_e64:
206 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_vgprcd_e64:
207 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_gfx940_acd:
208 case V_MFMA_F32_32X32X64_F8F6F4_f4_f6_gfx940_vcd:
209 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_e64:
210 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_mac_e64:
211 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_mac_vgprcd_e64:
212 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_vgprcd_e64:
213 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_gfx940_acd:
214 case V_MFMA_F32_32X32X64_F8F6F4_f6_f4_gfx940_vcd:
215 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_e64:
216 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_mac_e64:
217 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_mac_vgprcd_e64:
218 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_vgprcd_e64:
219 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_gfx940_acd:
220 case V_MFMA_F32_32X32X64_F8F6F4_f6_f6_gfx940_vcd:
221 case V_MFMA_F32_32X32X16_BF16_e64:
222 case V_MFMA_F32_32X32X16_BF16_mac_e64:
223 case V_MFMA_F32_32X32X16_BF16_mac_vgprcd_e64:
224 case V_MFMA_F32_32X32X16_BF16_vgprcd_e64:
225 case V_MFMA_F32_32X32X16_BF16_gfx940_acd:
226 case V_MFMA_F32_32X32X16_BF16_gfx940_vcd:
227 case V_MFMA_I32_32X32X32_I8_e64:
228 case V_MFMA_I32_32X32X32_I8_mac_e64:
229 case V_MFMA_I32_32X32X32_I8_mac_vgprcd_e64:
230 case V_MFMA_I32_32X32X32_I8_vgprcd_e64:
231 case V_MFMA_I32_32X32X32_I8_gfx940_acd:
232 case V_MFMA_I32_32X32X32_I8_gfx940_vcd:
233 case V_MFMA_F32_32X32X16_F16_e64:
234 case V_MFMA_F32_32X32X16_F16_mac_e64:
235 case V_MFMA_F32_32X32X16_F16_mac_vgprcd_e64:
236 case V_MFMA_F32_32X32X16_F16_vgprcd_e64:
237 case V_MFMA_F32_32X32X16_F16_gfx940_acd:
238 case V_MFMA_F32_32X32X16_F16_gfx940_vcd:
239 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_gfx940_acd:
240 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_gfx940_vcd:
241 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_gfx940_acd:
242 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_gfx940_vcd:
243 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_gfx940_acd:
244 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_gfx940_vcd:
245 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_gfx940_acd:
246 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_gfx940_vcd:
247 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_e64:
248 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_e64:
249 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_e64:
250 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_e64:
251 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_vgprcd_e64:
252 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_vgprcd_e64:
253 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_vgprcd_e64:
254 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_vgprcd_e64:
255 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_mac_e64:
256 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_mac_e64:
257 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_mac_e64:
258 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_mac_e64:
259 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f4_mac_vgprcd_e64:
260 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6_mac_vgprcd_e64:
261 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4_mac_vgprcd_e64:
262 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6_mac_vgprcd_e64:
263 Res.TotalWindow = 8;
264 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
265 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, StartIndex: 2);
266 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 4);
267 return Res;
268
269 // 16-cycle occupancy, 20-cycle window.
270 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_e64:
271 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_mac_e64:
272 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_mac_vgprcd_e64:
273 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_vgprcd_e64:
274 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_gfx940_acd:
275 case V_MFMA_F32_32X32X64_F8F6F4_f4_f8_gfx940_vcd:
276 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_e64:
277 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_mac_e64:
278 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_mac_vgprcd_e64:
279 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_vgprcd_e64:
280 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_gfx940_acd:
281 case V_MFMA_F32_32X32X64_F8F6F4_f6_f8_gfx940_vcd:
282 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_e64:
283 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_mac_e64:
284 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_mac_vgprcd_e64:
285 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_vgprcd_e64:
286 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_gfx940_acd:
287 case V_MFMA_F32_32X32X64_F8F6F4_f8_f4_gfx940_vcd:
288 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_e64:
289 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_mac_e64:
290 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_mac_vgprcd_e64:
291 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_vgprcd_e64:
292 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_gfx940_acd:
293 case V_MFMA_F32_32X32X64_F8F6F4_f8_f6_gfx940_vcd:
294 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_e64:
295 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_e64:
296 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64:
297 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_vgprcd_e64:
298 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_gfx940_acd:
299 case V_MFMA_F32_32X32X64_F8F6F4_f8_f8_gfx940_vcd:
300 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_e64:
301 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_e64:
302 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_e64:
303 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_e64:
304 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_e64:
305 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_vgprcd_e64:
306 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_vgprcd_e64:
307 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_vgprcd_e64:
308 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_vgprcd_e64:
309 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_vgprcd_e64:
310 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_mac_e64:
311 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_mac_e64:
312 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_mac_e64:
313 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_mac_e64:
314 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_mac_e64:
315 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_mac_vgprcd_e64:
316 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_mac_vgprcd_e64:
317 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_mac_vgprcd_e64:
318 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_mac_vgprcd_e64:
319 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_mac_vgprcd_e64:
320 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_gfx940_acd:
321 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f8_gfx940_vcd:
322 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_gfx940_acd:
323 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f8_gfx940_vcd:
324 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_gfx940_acd:
325 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4_gfx940_vcd:
326 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_gfx940_acd:
327 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6_gfx940_vcd:
328 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_gfx940_acd:
329 case V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8_gfx940_vcd:
330 Res.TotalWindow = 20;
331 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
332 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VMEM, StartIndex: 2);
333 allowCoExec(Info&: Res, ExtraBits: CoExecMask::VALU, StartIndex: 3);
334 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 16);
335 return Res;
336
337 // 9-cycle occupancy, 12-cycle window.
338 case V_SMFMAC_F32_32X32X32_BF16_e64:
339 case V_SMFMAC_F32_32X32X32_BF16_gfx940:
340 case V_SMFMAC_I32_32X32X64_I8_e64:
341 case V_SMFMAC_I32_32X32X64_I8_gfx940:
342 case V_SMFMAC_F32_32X32X64_BF8_BF8_e64:
343 case V_SMFMAC_F32_32X32X64_BF8_BF8_gfx940:
344 case V_SMFMAC_F32_32X32X64_BF8_FP8_e64:
345 case V_SMFMAC_F32_32X32X64_BF8_FP8_gfx940:
346 case V_SMFMAC_F32_32X32X64_FP8_BF8_e64:
347 case V_SMFMAC_F32_32X32X64_FP8_BF8_gfx940:
348 case V_SMFMAC_F32_32X32X64_FP8_FP8_e64:
349 case V_SMFMAC_F32_32X32X64_FP8_FP8_gfx940:
350 case V_SMFMAC_F32_32X32X32_F16_e64:
351 case V_SMFMAC_F32_32X32X32_F16_gfx940:
352 Res.TotalWindow = 12;
353 allowCoExec(Info&: Res, ExtraBits: CoExecMask::SALU, StartIndex: 1);
354 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, StartIndex: 4);
355 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA, StartIndex: 9);
356 return Res;
357
358 // 18-cycle occupancy, 19-cycle window.
359 case V_MFMA_F64_16X16X4F64_e64:
360 case V_MFMA_F64_16X16X4F64_mac_e64:
361 case V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
362 case V_MFMA_F64_16X16X4F64_vgprcd_e64:
363 Res.TotalWindow = 19;
364 allowCoExec(Info&: Res, ExtraBits: CoExecMask::DS | CoExecMask::SALU | CoExecMask::VMEM, StartIndex: 0);
365 allowCoExec(Info&: Res, ExtraBits: CoExecMask::WMMA | CoExecMask::VALU, StartIndex: 18);
366 return Res;
367
368 default:
369 // Default fallback: permissive 8-cycle pattern
370 return CoExecInfo::build(UnitOccupancy: 0, TotalWindow: 9, Pattern: "AAAAAAAAA");
371 }
372}
373
374InstructionFlavor llvm::AMDGPU::classifyFlavor(const MachineInstr &MI,
375 const SIInstrInfo &SII) {
376 if (MI.isDebugInstr())
377 return InstructionFlavor::Other;
378
379 unsigned Opc = MI.getOpcode();
380
381 // Check for specific opcodes first.
382 if (Opc == AMDGPU::ATOMIC_FENCE || Opc == AMDGPU::S_BARRIER_WAIT ||
383 Opc == AMDGPU::S_BARRIER_SIGNAL_IMM || SII.isWaitcnt(Opcode: Opc))
384 return InstructionFlavor::Fence;
385
386 if (SII.isLDSDMA(MI))
387 return InstructionFlavor::DMA;
388
389 if (SII.isMFMA(MI)) {
390 // TODO: Consider further sub-classifying this (XDL, XDL2x, S/DGEMM).
391 // GFX9 SPG sub-classifies MFMA into XDL, XDL2x and S/DGEMM, because only
392 // certain sub-classes can be co-executed in certain slots. For now, we
393 // simply treat them all as one to simplify the change and leave the rest
394 // to a follow-up fine-tuning.
395 return InstructionFlavor::WMMA;
396 }
397
398 if (SII.isWMMA(MI) || SII.isSWMMAC(MI))
399 return InstructionFlavor::WMMA;
400
401 if (SII.isTRANS(MI))
402 return InstructionFlavor::TRANS;
403
404 if (SII.isVALU(MI, /*AllowLDSDMA=*/false)) {
405 if (SII.getBlockingCycles(MI) > 1)
406 return InstructionFlavor::MultiCycleVALU;
407
408 return InstructionFlavor::SingleCycleVALU;
409 }
410
411 if (SII.isSMRD(MI))
412 return InstructionFlavor::SMEM;
413
414 if (SII.isDS(MI))
415 return InstructionFlavor::DS;
416
417 if (SII.isVMEM(MI))
418 return InstructionFlavor::VMEM;
419
420 if (SII.isSALU(MI))
421 return InstructionFlavor::SALU;
422
423 return InstructionFlavor::Other;
424}
425
426SUnit *HardwareUnitInfo::getNextTargetSU(bool LookDeep) const {
427 for (SUnit *PrioritySU : PrioritySUs) {
428 if (!PrioritySU->isTopReady())
429 return PrioritySU;
430 }
431
432 if (!LookDeep)
433 return nullptr;
434
435 unsigned MinDepth = std::numeric_limits<unsigned int>::max();
436 SUnit *TargetSU = nullptr;
437 for (auto *SU : AllSUs) {
438 if (SU->isScheduled)
439 continue;
440
441 if (SU->isTopReady())
442 continue;
443
444 if (SU->getDepth() < MinDepth) {
445 MinDepth = SU->getDepth();
446 TargetSU = SU;
447 }
448 }
449 return TargetSU;
450}
451
452void HardwareUnitInfo::insert(SUnit *SU, unsigned BlockingCycles) {
453 if (!AllSUs.insert(X: SU))
454 llvm_unreachable("HardwareUnit already contains SU!");
455
456 TotalCycles += BlockingCycles;
457
458 if (PrioritySUs.empty()) {
459 PrioritySUs.insert(X: SU);
460 return;
461 }
462 unsigned SUDepth = SU->getDepth();
463 unsigned CurrDepth = (*PrioritySUs.begin())->getDepth();
464 if (SUDepth > CurrDepth)
465 return;
466
467 if (SUDepth == CurrDepth) {
468 PrioritySUs.insert(X: SU);
469 return;
470 }
471
472 // SU is lower depth and should be prioritized.
473 PrioritySUs.clear();
474 PrioritySUs.insert(X: SU);
475}
476
477void HardwareUnitInfo::markScheduled(SUnit *SU, unsigned BlockingCycles) {
478 // We may want to ignore some HWUIs (e.g. InstructionFlavor::Other). To do so,
479 // we just clear the HWUI. However, we still have instructions which map to
480 // this HWUI. Don't bother managing the state for these HWUI.
481 if (TotalCycles == 0)
482 return;
483
484 ScheduledSUs.push_back(Elt: SU);
485 AllSUs.remove(X: SU);
486 PrioritySUs.remove(X: SU);
487
488 // BufferSize 0 is unlimited, while size 1 has no parallel buffering. In
489 // either case, each SU uses the HardwareUnit for BlockingCycles.
490 if (BufferSize <= 1 || (ScheduledSUs.size() % BufferSize == 0))
491 TotalCycles -= std::min(a: TotalCycles, b: BlockingCycles);
492
493 if (AllSUs.empty())
494 return;
495 if (PrioritySUs.empty()) {
496 for (auto SU : AllSUs) {
497 if (PrioritySUs.empty()) {
498 PrioritySUs.insert(X: SU);
499 continue;
500 }
501 unsigned SUDepth = SU->getDepth();
502 unsigned CurrDepth = (*PrioritySUs.begin())->getDepth();
503 if (SUDepth > CurrDepth)
504 continue;
505
506 if (SUDepth == CurrDepth) {
507 PrioritySUs.insert(X: SU);
508 continue;
509 }
510
511 // SU is lower depth and should be prioritized.
512 PrioritySUs.clear();
513 PrioritySUs.insert(X: SU);
514 }
515 }
516}
517
518void HardwareUnitInfo::finalizeCycles() {
519 if (BufferSize == 0 || AllSUs.empty())
520 return;
521
522 // We estimate the amount of cycles it takes to free up a slot in the buffer
523 // as the average cycles per SU.
524 BufferCycles = static_cast<unsigned>(TotalCycles / AllSUs.size());
525 // A single-entry buffer does not reduce TotalCycles.
526 if (BufferSize == 1)
527 return;
528
529 // The TotalCycles is normalized against the BufferSize.
530 // This provides an estimate of the TotalCycles which is not always accurate
531 // -- particularly in cases where we have fewer instructions than the
532 // BufferSize. For example, if we have 2 instructions which each take 50
533 // cycles and a BufferSize of 16, then a TotalCycles of 51 cycles would be
534 // somewhat accurate. This normalization calculates TotalCycles as 6. However,
535 // if we have 64 of these instructions, our normalized estimate of 200 is more
536 // reasonable, given the more accurate measure is 264. Having a completely
537 // accurate measure is not very important, since this metric is mainly used to
538 // compare the relative demand per HardwareUnit across the region. The simpler
539 // estimate makes managing the metric incrementally during scheduling much
540 // simpler.
541 TotalCycles /= BufferSize;
542}
543
544HardwareUnitInfo *
545CandidateHeuristics::getHWUIFromFlavor(InstructionFlavor Flavor) {
546 for (HardwareUnitInfo &HWUICand : HWUInfo) {
547 if (HWUICand.getType() == Flavor) {
548 return &HWUICand;
549 }
550 }
551 return nullptr;
552}
553
554unsigned CandidateHeuristics::getMaxBlockingCycles(const MCSchedClassDesc *SC,
555 const MachineInstr *MI) {
556 unsigned ReleaseAtCycle = 0;
557 for (TargetSchedModel::ProcResIter PI = SchedModel->getWriteProcResBegin(SC),
558 PE = SchedModel->getWriteProcResEnd(SC);
559 PI != PE; ++PI) {
560 ReleaseAtCycle =
561 std::max(a: ReleaseAtCycle, b: static_cast<unsigned>(PI->ReleaseAtCycle));
562 }
563 ReleaseAtCycle = std::max(a: ReleaseAtCycle, b: SII->getBlockingCycles(MI: *MI));
564 return ReleaseAtCycle;
565}
566
567unsigned CandidateHeuristics::getHWUICyclesForMI(MachineInstr *MI) {
568 assert(SchedModel && SchedModel->hasInstrSchedModel());
569 if (MI->mayLoadOrStore())
570 return SchedModel->computeInstrLatency(MI, UseDefaultDefLatency: false);
571 return getMaxBlockingCycles(SC: SchedModel->resolveSchedClass(MI), MI);
572}
573
574void CandidateHeuristics::updateForScheduling(SUnit *SU) {
575 MachineInstr *MI = SU->getInstr();
576 HardwareUnitInfo *HWUI = getHWUIFromFlavor(Flavor: classifyFlavor(MI: *MI, SII: *SII));
577 assert(HWUI);
578 HWUI->markScheduled(SU, BlockingCycles: getHWUICyclesForMI(MI));
579}
580
581void CandidateHeuristics::initialize(ScheduleDAGMI *SchedDAG,
582 const TargetSchedModel *TargetSchedModel,
583 const TargetRegisterInfo *TRI,
584 const MachineLoopInfo *LoopInfo) {
585 DAG = SchedDAG;
586 SchedModel = TargetSchedModel;
587 MLI = LoopInfo;
588 assert(SchedModel && SchedModel->hasInstrSchedModel());
589
590 SRI = static_cast<const SIRegisterInfo *>(TRI);
591 SII = static_cast<const SIInstrInfo *>(DAG->TII);
592
593 HWUInfo.resize(N: static_cast<int>(InstructionFlavor::NUM_FLAVORS));
594
595 for (unsigned I = 0; I < HWUInfo.size(); I++) {
596 HWUInfo[I].reset();
597 HWUInfo[I].setType(I);
598 }
599
600 HWUInfo[static_cast<int>(InstructionFlavor::WMMA)].setProducesCoexecWindow(
601 true);
602 HWUInfo[static_cast<int>(InstructionFlavor::MultiCycleVALU)]
603 .setProducesCoexecWindow(true);
604 HWUInfo[static_cast<int>(InstructionFlavor::TRANS)].setProducesCoexecWindow(
605 true);
606 HWUInfo[static_cast<int>(InstructionFlavor::DS)].setBufferSize(
607 DefaultBufferSizes::DS);
608
609 collectRegionSummary();
610}
611
612unsigned CandidateHeuristics::getCarriedLatency(SUnit *SU) {
613 if (RegionCarriedLatency == CarriedLatency::Off)
614 return 0;
615
616 MachineInstr *MI = SU->getInstr();
617 unsigned CarriedLatency = 0;
618 const InstructionFlavor Flavor = classifyFlavor(MI: *MI, SII: *SII);
619 if (Flavor == InstructionFlavor::Fence) {
620 MachineBasicBlock *MBB = MI->getParent();
621 // Scan each direct predecessor back to its nearest Fence or block start for
622 // DS instructions.
623 for (auto PredMBB : MBB->predecessors()) {
624 auto I = PredMBB->rbegin();
625 auto E = PredMBB->rend();
626 for (; I != E; I++) {
627 const InstructionFlavor ItFlavor = classifyFlavor(MI: *I, SII: *SII);
628 if (ItFlavor == InstructionFlavor::Fence)
629 break;
630
631 // Found carried latency.
632 if (ItFlavor == InstructionFlavor::DS)
633 CarriedLatency = std::max(a: CarriedLatency, b: getHWUICyclesForMI(MI: &*I));
634 }
635 }
636 }
637
638 if (RegionCarriedLatency == CarriedLatency::Fence)
639 return CarriedLatency;
640
641 for (MachineOperand &Op : MI->all_uses()) {
642 auto Reg = Op.getReg();
643 if (!Reg.isVirtual())
644 continue;
645
646 for (MachineInstr &Def : DAG->MRI.def_instructions(Reg)) {
647 // We don't have the proper modelling to accurately measure all carried
648 // latency. Just try to measure carried latency for long latency loads to
649 // avoid long stalls.
650 if (!Def.mayLoad())
651 continue;
652
653 unsigned Latency = getHWUICyclesForMI(MI: &Def);
654
655 // Load is carried across block.
656 if (Def.getParent() != MI->getParent()) {
657 bool FoundUseInDefBlock = false;
658 for (MachineInstr &Use : DAG->MRI.use_nodbg_instructions(Reg)) {
659 if (Use.getParent() != Def.getParent())
660 continue;
661
662 SlotIndex DefIdx = DAG->getLIS()->getInstructionIndex(Instr: Def);
663 SlotIndex UseIdx = DAG->getLIS()->getInstructionIndex(Instr: Use);
664 // We have a use of this load in the def block that occurs after the
665 // load. In this case we must wait for the load in the def block, and
666 // we do not have any carried latency from this load.
667 if (SlotIndex::isEarlierInstr(A: DefIdx, B: UseIdx)) {
668 FoundUseInDefBlock = true;
669 break;
670 }
671 }
672 if (!FoundUseInDefBlock)
673 CarriedLatency = std::max(a: Latency, b: CarriedLatency);
674
675 continue;
676 }
677
678 assert(Def.getParent() == MI->getParent());
679 // Load is in the same block.
680 SlotIndex LoadIdx = DAG->getLIS()->getInstructionIndex(Instr: Def);
681 SlotIndex UseIdx = DAG->getLIS()->getInstructionIndex(Instr: *MI);
682 // The load occurs after this use -- the latency is carried across loop
683 // backedge.
684 if (SlotIndex::isEarlierInstr(A: UseIdx, B: LoadIdx))
685 CarriedLatency = std::max(a: Latency, b: CarriedLatency);
686 }
687 }
688 return CarriedLatency;
689}
690
691void CandidateHeuristics::collectRegionSummary() {
692 CarriedLatencies.clear();
693 RegionCarriedLatency = CarriedLatency::Off;
694
695 if (!SchedModel || !SchedModel->hasInstrSchedModel())
696 return;
697
698 // In some cases, we may end up with loads at the end of a predecssor block.
699 // In these cases, it is preferable to defer scheduling a fence which
700 // corresponds with a waitcnt for those loads until the latency of the load
701 // has cleared. Unfortunately, there is no reliable way to determine whether
702 // or not a predecessor block will end up scheduling loads at the end. Here,
703 // we inspect the dependency structure to define a rough heuristic: if we
704 // must schedule ds_loads after wmma, then we carry the latency of ds_loads
705 // to successor block fences. Currently this only checks for the loop-carried
706 // case, as this will generally have the largest impact on performance.
707 // TODO: 1. extend to different memory instructions, 2. teach carried
708 // latencies about fence legalization.
709 auto mustHaveDSAfter = [this]() {
710 if (!DAG->SUnits.size())
711 return false;
712
713 MachineBasicBlock *MBB = DAG->begin()->getParent();
714 MachineLoop *Loop = MLI->getLoopFor(BB: MBB);
715 if (!Loop || Loop->getNumBlocks() != 1)
716 return false;
717
718 SmallVector<SUnit *, 16> RegionWMMAs;
719
720 for (auto &SU : DAG->SUnits) {
721 if (!SU.getInstr())
722 continue;
723 MachineInstr *MI = SU.getInstr();
724 const InstructionFlavor Flavor = classifyFlavor(MI: *MI, SII: *SII);
725 if (Flavor == InstructionFlavor::WMMA)
726 RegionWMMAs.push_back(Elt: &SU);
727 }
728
729 bool MustHaveDSAfter = RegionWMMAs.size();
730
731 for (SUnit *SU : RegionWMMAs) {
732 bool HasDSSucc = false;
733 for (auto &Succ : SU->Succs) {
734 if (!Succ.getSUnit()->getInstr())
735 continue;
736 if (classifyFlavor(MI: *Succ.getSUnit()->getInstr(), SII: *SII) ==
737 InstructionFlavor::DS) {
738 HasDSSucc = true;
739 break;
740 }
741 }
742
743 if (!HasDSSucc) {
744 MustHaveDSAfter = false;
745 break;
746 }
747 }
748
749 return MustHaveDSAfter;
750 };
751
752 std::optional<CarriedLatency> BlockCarriedLatency =
753 DAG->MF.getSubtarget<GCNSubtarget>().getCLOpts().block_carried_latency;
754 RegionCarriedLatency = BlockCarriedLatency ? *BlockCarriedLatency
755 : mustHaveDSAfter() ? CarriedLatency::Fence
756 : CarriedLatency::Off;
757
758 for (auto &SU : DAG->SUnits) {
759 MachineInstr *MI = SU.getInstr();
760 const InstructionFlavor Flavor = classifyFlavor(MI: *MI, SII: *SII);
761 HWUInfo[static_cast<int>(Flavor)].insert(SU: &SU, BlockingCycles: getHWUICyclesForMI(MI));
762 unsigned CarriedLatency = getCarriedLatency(SU: &SU);
763 if (CarriedLatency)
764 CarriedLatencies[MI] = CarriedLatency;
765 }
766
767 for (auto &HWUI : HWUInfo)
768 HWUI.finalizeCycles();
769
770 LLVM_DEBUG(dumpRegionSummary());
771}
772
773void CandidateHeuristics::dumpRegionSummary() {
774 MachineBasicBlock *BB = DAG->begin()->getParent();
775 dbgs() << "\n=== Region: " << DAG->MF.getName() << " BB" << BB->getNumber()
776 << " (" << DAG->SUnits.size() << " SUs) ===\n";
777
778 dbgs() << "\nHWUI Resource Pressure:\n";
779 for (auto &HWUI : HWUInfo) {
780 if (HWUI.getTotalCycles() == 0)
781 continue;
782
783 StringRef Name = getFlavorName(F: HWUI.getType());
784 dbgs() << " " << Name << ": " << HWUI.getTotalCycles() << " cycles, "
785 << HWUI.size() << " instrs\n";
786 }
787 dbgs() << "\n";
788}
789
790void CandidateHeuristics::sortHWUIResources() {
791 // Highest priority should be first.
792 llvm::sort(C&: HWUInfo, Comp: [](HardwareUnitInfo &A, HardwareUnitInfo &B) {
793 // Prefer CoexecWindow producers
794 if (A.producesCoexecWindow() != B.producesCoexecWindow())
795 return A.producesCoexecWindow();
796
797 // Prefer more demanded resources
798 if (A.getTotalCycles() != B.getTotalCycles())
799 return A.getTotalCycles() > B.getTotalCycles();
800
801 // In ties -- prefer the resource with more instructions
802 if (A.size() != B.size())
803 return A.size() < B.size();
804
805 // Default to Flavor order
806 return static_cast<unsigned>(A.getType()) <
807 static_cast<unsigned>(B.getType());
808 });
809}
810
811unsigned CandidateHeuristics::getStructuralStallCycles(SchedBoundary &Zone,
812 SUnit *SU) {
813 assert(Zone.isTop() && "effective stall comparison requires top boundary");
814 if (!SU)
815 return 0;
816
817 MachineInstr *MI = SU->getInstr();
818 unsigned CurrCycle = Zone.getCurrCycle();
819 unsigned Stall = 0;
820
821 // Query SchedModel for resource stalls (unbuffered resources).
822 if (SchedModel->hasInstrSchedModel() && SU->hasReservedResource) {
823 const MCSchedClassDesc *SC = DAG->getSchedClass(SU);
824 for (const MCWriteProcResEntry &PE :
825 make_range(x: SchedModel->getWriteProcResBegin(SC),
826 y: SchedModel->getWriteProcResEnd(SC))) {
827 unsigned NextAvail =
828 Zone.getNextResourceCycle(SC, PIdx: PE.ProcResourceIdx, ReleaseAtCycle: PE.ReleaseAtCycle,
829 AcquireAtCycle: PE.AcquireAtCycle)
830 .first;
831 if (NextAvail > CurrCycle)
832 Stall = std::max(a: Stall, b: NextAvail - CurrCycle);
833 }
834 }
835
836 // Query HazardRecognizer for sequence-dependent hazard penalties.
837 if (Zone.HazardRec && Zone.HazardRec->isEnabled()) {
838 auto *HR = static_cast<GCNHazardRecognizer *>(Zone.HazardRec.get());
839 Stall = std::max(a: Stall, b: HR->getHazardWaitStates(MI));
840 }
841
842 return Stall;
843}
844
845CandidateHeuristics::StallCosts
846CandidateHeuristics::getStallCosts(SUnit *SU, SchedBoundary &Zone) {
847 if (!Zone.isTop())
848 return {};
849
850 auto getBufferFullStalls = [this, &Zone](SUnit *SU) -> unsigned {
851 InstructionFlavor Flavor = classifyFlavor(
852 MI: *SU->getInstr(), SII: *static_cast<const SIInstrInfo *>(DAG->TII));
853 HardwareUnitInfo *HWUI = getHWUIFromFlavor(Flavor);
854
855 // A BufferSize of 0 means "unlimited" buffer, thus we will never fill it.
856 if (HWUI->getBufferSize() == 0)
857 return 0;
858
859 unsigned CurrCycle = Zone.getCurrCycle();
860 unsigned BufferReadyCycle = HWUI->getBufferAvailableCycle(CurrCycle);
861 if (BufferReadyCycle <= CurrCycle)
862 return 0;
863
864 return BufferReadyCycle - CurrCycle;
865 };
866
867 unsigned CurrCycle = Zone.getCurrCycle();
868
869 auto getFenceStalls = [this, &CurrCycle](SUnit *SU) -> unsigned {
870 InstructionFlavor Flavor = classifyFlavor(
871 MI: *SU->getInstr(), SII: *static_cast<const SIInstrInfo *>(DAG->TII));
872
873 if (Flavor != InstructionFlavor::Fence)
874 return 0;
875
876 HardwareUnitInfo *ConsumerHWUI = getHWUIFromFlavor(Flavor);
877 HardwareUnitInfo *ProducerHWUI = getHWUIFromFlavor(Flavor: InstructionFlavor::DS);
878
879 SUnit *LastProducer = ProducerHWUI->getLastScheduledSU();
880 if (!LastProducer)
881 return 0;
882
883 SUnit *LastConsumer = ConsumerHWUI->getLastScheduledSU();
884 unsigned LastConsumerCycle = LastConsumer ? LastConsumer->TopReadyCycle : 0;
885 unsigned LastProducerCycle = LastProducer->TopReadyCycle;
886
887 if (LastProducerCycle < LastConsumerCycle)
888 return 0;
889
890 unsigned FenceStallFinish =
891 LastProducerCycle + getHWUICyclesForMI(MI: LastProducer->getInstr());
892 return FenceStallFinish <= CurrCycle ? 0 : FenceStallFinish - CurrCycle;
893 };
894
895 unsigned ReadyCycle = SU->TopReadyCycle;
896 StallCosts Costs;
897 Costs.Ready = ReadyCycle > CurrCycle ? ReadyCycle - CurrCycle : 0;
898 Costs.Structural = getStructuralStallCycles(Zone, SU);
899 Costs.Latency = Zone.getLatencyStallCycles(SU);
900 unsigned CarriedLatency = CarriedLatencies.lookup_or(Val: SU->getInstr(), Default: 0);
901 Costs.Carried = CarriedLatency > CurrCycle ? CarriedLatency - CurrCycle : 0;
902 Costs.Buffer = getBufferFullStalls(SU);
903 Costs.Fence = getFenceStalls(SU);
904 Costs.Effective = std::max(l: {Costs.Ready, Costs.Structural, Costs.Latency,
905 Costs.Carried, Costs.Buffer, Costs.Fence});
906 return Costs;
907}
908
909bool CandidateHeuristics::tryEffectiveStall(
910 GenericSchedulerBase::SchedCandidate &TryCand,
911 GenericSchedulerBase::SchedCandidate &Cand, SchedBoundary &Zone) {
912 // Only implemented for top-down scheduling
913 if (!Zone.isTop())
914 return 0;
915
916 StallCosts TryCosts = getStallCosts(SU: TryCand.SU, Zone);
917 StallCosts CandCosts = getStallCosts(SU: Cand.SU, Zone);
918
919 LLVM_DEBUG(if (TryCosts.Effective || CandCosts.Effective) {
920 dbgs() << "Effective stalls: try=" << TryCosts.Effective
921 << " (ready=" << TryCosts.Ready << ", struct=" << TryCosts.Structural
922 << ", lat=" << TryCosts.Latency << ", carried=" << TryCosts.Carried
923 << ", buffer=" << TryCosts.Buffer << ", fence=" << TryCosts.Fence
924 << ") cand=" << CandCosts.Effective << " (ready=" << CandCosts.Ready
925 << ", struct=" << CandCosts.Structural
926 << ", lat=" << CandCosts.Latency << ", carried=" << CandCosts.Carried
927 << ", buffer=" << CandCosts.Buffer << ", fence=" << CandCosts.Fence
928 << ")\n";
929 });
930
931 return tryLess(TryVal: TryCosts.Effective, CandVal: CandCosts.Effective, TryCand, Cand,
932 Reason: AMDGPUCoExecSchedStrategy::Stall);
933}
934
935bool CandidateHeuristics::tryMemoryPipeline(
936 GenericSchedulerBase::SchedCandidate &TryCand,
937 GenericSchedulerBase::SchedCandidate &Cand, SchedBoundary &Zone) {
938
939 InstructionFlavor TryFlavor = classifyFlavor(MI: *TryCand.SU->getInstr(), SII: *SII);
940
941 InstructionFlavor CandFlavor = classifyFlavor(MI: *Cand.SU->getInstr(), SII: *SII);
942
943 bool TryIsMemoryPipeline = TryFlavor == InstructionFlavor::DMA ||
944 TryFlavor == InstructionFlavor::Fence;
945 bool CandIsMemoryPipeline = CandFlavor == InstructionFlavor::DMA ||
946 CandFlavor == InstructionFlavor::Fence;
947
948 if (!(TryIsMemoryPipeline || CandIsMemoryPipeline))
949 return false;
950
951 if (TryIsMemoryPipeline)
952 TryIsMemoryPipeline &= getStallCosts(SU: TryCand.SU, Zone).Effective == 0;
953
954 if (CandIsMemoryPipeline)
955 CandIsMemoryPipeline &= getStallCosts(SU: Cand.SU, Zone).Effective == 0;
956
957 if (TryIsMemoryPipeline == CandIsMemoryPipeline)
958 return false;
959
960 if (CandIsMemoryPipeline) {
961 if (Cand.Reason > GenericSchedulerBase::RegCritical)
962 Cand.Reason = GenericSchedulerBase::RegCritical;
963
964 return true;
965 }
966
967 TryCand.Reason = GenericSchedulerBase::RegCritical;
968 return true;
969}
970
971bool CandidateHeuristics::tryCriticalResourceDependency(
972 GenericSchedulerBase::SchedCandidate &TryCand,
973 GenericSchedulerBase::SchedCandidate &Cand, SchedBoundary *Zone) const {
974
975 auto HasPrioritySU = [this, &Cand, &TryCand](unsigned ResourceIdx) {
976 const HardwareUnitInfo &HWUI = HWUInfo[ResourceIdx];
977
978 auto CandFlavor = classifyFlavor(MI: *Cand.SU->getInstr(), SII: *SII);
979 auto TryCandFlavor = classifyFlavor(MI: *TryCand.SU->getInstr(), SII: *SII);
980 bool LookDeep = (CandFlavor == InstructionFlavor::DS ||
981 TryCandFlavor == InstructionFlavor::DS) &&
982 HWUI.getType() == InstructionFlavor::WMMA;
983 auto *TargetSU = HWUI.getNextTargetSU(LookDeep);
984
985 // If we do not have a TargetSU for this resource, then it is not critical.
986 if (!TargetSU)
987 return false;
988
989 return true;
990 };
991
992 auto TryEnablesResource = [&Cand, &TryCand, this](unsigned ResourceIdx) {
993 const HardwareUnitInfo &HWUI = HWUInfo[ResourceIdx];
994 auto CandFlavor = classifyFlavor(MI: *Cand.SU->getInstr(), SII: *SII);
995
996 // We want to ensure our DS order matches WMMA order.
997 bool LookDeep = CandFlavor == InstructionFlavor::DS &&
998 HWUI.getType() == InstructionFlavor::WMMA;
999 auto *TargetSU = HWUI.getNextTargetSU(LookDeep);
1000
1001 bool CandEnables =
1002 TargetSU != Cand.SU && DAG->IsReachable(SU: TargetSU, TargetSU: Cand.SU);
1003 bool TryCandEnables =
1004 TargetSU != TryCand.SU && DAG->IsReachable(SU: TargetSU, TargetSU: TryCand.SU);
1005
1006 if (!CandEnables && !TryCandEnables)
1007 return false;
1008
1009 if (CandEnables && !TryCandEnables) {
1010 if (Cand.Reason > GenericSchedulerBase::RegCritical)
1011 Cand.Reason = GenericSchedulerBase::RegCritical;
1012
1013 return true;
1014 }
1015
1016 if (!CandEnables && TryCandEnables) {
1017 TryCand.Reason = GenericSchedulerBase::RegCritical;
1018 return true;
1019 }
1020
1021 // Both enable, prefer the critical path.
1022 unsigned CandHeight = Cand.SU->getHeight();
1023 unsigned TryCandHeight = TryCand.SU->getHeight();
1024
1025 if (CandHeight > TryCandHeight) {
1026 if (Cand.Reason > GenericSchedulerBase::RegCritical)
1027 Cand.Reason = GenericSchedulerBase::RegCritical;
1028
1029 return true;
1030 }
1031
1032 if (CandHeight < TryCandHeight) {
1033 TryCand.Reason = GenericSchedulerBase::RegCritical;
1034 return true;
1035 }
1036
1037 // Same critical path, just prefer original candidate.
1038 if (Cand.Reason > GenericSchedulerBase::RegCritical)
1039 Cand.Reason = GenericSchedulerBase::RegCritical;
1040
1041 return true;
1042 };
1043
1044 for (unsigned I = 0; I < HWUInfo.size(); I++) {
1045 // If we have encountered a resource that is not critical, then neither
1046 // candidate enables a critical resource
1047 if (!HasPrioritySU(I))
1048 continue;
1049
1050 bool Enabled = TryEnablesResource(I);
1051 // If neither has enabled the resource, continue to the next resource
1052 if (Enabled)
1053 return true;
1054 }
1055 return false;
1056}
1057
1058bool CandidateHeuristics::tryCriticalResource(
1059 GenericSchedulerBase::SchedCandidate &TryCand,
1060 GenericSchedulerBase::SchedCandidate &Cand, SchedBoundary *Zone) const {
1061 for (unsigned I = 0; I < HWUInfo.size(); I++) {
1062 const HardwareUnitInfo &HWUI = HWUInfo[I];
1063
1064 bool CandUsesCrit = HWUI.contains(SU: Cand.SU);
1065 bool TryCandUsesCrit = HWUI.contains(SU: TryCand.SU);
1066
1067 if (!CandUsesCrit && !TryCandUsesCrit)
1068 continue;
1069
1070 if (CandUsesCrit != TryCandUsesCrit) {
1071 if (CandUsesCrit) {
1072 if (Cand.Reason > GenericSchedulerBase::RegCritical)
1073 Cand.Reason = GenericSchedulerBase::RegCritical;
1074 return true;
1075 }
1076 TryCand.Reason = GenericSchedulerBase::RegCritical;
1077 return true;
1078 }
1079
1080 // Otherwise, both use the critical resource
1081 // For longer latency InstructionFlavors, we should prioritize first by
1082 // their enablement of critical resources
1083 if (HWUI.getType() == InstructionFlavor::DS) {
1084 if (tryCriticalResourceDependency(TryCand, Cand, Zone))
1085 return true;
1086 }
1087
1088 // Prioritize based on HWUI priorities.
1089 SUnit *Match = HWUI.getHigherPriority(SU: Cand.SU, Other: TryCand.SU);
1090 if (Match) {
1091 if (Match == Cand.SU) {
1092 if (Cand.Reason > GenericSchedulerBase::RegCritical)
1093 Cand.Reason = GenericSchedulerBase::RegCritical;
1094 return true;
1095 }
1096 TryCand.Reason = GenericSchedulerBase::RegCritical;
1097 return true;
1098 }
1099 }
1100
1101 return false;
1102}
1103
1104AMDGPUCoExecSchedStrategy::AMDGPUCoExecSchedStrategy(
1105 const MachineSchedContext *C)
1106 : GCNSchedStrategy(C) {
1107 SchedStages.push_back(Elt: GCNSchedStageID::ILPInitialSchedule);
1108 SchedStages.push_back(Elt: GCNSchedStageID::RewriteMFMAForm);
1109 SchedStages.push_back(Elt: GCNSchedStageID::LiveIntervalRPReschedule);
1110 SchedStages.push_back(Elt: GCNSchedStageID::PreRARematerialize);
1111 // Use more accurate GCN pressure trackers.
1112 UseGCNTrackers = true;
1113
1114 if (!VGPRThresholdPercentOpt.getNumOccurrences())
1115 VGPRThresholdPercent = DefaultCoExecVGPRThresholdPercent;
1116}
1117
1118void AMDGPUCoExecSchedStrategy::initPolicy(MachineBasicBlock::iterator Begin,
1119 MachineBasicBlock::iterator End,
1120 unsigned NumRegionInstrs) {
1121 GCNSchedStrategy::initPolicy(Begin, End, NumRegionInstrs);
1122 if (getPreRADirection() == MISched::BottomUp ||
1123 getPreRADirection() == MISched::Bidirectional)
1124 report_fatal_error(reason: "CoExecSchedStrategy only support TopDown scheduling.");
1125 RegionPolicy.OnlyTopDown = true;
1126 RegionPolicy.OnlyBottomUp = false;
1127 RegionPolicy.ShouldTrackLaneMasks = true;
1128}
1129
1130void AMDGPUCoExecSchedStrategy::initialize(ScheduleDAGMI *DAG) {
1131 // Coexecution scheduling strategy is only done top-down to support new
1132 // resource balancing heuristics.
1133 RegionPolicy.OnlyTopDown = true;
1134 RegionPolicy.OnlyBottomUp = false;
1135
1136 GCNSchedStrategy::initialize(DAG);
1137 Heurs.initialize(SchedDAG: DAG, TargetSchedModel: SchedModel, TRI, LoopInfo: Context->MLI);
1138
1139 // Replace the default hazard recognizer with our PreRA one so that pre-RA
1140 // scheduling accounts for WMMA co-execution slot constraints. This must
1141 // happen after GCNSchedStrategy::initialize() because
1142 // GenericScheduler::initialize() calls SchedBoundary::reset(), which deletes
1143 // and recreates the hazard recognizer each region.
1144 Top.HazardRec = std::make_unique<GCNHazardRecognizer>(
1145 args&: DAG->MF, args: GCNHazardRecognizer::OperatingMode::PreRA);
1146}
1147
1148void AMDGPUCoExecSchedStrategy::schedNode(SUnit *SU, bool IsTopNode) {
1149 Heurs.updateForScheduling(SU);
1150 GCNSchedStrategy::schedNode(SU, IsTopNode);
1151}
1152
1153SUnit *AMDGPUCoExecSchedStrategy::pickNode(bool &IsTopNode) {
1154 assert(RegionPolicy.OnlyTopDown && !RegionPolicy.OnlyBottomUp &&
1155 "coexec scheduler only supports top-down scheduling");
1156
1157 if (DAG->top() == DAG->bottom()) {
1158 assert(Top.Available.empty() && Top.Pending.empty() &&
1159 Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage");
1160 return nullptr;
1161 }
1162
1163 bool PickedPending = false;
1164 SUnit *SU = nullptr;
1165#ifndef NDEBUG
1166 SchedCandidate *PickedCand = nullptr;
1167#endif
1168 do {
1169 PickedPending = false;
1170 SU = pickOnlyChoice(Zone&: Top);
1171 if (!SU) {
1172 CandPolicy NoPolicy;
1173 TopCand.reset(NewPolicy: NoPolicy);
1174 pickNodeFromQueue(Zone&: Top, ZonePolicy: NoPolicy, RPTracker: DAG->getTopRPTracker(), Cand&: TopCand,
1175 PickedPending, /*IsBottomUp=*/false);
1176 assert(TopCand.Reason != NoCand && "failed to find a candidate");
1177 SU = TopCand.SU;
1178#ifndef NDEBUG
1179 PickedCand = &TopCand;
1180#endif
1181 }
1182 IsTopNode = true;
1183 } while (SU->isScheduled);
1184
1185 LLVM_DEBUG(if (PickedCand) dumpPickSummary(SU, IsTopNode, *PickedCand));
1186
1187 if (PickedPending) {
1188 unsigned ReadyCycle = SU->TopReadyCycle;
1189 unsigned CurrentCycle = Top.getCurrCycle();
1190 if (ReadyCycle > CurrentCycle)
1191 Top.bumpCycle(NextCycle: ReadyCycle);
1192
1193 // checkHazard() does not expose the exact cycle where the hazard clears.
1194 while (Top.checkHazard(SU))
1195 Top.bumpCycle(NextCycle: Top.getCurrCycle() + 1);
1196
1197 Top.releasePending();
1198 }
1199
1200 if (SU->isTopReady())
1201 Top.removeReady(SU);
1202 if (SU->isBottomReady())
1203 Bot.removeReady(SU);
1204
1205 LLVM_DEBUG(dbgs() << "Scheduling " << *SU << " " << *SU->getInstr());
1206
1207 assert(IsTopNode && "coexec scheduler must only schedule from top boundary");
1208 return SU;
1209}
1210
1211void AMDGPUCoExecSchedStrategy::pickNodeFromQueue(
1212 SchedBoundary &Zone, const CandPolicy &ZonePolicy,
1213 const RegPressureTracker &RPTracker, SchedCandidate &Cand,
1214 bool &PickedPending, bool IsBottomUp) {
1215 assert(Zone.isTop() && "coexec scheduler only supports top boundary");
1216 assert(!IsBottomUp && "coexec scheduler only supports top-down scheduling");
1217
1218 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo *>(TRI);
1219 ArrayRef<unsigned> Pressure = RPTracker.getRegSetPressureAtPos();
1220 unsigned SGPRPressure = 0;
1221 unsigned VGPRPressure = 0;
1222 unsigned AGPRPressure = 0;
1223 PickedPending = false;
1224 if (DAG->isTrackingPressure()) {
1225 if (!useGCNTrackers()) {
1226 SGPRPressure = Pressure[AMDGPU::RegisterPressureSets::SReg_32];
1227 VGPRPressure = Pressure[AMDGPU::RegisterPressureSets::VGPR_32];
1228 AGPRPressure = Pressure[AMDGPU::RegisterPressureSets::AGPR_32];
1229 } else {
1230 SGPRPressure = DownwardTracker.getPressure().getSGPRNum();
1231 std::tie(args&: VGPRPressure, args&: AGPRPressure) =
1232 getBankedPressure(RP: DownwardTracker.getPressure());
1233 }
1234 }
1235
1236 auto EvaluateQueue = [&](ReadyQueue &Q, bool FromPending) {
1237 for (SUnit *SU : Q) {
1238 SchedCandidate TryCand(ZonePolicy);
1239 initCandidate(Cand&: TryCand, SU, AtTop: Zone.isTop(), RPTracker, SRI, SGPRPressure,
1240 VGPRPressure, AGPRPressure, IsBottomUp);
1241 SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
1242 tryCandidateCoexec(Cand, TryCand, Zone: ZoneArg);
1243 if (TryCand.Reason != NoCand) {
1244 if (TryCand.ResDelta == SchedResourceDelta())
1245 TryCand.initResourceDelta(DAG: Zone.DAG, SchedModel);
1246 LLVM_DEBUG(printCandidateDecision(Cand, TryCand));
1247 PickedPending = FromPending;
1248 Cand.setBest(TryCand);
1249 } else {
1250 LLVM_DEBUG(printCandidateDecision(TryCand, Cand));
1251 }
1252 }
1253 };
1254
1255 LLVM_DEBUG(dbgs() << "Available Q:\n");
1256 EvaluateQueue(Zone.Available, /*FromPending=*/false);
1257
1258 LLVM_DEBUG(dbgs() << "Pending Q:\n");
1259 EvaluateQueue(Zone.Pending, /*FromPending=*/true);
1260}
1261
1262#ifndef NDEBUG
1263void AMDGPUCoExecSchedStrategy::dumpPickSummary(SUnit *SU, bool IsTopNode,
1264 SchedCandidate &Cand) {
1265 const SIInstrInfo *SII = static_cast<const SIInstrInfo *>(DAG->TII);
1266 unsigned Cycle = IsTopNode ? Top.getCurrCycle() : Bot.getCurrCycle();
1267
1268 dbgs() << "=== Pick @ Cycle " << Cycle << " ===\n";
1269
1270 const InstructionFlavor Flavor = classifyFlavor(*SU->getInstr(), *SII);
1271 dbgs() << "Picked: " << *SU << " ";
1272 SU->getInstr()->print(dbgs(), /*IsStandalone=*/true, /*SkipOpers=*/false,
1273 /*SkipDebugLoc=*/true);
1274 dbgs() << " [" << getFlavorName(Flavor) << "]\n";
1275
1276 dbgs() << " Reason: ";
1277 if (LastAMDGPUReason != AMDGPUSchedReason::None)
1278 dbgs() << getReasonName(LastAMDGPUReason);
1279 else if (Cand.Reason != NoCand)
1280 dbgs() << GenericSchedulerBase::getReasonStr(Cand.Reason);
1281 else
1282 dbgs() << "Unknown";
1283 dbgs() << "\n\n";
1284
1285 LastAMDGPUReason = AMDGPUSchedReason::None;
1286}
1287#endif
1288
1289bool AMDGPUCoExecSchedStrategy::tryCandidateCoexec(SchedCandidate &Cand,
1290 SchedCandidate &TryCand,
1291 SchedBoundary *Zone) {
1292 // Initialize the candidate if needed.
1293 if (!Cand.isValid()) {
1294 TryCand.Reason = FirstValid;
1295 return true;
1296 }
1297
1298 // Bias PhysReg Defs and copies to their uses and defined respectively.
1299 if (tryGreater(TryVal: biasPhysReg(SU: TryCand.SU, isTop: TryCand.AtTop),
1300 CandVal: biasPhysReg(SU: Cand.SU, isTop: Cand.AtTop), TryCand, Cand, Reason: PhysReg))
1301 return TryCand.Reason != NoCand;
1302
1303 // Avoid exceeding the target's limit.
1304 if (DAG->isTrackingPressure() &&
1305 tryPressure(TryP: TryCand.RPDelta.Excess, CandP: Cand.RPDelta.Excess, TryCand, Cand,
1306 Reason: RegExcess, TRI, MF: DAG->MF))
1307 return TryCand.Reason != NoCand;
1308
1309 // We only compare a subset of features when comparing nodes between
1310 // Top and Bottom boundary. Some properties are simply incomparable, in many
1311 // other instances we should only override the other boundary if something
1312 // is a clear good pick on one boundary. Skip heuristics that are more
1313 // "tie-breaking" in nature.
1314 bool SameBoundary = Zone != nullptr;
1315 if (SameBoundary) {
1316 // Compare candidates by the stall they would introduce if
1317 // scheduled in the current cycle.
1318 if (Heurs.tryEffectiveStall(TryCand, Cand, Zone&: *Zone)) {
1319 LastAMDGPUReason = AMDGPUSchedReason::Stall;
1320 return TryCand.Reason != NoCand;
1321 }
1322
1323 if (Heurs.tryMemoryPipeline(TryCand, Cand, Zone&: *Zone)) {
1324 LastAMDGPUReason = AMDGPUSchedReason::MemoryPipeline;
1325 return TryCand.Reason != NoCand;
1326 }
1327
1328 Heurs.sortHWUIResources();
1329 if (Heurs.tryCriticalResource(TryCand, Cand, Zone)) {
1330 LastAMDGPUReason = AMDGPUSchedReason::CritResourceBalance;
1331 return TryCand.Reason != NoCand;
1332 }
1333
1334 if (Heurs.tryCriticalResourceDependency(TryCand, Cand, Zone)) {
1335 LastAMDGPUReason = AMDGPUSchedReason::CritResourceDep;
1336 return TryCand.Reason != NoCand;
1337 }
1338 }
1339
1340 // Keep clustered nodes together to encourage downstream peephole
1341 // optimizations which may reduce resource requirements.
1342 //
1343 // This is a best effort to set things up for a post-RA pass. Optimizations
1344 // like generating loads of multiple registers should ideally be done within
1345 // the scheduler pass by combining the loads during DAG postprocessing.
1346 unsigned CandZoneCluster = Cand.AtTop ? TopClusterID : BotClusterID;
1347 unsigned TryCandZoneCluster = TryCand.AtTop ? TopClusterID : BotClusterID;
1348 bool CandIsClusterSucc =
1349 isTheSameCluster(A: CandZoneCluster, B: Cand.SU->ParentClusterIdx);
1350 bool TryCandIsClusterSucc =
1351 isTheSameCluster(A: TryCandZoneCluster, B: TryCand.SU->ParentClusterIdx);
1352
1353 if (tryGreater(TryVal: TryCandIsClusterSucc, CandVal: CandIsClusterSucc, TryCand, Cand,
1354 Reason: Cluster))
1355 return TryCand.Reason != NoCand;
1356
1357 if (SameBoundary) {
1358 // Weak edges are for clustering and other constraints.
1359 if (tryLess(TryVal: getWeakLeft(SU: TryCand.SU, isTop: TryCand.AtTop),
1360 CandVal: getWeakLeft(SU: Cand.SU, isTop: Cand.AtTop), TryCand, Cand, Reason: Weak))
1361 return TryCand.Reason != NoCand;
1362 }
1363
1364 // Avoid increasing the max pressure of the entire region.
1365 if (DAG->isTrackingPressure() &&
1366 tryPressure(TryP: TryCand.RPDelta.CurrentMax, CandP: Cand.RPDelta.CurrentMax, TryCand,
1367 Cand, Reason: RegMax, TRI, MF: DAG->MF))
1368 return TryCand.Reason != NoCand;
1369
1370 if (SameBoundary) {
1371 // Avoid serializing long latency dependence chains.
1372 // For acyclic path limited loops, latency was already checked above.
1373 if (!RegionPolicy.DisableLatencyHeuristic && TryCand.Policy.ReduceLatency &&
1374 !Rem.IsAcyclicLatencyLimited && tryLatency(TryCand, Cand, Zone&: *Zone))
1375 return TryCand.Reason != NoCand;
1376
1377 // Fall through to original instruction order.
1378 if ((Zone->isTop() && TryCand.SU->NodeNum < Cand.SU->NodeNum) ||
1379 (!Zone->isTop() && TryCand.SU->NodeNum > Cand.SU->NodeNum)) {
1380 TryCand.Reason = NodeOrder;
1381 return true;
1382 }
1383 }
1384
1385 return false;
1386}
1387
1388ScheduleDAGInstrs *
1389llvm::createGCNCoExecMachineScheduler(MachineSchedContext *C) {
1390 LLVM_DEBUG(dbgs() << "AMDGPU coexec preRA scheduler selected for "
1391 << C->MF->getName() << '\n');
1392 ScheduleDAGMILive *DAG = new GCNScheduleDAGMILive(
1393 C, std::make_unique<AMDGPUCoExecSchedStrategy>(args&: C));
1394 DAG->addMutation(Mutation: createIGroupLPDAGMutation(Phase: AMDGPU::SchedulingPhase::Initial));
1395 DAG->addMutation(Mutation: createAMDGPUBarrierLatencyDAGMutation(MF: C->MF));
1396 return DAG;
1397}
1398
1399ScheduleDAGInstrs *
1400llvm::createGCNNoopPostMachineScheduler(MachineSchedContext *C) {
1401 LLVM_DEBUG(dbgs() << "AMDGPU nop postRA scheduler selected for "
1402 << C->MF->getName() << '\n');
1403 return new GCNNoopPostScheduleDAG(C);
1404}
1405