1//===-- AMDGPUCoExecInfo.h - Co-execution info ------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Shared types for co-execution modeling used by GCNHazardRecognizer and the
11/// schedulers.
12///
13/// Multi-cycle instructions (WMMA, TRANS, etc.) have execution windows where
14/// other instruction types can co-execute. For WMMA, slot patterns depend on
15/// the variant:
16///
17/// E0 (Issue): Control instructions only (s_delay_alu, s_set_vgpr_msb)
18/// E (External): Memory and SALU can co-execute, no VALU
19/// I (Internal): VALU, TRANS, memory, and SALU can all co-execute
20/// V (Vacant): Memory/SALU/next-WMMA ok, NO VALU/TRANS
21///
22//===----------------------------------------------------------------------===//
23
24#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUCOEXECINFO_H
25#define LLVM_LIB_TARGET_AMDGPU_AMDGPUCOEXECINFO_H
26
27#include "SIDefines.h"
28#include "SIInstrInfo.h"
29#include "llvm/ADT/BitmaskEnum.h"
30#include "llvm/ADT/StringRef.h"
31#include "llvm/Support/ErrorHandling.h"
32#include <cassert>
33#include <cstdint>
34#include <optional>
35
36namespace llvm {
37
38namespace AMDGPU {
39
40//===----------------------------------------------------------------------===//
41// Co-execution Bitmasks
42//===----------------------------------------------------------------------===//
43
44/// Bitmask for instruction types allowed to co-execute at a stage.
45enum class CoExecMask : uint16_t {
46 None = 0,
47 CTRL = 1 << 0, // Control: s_delay_alu, s_set_vgpr_msb
48 VALU = 1 << 1, // Vector ALU
49 TRANS = 1 << 2, // Transcendentals (V_EXP etc)
50 SALU = 1 << 3, // Scalar ALU
51 DS = 1 << 4, // LDS read/write
52 VMEM = 1 << 5, // Global memory
53 SMEM = 1 << 6, // Scalar memory
54 WMMA = 1 << 7, // Next WMMA (V stages only), or MFMA
55 All = 0xFFFF,
56
57 MEM = DS | VMEM | SMEM,
58 StageE0 = CTRL, // Issue: control only
59 StageE = CTRL | SALU | MEM, // External: mem/salu
60 StageI = CTRL | SALU | MEM | VALU | TRANS, // Internal: all ALU
61 // Internal + scaled-WMMA absorb: same as StageI but the next scaled
62 // WMMA may issue here - its LD_SCALE consumes the I cycle and the matrix
63 // multiply lands in the V slot that follows. Used for the last I before
64 // V of scaled patterns.
65 StageIS = StageI | WMMA,
66 StageV = CTRL | SALU | MEM | WMMA, // Vacant: no valu/trans
67 StageTR = All & ~TRANS, // TRANS co-exec: no TRANS
68
69 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/All)
70};
71
72using CoExecMaskT = CoExecMask;
73
74//===----------------------------------------------------------------------===//
75// Instruction Flavor Classification
76//===----------------------------------------------------------------------===//
77
78/// Classification of instructions by execution characteristics.
79/// Used for scheduling decisions and co-execution slot preferences.
80enum class InstructionFlavor : uint8_t {
81 WMMA, // WMMA/MFMA matrix operations
82 SingleCycleVALU, // Single-cycle VALU (not TRANS, not multi-cycle CVT)
83 TRANS, // Transcendental ops (v_exp, v_log, etc.)
84 MultiCycleVALU, // VALU instructions with repeat rate > 1
85 VMEM, // FLAT/GLOBAL memory operations
86 SMEM, // Scalar memory operations
87 DS, // LDS/GDS operations
88 SALU, // Scalar ALU
89 DMA, // Tensor DMA operations
90 Fence, // Fences and waits
91 Other, // Everything else
92 NUM_FLAVORS
93};
94
95constexpr StringRef getFlavorName(InstructionFlavor F) {
96 switch (F) {
97 case InstructionFlavor::WMMA:
98 return "WMMA";
99 case InstructionFlavor::SingleCycleVALU:
100 return "VALU(1c)";
101 case InstructionFlavor::TRANS:
102 return "TRANS";
103 case InstructionFlavor::MultiCycleVALU:
104 return "VALU(Nc)";
105 case InstructionFlavor::VMEM:
106 return "VMEM";
107 case InstructionFlavor::SMEM:
108 return "SMEM";
109 case InstructionFlavor::DS:
110 return "DS";
111 case InstructionFlavor::SALU:
112 return "SALU";
113 case InstructionFlavor::DMA:
114 return "DMA";
115 case InstructionFlavor::Fence:
116 return "Fence";
117 case InstructionFlavor::Other:
118 return "Other";
119 case InstructionFlavor::NUM_FLAVORS:
120 return "???";
121 }
122 llvm_unreachable("Unknown InstructionFlavor");
123}
124
125/// Classify \p MI into the execution flavor that drives both the scheduler's
126/// slot preferences and the hazard recognizer's co-execution masks.
127InstructionFlavor classifyFlavor(const MachineInstr &MI,
128 const SIInstrInfo &SII);
129
130/// Map a flavor to the co-execution class it occupies in a window slot.
131constexpr CoExecMaskT getCoExecMask(InstructionFlavor F) {
132 switch (F) {
133 case InstructionFlavor::WMMA:
134 return CoExecMask::WMMA;
135 case InstructionFlavor::TRANS:
136 return CoExecMask::TRANS;
137 case InstructionFlavor::SingleCycleVALU:
138 case InstructionFlavor::MultiCycleVALU:
139 // LDS DMA and tensor DMA issue on the VALU pipe.
140 case InstructionFlavor::DMA:
141 return CoExecMask::VALU;
142 case InstructionFlavor::DS:
143 return CoExecMask::DS;
144 case InstructionFlavor::VMEM:
145 return CoExecMask::VMEM;
146 case InstructionFlavor::SMEM:
147 return CoExecMask::SMEM;
148 case InstructionFlavor::SALU:
149 // Fences are s_barrier_*/s_wait_*, which issue on the scalar pipe.
150 case InstructionFlavor::Fence:
151 return CoExecMask::SALU;
152 case InstructionFlavor::Other:
153 return CoExecMask::CTRL;
154 case InstructionFlavor::NUM_FLAVORS:
155 break;
156 }
157 llvm_unreachable("Unknown InstructionFlavor");
158}
159
160//===----------------------------------------------------------------------===//
161// Co-execution Stage Type
162//===----------------------------------------------------------------------===//
163
164/// Stage type for co-execution (for annotation/display).
165enum class CoExecStageType : uint8_t {
166 NONE = 0, // Not in co-exec window
167 E0, // Issue cycle - control only
168 E, // External - MEM/SALU allowed
169 I, // Internal - MEM/SALU/VALU allowed
170 IS, // Internal + scaled-WMMA absorb (I plus next-WMMA issue)
171 V, // Vacant - MEM/SALU/WMMA allowed, no VALU
172 TR // TRANS co-exec - everything except TRANS
173};
174
175inline const char *getStageTypeName(CoExecStageType T) {
176 switch (T) {
177 case CoExecStageType::NONE:
178 return "--";
179 case CoExecStageType::E0:
180 return "E0";
181 case CoExecStageType::E:
182 return "E";
183 case CoExecStageType::I:
184 return "I";
185 case CoExecStageType::IS:
186 return "IS";
187 case CoExecStageType::V:
188 return "V";
189 case CoExecStageType::TR:
190 return "TR";
191 }
192 llvm_unreachable("Unknown CoExecStageType");
193}
194
195/// Return a human-readable name for a mask holding a single instruction class,
196/// as produced by getCoExecMask().
197inline const char *getCoExecMaskName(CoExecMaskT Mask) {
198 switch (Mask) {
199 case CoExecMask::CTRL:
200 return "CTRL";
201 case CoExecMask::VALU:
202 return "VALU";
203 case CoExecMask::TRANS:
204 return "TRANS";
205 case CoExecMask::SALU:
206 return "SALU";
207 case CoExecMask::DS:
208 return "DS";
209 case CoExecMask::VMEM:
210 return "VMEM";
211 case CoExecMask::SMEM:
212 return "SMEM";
213 case CoExecMask::WMMA:
214 return "WMMA";
215 default:
216 llvm_unreachable("Not a single instruction class");
217 }
218}
219
220/// Max stages: INT8 16x16x64 = 17 cycles, round up for safety.
221constexpr unsigned MaxCoExecStages = 32;
222
223//===----------------------------------------------------------------------===//
224// Co-execution Slot Info
225//===----------------------------------------------------------------------===//
226
227/// Per-slot info: which instruction classes may co-execute here.
228struct CoExecSlotInfo {
229 CoExecMaskT Mask = CoExecMask::All; // What CAN execute (correctness)
230};
231
232//===----------------------------------------------------------------------===//
233// Co-execution Info
234//===----------------------------------------------------------------------===//
235
236/// Co-execution characteristics for a multi-cycle instruction.
237struct CoExecInfo {
238 /// Number of cycles for which the producing instruction occupies its
239 /// execution unit.
240 unsigned UnitOccupancy = 0;
241 /// Number of cycles in the co-execution window, counting any trailing
242 /// vacant stages.
243 unsigned TotalWindow = 0;
244 /// Per-stage slot info (capability mask).
245 CoExecSlotInfo Slots[MaxCoExecStages];
246 /// Pattern string for display (e.g., "0EIIEEIIV").
247 StringRef Pattern;
248
249 /// Default constructor - initialize to safe defaults.
250 CoExecInfo() {
251 for (unsigned I = 0; I < MaxCoExecStages; ++I)
252 Slots[I].Mask = CoExecMask::All; // Default: permissive
253 }
254
255 /// Get capability mask for a stage.
256 CoExecMaskT getMask(unsigned Stage) const {
257 return Stage < TotalWindow ? Slots[Stage].Mask : CoExecMask::All;
258 }
259
260 /// Check if an instruction class mask can co-execute at a given stage.
261 bool canCoExec(CoExecMaskT InstMask, unsigned Stage) const {
262 if (Stage >= TotalWindow)
263 return true;
264 return any(Val: Slots[Stage].Mask & InstMask);
265 }
266
267 /// Find next stage where the instruction class is allowed.
268 std::optional<unsigned> findNextAllowedStage(CoExecMaskT InstMask,
269 unsigned FromStage) const {
270 for (unsigned I = FromStage; I < TotalWindow; ++I) {
271 if (any(Val: Slots[I].Mask & InstMask))
272 return I;
273 }
274 return std::nullopt;
275 }
276
277 /// Return the number of cycles an instruction matching \p InstMask must wait
278 /// at \p Stage. Returns zero when no stall is required.
279 unsigned getStallCycles(CoExecMaskT InstMask, unsigned Stage) const {
280 if (canCoExec(InstMask, Stage))
281 return 0;
282
283 // Stall until the next stage that accepts the instruction.
284 if (std::optional<unsigned> Next =
285 findNextAllowedStage(InstMask, FromStage: Stage + 1))
286 return *Next - Stage;
287
288 // Stall until the window ends if no later stage accepts the instruction.
289 return TotalWindow - Stage;
290 }
291
292 /// Get stage type from mask for display.
293 static CoExecStageType getStageType(CoExecMaskT Mask) {
294 if (Mask == CoExecMask::StageE0)
295 return CoExecStageType::E0;
296 if (Mask == CoExecMask::StageE)
297 return CoExecStageType::E;
298 if (Mask == CoExecMask::StageIS)
299 return CoExecStageType::IS;
300 if (Mask == CoExecMask::StageI)
301 return CoExecStageType::I;
302 if (Mask == CoExecMask::StageV)
303 return CoExecStageType::V;
304 if (Mask == CoExecMask::StageTR)
305 return CoExecStageType::TR;
306 // For 'All' or unknown, return based on what's allowed.
307 if (any(Val: Mask & CoExecMask::VALU))
308 return CoExecStageType::I; // If VALU allowed, it's I-like
309 if (any(Val: Mask & CoExecMask::WMMA))
310 return CoExecStageType::V; // If WMMA allowed (not VALU), V-like
311 return CoExecStageType::E; // Default to E
312 }
313
314 /// Get stage type for a specific stage.
315 CoExecStageType getType(unsigned Stage) const {
316 return getStageType(Mask: getMask(Stage));
317 }
318
319 /// Build a CoExecInfo from an occupancy and stage pattern.
320 static CoExecInfo build(unsigned UnitOccupancy, unsigned TotalWindow,
321 const char *Pattern);
322};
323
324//===----------------------------------------------------------------------===//
325// Co-execution Info Construction
326//===----------------------------------------------------------------------===//
327
328/// Build CoExecInfo from a pattern string.
329/// Pattern chars: '0'=E0, 'E'=External, 'I'=Internal, 'V'=Vacant,
330/// 'S'=Internal+ScaleWMMAAbsorb (I plus next scaled WMMA),
331/// 'T'=TRANS co-exec (all except TRANS), 'A'=Any
332inline CoExecInfo CoExecInfo::build(unsigned UnitOccupancy,
333 unsigned TotalWindow, const char *Pattern) {
334 CoExecInfo Info;
335 Info.UnitOccupancy = UnitOccupancy;
336 Info.TotalWindow = TotalWindow;
337 Info.Pattern = Pattern;
338 assert(Info.Pattern.size() == TotalWindow &&
339 "Pattern must describe every cycle of the co-execution window");
340 assert(TotalWindow <= MaxCoExecStages && "Co-execution window is too long");
341
342 for (unsigned I = 0; I < Info.TotalWindow; ++I) {
343 switch (Pattern[I]) {
344 case '0':
345 Info.Slots[I].Mask = CoExecMask::StageE0;
346 break;
347 case 'E':
348 Info.Slots[I].Mask = CoExecMask::StageE;
349 break;
350 case 'I':
351 Info.Slots[I].Mask = CoExecMask::StageI;
352 break;
353 case 'S':
354 Info.Slots[I].Mask = CoExecMask::StageIS;
355 break;
356 case 'V':
357 Info.Slots[I].Mask = CoExecMask::StageV;
358 break;
359 case 'T':
360 Info.Slots[I].Mask = CoExecMask::StageTR;
361 break;
362 case 'A':
363 default:
364 Info.Slots[I].Mask = CoExecMask::All;
365 break;
366 }
367 }
368 return Info;
369}
370
371/// Get co-execution info for a gfx950 MFMA instruction.
372CoExecInfo getMFMACoExecInfo(unsigned Opcode);
373
374/// Get co-execution info for a WMMA instruction, selecting the per-cycle slot
375/// pattern from the opcode (and operand formats for the F8F6F4 variants).
376inline CoExecInfo getCoExecInfo(const MachineInstr &MI,
377 const SIInstrInfo &TII) {
378 unsigned Opc = MI.getOpcode();
379 const WMMAInstInfo *InstInfo = getWMMAInstInfoHelper(Opcode: Opc);
380 WMMAVariant Variant =
381 InstInfo ? InstInfo->CoExecVariant : WMMAVariant::Unknown;
382
383 if (TII.isMFMA(Opcode: Opc))
384 return getMFMACoExecInfo(Opcode: Opc);
385
386 // Scaled variants (LD_SCALE rule) absorb the next WMMA in the last I slot.
387 bool HasScaling = InstInfo && InstInfo->HasMatrixScale;
388
389 // The F8F6F4 family is the only WMMA carrying matrix format operands, and its
390 // window depends on them: both inputs f4 issue in 4 cycles, anything wider in
391 // 8. This matches the PredIsNotBothF4_WMMA_SCALE latency variant.
392 if (const MachineOperand *FmtA =
393 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::matrix_a_fmt)) {
394 const MachineOperand *FmtB =
395 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::matrix_b_fmt);
396 bool BothF4 = FmtB && FmtA->getImm() == AMDGPU::WMMA::MATRIX_FMT_FP4 &&
397 FmtB->getImm() == AMDGPU::WMMA::MATRIX_FMT_FP4;
398 if (BothF4)
399 Variant = WMMAVariant::F8F6F4_16x16x128_BothF4;
400 }
401
402 switch (Variant) {
403 // 16x16x64 IU8: 16-cycle occupancy, 17-cycle window.
404 case WMMAVariant::IU8_16x16x64:
405 return CoExecInfo::build(UnitOccupancy: 16, TotalWindow: 17, Pattern: "0EIIEEIIEEIIEEIIV");
406
407 // 16x16x128 F8/F6/F4 has occupancy of 8 cycles and a window of 10 cycles.
408 case WMMAVariant::F8F6F4_16x16x128:
409 return CoExecInfo::build(UnitOccupancy: 8, TotalWindow: 10, Pattern: HasScaling ? "0EEIEEISVV" : "0EEIEEIIVV");
410
411 // 16x16x128 with two F4 inputs has occupancy of 4 cycles and a window of 6
412 // cycles.
413 case WMMAVariant::F8F6F4_16x16x128_BothF4:
414 return CoExecInfo::build(UnitOccupancy: 4, TotalWindow: 6, Pattern: HasScaling ? "0EESVV" : "0EEIVV");
415
416 // 16x16x64 FP8/BF8: 4-cycle occupancy, 6-cycle window.
417 case WMMAVariant::FP8BF8_16x16x64:
418 return CoExecInfo::build(UnitOccupancy: 4, TotalWindow: 6, Pattern: "0EEIVV");
419
420 // 16x16x32 F16/BF16: 8-cycle occupancy, 9-cycle window.
421 case WMMAVariant::F16BF16_16x16x32:
422 return CoExecInfo::build(UnitOccupancy: 8, TotalWindow: 9, Pattern: "0EIIEEIIV");
423
424 // 16x16x128 FP8/BF8: 8-cycle occupancy, 10-cycle window.
425 case WMMAVariant::FP8BF8_16x16x128:
426 return CoExecInfo::build(UnitOccupancy: 8, TotalWindow: 10, Pattern: "0EEIEEIIVV");
427
428 // 32x16x128 F4: 8-cycle occupancy, 10-cycle window.
429 case WMMAVariant::F4_32x16x128:
430 return CoExecInfo::build(UnitOccupancy: 8, TotalWindow: 10, Pattern: HasScaling ? "0EEIEIESVV" : "0EEIEIEIVV");
431
432 case WMMAVariant::Unknown:
433 // Permissive window for variants without a modeled slot pattern.
434 return CoExecInfo::build(UnitOccupancy: 0, TotalWindow: 9, Pattern: "AAAAAAAAA");
435 }
436 llvm_unreachable("unknown WMMA variant");
437}
438
439} // namespace AMDGPU
440} // namespace llvm
441
442#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUCOEXECINFO_H
443