1//===-- AMDGPUCoExecInfo.h - Co-execution info ------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Shared types for co-execution modeling used by GCNHazardRecognizer and the
11/// schedulers.
12///
13/// Multi-cycle instructions (WMMA, TRANS, etc.) have execution windows where
14/// other instruction types can co-execute. For WMMA, slot patterns depend on
15/// the variant:
16///
17/// E0 (Issue): Control instructions only (s_delay_alu, s_set_vgpr_msb)
18/// E (External): Memory and SALU can co-execute, no VALU
19/// I (Internal): VALU, TRANS, memory, and SALU can all co-execute
20/// V (Vacant): Memory/SALU/next-WMMA ok, NO VALU/TRANS
21///
22//===----------------------------------------------------------------------===//
23
24#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUCOEXECINFO_H
25#define LLVM_LIB_TARGET_AMDGPU_AMDGPUCOEXECINFO_H
26
27#include "SIDefines.h"
28#include "SIInstrInfo.h"
29#include "llvm/ADT/BitmaskEnum.h"
30#include "llvm/ADT/StringRef.h"
31#include <cassert>
32#include <cstdint>
33#include <optional>
34
35namespace llvm {
36
37namespace AMDGPU {
38
39//===----------------------------------------------------------------------===//
40// Co-execution Bitmasks
41//===----------------------------------------------------------------------===//
42
43/// Bitmask for instruction types allowed to co-execute at a stage.
44enum class CoExecMask : uint16_t {
45 None = 0,
46 CTRL = 1 << 0, // Control: s_delay_alu, s_set_vgpr_msb
47 VALU = 1 << 1, // Vector ALU
48 TRANS = 1 << 2, // Transcendentals (V_EXP etc)
49 SALU = 1 << 3, // Scalar ALU
50 DS = 1 << 4, // LDS read/write
51 VMEM = 1 << 5, // Global memory
52 SMEM = 1 << 6, // Scalar memory
53 WMMA = 1 << 7, // Next WMMA (V stages only)
54 All = 0xFFFF,
55
56 MEM = DS | VMEM | SMEM,
57 StageE0 = CTRL, // Issue: control only
58 StageE = CTRL | SALU | MEM, // External: mem/salu
59 StageI = CTRL | SALU | MEM | VALU | TRANS, // Internal: all ALU
60 // Internal + scaled-WMMA absorb: same as StageI but the next scaled
61 // WMMA may issue here - its LD_SCALE consumes the I cycle and the matrix
62 // multiply lands in the V slot that follows. Used for the last I before
63 // V of scaled patterns.
64 StageIS = StageI | WMMA,
65 StageV = CTRL | SALU | MEM | WMMA, // Vacant: no valu/trans
66 StageTR = All & ~TRANS, // TRANS co-exec: no TRANS
67
68 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/All)
69};
70
71using CoExecMaskT = CoExecMask;
72
73//===----------------------------------------------------------------------===//
74// Instruction Flavor Classification
75//===----------------------------------------------------------------------===//
76
77/// Classification of instructions by execution characteristics.
78/// Used for scheduling decisions and co-execution slot preferences.
79enum class InstructionFlavor : uint8_t {
80 WMMA, // WMMA/MFMA matrix operations
81 SingleCycleVALU, // Single-cycle VALU (not TRANS, not multi-cycle CVT)
82 TRANS, // Transcendental ops (v_exp, v_log, etc.)
83 MultiCycleVALU, // VALU instructions with repeat rate > 1
84 VMEM, // FLAT/GLOBAL memory operations
85 SMEM, // Scalar memory operations
86 DS, // LDS/GDS operations
87 SALU, // Scalar ALU
88 DMA, // Tensor DMA operations
89 Fence, // Fences and waits
90 Other, // Everything else
91 NUM_FLAVORS
92};
93
94constexpr StringRef getFlavorName(InstructionFlavor F) {
95 switch (F) {
96 case InstructionFlavor::WMMA:
97 return "WMMA";
98 case InstructionFlavor::SingleCycleVALU:
99 return "VALU(1c)";
100 case InstructionFlavor::TRANS:
101 return "TRANS";
102 case InstructionFlavor::MultiCycleVALU:
103 return "VALU(Nc)";
104 case InstructionFlavor::VMEM:
105 return "VMEM";
106 case InstructionFlavor::SMEM:
107 return "SMEM";
108 case InstructionFlavor::DS:
109 return "DS";
110 case InstructionFlavor::SALU:
111 return "SALU";
112 case InstructionFlavor::DMA:
113 return "DMA";
114 case InstructionFlavor::Fence:
115 return "Fence";
116 case InstructionFlavor::Other:
117 return "Other";
118 case InstructionFlavor::NUM_FLAVORS:
119 return "???";
120 }
121 llvm_unreachable("Unknown InstructionFlavor");
122}
123
124/// Classify \p MI into the execution flavor that drives both the scheduler's
125/// slot preferences and the hazard recognizer's co-execution masks.
126InstructionFlavor classifyFlavor(const MachineInstr &MI,
127 const SIInstrInfo &SII);
128
129/// Map a flavor to the co-execution class it occupies in a window slot.
130constexpr CoExecMaskT getCoExecMask(InstructionFlavor F) {
131 switch (F) {
132 case InstructionFlavor::WMMA:
133 return CoExecMask::WMMA;
134 case InstructionFlavor::TRANS:
135 return CoExecMask::TRANS;
136 case InstructionFlavor::SingleCycleVALU:
137 case InstructionFlavor::MultiCycleVALU:
138 // LDS DMA and tensor DMA issue on the VALU pipe.
139 case InstructionFlavor::DMA:
140 return CoExecMask::VALU;
141 case InstructionFlavor::DS:
142 return CoExecMask::DS;
143 case InstructionFlavor::VMEM:
144 return CoExecMask::VMEM;
145 case InstructionFlavor::SMEM:
146 return CoExecMask::SMEM;
147 case InstructionFlavor::SALU:
148 // Fences are s_barrier_*/s_wait_*, which issue on the scalar pipe.
149 case InstructionFlavor::Fence:
150 return CoExecMask::SALU;
151 case InstructionFlavor::Other:
152 return CoExecMask::CTRL;
153 case InstructionFlavor::NUM_FLAVORS:
154 break;
155 }
156 llvm_unreachable("Unknown InstructionFlavor");
157}
158
159//===----------------------------------------------------------------------===//
160// Co-execution Stage Type
161//===----------------------------------------------------------------------===//
162
163/// Stage type for co-execution (for annotation/display).
164enum class CoExecStageType : uint8_t {
165 NONE = 0, // Not in co-exec window
166 E0, // Issue cycle - control only
167 E, // External - MEM/SALU allowed
168 I, // Internal - MEM/SALU/VALU allowed
169 IS, // Internal + scaled-WMMA absorb (I plus next-WMMA issue)
170 V, // Vacant - MEM/SALU/WMMA allowed, no VALU
171 TR // TRANS co-exec - everything except TRANS
172};
173
174inline const char *getStageTypeName(CoExecStageType T) {
175 switch (T) {
176 case CoExecStageType::NONE:
177 return "--";
178 case CoExecStageType::E0:
179 return "E0";
180 case CoExecStageType::E:
181 return "E";
182 case CoExecStageType::I:
183 return "I";
184 case CoExecStageType::IS:
185 return "IS";
186 case CoExecStageType::V:
187 return "V";
188 case CoExecStageType::TR:
189 return "TR";
190 }
191 llvm_unreachable("Unknown CoExecStageType");
192}
193
194/// Return a human-readable name for a mask holding a single instruction class,
195/// as produced by getCoExecMask().
196inline const char *getCoExecMaskName(CoExecMaskT Mask) {
197 switch (Mask) {
198 case CoExecMask::CTRL:
199 return "CTRL";
200 case CoExecMask::VALU:
201 return "VALU";
202 case CoExecMask::TRANS:
203 return "TRANS";
204 case CoExecMask::SALU:
205 return "SALU";
206 case CoExecMask::DS:
207 return "DS";
208 case CoExecMask::VMEM:
209 return "VMEM";
210 case CoExecMask::SMEM:
211 return "SMEM";
212 case CoExecMask::WMMA:
213 return "WMMA";
214 default:
215 llvm_unreachable("Not a single instruction class");
216 }
217}
218
219/// Max stages: INT8 16x16x64 = 17 cycles, round up for safety.
220constexpr unsigned MaxCoExecStages = 32;
221
222//===----------------------------------------------------------------------===//
223// Co-execution Slot Info
224//===----------------------------------------------------------------------===//
225
226/// Per-slot info: which instruction classes may co-execute here.
227struct CoExecSlotInfo {
228 CoExecMaskT Mask = CoExecMask::All; // What CAN execute (correctness)
229};
230
231//===----------------------------------------------------------------------===//
232// Co-execution Info
233//===----------------------------------------------------------------------===//
234
235/// Co-execution characteristics for a multi-cycle instruction.
236struct CoExecInfo {
237 /// Number of cycles in the co-execution window, counting any trailing
238 /// vacant stages.
239 unsigned TotalWindow = 0;
240 /// Per-stage slot info (capability mask).
241 CoExecSlotInfo Slots[MaxCoExecStages];
242 /// Pattern string for display (e.g., "0EIIEEIIV").
243 StringRef Pattern;
244
245 /// Default constructor - initialize to safe defaults.
246 CoExecInfo() {
247 for (unsigned I = 0; I < MaxCoExecStages; ++I)
248 Slots[I].Mask = CoExecMask::All; // Default: permissive
249 }
250
251 /// Get capability mask for a stage.
252 CoExecMaskT getMask(unsigned Stage) const {
253 return Stage < TotalWindow ? Slots[Stage].Mask : CoExecMask::All;
254 }
255
256 /// Check if an instruction class mask can co-execute at a given stage.
257 bool canCoExec(CoExecMaskT InstMask, unsigned Stage) const {
258 if (Stage >= TotalWindow)
259 return true;
260 return any(Val: Slots[Stage].Mask & InstMask);
261 }
262
263 /// Find next stage where the instruction class is allowed.
264 std::optional<unsigned> findNextAllowedStage(CoExecMaskT InstMask,
265 unsigned FromStage) const {
266 for (unsigned I = FromStage; I < TotalWindow; ++I) {
267 if (any(Val: Slots[I].Mask & InstMask))
268 return I;
269 }
270 return std::nullopt;
271 }
272
273 /// Get stage type from mask for display.
274 static CoExecStageType getStageType(CoExecMaskT Mask) {
275 if (Mask == CoExecMask::StageE0)
276 return CoExecStageType::E0;
277 if (Mask == CoExecMask::StageE)
278 return CoExecStageType::E;
279 if (Mask == CoExecMask::StageIS)
280 return CoExecStageType::IS;
281 if (Mask == CoExecMask::StageI)
282 return CoExecStageType::I;
283 if (Mask == CoExecMask::StageV)
284 return CoExecStageType::V;
285 if (Mask == CoExecMask::StageTR)
286 return CoExecStageType::TR;
287 // For 'All' or unknown, return based on what's allowed.
288 if (any(Val: Mask & CoExecMask::VALU))
289 return CoExecStageType::I; // If VALU allowed, it's I-like
290 if (any(Val: Mask & CoExecMask::WMMA))
291 return CoExecStageType::V; // If WMMA allowed (not VALU), V-like
292 return CoExecStageType::E; // Default to E
293 }
294
295 /// Get stage type for a specific stage.
296 CoExecStageType getType(unsigned Stage) const {
297 return getStageType(Mask: getMask(Stage));
298 }
299
300 /// Build a CoExecInfo from a pattern string.
301 static CoExecInfo build(unsigned TotalWindow, const char *Pattern);
302};
303
304//===----------------------------------------------------------------------===//
305// Co-execution Info Construction
306//===----------------------------------------------------------------------===//
307
308/// Build CoExecInfo from a pattern string.
309/// Pattern chars: '0'=E0, 'E'=External, 'I'=Internal, 'V'=Vacant,
310/// 'S'=Internal+ScaleWMMAAbsorb (I plus next scaled WMMA),
311/// 'T'=TRANS co-exec (all except TRANS), 'A'=Any
312inline CoExecInfo CoExecInfo::build(unsigned TotalWindow, const char *Pattern) {
313 CoExecInfo Info;
314 Info.TotalWindow = TotalWindow;
315 Info.Pattern = Pattern;
316 assert(Info.Pattern.size() == TotalWindow &&
317 "Pattern must describe every cycle of the co-execution window");
318 assert(TotalWindow <= MaxCoExecStages && "Co-execution window is too long");
319
320 for (unsigned I = 0; I < Info.TotalWindow; ++I) {
321 switch (Pattern[I]) {
322 case '0':
323 Info.Slots[I].Mask = CoExecMask::StageE0;
324 break;
325 case 'E':
326 Info.Slots[I].Mask = CoExecMask::StageE;
327 break;
328 case 'I':
329 Info.Slots[I].Mask = CoExecMask::StageI;
330 break;
331 case 'S':
332 Info.Slots[I].Mask = CoExecMask::StageIS;
333 break;
334 case 'V':
335 Info.Slots[I].Mask = CoExecMask::StageV;
336 break;
337 case 'T':
338 Info.Slots[I].Mask = CoExecMask::StageTR;
339 break;
340 case 'A':
341 default:
342 Info.Slots[I].Mask = CoExecMask::All;
343 break;
344 }
345 }
346 return Info;
347}
348
349/// Get co-execution info for a WMMA instruction, selecting the per-cycle slot
350/// pattern from the opcode (and operand formats for the F8F6F4 variants).
351inline CoExecInfo getCoExecInfo(const MachineInstr &MI,
352 const SIInstrInfo &TII) {
353 unsigned Opc = MI.getOpcode();
354
355 // Scaled variants (LD_SCALE rule) absorb the next WMMA in the last I slot.
356 bool HasScaling = AMDGPU::getHasMatrixScale(Opc);
357
358 // The F8F6F4 family is the only WMMA carrying matrix format operands, and its
359 // window depends on them: both inputs f4 issue in 4 cycles, anything wider in
360 // 8. This matches the PredIsNotBothF4_WMMA_SCALE latency variant.
361 if (const MachineOperand *FmtA =
362 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::matrix_a_fmt)) {
363 const MachineOperand *FmtB =
364 TII.getNamedOperand(MI, OperandName: AMDGPU::OpName::matrix_b_fmt);
365 bool BothF4 = FmtB && FmtA->getImm() == AMDGPU::WMMA::MATRIX_FMT_FP4 &&
366 FmtB->getImm() == AMDGPU::WMMA::MATRIX_FMT_FP4;
367 if (BothF4)
368 return CoExecInfo::build(TotalWindow: 6, Pattern: HasScaling ? "0EESVV" : "0EEIVV");
369 return CoExecInfo::build(TotalWindow: 10, Pattern: HasScaling ? "0EEIEEISVV" : "0EEIEEIIVV");
370 }
371
372 switch (Opc) {
373 // 16x16x64 IU8: 16-cycle occupancy, 17-cycle window.
374 case AMDGPU::V_WMMA_I32_16X16X64_IU8_w32_threeaddr:
375 case AMDGPU::V_WMMA_I32_16X16X64_IU8_w32_twoaddr:
376 return CoExecInfo::build(TotalWindow: 17, Pattern: "0EIIEEIIEEIIEEIIV");
377
378 // 16x16x64 FP8/BF8: 4-cycle occupancy, 6-cycle window.
379 case AMDGPU::V_WMMA_F16_16X16X64_BF8_BF8_w32_threeaddr:
380 case AMDGPU::V_WMMA_F16_16X16X64_BF8_BF8_w32_twoaddr:
381 case AMDGPU::V_WMMA_F16_16X16X64_BF8_FP8_w32_threeaddr:
382 case AMDGPU::V_WMMA_F16_16X16X64_BF8_FP8_w32_twoaddr:
383 case AMDGPU::V_WMMA_F16_16X16X64_FP8_BF8_w32_threeaddr:
384 case AMDGPU::V_WMMA_F16_16X16X64_FP8_BF8_w32_twoaddr:
385 case AMDGPU::V_WMMA_F16_16X16X64_FP8_FP8_w32_threeaddr:
386 case AMDGPU::V_WMMA_F16_16X16X64_FP8_FP8_w32_twoaddr:
387 case AMDGPU::V_WMMA_F32_16X16X64_BF8_BF8_w32_threeaddr:
388 case AMDGPU::V_WMMA_F32_16X16X64_BF8_BF8_w32_twoaddr:
389 case AMDGPU::V_WMMA_F32_16X16X64_BF8_FP8_w32_threeaddr:
390 case AMDGPU::V_WMMA_F32_16X16X64_BF8_FP8_w32_twoaddr:
391 case AMDGPU::V_WMMA_F32_16X16X64_FP8_BF8_w32_threeaddr:
392 case AMDGPU::V_WMMA_F32_16X16X64_FP8_BF8_w32_twoaddr:
393 case AMDGPU::V_WMMA_F32_16X16X64_FP8_FP8_w32_threeaddr:
394 case AMDGPU::V_WMMA_F32_16X16X64_FP8_FP8_w32_twoaddr:
395 return CoExecInfo::build(TotalWindow: 6, Pattern: "0EEIVV");
396
397 // 16x16x32 F16/BF16: 8-cycle occupancy, 9-cycle window.
398 case AMDGPU::V_SWMMAC_BF16_16X16X32_BF16_w32_twoaddr:
399 case AMDGPU::V_SWMMAC_BF16_16X16X32_BF16_w64_twoaddr:
400 case AMDGPU::V_SWMMAC_F16_16X16X32_F16_w32_twoaddr:
401 case AMDGPU::V_SWMMAC_F16_16X16X32_F16_w64_twoaddr:
402 case AMDGPU::V_SWMMAC_F32_16X16X32_BF16_w32_twoaddr:
403 case AMDGPU::V_SWMMAC_F32_16X16X32_BF16_w64_twoaddr:
404 case AMDGPU::V_SWMMAC_F32_16X16X32_F16_w32_twoaddr:
405 case AMDGPU::V_SWMMAC_F32_16X16X32_F16_w64_twoaddr:
406 case AMDGPU::V_WMMA_BF16F32_16X16X32_BF16_w32_threeaddr:
407 case AMDGPU::V_WMMA_BF16F32_16X16X32_BF16_w32_twoaddr:
408 case AMDGPU::V_WMMA_BF16_16X16X32_BF16_w32_threeaddr:
409 case AMDGPU::V_WMMA_BF16_16X16X32_BF16_w32_twoaddr:
410 case AMDGPU::V_WMMA_F16_16X16X32_F16_w32_threeaddr:
411 case AMDGPU::V_WMMA_F16_16X16X32_F16_w32_twoaddr:
412 case AMDGPU::V_WMMA_F32_16X16X32_BF16_w32_threeaddr:
413 case AMDGPU::V_WMMA_F32_16X16X32_BF16_w32_twoaddr:
414 case AMDGPU::V_WMMA_F32_16X16X32_F16_w32_threeaddr:
415 case AMDGPU::V_WMMA_F32_16X16X32_F16_w32_twoaddr:
416 return CoExecInfo::build(TotalWindow: 9, Pattern: "0EIIEEIIV");
417
418 // 16x16x128 FP8/BF8: 8-cycle occupancy, 10-cycle window.
419 case AMDGPU::V_SWMMAC_F16_16X16X128_BF8_BF8_w32_twoaddr:
420 case AMDGPU::V_SWMMAC_F16_16X16X128_BF8_FP8_w32_twoaddr:
421 case AMDGPU::V_SWMMAC_F16_16X16X128_FP8_BF8_w32_twoaddr:
422 case AMDGPU::V_SWMMAC_F16_16X16X128_FP8_FP8_w32_twoaddr:
423 case AMDGPU::V_SWMMAC_F32_16X16X128_BF8_BF8_w32_twoaddr:
424 case AMDGPU::V_SWMMAC_F32_16X16X128_BF8_FP8_w32_twoaddr:
425 case AMDGPU::V_SWMMAC_F32_16X16X128_FP8_BF8_w32_twoaddr:
426 case AMDGPU::V_SWMMAC_F32_16X16X128_FP8_FP8_w32_twoaddr:
427 case AMDGPU::V_WMMA_F16_16X16X128_BF8_BF8_w32_threeaddr:
428 case AMDGPU::V_WMMA_F16_16X16X128_BF8_BF8_w32_twoaddr:
429 case AMDGPU::V_WMMA_F16_16X16X128_BF8_FP8_w32_threeaddr:
430 case AMDGPU::V_WMMA_F16_16X16X128_BF8_FP8_w32_twoaddr:
431 case AMDGPU::V_WMMA_F16_16X16X128_FP8_BF8_w32_threeaddr:
432 case AMDGPU::V_WMMA_F16_16X16X128_FP8_BF8_w32_twoaddr:
433 case AMDGPU::V_WMMA_F16_16X16X128_FP8_FP8_w32_threeaddr:
434 case AMDGPU::V_WMMA_F16_16X16X128_FP8_FP8_w32_twoaddr:
435 case AMDGPU::V_WMMA_F32_16X16X128_BF8_BF8_w32_threeaddr:
436 case AMDGPU::V_WMMA_F32_16X16X128_BF8_BF8_w32_twoaddr:
437 case AMDGPU::V_WMMA_F32_16X16X128_BF8_FP8_w32_threeaddr:
438 case AMDGPU::V_WMMA_F32_16X16X128_BF8_FP8_w32_twoaddr:
439 case AMDGPU::V_WMMA_F32_16X16X128_FP8_BF8_w32_threeaddr:
440 case AMDGPU::V_WMMA_F32_16X16X128_FP8_BF8_w32_twoaddr:
441 case AMDGPU::V_WMMA_F32_16X16X128_FP8_FP8_w32_threeaddr:
442 case AMDGPU::V_WMMA_F32_16X16X128_FP8_FP8_w32_twoaddr:
443 return CoExecInfo::build(TotalWindow: 10, Pattern: "0EEIEEIIVV");
444
445 // 32x16x128 F4: 8-cycle occupancy, 10-cycle window.
446 case AMDGPU::V_WMMA_F32_32X16X128_F4_w32_threeaddr:
447 case AMDGPU::V_WMMA_F32_32X16X128_F4_w32_twoaddr:
448 case AMDGPU::V_WMMA_SCALE16_F32_32X16X128_F4_w32_threeaddr:
449 case AMDGPU::V_WMMA_SCALE16_F32_32X16X128_F4_w32_twoaddr:
450 case AMDGPU::V_WMMA_SCALE_F32_32X16X128_F4_w32_threeaddr:
451 case AMDGPU::V_WMMA_SCALE_F32_32X16X128_F4_w32_twoaddr:
452 return CoExecInfo::build(TotalWindow: 10, Pattern: HasScaling ? "0EEIEIESVV" : "0EEIEIEIVV");
453
454 default:
455 // Permissive window for variants without a modeled slot pattern.
456 return CoExecInfo::build(TotalWindow: 9, Pattern: "AAAAAAAAA");
457 }
458}
459
460} // namespace AMDGPU
461} // namespace llvm
462
463#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUCOEXECINFO_H
464