1//===- AMDGPUMIRFormatter.cpp ---------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Implementation of AMDGPU overrides of MIRFormatter.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUMIRFormatter.h"
15#include "AMDGPUWaitcntUtils.h"
16#include "SIMachineFunctionInfo.h"
17#include "llvm/TargetParser/AMDGPUTargetParser.h"
18
19using namespace llvm;
20
21const char SWaitAluImmPrefix = '.';
22StringLiteral SWaitAluDelim = "_";
23
24StringLiteral VaVdstName = "VaVdst";
25StringLiteral VaSdstName = "VaSdst";
26StringLiteral VaSsrcName = "VaSsrc";
27StringLiteral HoldCntName = "HoldCnt";
28StringLiteral VmVsrcName = "VmVsrc";
29StringLiteral VaVccName = "VaVcc";
30StringLiteral SaSdstName = "SaSdst";
31
32StringLiteral AllOff = "AllOff";
33
34StringLiteral VmcntName = "Vmcnt";
35StringLiteral ExpcntName = "Expcnt";
36StringLiteral LgkmcntName = "Lgkmcnt";
37
38StringLiteral LoadcntName = "Loadcnt";
39StringLiteral DscntName = "Dscnt";
40
41void AMDGPUMIRFormatter::printSWaitAluImm(uint64_t Imm, raw_ostream &OS) const {
42 bool NonePrinted = true;
43 ListSeparator Delim(SWaitAluDelim);
44 auto PrintFieldIfNotMax = [&](StringRef Descr, uint64_t Num, unsigned Max) {
45 if (Num != Max) {
46 OS << Delim << Descr << SWaitAluDelim << Num;
47 NonePrinted = false;
48 }
49 };
50 OS << SWaitAluImmPrefix;
51 PrintFieldIfNotMax(VaVdstName, AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: Imm),
52 AMDGPU::DepCtr::getVaVdstBitMask());
53 PrintFieldIfNotMax(VaSdstName, AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Imm),
54 AMDGPU::DepCtr::getVaSdstBitMask());
55 PrintFieldIfNotMax(VaSsrcName, AMDGPU::DepCtr::decodeFieldVaSsrc(Encoded: Imm),
56 AMDGPU::DepCtr::getVaSsrcBitMask());
57 PrintFieldIfNotMax(
58 HoldCntName,
59 AMDGPU::DepCtr::decodeFieldHoldCnt(Encoded: Imm,
60 Version: AMDGPU::getIsaVersion(GPU: STI.getCPU())),
61 AMDGPU::DepCtr::getHoldCntBitMask(Version: AMDGPU::getIsaVersion(GPU: STI.getCPU())));
62 PrintFieldIfNotMax(VmVsrcName, AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: Imm),
63 AMDGPU::DepCtr::getVmVsrcBitMask());
64 PrintFieldIfNotMax(VaVccName, AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Imm),
65 AMDGPU::DepCtr::getVaVccBitMask());
66 PrintFieldIfNotMax(SaSdstName, AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Imm),
67 AMDGPU::DepCtr::getSaSdstBitMask());
68 if (NonePrinted)
69 OS << AllOff;
70}
71
72void AMDGPUMIRFormatter::printSWaitcntImm(uint64_t Imm, raw_ostream &OS) const {
73 const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU());
74 bool NonePrinted = true;
75 ListSeparator Delim(SWaitAluDelim);
76 auto PrintFieldIfNotMax = [&](StringRef Descr, uint64_t Num, unsigned Max) {
77 if (Num != Max) {
78 OS << Delim << Descr << SWaitAluDelim << Num;
79 NonePrinted = false;
80 }
81 };
82 OS << SWaitAluImmPrefix;
83 PrintFieldIfNotMax(VmcntName, AMDGPU::decodeVmcnt(Version, Waitcnt: Imm),
84 AMDGPU::getVmcntBitMask(Version));
85 PrintFieldIfNotMax(ExpcntName, AMDGPU::decodeExpcnt(Version, Waitcnt: Imm),
86 AMDGPU::getExpcntBitMask(Version));
87 PrintFieldIfNotMax(LgkmcntName, AMDGPU::decodeLgkmcnt(Version, Waitcnt: Imm),
88 AMDGPU::getLgkmcntBitMask(Version));
89 if (NonePrinted)
90 OS << AllOff;
91}
92
93void AMDGPUMIRFormatter::printSWaitLoadcntDscntImm(uint64_t Imm,
94 raw_ostream &OS) const {
95 const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU());
96 bool NonePrinted = true;
97 ListSeparator Delim(SWaitAluDelim);
98 auto PrintFieldIfNotMax = [&](StringRef Descr, uint64_t Num, unsigned Max) {
99 if (Num != Max) {
100 OS << Delim << Descr << SWaitAluDelim << Num;
101 NonePrinted = false;
102 }
103 };
104 OS << SWaitAluImmPrefix;
105 PrintFieldIfNotMax(LoadcntName, AMDGPU::decodeLoadcnt(Version, Waitcnt: Imm),
106 AMDGPU::getLoadcntBitMask(Version));
107 PrintFieldIfNotMax(DscntName, AMDGPU::decodeDscnt(Version, Waitcnt: Imm),
108 AMDGPU::getDscntBitMask(Version));
109 if (NonePrinted)
110 OS << AllOff;
111}
112
113void AMDGPUMIRFormatter::printImm(raw_ostream &OS, const MachineInstr &MI,
114 std::optional<unsigned int> OpIdx, int64_t Imm) const {
115
116 switch (MI.getOpcode()) {
117 case AMDGPU::S_WAITCNT:
118 case AMDGPU::S_WAITCNT_soft:
119 printSWaitcntImm(Imm, OS);
120 break;
121 case AMDGPU::S_WAIT_LOADCNT_DSCNT:
122 printSWaitLoadcntDscntImm(Imm, OS);
123 break;
124 case AMDGPU::S_WAITCNT_DEPCTR:
125 printSWaitAluImm(Imm, OS);
126 break;
127 case AMDGPU::S_DELAY_ALU:
128 assert(OpIdx == 0);
129 printSDelayAluImm(Imm, OS);
130 break;
131 default:
132 MIRFormatter::printImm(OS, MI, OpIdx, Imm);
133 break;
134 }
135}
136
137/// Implement target specific parsing of immediate mnemonics. The mnemonic is
138/// a string with a leading dot.
139bool AMDGPUMIRFormatter::parseImmMnemonic(const unsigned OpCode,
140 const unsigned OpIdx,
141 StringRef Src, int64_t &Imm,
142 ErrorCallbackType ErrorCallback) const
143{
144
145 switch (OpCode) {
146 case AMDGPU::S_WAITCNT:
147 case AMDGPU::S_WAITCNT_soft:
148 return parseSWaitcntImmMnemonic(OpIdx, Imm, Src, ErrorCallback);
149 case AMDGPU::S_WAIT_LOADCNT_DSCNT:
150 return parseSWaitLoadcntDscntImmMnemonic(OpIdx, Imm, Src, ErrorCallback);
151 case AMDGPU::S_WAITCNT_DEPCTR:
152 return parseSWaitAluImmMnemonic(OpIdx, Imm, Src, ErrorCallback);
153 case AMDGPU::S_DELAY_ALU:
154 return parseSDelayAluImmMnemonic(OpIdx, Imm, Src, ErrorCallback);
155 default:
156 break;
157 }
158 return true; // Don't know what this is
159}
160
161void AMDGPUMIRFormatter::printSDelayAluImm(int64_t Imm,
162 llvm::raw_ostream &OS) const {
163 // Construct an immediate string to represent the information encoded in the
164 // s_delay_alu immediate.
165 // .id0_<dep>[_skip_<count>_id1<dep>]
166 constexpr int64_t None = 0;
167 constexpr int64_t Same = 0;
168
169 uint64_t Id0 = (Imm & 0xF);
170 uint64_t Skip = ((Imm >> 4) & 0x7);
171 uint64_t Id1 = ((Imm >> 7) & 0xF);
172 auto Outdep = [&](uint64_t Id) {
173 if (Id == None)
174 OS << "NONE";
175 else if (Id < 5)
176 OS << "VALU_DEP_" << Id;
177 else if (Id < 8)
178 OS << "TRANS32_DEP_" << Id - 4;
179 else
180 OS << "SALU_CYCLE_" << Id - 8;
181 };
182
183 OS << ".id0_";
184 Outdep(Id0);
185
186 // If the second inst is "same" and "none", no need to print the rest of the
187 // string.
188 if (Skip == Same && Id1 == None)
189 return;
190
191 // Encode the second delay specification.
192 OS << "_skip_";
193 if (Skip == 0)
194 OS << "SAME";
195 else if (Skip == 1)
196 OS << "NEXT";
197 else
198 OS << "SKIP_" << Skip - 1;
199
200 OS << "_id1_";
201 Outdep(Id1);
202}
203
204bool AMDGPUMIRFormatter::parseSWaitcntImmMnemonic(
205 const unsigned int OpIdx, int64_t &Imm, StringRef &Src,
206 MIRFormatter::ErrorCallbackType &ErrorCallback) const {
207 const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU());
208
209 // Accept integer masks for compatibility with old MIR.
210 if (!Src.consumeInteger(Radix: 10, Result&: Imm))
211 return false;
212
213 // Initialize with all counters at max (no wait).
214 unsigned Vmcnt = AMDGPU::getVmcntBitMask(Version);
215 unsigned Expcnt = AMDGPU::getExpcntBitMask(Version);
216 unsigned Lgkmcnt = AMDGPU::getLgkmcntBitMask(Version);
217
218 // The input is in the form: .Name1_Num1_Name2_Num2
219 // Drop the '.' prefix.
220 if (!Src.consume_front(Prefix: SWaitAluImmPrefix))
221 return ErrorCallback(Src.begin(), "expected prefix");
222 if (Src.empty())
223 return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>");
224
225 // Special case for all off (all counters at max).
226 if (Src == AllOff) {
227 Imm = AMDGPU::encodeWaitcnt(Version, Vmcnt, Expcnt, Lgkmcnt);
228 return false;
229 }
230
231 // Parse counter name, number pairs.
232 while (!Src.empty()) {
233 size_t DelimIdx = Src.find(Str: SWaitAluDelim);
234 if (DelimIdx == StringRef::npos)
235 return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>");
236 StringRef Name = Src.substr(Start: 0, N: DelimIdx);
237 StringRef::iterator NamePos = Src.begin();
238 Src.consume_front(Prefix: Name);
239 Src.consume_front(Prefix: SWaitAluDelim);
240
241 int64_t Num;
242 StringRef::iterator NumPos = Src.begin();
243 if (Src.consumeInteger(Radix: 10, Result&: Num) || Num < 0)
244 return ErrorCallback(NumPos,
245 "expected non-negative integer counter number");
246
247 unsigned Max;
248 if (Name == VmcntName) {
249 Max = AMDGPU::getVmcntBitMask(Version);
250 Vmcnt = Num;
251 } else if (Name == ExpcntName) {
252 Max = AMDGPU::getExpcntBitMask(Version);
253 Expcnt = Num;
254 } else if (Name == LgkmcntName) {
255 Max = AMDGPU::getLgkmcntBitMask(Version);
256 Lgkmcnt = Num;
257 } else {
258 return ErrorCallback(NamePos, "invalid counter name");
259 }
260 if (Num >= Max)
261 return ErrorCallback(NumPos, "counter value too large");
262
263 Src.consume_front(Prefix: SWaitAluDelim);
264 }
265
266 Imm = AMDGPU::encodeWaitcnt(Version, Vmcnt, Expcnt, Lgkmcnt);
267 return false;
268}
269
270bool AMDGPUMIRFormatter::parseSWaitLoadcntDscntImmMnemonic(
271 const unsigned int OpIdx, int64_t &Imm, StringRef &Src,
272 MIRFormatter::ErrorCallbackType &ErrorCallback) const {
273 const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU());
274
275 // Accept integer masks for compatibility with old MIR.
276 if (!Src.consumeInteger(Radix: 10, Result&: Imm))
277 return false;
278
279 // Initialize with all counters at max (no wait).
280 unsigned Loadcnt = AMDGPU::getLoadcntBitMask(Version);
281 unsigned Dscnt = AMDGPU::getDscntBitMask(Version);
282
283 // The input is in the form: .Name1_Num1_Name2_Num2
284 // Drop the '.' prefix.
285 if (!Src.consume_front(Prefix: SWaitAluImmPrefix))
286 return ErrorCallback(Src.begin(), "expected prefix");
287 if (Src.empty())
288 return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>");
289
290 // Special case for all off (all counters at max).
291 if (Src == AllOff) {
292 Imm = AMDGPU::encodeLoadcntDscnt(Version, Loadcnt, Dscnt);
293 return false;
294 }
295
296 // Parse counter name, number pairs.
297 while (!Src.empty()) {
298 size_t DelimIdx = Src.find(Str: SWaitAluDelim);
299 if (DelimIdx == StringRef::npos)
300 return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>");
301 StringRef Name = Src.substr(Start: 0, N: DelimIdx);
302 StringRef::iterator NamePos = Src.begin();
303 Src.consume_front(Prefix: Name);
304 Src.consume_front(Prefix: SWaitAluDelim);
305
306 int64_t Num;
307 StringRef::iterator NumPos = Src.begin();
308 if (Src.consumeInteger(Radix: 10, Result&: Num) || Num < 0)
309 return ErrorCallback(NumPos,
310 "expected non-negative integer counter number");
311
312 unsigned Max;
313 if (Name == LoadcntName) {
314 Max = AMDGPU::getLoadcntBitMask(Version);
315 Loadcnt = Num;
316 } else if (Name == DscntName) {
317 Max = AMDGPU::getDscntBitMask(Version);
318 Dscnt = Num;
319 } else {
320 return ErrorCallback(NamePos, "invalid counter name");
321 }
322 if (Num >= Max)
323 return ErrorCallback(NumPos, "counter value too large");
324
325 Src.consume_front(Prefix: SWaitAluDelim);
326 }
327
328 Imm = AMDGPU::encodeLoadcntDscnt(Version, Loadcnt, Dscnt);
329 return false;
330}
331
332bool AMDGPUMIRFormatter::parseSWaitAluImmMnemonic(
333 const unsigned int OpIdx, int64_t &Imm, StringRef &Src,
334 MIRFormatter::ErrorCallbackType &ErrorCallback) const {
335 // TODO: For now accept integer masks for compatibility with old MIR.
336 if (!Src.consumeInteger(Radix: 10, Result&: Imm))
337 return false;
338
339 // Initialize with all checks off.
340 Imm = AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI);
341 // The input is in the form: .Name1_Num1_Name2_Num2
342 // Drop the '.' prefix.
343 bool ConsumePrefix = Src.consume_front(Prefix: SWaitAluImmPrefix);
344 if (!ConsumePrefix)
345 return ErrorCallback(Src.begin(), "expected prefix");
346 if (Src.empty())
347 return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>");
348
349 // Special case for all off.
350 if (Src == AllOff)
351 return false;
352
353 // Parse a counter name, number pair in each iteration.
354 while (!Src.empty()) {
355 // Src: Name1_Num1_Name2_Num2
356 // ^
357 size_t DelimIdx = Src.find(Str: SWaitAluDelim);
358 if (DelimIdx == StringRef::npos)
359 return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>");
360 // Src: Name1_Num1_Name2_Num2
361 // ^^^^^
362 StringRef Name = Src.substr(Start: 0, N: DelimIdx);
363 // Save the position of the name for accurate error reporting.
364 StringRef::iterator NamePos = Src.begin();
365 [[maybe_unused]] bool ConsumeName = Src.consume_front(Prefix: Name);
366 assert(ConsumeName && "Expected name");
367 [[maybe_unused]] bool ConsumeDelim = Src.consume_front(Prefix: SWaitAluDelim);
368 assert(ConsumeDelim && "Expected delimiter");
369 // Src: Num1_Name2_Num2
370 // ^
371 DelimIdx = Src.find(Str: SWaitAluDelim);
372 // Src: Num1_Name2_Num2
373 // ^^^^
374 int64_t Num;
375 // Save the position of the number for accurate error reporting.
376 StringRef::iterator NumPos = Src.begin();
377 if (Src.consumeInteger(Radix: 10, Result&: Num) || Num < 0)
378 return ErrorCallback(NumPos,
379 "expected non-negative integer counter number");
380 unsigned Max;
381 if (Name == VaVdstName) {
382 Max = AMDGPU::DepCtr::getVaVdstBitMask();
383 Imm = AMDGPU::DepCtr::encodeFieldVaVdst(Encoded: Imm, VaVdst: Num);
384 } else if (Name == VmVsrcName) {
385 Max = AMDGPU::DepCtr::getVmVsrcBitMask();
386 Imm = AMDGPU::DepCtr::encodeFieldVmVsrc(Encoded: Imm, VmVsrc: Num);
387 } else if (Name == VaSdstName) {
388 Max = AMDGPU::DepCtr::getVaSdstBitMask();
389 Imm = AMDGPU::DepCtr::encodeFieldVaSdst(Encoded: Imm, VaSdst: Num);
390 } else if (Name == VaSsrcName) {
391 Max = AMDGPU::DepCtr::getVaSsrcBitMask();
392 Imm = AMDGPU::DepCtr::encodeFieldVaSsrc(Encoded: Imm, VaSsrc: Num);
393 } else if (Name == HoldCntName) {
394 const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU());
395 Max = AMDGPU::DepCtr::getHoldCntBitMask(Version);
396 Imm = AMDGPU::DepCtr::encodeFieldHoldCnt(Encoded: Imm, HoldCnt: Num, Version);
397 } else if (Name == VaVccName) {
398 Max = AMDGPU::DepCtr::getVaVccBitMask();
399 Imm = AMDGPU::DepCtr::encodeFieldVaVcc(Encoded: Imm, VaVcc: Num);
400 } else if (Name == SaSdstName) {
401 Max = AMDGPU::DepCtr::getSaSdstBitMask();
402 Imm = AMDGPU::DepCtr::encodeFieldSaSdst(Encoded: Imm, SaSdst: Num);
403 } else {
404 return ErrorCallback(NamePos, "invalid counter name");
405 }
406 // Don't allow the values to reach their maximum value.
407 if (Num >= Max)
408 return ErrorCallback(NumPos, "counter value too large");
409 // Src: Name2_Num2
410 Src.consume_front(Prefix: SWaitAluDelim);
411 }
412 return false;
413}
414
415bool AMDGPUMIRFormatter::parseSDelayAluImmMnemonic(
416 const unsigned int OpIdx, int64_t &Imm, llvm::StringRef &Src,
417 llvm::MIRFormatter::ErrorCallbackType &ErrorCallback) const
418{
419 assert(OpIdx == 0);
420
421 Imm = 0;
422 bool Expected = Src.consume_front(Prefix: ".id0_");
423 if (!Expected)
424 return ErrorCallback(Src.begin(), "Expected .id0_");
425
426 auto ExpectInt = [&](StringRef &Src, int64_t Offset) -> int64_t {
427 int64_t Dep;
428 if (!Src.consumeInteger(Radix: 10, Result&: Dep))
429 return Dep + Offset;
430
431 return -1;
432 };
433
434 auto DecodeDelay = [&](StringRef &Src) -> int64_t {
435 if (Src.consume_front(Prefix: "NONE"))
436 return 0;
437 if (Src.consume_front(Prefix: "VALU_DEP_"))
438 return ExpectInt(Src, 0);
439 if (Src.consume_front(Prefix: "TRANS32_DEP_"))
440 return ExpectInt(Src, 4);
441 if (Src.consume_front(Prefix: "SALU_CYCLE_"))
442 return ExpectInt(Src, 8);
443
444 return -1;
445 };
446
447 int64_t Delay0 = DecodeDelay(Src);
448 int64_t Skip = 0;
449 int64_t Delay1 = 0;
450 if (Delay0 == -1)
451 return ErrorCallback(Src.begin(), "Could not decode delay0");
452
453
454 // Set the Imm so far, to that early return has the correct value.
455 Imm = Delay0;
456
457 // If that was the end of the string, the second instruction is "same" and
458 // "none"
459 if (Src.begin() == Src.end())
460 return false;
461
462 Expected = Src.consume_front(Prefix: "_skip_");
463 if (!Expected)
464 return ErrorCallback(Src.begin(), "Expected _skip_");
465
466
467 if (Src.consume_front(Prefix: "SAME")) {
468 Skip = 0;
469 } else if (Src.consume_front(Prefix: "NEXT")) {
470 Skip = 1;
471 } else if (Src.consume_front(Prefix: "SKIP_")) {
472 if (Src.consumeInteger(Radix: 10, Result&: Skip)) {
473 return ErrorCallback(Src.begin(), "Expected integer Skip value");
474 }
475 Skip += 1;
476 } else {
477 ErrorCallback(Src.begin(), "Unexpected Skip Value");
478 }
479
480 Expected = Src.consume_front(Prefix: "_id1_");
481 if (!Expected)
482 return ErrorCallback(Src.begin(), "Expected _id1_");
483
484 Delay1 = DecodeDelay(Src);
485 if (Delay1 == -1)
486 return ErrorCallback(Src.begin(), "Could not decode delay1");
487
488 Imm = Imm | (Skip << 4) | (Delay1 << 7);
489 return false;
490}
491
492bool AMDGPUMIRFormatter::parseCustomPseudoSourceValue(
493 StringRef Src, MachineFunction &MF, PerFunctionMIParsingState &PFS,
494 const PseudoSourceValue *&PSV, ErrorCallbackType ErrorCallback) const {
495 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
496 const AMDGPUTargetMachine &TM =
497 static_cast<const AMDGPUTargetMachine &>(MF.getTarget());
498 if (Src == "GWSResource") {
499 PSV = MFI->getGWSPSV(TM);
500 return false;
501 }
502 llvm_unreachable("unknown MIR custom pseudo source value");
503}
504