1//===- AMDGPUMIRFormatter.cpp ---------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Implementation of AMDGPU overrides of MIRFormatter.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUMIRFormatter.h"
15#include "SIMachineFunctionInfo.h"
16#include "llvm/TargetParser/AMDGPUTargetParser.h"
17
18using namespace llvm;
19
20const char SWaitAluImmPrefix = '.';
21StringLiteral SWaitAluDelim = "_";
22
23StringLiteral VaVdstName = "VaVdst";
24StringLiteral VaSdstName = "VaSdst";
25StringLiteral VaSsrcName = "VaSsrc";
26StringLiteral HoldCntName = "HoldCnt";
27StringLiteral VmVsrcName = "VmVsrc";
28StringLiteral VaVccName = "VaVcc";
29StringLiteral SaSdstName = "SaSdst";
30
31StringLiteral AllOff = "AllOff";
32
33StringLiteral VmcntName = "Vmcnt";
34StringLiteral ExpcntName = "Expcnt";
35StringLiteral LgkmcntName = "Lgkmcnt";
36
37StringLiteral LoadcntName = "Loadcnt";
38StringLiteral DscntName = "Dscnt";
39
40void AMDGPUMIRFormatter::printSWaitAluImm(uint64_t Imm, raw_ostream &OS) const {
41 bool NonePrinted = true;
42 ListSeparator Delim(SWaitAluDelim);
43 auto PrintFieldIfNotMax = [&](StringRef Descr, uint64_t Num, unsigned Max) {
44 if (Num != Max) {
45 OS << Delim << Descr << SWaitAluDelim << Num;
46 NonePrinted = false;
47 }
48 };
49 OS << SWaitAluImmPrefix;
50 PrintFieldIfNotMax(VaVdstName, AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: Imm),
51 AMDGPU::DepCtr::getVaVdstBitMask());
52 PrintFieldIfNotMax(VaSdstName, AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Imm),
53 AMDGPU::DepCtr::getVaSdstBitMask());
54 PrintFieldIfNotMax(VaSsrcName, AMDGPU::DepCtr::decodeFieldVaSsrc(Encoded: Imm),
55 AMDGPU::DepCtr::getVaSsrcBitMask());
56 PrintFieldIfNotMax(
57 HoldCntName,
58 AMDGPU::DepCtr::decodeFieldHoldCnt(Encoded: Imm,
59 Version: AMDGPU::getIsaVersion(GPU: STI.getCPU())),
60 AMDGPU::DepCtr::getHoldCntBitMask(Version: AMDGPU::getIsaVersion(GPU: STI.getCPU())));
61 PrintFieldIfNotMax(VmVsrcName, AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: Imm),
62 AMDGPU::DepCtr::getVmVsrcBitMask());
63 PrintFieldIfNotMax(VaVccName, AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Imm),
64 AMDGPU::DepCtr::getVaVccBitMask());
65 PrintFieldIfNotMax(SaSdstName, AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Imm),
66 AMDGPU::DepCtr::getSaSdstBitMask());
67 if (NonePrinted)
68 OS << AllOff;
69}
70
71void AMDGPUMIRFormatter::printSWaitcntImm(uint64_t Imm, raw_ostream &OS) const {
72 const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU());
73 bool NonePrinted = true;
74 ListSeparator Delim(SWaitAluDelim);
75 auto PrintFieldIfNotMax = [&](StringRef Descr, uint64_t Num, unsigned Max) {
76 if (Num != Max) {
77 OS << Delim << Descr << SWaitAluDelim << Num;
78 NonePrinted = false;
79 }
80 };
81 OS << SWaitAluImmPrefix;
82 PrintFieldIfNotMax(VmcntName, AMDGPU::decodeVmcnt(Version, Waitcnt: Imm),
83 AMDGPU::getVmcntBitMask(Version));
84 PrintFieldIfNotMax(ExpcntName, AMDGPU::decodeExpcnt(Version, Waitcnt: Imm),
85 AMDGPU::getExpcntBitMask(Version));
86 PrintFieldIfNotMax(LgkmcntName, AMDGPU::decodeLgkmcnt(Version, Waitcnt: Imm),
87 AMDGPU::getLgkmcntBitMask(Version));
88 if (NonePrinted)
89 OS << AllOff;
90}
91
92void AMDGPUMIRFormatter::printSWaitLoadcntDscntImm(uint64_t Imm,
93 raw_ostream &OS) const {
94 const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU());
95 bool NonePrinted = true;
96 ListSeparator Delim(SWaitAluDelim);
97 auto PrintFieldIfNotMax = [&](StringRef Descr, uint64_t Num, unsigned Max) {
98 if (Num != Max) {
99 OS << Delim << Descr << SWaitAluDelim << Num;
100 NonePrinted = false;
101 }
102 };
103 OS << SWaitAluImmPrefix;
104 PrintFieldIfNotMax(LoadcntName, AMDGPU::decodeLoadcnt(Version, Waitcnt: Imm),
105 AMDGPU::getLoadcntBitMask(Version));
106 PrintFieldIfNotMax(DscntName, AMDGPU::decodeDscnt(Version, Waitcnt: Imm),
107 AMDGPU::getDscntBitMask(Version));
108 if (NonePrinted)
109 OS << AllOff;
110}
111
112void AMDGPUMIRFormatter::printImm(raw_ostream &OS, const MachineInstr &MI,
113 std::optional<unsigned int> OpIdx, int64_t Imm) const {
114
115 switch (MI.getOpcode()) {
116 case AMDGPU::S_WAITCNT:
117 case AMDGPU::S_WAITCNT_soft:
118 printSWaitcntImm(Imm, OS);
119 break;
120 case AMDGPU::S_WAIT_LOADCNT_DSCNT:
121 printSWaitLoadcntDscntImm(Imm, OS);
122 break;
123 case AMDGPU::S_WAITCNT_DEPCTR:
124 printSWaitAluImm(Imm, OS);
125 break;
126 case AMDGPU::S_DELAY_ALU:
127 assert(OpIdx == 0);
128 printSDelayAluImm(Imm, OS);
129 break;
130 default:
131 MIRFormatter::printImm(OS, MI, OpIdx, Imm);
132 break;
133 }
134}
135
136/// Implement target specific parsing of immediate mnemonics. The mnemonic is
137/// a string with a leading dot.
138bool AMDGPUMIRFormatter::parseImmMnemonic(const unsigned OpCode,
139 const unsigned OpIdx,
140 StringRef Src, int64_t &Imm,
141 ErrorCallbackType ErrorCallback) const
142{
143
144 switch (OpCode) {
145 case AMDGPU::S_WAITCNT:
146 case AMDGPU::S_WAITCNT_soft:
147 return parseSWaitcntImmMnemonic(OpIdx, Imm, Src, ErrorCallback);
148 case AMDGPU::S_WAIT_LOADCNT_DSCNT:
149 return parseSWaitLoadcntDscntImmMnemonic(OpIdx, Imm, Src, ErrorCallback);
150 case AMDGPU::S_WAITCNT_DEPCTR:
151 return parseSWaitAluImmMnemonic(OpIdx, Imm, Src, ErrorCallback);
152 case AMDGPU::S_DELAY_ALU:
153 return parseSDelayAluImmMnemonic(OpIdx, Imm, Src, ErrorCallback);
154 default:
155 break;
156 }
157 return true; // Don't know what this is
158}
159
160void AMDGPUMIRFormatter::printSDelayAluImm(int64_t Imm,
161 llvm::raw_ostream &OS) const {
162 // Construct an immediate string to represent the information encoded in the
163 // s_delay_alu immediate.
164 // .id0_<dep>[_skip_<count>_id1<dep>]
165 constexpr int64_t None = 0;
166 constexpr int64_t Same = 0;
167
168 uint64_t Id0 = (Imm & 0xF);
169 uint64_t Skip = ((Imm >> 4) & 0x7);
170 uint64_t Id1 = ((Imm >> 7) & 0xF);
171 auto Outdep = [&](uint64_t Id) {
172 if (Id == None)
173 OS << "NONE";
174 else if (Id < 5)
175 OS << "VALU_DEP_" << Id;
176 else if (Id < 8)
177 OS << "TRANS32_DEP_" << Id - 4;
178 else
179 OS << "SALU_CYCLE_" << Id - 8;
180 };
181
182 OS << ".id0_";
183 Outdep(Id0);
184
185 // If the second inst is "same" and "none", no need to print the rest of the
186 // string.
187 if (Skip == Same && Id1 == None)
188 return;
189
190 // Encode the second delay specification.
191 OS << "_skip_";
192 if (Skip == 0)
193 OS << "SAME";
194 else if (Skip == 1)
195 OS << "NEXT";
196 else
197 OS << "SKIP_" << Skip - 1;
198
199 OS << "_id1_";
200 Outdep(Id1);
201}
202
203bool AMDGPUMIRFormatter::parseSWaitcntImmMnemonic(
204 const unsigned int OpIdx, int64_t &Imm, StringRef &Src,
205 MIRFormatter::ErrorCallbackType &ErrorCallback) const {
206 const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU());
207
208 // Accept integer masks for compatibility with old MIR.
209 if (!Src.consumeInteger(Radix: 10, Result&: Imm))
210 return false;
211
212 // Initialize with all counters at max (no wait).
213 unsigned Vmcnt = AMDGPU::getVmcntBitMask(Version);
214 unsigned Expcnt = AMDGPU::getExpcntBitMask(Version);
215 unsigned Lgkmcnt = AMDGPU::getLgkmcntBitMask(Version);
216
217 // The input is in the form: .Name1_Num1_Name2_Num2
218 // Drop the '.' prefix.
219 if (!Src.consume_front(Prefix: SWaitAluImmPrefix))
220 return ErrorCallback(Src.begin(), "expected prefix");
221 if (Src.empty())
222 return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>");
223
224 // Special case for all off (all counters at max).
225 if (Src == AllOff) {
226 Imm = AMDGPU::encodeWaitcnt(Version, Vmcnt, Expcnt, Lgkmcnt);
227 return false;
228 }
229
230 // Parse counter name, number pairs.
231 while (!Src.empty()) {
232 size_t DelimIdx = Src.find(Str: SWaitAluDelim);
233 if (DelimIdx == StringRef::npos)
234 return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>");
235 StringRef Name = Src.substr(Start: 0, N: DelimIdx);
236 StringRef::iterator NamePos = Src.begin();
237 Src.consume_front(Prefix: Name);
238 Src.consume_front(Prefix: SWaitAluDelim);
239
240 int64_t Num;
241 StringRef::iterator NumPos = Src.begin();
242 if (Src.consumeInteger(Radix: 10, Result&: Num) || Num < 0)
243 return ErrorCallback(NumPos,
244 "expected non-negative integer counter number");
245
246 unsigned Max;
247 if (Name == VmcntName) {
248 Max = AMDGPU::getVmcntBitMask(Version);
249 Vmcnt = Num;
250 } else if (Name == ExpcntName) {
251 Max = AMDGPU::getExpcntBitMask(Version);
252 Expcnt = Num;
253 } else if (Name == LgkmcntName) {
254 Max = AMDGPU::getLgkmcntBitMask(Version);
255 Lgkmcnt = Num;
256 } else {
257 return ErrorCallback(NamePos, "invalid counter name");
258 }
259 if (Num >= Max)
260 return ErrorCallback(NumPos, "counter value too large");
261
262 Src.consume_front(Prefix: SWaitAluDelim);
263 }
264
265 Imm = AMDGPU::encodeWaitcnt(Version, Vmcnt, Expcnt, Lgkmcnt);
266 return false;
267}
268
269bool AMDGPUMIRFormatter::parseSWaitLoadcntDscntImmMnemonic(
270 const unsigned int OpIdx, int64_t &Imm, StringRef &Src,
271 MIRFormatter::ErrorCallbackType &ErrorCallback) const {
272 const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU());
273
274 // Accept integer masks for compatibility with old MIR.
275 if (!Src.consumeInteger(Radix: 10, Result&: Imm))
276 return false;
277
278 // Initialize with all counters at max (no wait).
279 unsigned Loadcnt = AMDGPU::getLoadcntBitMask(Version);
280 unsigned Dscnt = AMDGPU::getDscntBitMask(Version);
281
282 // The input is in the form: .Name1_Num1_Name2_Num2
283 // Drop the '.' prefix.
284 if (!Src.consume_front(Prefix: SWaitAluImmPrefix))
285 return ErrorCallback(Src.begin(), "expected prefix");
286 if (Src.empty())
287 return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>");
288
289 // Special case for all off (all counters at max).
290 if (Src == AllOff) {
291 Imm = AMDGPU::encodeLoadcntDscnt(Version, Loadcnt, Dscnt);
292 return false;
293 }
294
295 // Parse counter name, number pairs.
296 while (!Src.empty()) {
297 size_t DelimIdx = Src.find(Str: SWaitAluDelim);
298 if (DelimIdx == StringRef::npos)
299 return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>");
300 StringRef Name = Src.substr(Start: 0, N: DelimIdx);
301 StringRef::iterator NamePos = Src.begin();
302 Src.consume_front(Prefix: Name);
303 Src.consume_front(Prefix: SWaitAluDelim);
304
305 int64_t Num;
306 StringRef::iterator NumPos = Src.begin();
307 if (Src.consumeInteger(Radix: 10, Result&: Num) || Num < 0)
308 return ErrorCallback(NumPos,
309 "expected non-negative integer counter number");
310
311 unsigned Max;
312 if (Name == LoadcntName) {
313 Max = AMDGPU::getLoadcntBitMask(Version);
314 Loadcnt = Num;
315 } else if (Name == DscntName) {
316 Max = AMDGPU::getDscntBitMask(Version);
317 Dscnt = Num;
318 } else {
319 return ErrorCallback(NamePos, "invalid counter name");
320 }
321 if (Num >= Max)
322 return ErrorCallback(NumPos, "counter value too large");
323
324 Src.consume_front(Prefix: SWaitAluDelim);
325 }
326
327 Imm = AMDGPU::encodeLoadcntDscnt(Version, Loadcnt, Dscnt);
328 return false;
329}
330
331bool AMDGPUMIRFormatter::parseSWaitAluImmMnemonic(
332 const unsigned int OpIdx, int64_t &Imm, StringRef &Src,
333 MIRFormatter::ErrorCallbackType &ErrorCallback) const {
334 // TODO: For now accept integer masks for compatibility with old MIR.
335 if (!Src.consumeInteger(Radix: 10, Result&: Imm))
336 return false;
337
338 // Initialize with all checks off.
339 Imm = AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI);
340 // The input is in the form: .Name1_Num1_Name2_Num2
341 // Drop the '.' prefix.
342 bool ConsumePrefix = Src.consume_front(Prefix: SWaitAluImmPrefix);
343 if (!ConsumePrefix)
344 return ErrorCallback(Src.begin(), "expected prefix");
345 if (Src.empty())
346 return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>");
347
348 // Special case for all off.
349 if (Src == AllOff)
350 return false;
351
352 // Parse a counter name, number pair in each iteration.
353 while (!Src.empty()) {
354 // Src: Name1_Num1_Name2_Num2
355 // ^
356 size_t DelimIdx = Src.find(Str: SWaitAluDelim);
357 if (DelimIdx == StringRef::npos)
358 return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>");
359 // Src: Name1_Num1_Name2_Num2
360 // ^^^^^
361 StringRef Name = Src.substr(Start: 0, N: DelimIdx);
362 // Save the position of the name for accurate error reporting.
363 StringRef::iterator NamePos = Src.begin();
364 [[maybe_unused]] bool ConsumeName = Src.consume_front(Prefix: Name);
365 assert(ConsumeName && "Expected name");
366 [[maybe_unused]] bool ConsumeDelim = Src.consume_front(Prefix: SWaitAluDelim);
367 assert(ConsumeDelim && "Expected delimiter");
368 // Src: Num1_Name2_Num2
369 // ^
370 DelimIdx = Src.find(Str: SWaitAluDelim);
371 // Src: Num1_Name2_Num2
372 // ^^^^
373 int64_t Num;
374 // Save the position of the number for accurate error reporting.
375 StringRef::iterator NumPos = Src.begin();
376 if (Src.consumeInteger(Radix: 10, Result&: Num) || Num < 0)
377 return ErrorCallback(NumPos,
378 "expected non-negative integer counter number");
379 unsigned Max;
380 if (Name == VaVdstName) {
381 Max = AMDGPU::DepCtr::getVaVdstBitMask();
382 Imm = AMDGPU::DepCtr::encodeFieldVaVdst(Encoded: Imm, VaVdst: Num);
383 } else if (Name == VmVsrcName) {
384 Max = AMDGPU::DepCtr::getVmVsrcBitMask();
385 Imm = AMDGPU::DepCtr::encodeFieldVmVsrc(Encoded: Imm, VmVsrc: Num);
386 } else if (Name == VaSdstName) {
387 Max = AMDGPU::DepCtr::getVaSdstBitMask();
388 Imm = AMDGPU::DepCtr::encodeFieldVaSdst(Encoded: Imm, VaSdst: Num);
389 } else if (Name == VaSsrcName) {
390 Max = AMDGPU::DepCtr::getVaSsrcBitMask();
391 Imm = AMDGPU::DepCtr::encodeFieldVaSsrc(Encoded: Imm, VaSsrc: Num);
392 } else if (Name == HoldCntName) {
393 const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU());
394 Max = AMDGPU::DepCtr::getHoldCntBitMask(Version);
395 Imm = AMDGPU::DepCtr::encodeFieldHoldCnt(Encoded: Imm, HoldCnt: Num, Version);
396 } else if (Name == VaVccName) {
397 Max = AMDGPU::DepCtr::getVaVccBitMask();
398 Imm = AMDGPU::DepCtr::encodeFieldVaVcc(Encoded: Imm, VaVcc: Num);
399 } else if (Name == SaSdstName) {
400 Max = AMDGPU::DepCtr::getSaSdstBitMask();
401 Imm = AMDGPU::DepCtr::encodeFieldSaSdst(Encoded: Imm, SaSdst: Num);
402 } else {
403 return ErrorCallback(NamePos, "invalid counter name");
404 }
405 // Don't allow the values to reach their maximum value.
406 if (Num >= Max)
407 return ErrorCallback(NumPos, "counter value too large");
408 // Src: Name2_Num2
409 Src.consume_front(Prefix: SWaitAluDelim);
410 }
411 return false;
412}
413
414bool AMDGPUMIRFormatter::parseSDelayAluImmMnemonic(
415 const unsigned int OpIdx, int64_t &Imm, llvm::StringRef &Src,
416 llvm::MIRFormatter::ErrorCallbackType &ErrorCallback) const
417{
418 assert(OpIdx == 0);
419
420 Imm = 0;
421 bool Expected = Src.consume_front(Prefix: ".id0_");
422 if (!Expected)
423 return ErrorCallback(Src.begin(), "Expected .id0_");
424
425 auto ExpectInt = [&](StringRef &Src, int64_t Offset) -> int64_t {
426 int64_t Dep;
427 if (!Src.consumeInteger(Radix: 10, Result&: Dep))
428 return Dep + Offset;
429
430 return -1;
431 };
432
433 auto DecodeDelay = [&](StringRef &Src) -> int64_t {
434 if (Src.consume_front(Prefix: "NONE"))
435 return 0;
436 if (Src.consume_front(Prefix: "VALU_DEP_"))
437 return ExpectInt(Src, 0);
438 if (Src.consume_front(Prefix: "TRANS32_DEP_"))
439 return ExpectInt(Src, 4);
440 if (Src.consume_front(Prefix: "SALU_CYCLE_"))
441 return ExpectInt(Src, 8);
442
443 return -1;
444 };
445
446 int64_t Delay0 = DecodeDelay(Src);
447 int64_t Skip = 0;
448 int64_t Delay1 = 0;
449 if (Delay0 == -1)
450 return ErrorCallback(Src.begin(), "Could not decode delay0");
451
452
453 // Set the Imm so far, to that early return has the correct value.
454 Imm = Delay0;
455
456 // If that was the end of the string, the second instruction is "same" and
457 // "none"
458 if (Src.begin() == Src.end())
459 return false;
460
461 Expected = Src.consume_front(Prefix: "_skip_");
462 if (!Expected)
463 return ErrorCallback(Src.begin(), "Expected _skip_");
464
465
466 if (Src.consume_front(Prefix: "SAME")) {
467 Skip = 0;
468 } else if (Src.consume_front(Prefix: "NEXT")) {
469 Skip = 1;
470 } else if (Src.consume_front(Prefix: "SKIP_")) {
471 if (Src.consumeInteger(Radix: 10, Result&: Skip)) {
472 return ErrorCallback(Src.begin(), "Expected integer Skip value");
473 }
474 Skip += 1;
475 } else {
476 ErrorCallback(Src.begin(), "Unexpected Skip Value");
477 }
478
479 Expected = Src.consume_front(Prefix: "_id1_");
480 if (!Expected)
481 return ErrorCallback(Src.begin(), "Expected _id1_");
482
483 Delay1 = DecodeDelay(Src);
484 if (Delay1 == -1)
485 return ErrorCallback(Src.begin(), "Could not decode delay1");
486
487 Imm = Imm | (Skip << 4) | (Delay1 << 7);
488 return false;
489}
490
491bool AMDGPUMIRFormatter::parseCustomPseudoSourceValue(
492 StringRef Src, MachineFunction &MF, PerFunctionMIParsingState &PFS,
493 const PseudoSourceValue *&PSV, ErrorCallbackType ErrorCallback) const {
494 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
495 const AMDGPUTargetMachine &TM =
496 static_cast<const AMDGPUTargetMachine &>(MF.getTarget());
497 if (Src == "GWSResource") {
498 PSV = MFI->getGWSPSV(TM);
499 return false;
500 }
501 llvm_unreachable("unknown MIR custom pseudo source value");
502}
503