1//===- AMDGPUMIRFormatter.cpp ---------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Implementation of AMDGPU overrides of MIRFormatter.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUMIRFormatter.h"
15#include "SIMachineFunctionInfo.h"
16#include "llvm/Support/AMDGPUAsyncStages.h"
17#include "llvm/Support/Format.h"
18#include "llvm/TargetParser/AMDGPUTargetParser.h"
19
20using namespace llvm;
21
22const char SWaitAluImmPrefix = '.';
23StringLiteral SWaitAluDelim = "_";
24
25StringLiteral VaVdstName = "VaVdst";
26StringLiteral VaSdstName = "VaSdst";
27StringLiteral VaSsrcName = "VaSsrc";
28StringLiteral HoldCntName = "HoldCnt";
29StringLiteral VmVsrcName = "VmVsrc";
30StringLiteral VaVccName = "VaVcc";
31StringLiteral SaSdstName = "SaSdst";
32
33StringLiteral AllOff = "AllOff";
34
35StringLiteral VmcntName = "Vmcnt";
36StringLiteral ExpcntName = "Expcnt";
37StringLiteral LgkmcntName = "Lgkmcnt";
38
39StringLiteral LoadcntName = "Loadcnt";
40StringLiteral DscntName = "Dscnt";
41
42const char AsyncStageImmPrefix = '.';
43constexpr StringLiteral AsyncStageDelim(".");
44constexpr StringLiteral ReservedStagePrefix("RES");
45constexpr StringLiteral AllStages("AllStages");
46
47void AMDGPUMIRFormatter::printSWaitAluImm(unsigned Imm, raw_ostream &OS) const {
48 bool NonePrinted = true;
49 ListSeparator Delim(SWaitAluDelim);
50 auto PrintFieldIfNotMax = [&](StringRef Descr, uint64_t Num, unsigned Max) {
51 if (Num != Max) {
52 OS << Delim << Descr << SWaitAluDelim << Num;
53 NonePrinted = false;
54 }
55 };
56 OS << SWaitAluImmPrefix;
57 PrintFieldIfNotMax(VaVdstName, AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: Imm),
58 AMDGPU::DepCtr::getVaVdstBitMask());
59 PrintFieldIfNotMax(VaSdstName, AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Imm),
60 AMDGPU::DepCtr::getVaSdstBitMask());
61 PrintFieldIfNotMax(VaSsrcName, AMDGPU::DepCtr::decodeFieldVaSsrc(Encoded: Imm),
62 AMDGPU::DepCtr::getVaSsrcBitMask());
63 PrintFieldIfNotMax(
64 HoldCntName,
65 AMDGPU::DepCtr::decodeFieldHoldCnt(Encoded: Imm,
66 Version: AMDGPU::getIsaVersion(GPU: STI.getCPU())),
67 AMDGPU::DepCtr::getHoldCntBitMask(Version: AMDGPU::getIsaVersion(GPU: STI.getCPU())));
68 PrintFieldIfNotMax(VmVsrcName, AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: Imm),
69 AMDGPU::DepCtr::getVmVsrcBitMask());
70 PrintFieldIfNotMax(VaVccName, AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Imm),
71 AMDGPU::DepCtr::getVaVccBitMask());
72 PrintFieldIfNotMax(SaSdstName, AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Imm),
73 AMDGPU::DepCtr::getSaSdstBitMask());
74 if (NonePrinted)
75 OS << AllOff;
76}
77
78void AMDGPUMIRFormatter::printSWaitcntImm(unsigned Imm, raw_ostream &OS) const {
79 const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU());
80 bool NonePrinted = true;
81 ListSeparator Delim(SWaitAluDelim);
82 auto PrintFieldIfNotMax = [&](StringRef Descr, uint64_t Num, unsigned Max) {
83 if (Num != Max) {
84 OS << Delim << Descr << SWaitAluDelim << Num;
85 NonePrinted = false;
86 }
87 };
88 OS << SWaitAluImmPrefix;
89 PrintFieldIfNotMax(VmcntName, AMDGPU::decodeVmcnt(Version, Waitcnt: Imm),
90 AMDGPU::getVmcntBitMask(Version));
91 PrintFieldIfNotMax(ExpcntName, AMDGPU::decodeExpcnt(Version, Waitcnt: Imm),
92 AMDGPU::getExpcntBitMask(Version));
93 PrintFieldIfNotMax(LgkmcntName, AMDGPU::decodeLgkmcnt(Version, Waitcnt: Imm),
94 AMDGPU::getLgkmcntBitMask(Version));
95 if (NonePrinted)
96 OS << AllOff;
97}
98
99void AMDGPUMIRFormatter::printSWaitLoadcntDscntImm(unsigned Imm,
100 raw_ostream &OS) const {
101 const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU());
102 bool NonePrinted = true;
103 ListSeparator Delim(SWaitAluDelim);
104 auto PrintFieldIfNotMax = [&](StringRef Descr, uint64_t Num, unsigned Max) {
105 if (Num != Max) {
106 OS << Delim << Descr << SWaitAluDelim << Num;
107 NonePrinted = false;
108 }
109 };
110 OS << SWaitAluImmPrefix;
111 PrintFieldIfNotMax(LoadcntName, AMDGPU::decodeLoadcnt(Version, Waitcnt: Imm),
112 AMDGPU::getLoadcntBitMask(Version));
113 PrintFieldIfNotMax(DscntName, AMDGPU::decodeDscnt(Version, Waitcnt: Imm),
114 AMDGPU::getDscntBitMask(Version));
115 if (NonePrinted)
116 OS << AllOff;
117}
118
119void AMDGPUMIRFormatter::printAsyncStageMaskImm(int64_t Imm,
120 raw_ostream &OS) const {
121 if (!AMDGPU::isValidAsyncStageMask(Mask: Imm)) {
122 OS << Imm;
123 return;
124 }
125
126 OS << AsyncStageImmPrefix;
127 auto Mask = static_cast<AMDGPU::AsyncStages>(Imm);
128 // The zero mask names every stage. Spell that out rather than printing an
129 // empty list. The saturated mask means the same thing but is a different
130 // immediate, so it keeps its spelled-out list to stay round-trippable.
131 if (Mask == AMDGPU::AsyncStages::NONE) {
132 OS << AllStages;
133 return;
134 }
135
136 // Reserved stages have no meaningful name, collect them into a single
137 // hex bitmask instead.
138 ListSeparator Delim(AsyncStageDelim);
139 for (AMDGPU::AsyncStages S :
140 AMDGPU::asyncStages(Mask: Mask & ~AMDGPU::AsyncStages::RESERVED))
141 OS << Delim << AMDGPU::getAsyncStageName(Stage: S);
142 AMDGPU::AsyncStages Reserved = Mask & AMDGPU::AsyncStages::RESERVED;
143 if (any(Val: Reserved))
144 OS << Delim << ReservedStagePrefix
145 << format_hex_no_prefix(N: to_underlying(E: Reserved), Width: 1);
146}
147
148void AMDGPUMIRFormatter::printImm(raw_ostream &OS, const MachineInstr &MI,
149 std::optional<unsigned int> OpIdx, int64_t Imm) const {
150
151 switch (MI.getOpcode()) {
152 case AMDGPU::ASYNCMARK:
153 case AMDGPU::WAIT_ASYNCMARK:
154 if (OpIdx == MI.getNumExplicitOperands() - 1u)
155 printAsyncStageMaskImm(Imm, OS);
156 else
157 MIRFormatter::printImm(OS, MI, OpIdx, Imm);
158 break;
159 case AMDGPU::S_WAITCNT:
160 case AMDGPU::S_WAITCNT_soft:
161 printSWaitcntImm(Imm: static_cast<unsigned>(Imm), OS);
162 break;
163 case AMDGPU::S_WAIT_LOADCNT_DSCNT:
164 printSWaitLoadcntDscntImm(Imm: static_cast<unsigned>(Imm), OS);
165 break;
166 case AMDGPU::S_WAITCNT_DEPCTR:
167 printSWaitAluImm(Imm: static_cast<unsigned>(Imm), OS);
168 break;
169 case AMDGPU::S_DELAY_ALU:
170 assert(OpIdx == 0);
171 printSDelayAluImm(Imm, OS);
172 break;
173 default:
174 MIRFormatter::printImm(OS, MI, OpIdx, Imm);
175 break;
176 }
177}
178
179/// Implement target specific parsing of immediate mnemonics. The mnemonic is
180/// a string with a leading dot.
181bool AMDGPUMIRFormatter::parseImmMnemonic(const unsigned OpCode,
182 const unsigned OpIdx,
183 StringRef Src, int64_t &Imm,
184 ErrorCallbackType ErrorCallback) const
185{
186
187 switch (OpCode) {
188 case AMDGPU::S_WAITCNT:
189 case AMDGPU::S_WAITCNT_soft:
190 return parseSWaitcntImmMnemonic(OpIdx, Imm, Src, ErrorCallback);
191 case AMDGPU::S_WAIT_LOADCNT_DSCNT:
192 return parseSWaitLoadcntDscntImmMnemonic(OpIdx, Imm, Src, ErrorCallback);
193 case AMDGPU::S_WAITCNT_DEPCTR:
194 return parseSWaitAluImmMnemonic(OpIdx, Imm, Src, ErrorCallback);
195 case AMDGPU::S_DELAY_ALU:
196 return parseSDelayAluImmMnemonic(OpIdx, Imm, Src, ErrorCallback);
197 case AMDGPU::ASYNCMARK:
198 if (OpIdx == 0)
199 return parseAsyncStageMaskImmMnemonic(OpIdx, Imm, Src, ErrorCallback);
200 break;
201 case AMDGPU::WAIT_ASYNCMARK:
202 if (OpIdx == 1)
203 return parseAsyncStageMaskImmMnemonic(OpIdx, Imm, Src, ErrorCallback);
204 break;
205 default:
206 break;
207 }
208 return true; // Don't know what this is
209}
210
211void AMDGPUMIRFormatter::printSDelayAluImm(int64_t Imm,
212 llvm::raw_ostream &OS) const {
213 // Construct an immediate string to represent the information encoded in the
214 // s_delay_alu immediate.
215 // .id0_<dep>[_skip_<count>_id1<dep>]
216 constexpr int64_t None = 0;
217 constexpr int64_t Same = 0;
218
219 uint64_t Id0 = (Imm & 0xF);
220 uint64_t Skip = ((Imm >> 4) & 0x7);
221 uint64_t Id1 = ((Imm >> 7) & 0xF);
222 auto Outdep = [&](uint64_t Id) {
223 if (Id == None)
224 OS << "NONE";
225 else if (Id < 5)
226 OS << "VALU_DEP_" << Id;
227 else if (Id < 8)
228 OS << "TRANS32_DEP_" << Id - 4;
229 else
230 OS << "SALU_CYCLE_" << Id - 8;
231 };
232
233 OS << ".id0_";
234 Outdep(Id0);
235
236 // If the second inst is "same" and "none", no need to print the rest of the
237 // string.
238 if (Skip == Same && Id1 == None)
239 return;
240
241 // Encode the second delay specification.
242 OS << "_skip_";
243 if (Skip == 0)
244 OS << "SAME";
245 else if (Skip == 1)
246 OS << "NEXT";
247 else
248 OS << "SKIP_" << Skip - 1;
249
250 OS << "_id1_";
251 Outdep(Id1);
252}
253
254bool AMDGPUMIRFormatter::parseSWaitcntImmMnemonic(
255 const unsigned int OpIdx, int64_t &Imm, StringRef &Src,
256 MIRFormatter::ErrorCallbackType &ErrorCallback) const {
257 const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU());
258
259 // Accept integer masks for compatibility with old MIR.
260 if (!Src.consumeInteger(Radix: 10, Result&: Imm))
261 return false;
262
263 // Initialize with all counters at max (no wait).
264 unsigned Vmcnt = AMDGPU::getVmcntBitMask(Version);
265 unsigned Expcnt = AMDGPU::getExpcntBitMask(Version);
266 unsigned Lgkmcnt = AMDGPU::getLgkmcntBitMask(Version);
267
268 // The input is in the form: .Name1_Num1_Name2_Num2
269 // Drop the '.' prefix.
270 if (!Src.consume_front(Prefix: SWaitAluImmPrefix))
271 return ErrorCallback(Src.begin(), "expected prefix");
272 if (Src.empty())
273 return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>");
274
275 // Special case for all off (all counters at max).
276 if (Src == AllOff) {
277 Imm = AMDGPU::encodeWaitcnt(Version, Vmcnt, Expcnt, Lgkmcnt);
278 return false;
279 }
280
281 // Parse counter name, number pairs.
282 while (!Src.empty()) {
283 size_t DelimIdx = Src.find(Str: SWaitAluDelim);
284 if (DelimIdx == StringRef::npos)
285 return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>");
286 StringRef Name = Src.substr(Start: 0, N: DelimIdx);
287 StringRef::iterator NamePos = Src.begin();
288 Src.consume_front(Prefix: Name);
289 Src.consume_front(Prefix: SWaitAluDelim);
290
291 int64_t Num;
292 StringRef::iterator NumPos = Src.begin();
293 if (Src.consumeInteger(Radix: 10, Result&: Num) || Num < 0)
294 return ErrorCallback(NumPos,
295 "expected non-negative integer counter number");
296
297 unsigned Max;
298 if (Name == VmcntName) {
299 Max = AMDGPU::getVmcntBitMask(Version);
300 Vmcnt = Num;
301 } else if (Name == ExpcntName) {
302 Max = AMDGPU::getExpcntBitMask(Version);
303 Expcnt = Num;
304 } else if (Name == LgkmcntName) {
305 Max = AMDGPU::getLgkmcntBitMask(Version);
306 Lgkmcnt = Num;
307 } else {
308 return ErrorCallback(NamePos, "invalid counter name");
309 }
310 if (Num >= Max)
311 return ErrorCallback(NumPos, "counter value too large");
312
313 Src.consume_front(Prefix: SWaitAluDelim);
314 }
315
316 Imm = AMDGPU::encodeWaitcnt(Version, Vmcnt, Expcnt, Lgkmcnt);
317 return false;
318}
319
320bool AMDGPUMIRFormatter::parseSWaitLoadcntDscntImmMnemonic(
321 const unsigned int OpIdx, int64_t &Imm, StringRef &Src,
322 MIRFormatter::ErrorCallbackType &ErrorCallback) const {
323 const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU());
324
325 // Accept integer masks for compatibility with old MIR.
326 if (!Src.consumeInteger(Radix: 10, Result&: Imm))
327 return false;
328
329 // Initialize with all counters at max (no wait).
330 unsigned Loadcnt = AMDGPU::getLoadcntBitMask(Version);
331 unsigned Dscnt = AMDGPU::getDscntBitMask(Version);
332
333 // The input is in the form: .Name1_Num1_Name2_Num2
334 // Drop the '.' prefix.
335 if (!Src.consume_front(Prefix: SWaitAluImmPrefix))
336 return ErrorCallback(Src.begin(), "expected prefix");
337 if (Src.empty())
338 return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>");
339
340 // Special case for all off (all counters at max).
341 if (Src == AllOff) {
342 Imm = AMDGPU::encodeLoadcntDscnt(Version, Loadcnt, Dscnt);
343 return false;
344 }
345
346 // Parse counter name, number pairs.
347 while (!Src.empty()) {
348 size_t DelimIdx = Src.find(Str: SWaitAluDelim);
349 if (DelimIdx == StringRef::npos)
350 return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>");
351 StringRef Name = Src.substr(Start: 0, N: DelimIdx);
352 StringRef::iterator NamePos = Src.begin();
353 Src.consume_front(Prefix: Name);
354 Src.consume_front(Prefix: SWaitAluDelim);
355
356 int64_t Num;
357 StringRef::iterator NumPos = Src.begin();
358 if (Src.consumeInteger(Radix: 10, Result&: Num) || Num < 0)
359 return ErrorCallback(NumPos,
360 "expected non-negative integer counter number");
361
362 unsigned Max;
363 if (Name == LoadcntName) {
364 Max = AMDGPU::getLoadcntBitMask(Version);
365 Loadcnt = Num;
366 } else if (Name == DscntName) {
367 Max = AMDGPU::getDscntBitMask(Version);
368 Dscnt = Num;
369 } else {
370 return ErrorCallback(NamePos, "invalid counter name");
371 }
372 if (Num >= Max)
373 return ErrorCallback(NumPos, "counter value too large");
374
375 Src.consume_front(Prefix: SWaitAluDelim);
376 }
377
378 Imm = AMDGPU::encodeLoadcntDscnt(Version, Loadcnt, Dscnt);
379 return false;
380}
381
382bool AMDGPUMIRFormatter::parseSWaitAluImmMnemonic(
383 const unsigned int OpIdx, int64_t &Imm, StringRef &Src,
384 MIRFormatter::ErrorCallbackType &ErrorCallback) const {
385 // TODO: For now accept integer masks for compatibility with old MIR.
386 if (!Src.consumeInteger(Radix: 10, Result&: Imm))
387 return false;
388
389 // Initialize with all checks off.
390 unsigned ImmEnc = AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI);
391 Imm = ImmEnc;
392 // The input is in the form: .Name1_Num1_Name2_Num2
393 // Drop the '.' prefix.
394 bool ConsumePrefix = Src.consume_front(Prefix: SWaitAluImmPrefix);
395 if (!ConsumePrefix)
396 return ErrorCallback(Src.begin(), "expected prefix");
397 if (Src.empty())
398 return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>");
399
400 // Special case for all off.
401 if (Src == AllOff)
402 return false;
403
404 // Parse a counter name, number pair in each iteration.
405 while (!Src.empty()) {
406 // Src: Name1_Num1_Name2_Num2
407 // ^
408 size_t DelimIdx = Src.find(Str: SWaitAluDelim);
409 if (DelimIdx == StringRef::npos)
410 return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>");
411 // Src: Name1_Num1_Name2_Num2
412 // ^^^^^
413 StringRef Name = Src.substr(Start: 0, N: DelimIdx);
414 // Save the position of the name for accurate error reporting.
415 StringRef::iterator NamePos = Src.begin();
416 [[maybe_unused]] bool ConsumeName = Src.consume_front(Prefix: Name);
417 assert(ConsumeName && "Expected name");
418 [[maybe_unused]] bool ConsumeDelim = Src.consume_front(Prefix: SWaitAluDelim);
419 assert(ConsumeDelim && "Expected delimiter");
420 // Src: Num1_Name2_Num2
421 // ^
422 DelimIdx = Src.find(Str: SWaitAluDelim);
423 // Src: Num1_Name2_Num2
424 // ^^^^
425 int64_t Num;
426 // Save the position of the number for accurate error reporting.
427 StringRef::iterator NumPos = Src.begin();
428 if (Src.consumeInteger(Radix: 10, Result&: Num) || Num < 0)
429 return ErrorCallback(NumPos,
430 "expected non-negative integer counter number");
431 unsigned Max;
432 if (Name == VaVdstName) {
433 Max = AMDGPU::DepCtr::getVaVdstBitMask();
434 ImmEnc = AMDGPU::DepCtr::encodeFieldVaVdst(Encoded: ImmEnc, VaVdst: Num);
435 } else if (Name == VmVsrcName) {
436 Max = AMDGPU::DepCtr::getVmVsrcBitMask();
437 ImmEnc = AMDGPU::DepCtr::encodeFieldVmVsrc(Encoded: ImmEnc, VmVsrc: Num);
438 } else if (Name == VaSdstName) {
439 Max = AMDGPU::DepCtr::getVaSdstBitMask();
440 ImmEnc = AMDGPU::DepCtr::encodeFieldVaSdst(Encoded: ImmEnc, VaSdst: Num);
441 } else if (Name == VaSsrcName) {
442 Max = AMDGPU::DepCtr::getVaSsrcBitMask();
443 ImmEnc = AMDGPU::DepCtr::encodeFieldVaSsrc(Encoded: ImmEnc, VaSsrc: Num);
444 } else if (Name == HoldCntName) {
445 const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU());
446 Max = AMDGPU::DepCtr::getHoldCntBitMask(Version);
447 ImmEnc = AMDGPU::DepCtr::encodeFieldHoldCnt(Encoded: ImmEnc, HoldCnt: Num, Version);
448 } else if (Name == VaVccName) {
449 Max = AMDGPU::DepCtr::getVaVccBitMask();
450 ImmEnc = AMDGPU::DepCtr::encodeFieldVaVcc(Encoded: ImmEnc, VaVcc: Num);
451 } else if (Name == SaSdstName) {
452 Max = AMDGPU::DepCtr::getSaSdstBitMask();
453 ImmEnc = AMDGPU::DepCtr::encodeFieldSaSdst(Encoded: ImmEnc, SaSdst: Num);
454 } else {
455 return ErrorCallback(NamePos, "invalid counter name");
456 }
457 Imm = ImmEnc;
458 // Don't allow the values to reach their maximum value.
459 if (Num >= Max)
460 return ErrorCallback(NumPos, "counter value too large");
461 // Src: Name2_Num2
462 Src.consume_front(Prefix: SWaitAluDelim);
463 }
464 return false;
465}
466
467bool AMDGPUMIRFormatter::parseSDelayAluImmMnemonic(
468 const unsigned int OpIdx, int64_t &Imm, llvm::StringRef &Src,
469 llvm::MIRFormatter::ErrorCallbackType &ErrorCallback) const
470{
471 assert(OpIdx == 0);
472
473 Imm = 0;
474 bool Expected = Src.consume_front(Prefix: ".id0_");
475 if (!Expected)
476 return ErrorCallback(Src.begin(), "Expected .id0_");
477
478 auto ExpectInt = [&](StringRef &Src, int64_t Offset) -> int64_t {
479 int64_t Dep;
480 if (!Src.consumeInteger(Radix: 10, Result&: Dep))
481 return Dep + Offset;
482
483 return -1;
484 };
485
486 auto DecodeDelay = [&](StringRef &Src) -> int64_t {
487 if (Src.consume_front(Prefix: "NONE"))
488 return 0;
489 if (Src.consume_front(Prefix: "VALU_DEP_"))
490 return ExpectInt(Src, 0);
491 if (Src.consume_front(Prefix: "TRANS32_DEP_"))
492 return ExpectInt(Src, 4);
493 if (Src.consume_front(Prefix: "SALU_CYCLE_"))
494 return ExpectInt(Src, 8);
495
496 return -1;
497 };
498
499 int64_t Delay0 = DecodeDelay(Src);
500 int64_t Skip = 0;
501 int64_t Delay1 = 0;
502 if (Delay0 == -1)
503 return ErrorCallback(Src.begin(), "Could not decode delay0");
504
505
506 // Set the Imm so far, to that early return has the correct value.
507 Imm = Delay0;
508
509 // If that was the end of the string, the second instruction is "same" and
510 // "none"
511 if (Src.begin() == Src.end())
512 return false;
513
514 Expected = Src.consume_front(Prefix: "_skip_");
515 if (!Expected)
516 return ErrorCallback(Src.begin(), "Expected _skip_");
517
518
519 if (Src.consume_front(Prefix: "SAME")) {
520 Skip = 0;
521 } else if (Src.consume_front(Prefix: "NEXT")) {
522 Skip = 1;
523 } else if (Src.consume_front(Prefix: "SKIP_")) {
524 if (Src.consumeInteger(Radix: 10, Result&: Skip)) {
525 return ErrorCallback(Src.begin(), "Expected integer Skip value");
526 }
527 Skip += 1;
528 } else {
529 ErrorCallback(Src.begin(), "Unexpected Skip Value");
530 }
531
532 Expected = Src.consume_front(Prefix: "_id1_");
533 if (!Expected)
534 return ErrorCallback(Src.begin(), "Expected _id1_");
535
536 Delay1 = DecodeDelay(Src);
537 if (Delay1 == -1)
538 return ErrorCallback(Src.begin(), "Could not decode delay1");
539
540 Imm = Imm | (Skip << 4) | (Delay1 << 7);
541 return false;
542}
543
544// Parse the async stage mask of asyncmark/wait_asyncmark. The mnemonic names
545// the stages the mask names.
546bool AMDGPUMIRFormatter::parseAsyncStageMaskImmMnemonic(
547 const unsigned int OpIdx, int64_t &Imm, StringRef &Src,
548 MIRFormatter::ErrorCallbackType &ErrorCallback) const {
549 if (!Src.consumeInteger(Radix: 10, Result&: Imm))
550 return false;
551
552 if (!Src.consume_front(Prefix: AsyncStageImmPrefix))
553 return ErrorCallback(Src.begin(), "expected prefix");
554 if (Src.empty())
555 return ErrorCallback(Src.begin(), "expected <StageName>");
556
557 if (Src == AllStages) {
558 Imm = 0;
559 return false;
560 }
561
562 AMDGPU::AsyncStages StageMask = AMDGPU::AsyncStages::NONE;
563 while (!Src.empty()) {
564 StringRef Name = Src.substr(Start: 0, N: Src.find(Str: AsyncStageDelim));
565 StringRef::iterator NamePos = Src.begin();
566 Src.consume_front(Prefix: Name);
567 Src.consume_front(Prefix: AsyncStageDelim);
568
569 // The reserved stages arrive together as one hex bitmask.
570 StringRef Bits = Name;
571 if (Bits.consume_front(Prefix: ReservedStagePrefix)) {
572 uint32_t Mask;
573 if (Bits.getAsInteger(Radix: 16, Result&: Mask) || Bits.empty() || !Mask)
574 return ErrorCallback(NamePos, "invalid async stage mask");
575 if (!AMDGPU::isValidAsyncStageMask(Mask))
576 return ErrorCallback(NamePos, "async stage mask out of range");
577 if (Mask & ~to_underlying(E: AMDGPU::AsyncStages::RESERVED))
578 return ErrorCallback(NamePos,
579 "async stage mask names a non-reserved stage");
580 StageMask |= static_cast<AMDGPU::AsyncStages>(Mask);
581 continue;
582 }
583
584 const StringLiteral *It = find(Range: AMDGPU::AsyncStageNames, Val: Name);
585 if (It == std::end(arr: AMDGPU::AsyncStageNames))
586 return ErrorCallback(NamePos, "invalid async stage name");
587 StageMask |= static_cast<AMDGPU::AsyncStages>(
588 1u << (It - std::begin(arr: AMDGPU::AsyncStageNames)));
589 }
590
591 Imm = to_underlying(E: StageMask);
592 return false;
593}
594
595bool AMDGPUMIRFormatter::parseCustomPseudoSourceValue(
596 StringRef Src, MachineFunction &MF, PerFunctionMIParsingState &PFS,
597 const PseudoSourceValue *&PSV, ErrorCallbackType ErrorCallback) const {
598 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
599 const AMDGPUTargetMachine &TM =
600 static_cast<const AMDGPUTargetMachine &>(MF.getTarget());
601 if (Src == "GWSResource") {
602 PSV = MFI->getGWSPSV(TM);
603 return false;
604 }
605 llvm_unreachable("unknown MIR custom pseudo source value");
606}
607