| 1 | //===- AMDGPUMIRFormatter.cpp ---------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | /// \file |
| 10 | /// Implementation of AMDGPU overrides of MIRFormatter. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "AMDGPUMIRFormatter.h" |
| 15 | #include "SIMachineFunctionInfo.h" |
| 16 | #include "llvm/Support/AMDGPUAsyncStages.h" |
| 17 | #include "llvm/Support/Format.h" |
| 18 | #include "llvm/TargetParser/AMDGPUTargetParser.h" |
| 19 | |
| 20 | using namespace llvm; |
| 21 | |
| 22 | const char SWaitAluImmPrefix = '.'; |
| 23 | StringLiteral SWaitAluDelim = "_" ; |
| 24 | |
| 25 | StringLiteral VaVdstName = "VaVdst" ; |
| 26 | StringLiteral VaSdstName = "VaSdst" ; |
| 27 | StringLiteral VaSsrcName = "VaSsrc" ; |
| 28 | StringLiteral HoldCntName = "HoldCnt" ; |
| 29 | StringLiteral VmVsrcName = "VmVsrc" ; |
| 30 | StringLiteral VaVccName = "VaVcc" ; |
| 31 | StringLiteral SaSdstName = "SaSdst" ; |
| 32 | |
| 33 | StringLiteral AllOff = "AllOff" ; |
| 34 | |
| 35 | StringLiteral VmcntName = "Vmcnt" ; |
| 36 | StringLiteral ExpcntName = "Expcnt" ; |
| 37 | StringLiteral LgkmcntName = "Lgkmcnt" ; |
| 38 | |
| 39 | StringLiteral LoadcntName = "Loadcnt" ; |
| 40 | StringLiteral DscntName = "Dscnt" ; |
| 41 | |
| 42 | const char AsyncStageImmPrefix = '.'; |
| 43 | constexpr StringLiteral AsyncStageDelim("." ); |
| 44 | constexpr StringLiteral ReservedStagePrefix("RES" ); |
| 45 | constexpr StringLiteral AllStages("AllStages" ); |
| 46 | |
| 47 | void AMDGPUMIRFormatter::printSWaitAluImm(unsigned Imm, raw_ostream &OS) const { |
| 48 | bool NonePrinted = true; |
| 49 | ListSeparator Delim(SWaitAluDelim); |
| 50 | auto PrintFieldIfNotMax = [&](StringRef Descr, uint64_t Num, unsigned Max) { |
| 51 | if (Num != Max) { |
| 52 | OS << Delim << Descr << SWaitAluDelim << Num; |
| 53 | NonePrinted = false; |
| 54 | } |
| 55 | }; |
| 56 | OS << SWaitAluImmPrefix; |
| 57 | PrintFieldIfNotMax(VaVdstName, AMDGPU::DepCtr::decodeFieldVaVdst(Encoded: Imm), |
| 58 | AMDGPU::DepCtr::getVaVdstBitMask()); |
| 59 | PrintFieldIfNotMax(VaSdstName, AMDGPU::DepCtr::decodeFieldVaSdst(Encoded: Imm), |
| 60 | AMDGPU::DepCtr::getVaSdstBitMask()); |
| 61 | PrintFieldIfNotMax(VaSsrcName, AMDGPU::DepCtr::decodeFieldVaSsrc(Encoded: Imm), |
| 62 | AMDGPU::DepCtr::getVaSsrcBitMask()); |
| 63 | PrintFieldIfNotMax( |
| 64 | HoldCntName, |
| 65 | AMDGPU::DepCtr::decodeFieldHoldCnt(Encoded: Imm, |
| 66 | Version: AMDGPU::getIsaVersion(GPU: STI.getCPU())), |
| 67 | AMDGPU::DepCtr::getHoldCntBitMask(Version: AMDGPU::getIsaVersion(GPU: STI.getCPU()))); |
| 68 | PrintFieldIfNotMax(VmVsrcName, AMDGPU::DepCtr::decodeFieldVmVsrc(Encoded: Imm), |
| 69 | AMDGPU::DepCtr::getVmVsrcBitMask()); |
| 70 | PrintFieldIfNotMax(VaVccName, AMDGPU::DepCtr::decodeFieldVaVcc(Encoded: Imm), |
| 71 | AMDGPU::DepCtr::getVaVccBitMask()); |
| 72 | PrintFieldIfNotMax(SaSdstName, AMDGPU::DepCtr::decodeFieldSaSdst(Encoded: Imm), |
| 73 | AMDGPU::DepCtr::getSaSdstBitMask()); |
| 74 | if (NonePrinted) |
| 75 | OS << AllOff; |
| 76 | } |
| 77 | |
| 78 | void AMDGPUMIRFormatter::printSWaitcntImm(unsigned Imm, raw_ostream &OS) const { |
| 79 | const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU()); |
| 80 | bool NonePrinted = true; |
| 81 | ListSeparator Delim(SWaitAluDelim); |
| 82 | auto PrintFieldIfNotMax = [&](StringRef Descr, uint64_t Num, unsigned Max) { |
| 83 | if (Num != Max) { |
| 84 | OS << Delim << Descr << SWaitAluDelim << Num; |
| 85 | NonePrinted = false; |
| 86 | } |
| 87 | }; |
| 88 | OS << SWaitAluImmPrefix; |
| 89 | PrintFieldIfNotMax(VmcntName, AMDGPU::decodeVmcnt(Version, Waitcnt: Imm), |
| 90 | AMDGPU::getVmcntBitMask(Version)); |
| 91 | PrintFieldIfNotMax(ExpcntName, AMDGPU::decodeExpcnt(Version, Waitcnt: Imm), |
| 92 | AMDGPU::getExpcntBitMask(Version)); |
| 93 | PrintFieldIfNotMax(LgkmcntName, AMDGPU::decodeLgkmcnt(Version, Waitcnt: Imm), |
| 94 | AMDGPU::getLgkmcntBitMask(Version)); |
| 95 | if (NonePrinted) |
| 96 | OS << AllOff; |
| 97 | } |
| 98 | |
| 99 | void AMDGPUMIRFormatter::printSWaitLoadcntDscntImm(unsigned Imm, |
| 100 | raw_ostream &OS) const { |
| 101 | const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU()); |
| 102 | bool NonePrinted = true; |
| 103 | ListSeparator Delim(SWaitAluDelim); |
| 104 | auto PrintFieldIfNotMax = [&](StringRef Descr, uint64_t Num, unsigned Max) { |
| 105 | if (Num != Max) { |
| 106 | OS << Delim << Descr << SWaitAluDelim << Num; |
| 107 | NonePrinted = false; |
| 108 | } |
| 109 | }; |
| 110 | OS << SWaitAluImmPrefix; |
| 111 | PrintFieldIfNotMax(LoadcntName, AMDGPU::decodeLoadcnt(Version, Waitcnt: Imm), |
| 112 | AMDGPU::getLoadcntBitMask(Version)); |
| 113 | PrintFieldIfNotMax(DscntName, AMDGPU::decodeDscnt(Version, Waitcnt: Imm), |
| 114 | AMDGPU::getDscntBitMask(Version)); |
| 115 | if (NonePrinted) |
| 116 | OS << AllOff; |
| 117 | } |
| 118 | |
| 119 | void AMDGPUMIRFormatter::printAsyncStageMaskImm(int64_t Imm, |
| 120 | raw_ostream &OS) const { |
| 121 | if (!AMDGPU::isValidAsyncStageMask(Mask: Imm)) { |
| 122 | OS << Imm; |
| 123 | return; |
| 124 | } |
| 125 | |
| 126 | OS << AsyncStageImmPrefix; |
| 127 | auto Mask = static_cast<AMDGPU::AsyncStages>(Imm); |
| 128 | // The zero mask names every stage. Spell that out rather than printing an |
| 129 | // empty list. The saturated mask means the same thing but is a different |
| 130 | // immediate, so it keeps its spelled-out list to stay round-trippable. |
| 131 | if (Mask == AMDGPU::AsyncStages::NONE) { |
| 132 | OS << AllStages; |
| 133 | return; |
| 134 | } |
| 135 | |
| 136 | // Reserved stages have no meaningful name, collect them into a single |
| 137 | // hex bitmask instead. |
| 138 | ListSeparator Delim(AsyncStageDelim); |
| 139 | for (AMDGPU::AsyncStages S : |
| 140 | AMDGPU::asyncStages(Mask: Mask & ~AMDGPU::AsyncStages::RESERVED)) |
| 141 | OS << Delim << AMDGPU::getAsyncStageName(Stage: S); |
| 142 | AMDGPU::AsyncStages Reserved = Mask & AMDGPU::AsyncStages::RESERVED; |
| 143 | if (any(Val: Reserved)) |
| 144 | OS << Delim << ReservedStagePrefix |
| 145 | << format_hex_no_prefix(N: to_underlying(E: Reserved), Width: 1); |
| 146 | } |
| 147 | |
| 148 | void AMDGPUMIRFormatter::printImm(raw_ostream &OS, const MachineInstr &MI, |
| 149 | std::optional<unsigned int> OpIdx, int64_t Imm) const { |
| 150 | |
| 151 | switch (MI.getOpcode()) { |
| 152 | case AMDGPU::ASYNCMARK: |
| 153 | case AMDGPU::WAIT_ASYNCMARK: |
| 154 | if (OpIdx == MI.getNumExplicitOperands() - 1u) |
| 155 | printAsyncStageMaskImm(Imm, OS); |
| 156 | else |
| 157 | MIRFormatter::printImm(OS, MI, OpIdx, Imm); |
| 158 | break; |
| 159 | case AMDGPU::S_WAITCNT: |
| 160 | case AMDGPU::S_WAITCNT_soft: |
| 161 | printSWaitcntImm(Imm: static_cast<unsigned>(Imm), OS); |
| 162 | break; |
| 163 | case AMDGPU::S_WAIT_LOADCNT_DSCNT: |
| 164 | printSWaitLoadcntDscntImm(Imm: static_cast<unsigned>(Imm), OS); |
| 165 | break; |
| 166 | case AMDGPU::S_WAITCNT_DEPCTR: |
| 167 | printSWaitAluImm(Imm: static_cast<unsigned>(Imm), OS); |
| 168 | break; |
| 169 | case AMDGPU::S_DELAY_ALU: |
| 170 | assert(OpIdx == 0); |
| 171 | printSDelayAluImm(Imm, OS); |
| 172 | break; |
| 173 | default: |
| 174 | MIRFormatter::printImm(OS, MI, OpIdx, Imm); |
| 175 | break; |
| 176 | } |
| 177 | } |
| 178 | |
| 179 | /// Implement target specific parsing of immediate mnemonics. The mnemonic is |
| 180 | /// a string with a leading dot. |
| 181 | bool AMDGPUMIRFormatter::parseImmMnemonic(const unsigned OpCode, |
| 182 | const unsigned OpIdx, |
| 183 | StringRef Src, int64_t &Imm, |
| 184 | ErrorCallbackType ErrorCallback) const |
| 185 | { |
| 186 | |
| 187 | switch (OpCode) { |
| 188 | case AMDGPU::S_WAITCNT: |
| 189 | case AMDGPU::S_WAITCNT_soft: |
| 190 | return parseSWaitcntImmMnemonic(OpIdx, Imm, Src, ErrorCallback); |
| 191 | case AMDGPU::S_WAIT_LOADCNT_DSCNT: |
| 192 | return parseSWaitLoadcntDscntImmMnemonic(OpIdx, Imm, Src, ErrorCallback); |
| 193 | case AMDGPU::S_WAITCNT_DEPCTR: |
| 194 | return parseSWaitAluImmMnemonic(OpIdx, Imm, Src, ErrorCallback); |
| 195 | case AMDGPU::S_DELAY_ALU: |
| 196 | return parseSDelayAluImmMnemonic(OpIdx, Imm, Src, ErrorCallback); |
| 197 | case AMDGPU::ASYNCMARK: |
| 198 | if (OpIdx == 0) |
| 199 | return parseAsyncStageMaskImmMnemonic(OpIdx, Imm, Src, ErrorCallback); |
| 200 | break; |
| 201 | case AMDGPU::WAIT_ASYNCMARK: |
| 202 | if (OpIdx == 1) |
| 203 | return parseAsyncStageMaskImmMnemonic(OpIdx, Imm, Src, ErrorCallback); |
| 204 | break; |
| 205 | default: |
| 206 | break; |
| 207 | } |
| 208 | return true; // Don't know what this is |
| 209 | } |
| 210 | |
| 211 | void AMDGPUMIRFormatter::printSDelayAluImm(int64_t Imm, |
| 212 | llvm::raw_ostream &OS) const { |
| 213 | // Construct an immediate string to represent the information encoded in the |
| 214 | // s_delay_alu immediate. |
| 215 | // .id0_<dep>[_skip_<count>_id1<dep>] |
| 216 | constexpr int64_t None = 0; |
| 217 | constexpr int64_t Same = 0; |
| 218 | |
| 219 | uint64_t Id0 = (Imm & 0xF); |
| 220 | uint64_t Skip = ((Imm >> 4) & 0x7); |
| 221 | uint64_t Id1 = ((Imm >> 7) & 0xF); |
| 222 | auto Outdep = [&](uint64_t Id) { |
| 223 | if (Id == None) |
| 224 | OS << "NONE" ; |
| 225 | else if (Id < 5) |
| 226 | OS << "VALU_DEP_" << Id; |
| 227 | else if (Id < 8) |
| 228 | OS << "TRANS32_DEP_" << Id - 4; |
| 229 | else |
| 230 | OS << "SALU_CYCLE_" << Id - 8; |
| 231 | }; |
| 232 | |
| 233 | OS << ".id0_" ; |
| 234 | Outdep(Id0); |
| 235 | |
| 236 | // If the second inst is "same" and "none", no need to print the rest of the |
| 237 | // string. |
| 238 | if (Skip == Same && Id1 == None) |
| 239 | return; |
| 240 | |
| 241 | // Encode the second delay specification. |
| 242 | OS << "_skip_" ; |
| 243 | if (Skip == 0) |
| 244 | OS << "SAME" ; |
| 245 | else if (Skip == 1) |
| 246 | OS << "NEXT" ; |
| 247 | else |
| 248 | OS << "SKIP_" << Skip - 1; |
| 249 | |
| 250 | OS << "_id1_" ; |
| 251 | Outdep(Id1); |
| 252 | } |
| 253 | |
| 254 | bool AMDGPUMIRFormatter::parseSWaitcntImmMnemonic( |
| 255 | const unsigned int OpIdx, int64_t &Imm, StringRef &Src, |
| 256 | MIRFormatter::ErrorCallbackType &ErrorCallback) const { |
| 257 | const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU()); |
| 258 | |
| 259 | // Accept integer masks for compatibility with old MIR. |
| 260 | if (!Src.consumeInteger(Radix: 10, Result&: Imm)) |
| 261 | return false; |
| 262 | |
| 263 | // Initialize with all counters at max (no wait). |
| 264 | unsigned Vmcnt = AMDGPU::getVmcntBitMask(Version); |
| 265 | unsigned Expcnt = AMDGPU::getExpcntBitMask(Version); |
| 266 | unsigned Lgkmcnt = AMDGPU::getLgkmcntBitMask(Version); |
| 267 | |
| 268 | // The input is in the form: .Name1_Num1_Name2_Num2 |
| 269 | // Drop the '.' prefix. |
| 270 | if (!Src.consume_front(Prefix: SWaitAluImmPrefix)) |
| 271 | return ErrorCallback(Src.begin(), "expected prefix" ); |
| 272 | if (Src.empty()) |
| 273 | return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>" ); |
| 274 | |
| 275 | // Special case for all off (all counters at max). |
| 276 | if (Src == AllOff) { |
| 277 | Imm = AMDGPU::encodeWaitcnt(Version, Vmcnt, Expcnt, Lgkmcnt); |
| 278 | return false; |
| 279 | } |
| 280 | |
| 281 | // Parse counter name, number pairs. |
| 282 | while (!Src.empty()) { |
| 283 | size_t DelimIdx = Src.find(Str: SWaitAluDelim); |
| 284 | if (DelimIdx == StringRef::npos) |
| 285 | return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>" ); |
| 286 | StringRef Name = Src.substr(Start: 0, N: DelimIdx); |
| 287 | StringRef::iterator NamePos = Src.begin(); |
| 288 | Src.consume_front(Prefix: Name); |
| 289 | Src.consume_front(Prefix: SWaitAluDelim); |
| 290 | |
| 291 | int64_t Num; |
| 292 | StringRef::iterator NumPos = Src.begin(); |
| 293 | if (Src.consumeInteger(Radix: 10, Result&: Num) || Num < 0) |
| 294 | return ErrorCallback(NumPos, |
| 295 | "expected non-negative integer counter number" ); |
| 296 | |
| 297 | unsigned Max; |
| 298 | if (Name == VmcntName) { |
| 299 | Max = AMDGPU::getVmcntBitMask(Version); |
| 300 | Vmcnt = Num; |
| 301 | } else if (Name == ExpcntName) { |
| 302 | Max = AMDGPU::getExpcntBitMask(Version); |
| 303 | Expcnt = Num; |
| 304 | } else if (Name == LgkmcntName) { |
| 305 | Max = AMDGPU::getLgkmcntBitMask(Version); |
| 306 | Lgkmcnt = Num; |
| 307 | } else { |
| 308 | return ErrorCallback(NamePos, "invalid counter name" ); |
| 309 | } |
| 310 | if (Num >= Max) |
| 311 | return ErrorCallback(NumPos, "counter value too large" ); |
| 312 | |
| 313 | Src.consume_front(Prefix: SWaitAluDelim); |
| 314 | } |
| 315 | |
| 316 | Imm = AMDGPU::encodeWaitcnt(Version, Vmcnt, Expcnt, Lgkmcnt); |
| 317 | return false; |
| 318 | } |
| 319 | |
| 320 | bool AMDGPUMIRFormatter::parseSWaitLoadcntDscntImmMnemonic( |
| 321 | const unsigned int OpIdx, int64_t &Imm, StringRef &Src, |
| 322 | MIRFormatter::ErrorCallbackType &ErrorCallback) const { |
| 323 | const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU()); |
| 324 | |
| 325 | // Accept integer masks for compatibility with old MIR. |
| 326 | if (!Src.consumeInteger(Radix: 10, Result&: Imm)) |
| 327 | return false; |
| 328 | |
| 329 | // Initialize with all counters at max (no wait). |
| 330 | unsigned Loadcnt = AMDGPU::getLoadcntBitMask(Version); |
| 331 | unsigned Dscnt = AMDGPU::getDscntBitMask(Version); |
| 332 | |
| 333 | // The input is in the form: .Name1_Num1_Name2_Num2 |
| 334 | // Drop the '.' prefix. |
| 335 | if (!Src.consume_front(Prefix: SWaitAluImmPrefix)) |
| 336 | return ErrorCallback(Src.begin(), "expected prefix" ); |
| 337 | if (Src.empty()) |
| 338 | return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>" ); |
| 339 | |
| 340 | // Special case for all off (all counters at max). |
| 341 | if (Src == AllOff) { |
| 342 | Imm = AMDGPU::encodeLoadcntDscnt(Version, Loadcnt, Dscnt); |
| 343 | return false; |
| 344 | } |
| 345 | |
| 346 | // Parse counter name, number pairs. |
| 347 | while (!Src.empty()) { |
| 348 | size_t DelimIdx = Src.find(Str: SWaitAluDelim); |
| 349 | if (DelimIdx == StringRef::npos) |
| 350 | return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>" ); |
| 351 | StringRef Name = Src.substr(Start: 0, N: DelimIdx); |
| 352 | StringRef::iterator NamePos = Src.begin(); |
| 353 | Src.consume_front(Prefix: Name); |
| 354 | Src.consume_front(Prefix: SWaitAluDelim); |
| 355 | |
| 356 | int64_t Num; |
| 357 | StringRef::iterator NumPos = Src.begin(); |
| 358 | if (Src.consumeInteger(Radix: 10, Result&: Num) || Num < 0) |
| 359 | return ErrorCallback(NumPos, |
| 360 | "expected non-negative integer counter number" ); |
| 361 | |
| 362 | unsigned Max; |
| 363 | if (Name == LoadcntName) { |
| 364 | Max = AMDGPU::getLoadcntBitMask(Version); |
| 365 | Loadcnt = Num; |
| 366 | } else if (Name == DscntName) { |
| 367 | Max = AMDGPU::getDscntBitMask(Version); |
| 368 | Dscnt = Num; |
| 369 | } else { |
| 370 | return ErrorCallback(NamePos, "invalid counter name" ); |
| 371 | } |
| 372 | if (Num >= Max) |
| 373 | return ErrorCallback(NumPos, "counter value too large" ); |
| 374 | |
| 375 | Src.consume_front(Prefix: SWaitAluDelim); |
| 376 | } |
| 377 | |
| 378 | Imm = AMDGPU::encodeLoadcntDscnt(Version, Loadcnt, Dscnt); |
| 379 | return false; |
| 380 | } |
| 381 | |
| 382 | bool AMDGPUMIRFormatter::parseSWaitAluImmMnemonic( |
| 383 | const unsigned int OpIdx, int64_t &Imm, StringRef &Src, |
| 384 | MIRFormatter::ErrorCallbackType &ErrorCallback) const { |
| 385 | // TODO: For now accept integer masks for compatibility with old MIR. |
| 386 | if (!Src.consumeInteger(Radix: 10, Result&: Imm)) |
| 387 | return false; |
| 388 | |
| 389 | // Initialize with all checks off. |
| 390 | unsigned ImmEnc = AMDGPU::DepCtr::getDefaultDepCtrEncoding(STI); |
| 391 | Imm = ImmEnc; |
| 392 | // The input is in the form: .Name1_Num1_Name2_Num2 |
| 393 | // Drop the '.' prefix. |
| 394 | bool ConsumePrefix = Src.consume_front(Prefix: SWaitAluImmPrefix); |
| 395 | if (!ConsumePrefix) |
| 396 | return ErrorCallback(Src.begin(), "expected prefix" ); |
| 397 | if (Src.empty()) |
| 398 | return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>" ); |
| 399 | |
| 400 | // Special case for all off. |
| 401 | if (Src == AllOff) |
| 402 | return false; |
| 403 | |
| 404 | // Parse a counter name, number pair in each iteration. |
| 405 | while (!Src.empty()) { |
| 406 | // Src: Name1_Num1_Name2_Num2 |
| 407 | // ^ |
| 408 | size_t DelimIdx = Src.find(Str: SWaitAluDelim); |
| 409 | if (DelimIdx == StringRef::npos) |
| 410 | return ErrorCallback(Src.begin(), "expected <CounterName>_<CounterNum>" ); |
| 411 | // Src: Name1_Num1_Name2_Num2 |
| 412 | // ^^^^^ |
| 413 | StringRef Name = Src.substr(Start: 0, N: DelimIdx); |
| 414 | // Save the position of the name for accurate error reporting. |
| 415 | StringRef::iterator NamePos = Src.begin(); |
| 416 | [[maybe_unused]] bool ConsumeName = Src.consume_front(Prefix: Name); |
| 417 | assert(ConsumeName && "Expected name" ); |
| 418 | [[maybe_unused]] bool ConsumeDelim = Src.consume_front(Prefix: SWaitAluDelim); |
| 419 | assert(ConsumeDelim && "Expected delimiter" ); |
| 420 | // Src: Num1_Name2_Num2 |
| 421 | // ^ |
| 422 | DelimIdx = Src.find(Str: SWaitAluDelim); |
| 423 | // Src: Num1_Name2_Num2 |
| 424 | // ^^^^ |
| 425 | int64_t Num; |
| 426 | // Save the position of the number for accurate error reporting. |
| 427 | StringRef::iterator NumPos = Src.begin(); |
| 428 | if (Src.consumeInteger(Radix: 10, Result&: Num) || Num < 0) |
| 429 | return ErrorCallback(NumPos, |
| 430 | "expected non-negative integer counter number" ); |
| 431 | unsigned Max; |
| 432 | if (Name == VaVdstName) { |
| 433 | Max = AMDGPU::DepCtr::getVaVdstBitMask(); |
| 434 | ImmEnc = AMDGPU::DepCtr::encodeFieldVaVdst(Encoded: ImmEnc, VaVdst: Num); |
| 435 | } else if (Name == VmVsrcName) { |
| 436 | Max = AMDGPU::DepCtr::getVmVsrcBitMask(); |
| 437 | ImmEnc = AMDGPU::DepCtr::encodeFieldVmVsrc(Encoded: ImmEnc, VmVsrc: Num); |
| 438 | } else if (Name == VaSdstName) { |
| 439 | Max = AMDGPU::DepCtr::getVaSdstBitMask(); |
| 440 | ImmEnc = AMDGPU::DepCtr::encodeFieldVaSdst(Encoded: ImmEnc, VaSdst: Num); |
| 441 | } else if (Name == VaSsrcName) { |
| 442 | Max = AMDGPU::DepCtr::getVaSsrcBitMask(); |
| 443 | ImmEnc = AMDGPU::DepCtr::encodeFieldVaSsrc(Encoded: ImmEnc, VaSsrc: Num); |
| 444 | } else if (Name == HoldCntName) { |
| 445 | const AMDGPU::IsaVersion &Version = AMDGPU::getIsaVersion(GPU: STI.getCPU()); |
| 446 | Max = AMDGPU::DepCtr::getHoldCntBitMask(Version); |
| 447 | ImmEnc = AMDGPU::DepCtr::encodeFieldHoldCnt(Encoded: ImmEnc, HoldCnt: Num, Version); |
| 448 | } else if (Name == VaVccName) { |
| 449 | Max = AMDGPU::DepCtr::getVaVccBitMask(); |
| 450 | ImmEnc = AMDGPU::DepCtr::encodeFieldVaVcc(Encoded: ImmEnc, VaVcc: Num); |
| 451 | } else if (Name == SaSdstName) { |
| 452 | Max = AMDGPU::DepCtr::getSaSdstBitMask(); |
| 453 | ImmEnc = AMDGPU::DepCtr::encodeFieldSaSdst(Encoded: ImmEnc, SaSdst: Num); |
| 454 | } else { |
| 455 | return ErrorCallback(NamePos, "invalid counter name" ); |
| 456 | } |
| 457 | Imm = ImmEnc; |
| 458 | // Don't allow the values to reach their maximum value. |
| 459 | if (Num >= Max) |
| 460 | return ErrorCallback(NumPos, "counter value too large" ); |
| 461 | // Src: Name2_Num2 |
| 462 | Src.consume_front(Prefix: SWaitAluDelim); |
| 463 | } |
| 464 | return false; |
| 465 | } |
| 466 | |
| 467 | bool AMDGPUMIRFormatter::parseSDelayAluImmMnemonic( |
| 468 | const unsigned int OpIdx, int64_t &Imm, llvm::StringRef &Src, |
| 469 | llvm::MIRFormatter::ErrorCallbackType &ErrorCallback) const |
| 470 | { |
| 471 | assert(OpIdx == 0); |
| 472 | |
| 473 | Imm = 0; |
| 474 | bool Expected = Src.consume_front(Prefix: ".id0_" ); |
| 475 | if (!Expected) |
| 476 | return ErrorCallback(Src.begin(), "Expected .id0_" ); |
| 477 | |
| 478 | auto ExpectInt = [&](StringRef &Src, int64_t Offset) -> int64_t { |
| 479 | int64_t Dep; |
| 480 | if (!Src.consumeInteger(Radix: 10, Result&: Dep)) |
| 481 | return Dep + Offset; |
| 482 | |
| 483 | return -1; |
| 484 | }; |
| 485 | |
| 486 | auto DecodeDelay = [&](StringRef &Src) -> int64_t { |
| 487 | if (Src.consume_front(Prefix: "NONE" )) |
| 488 | return 0; |
| 489 | if (Src.consume_front(Prefix: "VALU_DEP_" )) |
| 490 | return ExpectInt(Src, 0); |
| 491 | if (Src.consume_front(Prefix: "TRANS32_DEP_" )) |
| 492 | return ExpectInt(Src, 4); |
| 493 | if (Src.consume_front(Prefix: "SALU_CYCLE_" )) |
| 494 | return ExpectInt(Src, 8); |
| 495 | |
| 496 | return -1; |
| 497 | }; |
| 498 | |
| 499 | int64_t Delay0 = DecodeDelay(Src); |
| 500 | int64_t Skip = 0; |
| 501 | int64_t Delay1 = 0; |
| 502 | if (Delay0 == -1) |
| 503 | return ErrorCallback(Src.begin(), "Could not decode delay0" ); |
| 504 | |
| 505 | |
| 506 | // Set the Imm so far, to that early return has the correct value. |
| 507 | Imm = Delay0; |
| 508 | |
| 509 | // If that was the end of the string, the second instruction is "same" and |
| 510 | // "none" |
| 511 | if (Src.begin() == Src.end()) |
| 512 | return false; |
| 513 | |
| 514 | Expected = Src.consume_front(Prefix: "_skip_" ); |
| 515 | if (!Expected) |
| 516 | return ErrorCallback(Src.begin(), "Expected _skip_" ); |
| 517 | |
| 518 | |
| 519 | if (Src.consume_front(Prefix: "SAME" )) { |
| 520 | Skip = 0; |
| 521 | } else if (Src.consume_front(Prefix: "NEXT" )) { |
| 522 | Skip = 1; |
| 523 | } else if (Src.consume_front(Prefix: "SKIP_" )) { |
| 524 | if (Src.consumeInteger(Radix: 10, Result&: Skip)) { |
| 525 | return ErrorCallback(Src.begin(), "Expected integer Skip value" ); |
| 526 | } |
| 527 | Skip += 1; |
| 528 | } else { |
| 529 | ErrorCallback(Src.begin(), "Unexpected Skip Value" ); |
| 530 | } |
| 531 | |
| 532 | Expected = Src.consume_front(Prefix: "_id1_" ); |
| 533 | if (!Expected) |
| 534 | return ErrorCallback(Src.begin(), "Expected _id1_" ); |
| 535 | |
| 536 | Delay1 = DecodeDelay(Src); |
| 537 | if (Delay1 == -1) |
| 538 | return ErrorCallback(Src.begin(), "Could not decode delay1" ); |
| 539 | |
| 540 | Imm = Imm | (Skip << 4) | (Delay1 << 7); |
| 541 | return false; |
| 542 | } |
| 543 | |
| 544 | // Parse the async stage mask of asyncmark/wait_asyncmark. The mnemonic names |
| 545 | // the stages the mask names. |
| 546 | bool AMDGPUMIRFormatter::parseAsyncStageMaskImmMnemonic( |
| 547 | const unsigned int OpIdx, int64_t &Imm, StringRef &Src, |
| 548 | MIRFormatter::ErrorCallbackType &ErrorCallback) const { |
| 549 | if (!Src.consumeInteger(Radix: 10, Result&: Imm)) |
| 550 | return false; |
| 551 | |
| 552 | if (!Src.consume_front(Prefix: AsyncStageImmPrefix)) |
| 553 | return ErrorCallback(Src.begin(), "expected prefix" ); |
| 554 | if (Src.empty()) |
| 555 | return ErrorCallback(Src.begin(), "expected <StageName>" ); |
| 556 | |
| 557 | if (Src == AllStages) { |
| 558 | Imm = 0; |
| 559 | return false; |
| 560 | } |
| 561 | |
| 562 | AMDGPU::AsyncStages StageMask = AMDGPU::AsyncStages::NONE; |
| 563 | while (!Src.empty()) { |
| 564 | StringRef Name = Src.substr(Start: 0, N: Src.find(Str: AsyncStageDelim)); |
| 565 | StringRef::iterator NamePos = Src.begin(); |
| 566 | Src.consume_front(Prefix: Name); |
| 567 | Src.consume_front(Prefix: AsyncStageDelim); |
| 568 | |
| 569 | // The reserved stages arrive together as one hex bitmask. |
| 570 | StringRef Bits = Name; |
| 571 | if (Bits.consume_front(Prefix: ReservedStagePrefix)) { |
| 572 | uint32_t Mask; |
| 573 | if (Bits.getAsInteger(Radix: 16, Result&: Mask) || Bits.empty() || !Mask) |
| 574 | return ErrorCallback(NamePos, "invalid async stage mask" ); |
| 575 | if (!AMDGPU::isValidAsyncStageMask(Mask)) |
| 576 | return ErrorCallback(NamePos, "async stage mask out of range" ); |
| 577 | if (Mask & ~to_underlying(E: AMDGPU::AsyncStages::RESERVED)) |
| 578 | return ErrorCallback(NamePos, |
| 579 | "async stage mask names a non-reserved stage" ); |
| 580 | StageMask |= static_cast<AMDGPU::AsyncStages>(Mask); |
| 581 | continue; |
| 582 | } |
| 583 | |
| 584 | const StringLiteral *It = find(Range: AMDGPU::AsyncStageNames, Val: Name); |
| 585 | if (It == std::end(arr: AMDGPU::AsyncStageNames)) |
| 586 | return ErrorCallback(NamePos, "invalid async stage name" ); |
| 587 | StageMask |= static_cast<AMDGPU::AsyncStages>( |
| 588 | 1u << (It - std::begin(arr: AMDGPU::AsyncStageNames))); |
| 589 | } |
| 590 | |
| 591 | Imm = to_underlying(E: StageMask); |
| 592 | return false; |
| 593 | } |
| 594 | |
| 595 | bool AMDGPUMIRFormatter::parseCustomPseudoSourceValue( |
| 596 | StringRef Src, MachineFunction &MF, PerFunctionMIParsingState &PFS, |
| 597 | const PseudoSourceValue *&PSV, ErrorCallbackType ErrorCallback) const { |
| 598 | SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); |
| 599 | const AMDGPUTargetMachine &TM = |
| 600 | static_cast<const AMDGPUTargetMachine &>(MF.getTarget()); |
| 601 | if (Src == "GWSResource" ) { |
| 602 | PSV = MFI->getGWSPSV(TM); |
| 603 | return false; |
| 604 | } |
| 605 | llvm_unreachable("unknown MIR custom pseudo source value" ); |
| 606 | } |
| 607 | |