| 1 | //===- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "AMDKernelCodeT.h" |
| 10 | #include "MCTargetDesc/AMDGPUInstPrinter.h" |
| 11 | #include "MCTargetDesc/AMDGPUMCAsmInfo.h" |
| 12 | #include "MCTargetDesc/AMDGPUMCExpr.h" |
| 13 | #include "MCTargetDesc/AMDGPUMCKernelDescriptor.h" |
| 14 | #include "MCTargetDesc/AMDGPUMCTargetDesc.h" |
| 15 | #include "MCTargetDesc/AMDGPUTargetStreamer.h" |
| 16 | #include "SIDefines.h" |
| 17 | #include "SIInstrInfo.h" |
| 18 | #include "TargetInfo/AMDGPUTargetInfo.h" |
| 19 | #include "Utils/AMDGPUAsmUtils.h" |
| 20 | #include "Utils/AMDGPUBaseInfo.h" |
| 21 | #include "Utils/AMDKernelCodeTUtils.h" |
| 22 | #include "llvm/ADT/APFloat.h" |
| 23 | #include "llvm/ADT/SmallBitVector.h" |
| 24 | #include "llvm/ADT/Twine.h" |
| 25 | #include "llvm/BinaryFormat/ELF.h" |
| 26 | #include "llvm/CodeGenTypes/MachineValueType.h" |
| 27 | #include "llvm/MC/MCAsmInfo.h" |
| 28 | #include "llvm/MC/MCContext.h" |
| 29 | #include "llvm/MC/MCExpr.h" |
| 30 | #include "llvm/MC/MCInst.h" |
| 31 | #include "llvm/MC/MCInstrDesc.h" |
| 32 | #include "llvm/MC/MCParser/AsmLexer.h" |
| 33 | #include "llvm/MC/MCParser/MCAsmParser.h" |
| 34 | #include "llvm/MC/MCParser/MCParsedAsmOperand.h" |
| 35 | #include "llvm/MC/MCParser/MCTargetAsmParser.h" |
| 36 | #include "llvm/MC/MCRegisterInfo.h" |
| 37 | #include "llvm/MC/MCSymbol.h" |
| 38 | #include "llvm/MC/TargetRegistry.h" |
| 39 | #include "llvm/Support/AMDGPUMetadata.h" |
| 40 | #include "llvm/Support/AMDGPUObjLinkingInfo.h" |
| 41 | #include "llvm/Support/AMDHSAKernelDescriptor.h" |
| 42 | #include "llvm/Support/Casting.h" |
| 43 | #include "llvm/Support/Compiler.h" |
| 44 | #include "llvm/Support/MathExtras.h" |
| 45 | #include "llvm/TargetParser/AMDGPUTargetParser.h" |
| 46 | #include <optional> |
| 47 | |
| 48 | using namespace llvm; |
| 49 | using namespace llvm::AMDGPU; |
| 50 | using namespace llvm::amdhsa; |
| 51 | |
| 52 | namespace { |
| 53 | |
| 54 | class AMDGPUAsmParser; |
| 55 | |
| 56 | enum RegisterKind { |
| 57 | IS_UNKNOWN, |
| 58 | IS_VGPR, |
| 59 | IS_SGPR, |
| 60 | IS_AGPR, |
| 61 | IS_TTMP, |
| 62 | IS_SPECIAL |
| 63 | }; |
| 64 | |
| 65 | //===----------------------------------------------------------------------===// |
| 66 | // Operand |
| 67 | //===----------------------------------------------------------------------===// |
| 68 | |
| 69 | class AMDGPUOperand : public MCParsedAsmOperand { |
| 70 | enum KindTy { Token, Immediate, Register, Expression } Kind; |
| 71 | |
| 72 | SMLoc StartLoc, EndLoc; |
| 73 | const AMDGPUAsmParser *AsmParser; |
| 74 | |
| 75 | public: |
| 76 | AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_) |
| 77 | : Kind(Kind_), AsmParser(AsmParser_) {} |
| 78 | |
| 79 | using Ptr = std::unique_ptr<AMDGPUOperand>; |
| 80 | |
| 81 | struct Modifiers { |
| 82 | bool Abs = false; |
| 83 | bool Neg = false; |
| 84 | bool Sext = false; |
| 85 | LitModifier Lit = LitModifier::None; |
| 86 | |
| 87 | bool hasFPModifiers() const { return Abs || Neg; } |
| 88 | bool hasIntModifiers() const { return Sext; } |
| 89 | bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); } |
| 90 | bool isForcedLit() const { return Lit == LitModifier::Lit; } |
| 91 | bool isForcedLit64() const { return Lit == LitModifier::Lit64; } |
| 92 | |
| 93 | int64_t getFPModifiersOperand() const { |
| 94 | int64_t Operand = 0; |
| 95 | Operand |= Abs ? SISrcMods::ABS : 0u; |
| 96 | Operand |= Neg ? SISrcMods::NEG : 0u; |
| 97 | return Operand; |
| 98 | } |
| 99 | |
| 100 | int64_t getIntModifiersOperand() const { |
| 101 | int64_t Operand = 0; |
| 102 | Operand |= Sext ? SISrcMods::SEXT : 0u; |
| 103 | return Operand; |
| 104 | } |
| 105 | |
| 106 | int64_t getModifiersOperand() const { |
| 107 | assert(!(hasFPModifiers() && hasIntModifiers()) && |
| 108 | "fp and int modifiers should not be used simultaneously" ); |
| 109 | if (hasFPModifiers()) |
| 110 | return getFPModifiersOperand(); |
| 111 | if (hasIntModifiers()) |
| 112 | return getIntModifiersOperand(); |
| 113 | return 0; |
| 114 | } |
| 115 | |
| 116 | friend raw_ostream &operator<<(raw_ostream &OS, |
| 117 | AMDGPUOperand::Modifiers Mods); |
| 118 | }; |
| 119 | |
| 120 | enum ImmTy { |
| 121 | ImmTyNone, |
| 122 | ImmTyGDS, |
| 123 | ImmTyLDS, |
| 124 | ImmTyOffen, |
| 125 | ImmTyIdxen, |
| 126 | ImmTyAddr64, |
| 127 | ImmTyOffset, |
| 128 | ImmTyInstOffset, |
| 129 | ImmTyOffset0, |
| 130 | ImmTyOffset1, |
| 131 | ImmTySMEMOffsetMod, |
| 132 | ImmTyCPol, |
| 133 | ImmTyTFE, |
| 134 | ImmTyIsAsync, |
| 135 | ImmTyD16, |
| 136 | ImmTyClamp, |
| 137 | ImmTyOModSI, |
| 138 | ImmTySDWADstSel, |
| 139 | ImmTySDWASrc0Sel, |
| 140 | ImmTySDWASrc1Sel, |
| 141 | ImmTySDWADstUnused, |
| 142 | ImmTyDMask, |
| 143 | ImmTyDim, |
| 144 | ImmTyUNorm, |
| 145 | ImmTyDA, |
| 146 | ImmTyR128A16, |
| 147 | ImmTyA16, |
| 148 | ImmTyLWE, |
| 149 | ImmTyExpTgt, |
| 150 | ImmTyExpCompr, |
| 151 | ImmTyExpVM, |
| 152 | ImmTyDone, |
| 153 | ImmTyRowEn, |
| 154 | ImmTyFormat, |
| 155 | ImmTyFormatOp, |
| 156 | ImmTyHwreg, |
| 157 | ImmTyOff, |
| 158 | ImmTySendMsg, |
| 159 | ImmTyWaitEvent, |
| 160 | ImmTyInterpSlot, |
| 161 | ImmTyInterpAttr, |
| 162 | ImmTyInterpAttrChan, |
| 163 | ImmTyOpSel, |
| 164 | ImmTyOpSelHi, |
| 165 | ImmTyNegLo, |
| 166 | ImmTyNegHi, |
| 167 | ImmTyIndexKey8bit, |
| 168 | ImmTyIndexKey16bit, |
| 169 | ImmTyIndexKey32bit, |
| 170 | ImmTyDPP8, |
| 171 | ImmTyDppCtrl, |
| 172 | ImmTyDppRowMask, |
| 173 | ImmTyDppBankMask, |
| 174 | ImmTyDppBoundCtrl, |
| 175 | ImmTyDppFI, |
| 176 | ImmTySwizzle, |
| 177 | ImmTyGprIdxMode, |
| 178 | ImmTyHigh, |
| 179 | ImmTyBLGP, |
| 180 | ImmTyCBSZ, |
| 181 | ImmTyABID, |
| 182 | ImmTyEndpgm, |
| 183 | ImmTyWaitVDST, |
| 184 | ImmTyWaitEXP, |
| 185 | ImmTyWaitVAVDst, |
| 186 | ImmTyWaitVMVSrc, |
| 187 | ImmTyBitOp3, |
| 188 | ImmTyMatrixAFMT, |
| 189 | ImmTyMatrixBFMT, |
| 190 | ImmTyMatrixAScale, |
| 191 | ImmTyMatrixBScale, |
| 192 | ImmTyMatrixAScaleFmt, |
| 193 | ImmTyMatrixBScaleFmt, |
| 194 | ImmTyMatrixAReuse, |
| 195 | ImmTyMatrixBReuse, |
| 196 | ImmTyScaleSel, |
| 197 | ImmTyByteSel, |
| 198 | }; |
| 199 | |
| 200 | private: |
| 201 | struct TokOp { |
| 202 | const char *Data; |
| 203 | unsigned Length; |
| 204 | }; |
| 205 | |
| 206 | struct ImmOp { |
| 207 | int64_t Val; |
| 208 | ImmTy Type; |
| 209 | bool IsFPImm; |
| 210 | Modifiers Mods; |
| 211 | }; |
| 212 | |
| 213 | struct RegOp { |
| 214 | MCRegister RegNo; |
| 215 | Modifiers Mods; |
| 216 | }; |
| 217 | |
| 218 | union { |
| 219 | TokOp Tok; |
| 220 | ImmOp Imm; |
| 221 | RegOp Reg; |
| 222 | const MCExpr *Expr; |
| 223 | }; |
| 224 | |
| 225 | // The index of the associated MCInst operand. |
| 226 | mutable int MCOpIdx = -1; |
| 227 | |
| 228 | public: |
| 229 | bool isToken() const override { return Kind == Token; } |
| 230 | |
| 231 | bool isSymbolRefExpr() const { |
| 232 | return isExpr() && Expr && isa<MCSymbolRefExpr>(Val: Expr); |
| 233 | } |
| 234 | |
| 235 | bool isImm() const override { return Kind == Immediate; } |
| 236 | |
| 237 | bool isInlinableImm(MVT type) const; |
| 238 | bool isLiteralImm(MVT type) const; |
| 239 | |
| 240 | bool isRegKind() const { return Kind == Register; } |
| 241 | |
| 242 | bool isReg() const override { return isRegKind() && !hasModifiers(); } |
| 243 | |
| 244 | bool isRegOrInline(unsigned RCID, MVT type) const { |
| 245 | return isRegClass(RCID) || isInlinableImm(type); |
| 246 | } |
| 247 | |
| 248 | bool isRegOrInlineTarget(unsigned TargetRCIdx, MVT type) const { |
| 249 | return isRegClassTarget(TargetRCIdx) || isInlinableImm(type); |
| 250 | } |
| 251 | |
| 252 | bool isRegOrImmWithInputMods(unsigned RCID, MVT type) const { |
| 253 | return isRegOrInline(RCID, type) || isLiteralImm(type); |
| 254 | } |
| 255 | |
| 256 | bool isRegOrImmWithInputModsTarget(unsigned TargetRCIdx, MVT type) const { |
| 257 | return isRegOrInlineTarget(TargetRCIdx, type) || isLiteralImm(type); |
| 258 | } |
| 259 | |
| 260 | template <bool IsFake16> bool isRegOrImmWithIntT16InputMods() const { |
| 261 | return isRegOrImmWithInputMods( |
| 262 | RCID: IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, type: MVT::i16); |
| 263 | } |
| 264 | |
| 265 | bool isRegOrImmWithInt32InputMods() const { |
| 266 | return isRegOrImmWithInputMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::i32); |
| 267 | } |
| 268 | |
| 269 | template <bool IsFake16> bool isRegOrInlineImmWithIntT16InputMods() const { |
| 270 | return isRegOrInline( |
| 271 | RCID: IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, type: MVT::i16); |
| 272 | } |
| 273 | |
| 274 | bool isRegOrInlineImmWithInt32InputMods() const { |
| 275 | return isRegOrInline(RCID: AMDGPU::VS_32RegClassID, type: MVT::i32); |
| 276 | } |
| 277 | |
| 278 | bool isRegOrImmWithInt64InputMods() const { |
| 279 | return isRegOrImmWithInputModsTarget(TargetRCIdx: AMDGPU::VS_64_AlignTarget, type: MVT::i64); |
| 280 | } |
| 281 | |
| 282 | bool isRegOrImmWithFP16InputMods() const { |
| 283 | return isRegOrImmWithInputMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::f16); |
| 284 | } |
| 285 | |
| 286 | template <bool IsFake16> bool isRegOrImmWithFPT16InputMods() const { |
| 287 | return isRegOrImmWithInputMods( |
| 288 | RCID: IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, type: MVT::f16); |
| 289 | } |
| 290 | |
| 291 | bool isRegOrImmWithFPT16_LO16InputMods() const { |
| 292 | return isRegOrImmWithInputMods(RCID: AMDGPU::VS_16_LO16RegClassID, type: MVT::f16); |
| 293 | } |
| 294 | |
| 295 | bool isRegOrImmWithFP32InputMods() const { |
| 296 | return isRegOrImmWithInputMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::f32); |
| 297 | } |
| 298 | |
| 299 | bool isRegOrImmWithFP64InputMods() const { |
| 300 | return isRegOrImmWithInputModsTarget(TargetRCIdx: AMDGPU::VS_64_AlignTarget, type: MVT::f64); |
| 301 | } |
| 302 | |
| 303 | template <bool IsFake16> bool isRegOrInlineImmWithFP16InputMods() const { |
| 304 | return isRegOrInline( |
| 305 | RCID: IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, type: MVT::f16); |
| 306 | } |
| 307 | |
| 308 | bool isRegOrInlineImmWithFP32InputMods() const { |
| 309 | return isRegOrInline(RCID: AMDGPU::VS_32RegClassID, type: MVT::f32); |
| 310 | } |
| 311 | |
| 312 | bool isRegOrInlineImmWithFP64InputMods() const { |
| 313 | return isRegOrInlineTarget(TargetRCIdx: AMDGPU::VS_64_AlignTarget, type: MVT::f64); |
| 314 | } |
| 315 | |
| 316 | bool isVRegWithInputMods(unsigned RCID) const { return isRegClass(RCID); } |
| 317 | |
| 318 | bool isVRegWithFP32InputMods() const { |
| 319 | return isVRegWithInputMods(RCID: AMDGPU::VGPR_32RegClassID); |
| 320 | } |
| 321 | |
| 322 | bool isVRegWithFP64InputMods() const { |
| 323 | return isRegClassTarget(TargetRCIdx: AMDGPU::VReg_64_AlignTarget); |
| 324 | } |
| 325 | |
| 326 | bool isPackedFP16InputMods() const { |
| 327 | return isRegOrImmWithInputMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::v2f16); |
| 328 | } |
| 329 | |
| 330 | bool isPackedVGPRFP32InputMods() const { |
| 331 | return isRegOrImmWithInputMods(RCID: AMDGPU::VReg_64RegClassID, type: MVT::v2f32); |
| 332 | } |
| 333 | |
| 334 | bool isVReg32() const { return isRegClass(RCID: AMDGPU::VGPR_32RegClassID); } |
| 335 | |
| 336 | bool isVReg32OrOff() const { return isOff() || isVReg32(); } |
| 337 | |
| 338 | bool isRsrcReg32() const { return isRegClass(RCID: AMDGPU::RsrcReg32RegClassID); } |
| 339 | |
| 340 | bool isNull() const { return isRegKind() && getReg() == AMDGPU::SGPR_NULL; } |
| 341 | |
| 342 | bool isAV_LdSt_32_Align2_RegOp() const { |
| 343 | return isRegClass(RCID: AMDGPU::VGPR_32RegClassID) || |
| 344 | isRegClass(RCID: AMDGPU::AGPR_32RegClassID); |
| 345 | } |
| 346 | |
| 347 | bool isVRegWithInputMods() const; |
| 348 | template <bool IsFake16> bool isT16_Lo128VRegWithInputMods() const; |
| 349 | template <bool IsFake16> bool isT16VRegWithInputMods() const; |
| 350 | bool isT16_LO16VRegWithInputMods() const; |
| 351 | |
| 352 | bool isSDWAOperand(MVT type) const; |
| 353 | bool isSDWAFP16Operand() const; |
| 354 | bool isSDWAFP32Operand() const; |
| 355 | bool isSDWAInt16Operand() const; |
| 356 | bool isSDWAInt32Operand() const; |
| 357 | |
| 358 | bool isImmTy(ImmTy ImmT) const { return isImm() && Imm.Type == ImmT; } |
| 359 | |
| 360 | template <ImmTy Ty> bool isImmTy() const { return isImmTy(ImmT: Ty); } |
| 361 | |
| 362 | bool isImmLiteral() const { return isImmTy(ImmT: ImmTyNone); } |
| 363 | |
| 364 | bool isImmModifier() const { return isImm() && Imm.Type != ImmTyNone; } |
| 365 | |
| 366 | bool isOModSI() const { return isImmTy(ImmT: ImmTyOModSI); } |
| 367 | bool isDim() const { return isImmTy(ImmT: ImmTyDim); } |
| 368 | bool isR128A16() const { return isImmTy(ImmT: ImmTyR128A16); } |
| 369 | bool isOff() const { return isImmTy(ImmT: ImmTyOff); } |
| 370 | bool isExpTgt() const { return isImmTy(ImmT: ImmTyExpTgt); } |
| 371 | bool isOffen() const { return isImmTy(ImmT: ImmTyOffen); } |
| 372 | bool isIdxen() const { return isImmTy(ImmT: ImmTyIdxen); } |
| 373 | bool isAddr64() const { return isImmTy(ImmT: ImmTyAddr64); } |
| 374 | bool isSMEMOffsetMod() const { return isImmTy(ImmT: ImmTySMEMOffsetMod); } |
| 375 | bool isFlatOffset() const { |
| 376 | return isImmTy(ImmT: ImmTyOffset) || isImmTy(ImmT: ImmTyInstOffset); |
| 377 | } |
| 378 | bool isGDS() const { return isImmTy(ImmT: ImmTyGDS); } |
| 379 | bool isLDS() const { return isImmTy(ImmT: ImmTyLDS); } |
| 380 | bool isCPol() const { return isImmTy(ImmT: ImmTyCPol); } |
| 381 | bool isIndexKey8bit() const { return isImmTy(ImmT: ImmTyIndexKey8bit); } |
| 382 | bool isIndexKey16bit() const { return isImmTy(ImmT: ImmTyIndexKey16bit); } |
| 383 | bool isIndexKey32bit() const { return isImmTy(ImmT: ImmTyIndexKey32bit); } |
| 384 | bool isMatrixAFMT() const { return isImmTy(ImmT: ImmTyMatrixAFMT); } |
| 385 | bool isMatrixBFMT() const { return isImmTy(ImmT: ImmTyMatrixBFMT); } |
| 386 | bool isMatrixAScale() const { return isImmTy(ImmT: ImmTyMatrixAScale); } |
| 387 | bool isMatrixBScale() const { return isImmTy(ImmT: ImmTyMatrixBScale); } |
| 388 | bool isMatrixAScaleFmt() const { return isImmTy(ImmT: ImmTyMatrixAScaleFmt); } |
| 389 | bool isMatrixBScaleFmt() const { return isImmTy(ImmT: ImmTyMatrixBScaleFmt); } |
| 390 | bool isTFE() const { return isImmTy(ImmT: ImmTyTFE); } |
| 391 | bool isFormat() const { return isImmTy(ImmT: ImmTyFormat) && isUInt<7>(x: getImm()); } |
| 392 | bool isFormatOp() const { |
| 393 | return isImmTy(ImmT: ImmTyFormatOp) && isUInt<7>(x: getImm()); |
| 394 | } |
| 395 | bool isDppFI() const { return isImmTy(ImmT: ImmTyDppFI); } |
| 396 | bool isSDWADstSel() const { return isImmTy(ImmT: ImmTySDWADstSel); } |
| 397 | bool isSDWASrc0Sel() const { return isImmTy(ImmT: ImmTySDWASrc0Sel); } |
| 398 | bool isSDWASrc1Sel() const { return isImmTy(ImmT: ImmTySDWASrc1Sel); } |
| 399 | bool isSDWADstUnused() const { return isImmTy(ImmT: ImmTySDWADstUnused); } |
| 400 | bool isInterpSlot() const { return isImmTy(ImmT: ImmTyInterpSlot); } |
| 401 | bool isInterpAttr() const { return isImmTy(ImmT: ImmTyInterpAttr); } |
| 402 | bool isInterpAttrChan() const { return isImmTy(ImmT: ImmTyInterpAttrChan); } |
| 403 | bool isOpSel() const { return isImmTy(ImmT: ImmTyOpSel); } |
| 404 | bool isOpSelHi() const { return isImmTy(ImmT: ImmTyOpSelHi); } |
| 405 | bool isNegLo() const { return isImmTy(ImmT: ImmTyNegLo); } |
| 406 | bool isNegHi() const { return isImmTy(ImmT: ImmTyNegHi); } |
| 407 | bool isDone() const { return isImmTy(ImmT: ImmTyDone); } |
| 408 | bool isRowEn() const { return isImmTy(ImmT: ImmTyRowEn); } |
| 409 | |
| 410 | bool isRegOrImm() const { return isReg() || isImm(); } |
| 411 | |
| 412 | bool isRegClass(unsigned RCID) const; |
| 413 | |
| 414 | // Check the register against the HwMode-resolved operand class. |
| 415 | bool isRegClassTarget(unsigned TargetRCIdx) const; |
| 416 | |
| 417 | bool isInlineValue() const; |
| 418 | |
| 419 | bool isRegOrInlineNoMods(unsigned RCID, MVT type) const { |
| 420 | return isRegOrInline(RCID, type) && !hasModifiers(); |
| 421 | } |
| 422 | |
| 423 | bool isRegOrInlineNoModsTarget(unsigned TargetRCIdx, MVT type) const { |
| 424 | return isRegOrInlineTarget(TargetRCIdx, type) && !hasModifiers(); |
| 425 | } |
| 426 | |
| 427 | bool isSCSrcB16() const { |
| 428 | return isRegOrInlineNoMods(RCID: AMDGPU::SReg_32RegClassID, type: MVT::i16); |
| 429 | } |
| 430 | |
| 431 | bool isSCSrc_b32() const { |
| 432 | return isRegOrInlineNoMods(RCID: AMDGPU::SReg_32RegClassID, type: MVT::i32); |
| 433 | } |
| 434 | |
| 435 | bool isSCSrc_b64() const { |
| 436 | return isRegOrInlineNoMods(RCID: AMDGPU::SReg_64RegClassID, type: MVT::i64); |
| 437 | } |
| 438 | |
| 439 | bool isBoolReg() const; |
| 440 | |
| 441 | bool isSSrc_b32() const { |
| 442 | return isSCSrc_b32() || isLiteralImm(type: MVT::i32) || isExpr(); |
| 443 | } |
| 444 | |
| 445 | bool isSSrc_b16() const { return isSCSrcB16() || isLiteralImm(type: MVT::i16); } |
| 446 | |
| 447 | bool isSSrc_b64() const { |
| 448 | // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits. |
| 449 | // See isVSrc64(). |
| 450 | return isSCSrc_b64() || isLiteralImm(type: MVT::i64) || |
| 451 | (((const MCTargetAsmParser *)AsmParser) |
| 452 | ->getAvailableFeatures()[AMDGPU::Feature64BitLiterals] && |
| 453 | isExpr()); |
| 454 | } |
| 455 | |
| 456 | bool isSSrc_f32() const { |
| 457 | return isSCSrc_b32() || isLiteralImm(type: MVT::f32) || isExpr(); |
| 458 | } |
| 459 | |
| 460 | bool isSSrc_bf16() const { return isSCSrcB16() || isLiteralImm(type: MVT::bf16); } |
| 461 | |
| 462 | bool isSSrc_f16() const { return isSCSrcB16() || isLiteralImm(type: MVT::f16); } |
| 463 | |
| 464 | bool isSSrc_NoInline_f16() const { return isSSrc_f16(); } |
| 465 | |
| 466 | bool isSSrcOrLds_b32() const { |
| 467 | return isRegOrInlineNoMods(RCID: AMDGPU::SRegOrLds_32RegClassID, type: MVT::i32) || |
| 468 | isLiteralImm(type: MVT::i32) || isExpr(); |
| 469 | } |
| 470 | |
| 471 | bool isVCSrc_b32() const { |
| 472 | return isRegOrInlineNoMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::i32); |
| 473 | } |
| 474 | |
| 475 | bool isVCSrc_b32_Lo256() const { |
| 476 | return isRegOrInlineNoMods(RCID: AMDGPU::VS_32_Lo256RegClassID, type: MVT::i32); |
| 477 | } |
| 478 | |
| 479 | bool isVCSrc_b64_Lo256() const { |
| 480 | return isRegOrInlineNoMods(RCID: AMDGPU::VS_64_Lo256RegClassID, type: MVT::i64); |
| 481 | } |
| 482 | |
| 483 | bool isVCSrc_b64() const { |
| 484 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VS_64_AlignTarget, type: MVT::i64); |
| 485 | } |
| 486 | |
| 487 | bool isVCSrcT_b16() const { |
| 488 | return isRegOrInlineNoMods(RCID: AMDGPU::VS_16RegClassID, type: MVT::i16); |
| 489 | } |
| 490 | |
| 491 | bool isVCSrcTB16_Lo128() const { |
| 492 | return isRegOrInlineNoMods(RCID: AMDGPU::VS_16_Lo128RegClassID, type: MVT::i16); |
| 493 | } |
| 494 | |
| 495 | bool isVCSrcFake16B16_Lo128() const { |
| 496 | return isRegOrInlineNoMods(RCID: AMDGPU::VS_32_Lo128RegClassID, type: MVT::i16); |
| 497 | } |
| 498 | |
| 499 | bool isVCSrc_b16() const { |
| 500 | return isRegOrInlineNoMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::i16); |
| 501 | } |
| 502 | |
| 503 | bool isVCSrc_v2b16() const { return isVCSrc_b16(); } |
| 504 | |
| 505 | bool isVCSrc_f32() const { |
| 506 | return isRegOrInlineNoMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::f32); |
| 507 | } |
| 508 | |
| 509 | bool isVCSrc_f64() const { |
| 510 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VS_64_AlignTarget, type: MVT::f64); |
| 511 | } |
| 512 | |
| 513 | bool isVCSrcTBF16() const { |
| 514 | return isRegOrInlineNoMods(RCID: AMDGPU::VS_16RegClassID, type: MVT::bf16); |
| 515 | } |
| 516 | |
| 517 | bool isVCSrcT_f16() const { |
| 518 | return isRegOrInlineNoMods(RCID: AMDGPU::VS_16RegClassID, type: MVT::f16); |
| 519 | } |
| 520 | |
| 521 | bool isVCSrcT_bf16() const { |
| 522 | return isRegOrInlineNoMods(RCID: AMDGPU::VS_16RegClassID, type: MVT::f16); |
| 523 | } |
| 524 | |
| 525 | bool isVCSrcTF16_LO16() const { |
| 526 | return isRegOrInlineNoMods(RCID: AMDGPU::VS_16_LO16RegClassID, type: MVT::f16); |
| 527 | } |
| 528 | |
| 529 | bool isVCSrcTBF16_Lo128() const { |
| 530 | return isRegOrInlineNoMods(RCID: AMDGPU::VS_16_Lo128RegClassID, type: MVT::bf16); |
| 531 | } |
| 532 | |
| 533 | bool isVCSrcTF16_Lo128() const { |
| 534 | return isRegOrInlineNoMods(RCID: AMDGPU::VS_16_Lo128RegClassID, type: MVT::f16); |
| 535 | } |
| 536 | |
| 537 | bool isVCSrcFake16BF16_Lo128() const { |
| 538 | return isRegOrInlineNoMods(RCID: AMDGPU::VS_32_Lo128RegClassID, type: MVT::bf16); |
| 539 | } |
| 540 | |
| 541 | bool isVCSrcFake16F16_Lo128() const { |
| 542 | return isRegOrInlineNoMods(RCID: AMDGPU::VS_32_Lo128RegClassID, type: MVT::f16); |
| 543 | } |
| 544 | |
| 545 | bool isVCSrc_bf16() const { |
| 546 | return isRegOrInlineNoMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::bf16); |
| 547 | } |
| 548 | |
| 549 | bool isVCSrc_f16() const { |
| 550 | return isRegOrInlineNoMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::f16); |
| 551 | } |
| 552 | |
| 553 | bool isVCSrc_v2bf16() const { return isVCSrc_bf16(); } |
| 554 | |
| 555 | bool isVCSrc_v2f16() const { return isVCSrc_f16(); } |
| 556 | |
| 557 | bool isVSrc_b32() const { |
| 558 | return isVCSrc_f32() || isLiteralImm(type: MVT::i32) || isExpr(); |
| 559 | } |
| 560 | |
| 561 | bool isVSrc_b64() const { return isVCSrc_f64() || isLiteralImm(type: MVT::i64); } |
| 562 | |
| 563 | bool isVSrc_v2b64() const { |
| 564 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VS_128_AlignTarget, type: MVT::i64) || |
| 565 | isLiteralImm(type: MVT::i64); |
| 566 | } |
| 567 | |
| 568 | bool isVSrc_v2f64() const { |
| 569 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VS_128_AlignTarget, type: MVT::f64) || |
| 570 | isLiteralImm(type: MVT::f64); |
| 571 | } |
| 572 | |
| 573 | bool isVSrcT_b16() const { return isVCSrcT_b16() || isLiteralImm(type: MVT::i16); } |
| 574 | |
| 575 | bool isVSrcT_b16_Lo128() const { |
| 576 | return isVCSrcTB16_Lo128() || isLiteralImm(type: MVT::i16); |
| 577 | } |
| 578 | |
| 579 | bool isVSrcFake16_b16_Lo128() const { |
| 580 | return isVCSrcFake16B16_Lo128() || isLiteralImm(type: MVT::i16); |
| 581 | } |
| 582 | |
| 583 | bool isVSrc_b16() const { return isVCSrc_b16() || isLiteralImm(type: MVT::i16); } |
| 584 | |
| 585 | bool isVSrc_v2b16() const { return isVSrc_b16() || isLiteralImm(type: MVT::v2i16); } |
| 586 | |
| 587 | bool isVSrc_v2f32() const { return isVSrc_f64() || isLiteralImm(type: MVT::v2f32); } |
| 588 | |
| 589 | bool isVCSrc_v2b32() const { return isVCSrc_b64(); } |
| 590 | |
| 591 | bool isVSrc_v2b32() const { return isVSrc_b64() || isLiteralImm(type: MVT::v2i32); } |
| 592 | |
| 593 | bool isVSrc_f32() const { |
| 594 | return isVCSrc_f32() || isLiteralImm(type: MVT::f32) || isExpr(); |
| 595 | } |
| 596 | |
| 597 | bool isVSrc_f64() const { |
| 598 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VS_64_AlignTarget, type: MVT::f64) || |
| 599 | isLiteralImm(type: MVT::f64); |
| 600 | } |
| 601 | |
| 602 | bool isVSrcT_bf16() const { |
| 603 | return isVCSrcTBF16() || isLiteralImm(type: MVT::bf16); |
| 604 | } |
| 605 | |
| 606 | bool isVSrcT_f16() const { return isVCSrcT_f16() || isLiteralImm(type: MVT::f16); } |
| 607 | |
| 608 | bool isVSrcT_f16_LO16() const { |
| 609 | return isVCSrcTF16_LO16() || isLiteralImm(type: MVT::f16); |
| 610 | } |
| 611 | |
| 612 | bool isVSrcT_bf16_Lo128() const { |
| 613 | return isVCSrcTBF16_Lo128() || isLiteralImm(type: MVT::bf16); |
| 614 | } |
| 615 | |
| 616 | bool isVSrcT_f16_Lo128() const { |
| 617 | return isVCSrcTF16_Lo128() || isLiteralImm(type: MVT::f16); |
| 618 | } |
| 619 | |
| 620 | bool isVSrcFake16_bf16_Lo128() const { |
| 621 | return isVCSrcFake16BF16_Lo128() || isLiteralImm(type: MVT::bf16); |
| 622 | } |
| 623 | |
| 624 | bool isVSrcFake16_f16_Lo128() const { |
| 625 | return isVCSrcFake16F16_Lo128() || isLiteralImm(type: MVT::f16); |
| 626 | } |
| 627 | |
| 628 | bool isVSrc_bf16() const { return isVCSrc_bf16() || isLiteralImm(type: MVT::bf16); } |
| 629 | |
| 630 | bool isVSrc_f16() const { return isVCSrc_f16() || isLiteralImm(type: MVT::f16); } |
| 631 | |
| 632 | bool isVSrc_v2bf16() const { |
| 633 | return isVSrc_bf16() || isLiteralImm(type: MVT::v2bf16); |
| 634 | } |
| 635 | |
| 636 | bool isVSrc_v2f16() const { return isVSrc_f16() || isLiteralImm(type: MVT::v2f16); } |
| 637 | |
| 638 | bool isVSrc_v2f16_splat() const { return isVSrc_v2f16(); } |
| 639 | |
| 640 | bool isVSrc_NoInline_v2f16() const { return isVSrc_v2f16(); } |
| 641 | |
| 642 | bool isVISrc_64_bf16() const { |
| 643 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_64_AlignTarget, type: MVT::bf16); |
| 644 | } |
| 645 | |
| 646 | bool isVISrc_64_f16() const { |
| 647 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_64_AlignTarget, type: MVT::f16); |
| 648 | } |
| 649 | |
| 650 | bool isVISrc_64_b32() const { |
| 651 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_64_AlignTarget, type: MVT::i32); |
| 652 | } |
| 653 | |
| 654 | bool isVISrc_64_f64() const { |
| 655 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_64_AlignTarget, type: MVT::f64); |
| 656 | } |
| 657 | |
| 658 | bool isVISrc_256_b32() const { |
| 659 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_256_AlignTarget, type: MVT::i32); |
| 660 | } |
| 661 | |
| 662 | bool isVISrc_256_f32() const { |
| 663 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_256_AlignTarget, type: MVT::f32); |
| 664 | } |
| 665 | |
| 666 | bool isVISrc_256_f64() const { |
| 667 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_256_AlignTarget, type: MVT::f64); |
| 668 | } |
| 669 | |
| 670 | bool isVISrc_512_f64() const { |
| 671 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_512_AlignTarget, type: MVT::f64); |
| 672 | } |
| 673 | |
| 674 | bool isVISrc_128_b32() const { |
| 675 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_128_AlignTarget, type: MVT::i32); |
| 676 | } |
| 677 | |
| 678 | bool isVISrc_128_f32() const { |
| 679 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_128_AlignTarget, type: MVT::f32); |
| 680 | } |
| 681 | |
| 682 | bool isVISrc_512_b32() const { |
| 683 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_512_AlignTarget, type: MVT::i32); |
| 684 | } |
| 685 | |
| 686 | bool isVISrc_512_f32() const { |
| 687 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_512_AlignTarget, type: MVT::f32); |
| 688 | } |
| 689 | |
| 690 | bool isVISrc_1024_b32() const { |
| 691 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_1024_AlignTarget, type: MVT::i32); |
| 692 | } |
| 693 | |
| 694 | bool isVISrc_1024_f32() const { |
| 695 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_1024_AlignTarget, type: MVT::f32); |
| 696 | } |
| 697 | |
| 698 | bool isAISrc_64_f64() const { |
| 699 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::AReg_64_AlignTarget, type: MVT::f64); |
| 700 | } |
| 701 | |
| 702 | bool isAISrc_128_b32() const { |
| 703 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::AReg_128_AlignTarget, type: MVT::i32); |
| 704 | } |
| 705 | |
| 706 | bool isAISrc_128_f32() const { |
| 707 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::AReg_128_AlignTarget, type: MVT::f32); |
| 708 | } |
| 709 | |
| 710 | bool isVISrc_128_bf16() const { |
| 711 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_128_AlignTarget, type: MVT::bf16); |
| 712 | } |
| 713 | |
| 714 | bool isVISrc_128_f16() const { |
| 715 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_128_AlignTarget, type: MVT::f16); |
| 716 | } |
| 717 | |
| 718 | bool isAISrc_256_f64() const { |
| 719 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::AReg_256_AlignTarget, type: MVT::f64); |
| 720 | } |
| 721 | |
| 722 | bool isAISrc_512_b32() const { |
| 723 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::AReg_512_AlignTarget, type: MVT::i32); |
| 724 | } |
| 725 | |
| 726 | bool isAISrc_512_f32() const { |
| 727 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::AReg_512_AlignTarget, type: MVT::f32); |
| 728 | } |
| 729 | |
| 730 | bool isAISrc_1024_b32() const { |
| 731 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::AReg_1024_AlignTarget, type: MVT::i32); |
| 732 | } |
| 733 | |
| 734 | bool isAISrc_1024_f32() const { |
| 735 | return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::AReg_1024_AlignTarget, type: MVT::f32); |
| 736 | } |
| 737 | |
| 738 | bool isKImmFP32() const { return isLiteralImm(type: MVT::f32); } |
| 739 | |
| 740 | bool isKImmFP16() const { return isLiteralImm(type: MVT::f16); } |
| 741 | |
| 742 | bool isKImmFP64() const { return isLiteralImm(type: MVT::f64); } |
| 743 | |
| 744 | bool isMem() const override { return false; } |
| 745 | |
| 746 | bool isExpr() const { return Kind == Expression; } |
| 747 | |
| 748 | bool isSOPPBrTarget() const { return isExpr() || isImm(); } |
| 749 | |
| 750 | bool isSWaitCnt() const; |
| 751 | bool isDepCtr() const; |
| 752 | bool isSDelayALU() const; |
| 753 | bool isHwreg() const; |
| 754 | bool isSendMsg() const; |
| 755 | bool isWaitEvent() const; |
| 756 | bool isSplitBarrier() const; |
| 757 | bool isSwizzle() const; |
| 758 | bool isSMRDOffset8() const; |
| 759 | bool isSMEMOffset() const; |
| 760 | bool isSMRDLiteralOffset() const; |
| 761 | bool isDPP8() const; |
| 762 | bool isDPPCtrl() const; |
| 763 | bool isBLGP() const; |
| 764 | bool isGPRIdxMode() const; |
| 765 | bool isS16Imm() const; |
| 766 | bool isU16Imm() const; |
| 767 | bool isEndpgm() const; |
| 768 | |
| 769 | auto getPredicate(std::function<bool(const AMDGPUOperand &Op)> P) const { |
| 770 | return [this, P]() { return P(*this); }; |
| 771 | } |
| 772 | |
| 773 | StringRef getToken() const { |
| 774 | assert(isToken()); |
| 775 | return StringRef(Tok.Data, Tok.Length); |
| 776 | } |
| 777 | |
| 778 | int64_t getImm() const { |
| 779 | assert(isImm()); |
| 780 | return Imm.Val; |
| 781 | } |
| 782 | |
| 783 | ImmTy getImmTy() const { |
| 784 | assert(isImm()); |
| 785 | return Imm.Type; |
| 786 | } |
| 787 | |
| 788 | MCRegister getReg() const override { |
| 789 | assert(isRegKind()); |
| 790 | return Reg.RegNo; |
| 791 | } |
| 792 | |
| 793 | SMLoc getStartLoc() const override { return StartLoc; } |
| 794 | |
| 795 | SMLoc getEndLoc() const override { return EndLoc; } |
| 796 | |
| 797 | int getMCOpIdx() const { return MCOpIdx; } |
| 798 | |
| 799 | Modifiers getModifiers() const { |
| 800 | assert(isRegKind() || isImmTy(ImmTyNone)); |
| 801 | return isRegKind() ? Reg.Mods : Imm.Mods; |
| 802 | } |
| 803 | |
| 804 | void setModifiers(Modifiers Mods) { |
| 805 | assert(isRegKind() || isImmTy(ImmTyNone)); |
| 806 | if (isRegKind()) |
| 807 | Reg.Mods = Mods; |
| 808 | else |
| 809 | Imm.Mods = Mods; |
| 810 | } |
| 811 | |
| 812 | bool hasModifiers() const { return getModifiers().hasModifiers(); } |
| 813 | |
| 814 | bool hasFPModifiers() const { return getModifiers().hasFPModifiers(); } |
| 815 | |
| 816 | bool hasIntModifiers() const { return getModifiers().hasIntModifiers(); } |
| 817 | |
| 818 | bool isForcedLit() const { |
| 819 | return isImmLiteral() && getModifiers().isForcedLit(); |
| 820 | } |
| 821 | |
| 822 | bool isForcedLit64() const { |
| 823 | return isImmLiteral() && getModifiers().isForcedLit64(); |
| 824 | } |
| 825 | |
| 826 | uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const; |
| 827 | |
| 828 | void addImmOperands(MCInst &Inst, unsigned N, |
| 829 | bool ApplyModifiers = true) const; |
| 830 | |
| 831 | void addLiteralImmOperand(MCInst &Inst, int64_t Val, |
| 832 | bool ApplyModifiers) const; |
| 833 | |
| 834 | void addRegOperands(MCInst &Inst, unsigned N) const; |
| 835 | |
| 836 | void addRegOrImmOperands(MCInst &Inst, unsigned N) const { |
| 837 | if (isRegKind()) |
| 838 | addRegOperands(Inst, N); |
| 839 | else |
| 840 | addImmOperands(Inst, N); |
| 841 | } |
| 842 | |
| 843 | void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const { |
| 844 | Modifiers Mods = getModifiers(); |
| 845 | Inst.addOperand(Op: MCOperand::createImm(Val: Mods.getModifiersOperand())); |
| 846 | if (isRegKind()) { |
| 847 | addRegOperands(Inst, N); |
| 848 | } else { |
| 849 | addImmOperands(Inst, N, ApplyModifiers: false); |
| 850 | } |
| 851 | } |
| 852 | |
| 853 | void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const { |
| 854 | assert(!hasIntModifiers()); |
| 855 | addRegOrImmWithInputModsOperands(Inst, N); |
| 856 | } |
| 857 | |
| 858 | void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const { |
| 859 | Modifiers Mods = getModifiers(); |
| 860 | Inst.addOperand(Op: MCOperand::createImm(Val: Mods.getModifiersOperand())); |
| 861 | assert(isRegKind()); |
| 862 | addRegOperands(Inst, N); |
| 863 | } |
| 864 | |
| 865 | void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const { |
| 866 | assert(!hasIntModifiers()); |
| 867 | addRegWithInputModsOperands(Inst, N); |
| 868 | } |
| 869 | |
| 870 | static void printImmTy(raw_ostream &OS, ImmTy Type) { |
| 871 | // clang-format off |
| 872 | switch (Type) { |
| 873 | case ImmTyNone: OS << "None" ; break; |
| 874 | case ImmTyGDS: OS << "GDS" ; break; |
| 875 | case ImmTyLDS: OS << "LDS" ; break; |
| 876 | case ImmTyOffen: OS << "Offen" ; break; |
| 877 | case ImmTyIdxen: OS << "Idxen" ; break; |
| 878 | case ImmTyAddr64: OS << "Addr64" ; break; |
| 879 | case ImmTyOffset: OS << "Offset" ; break; |
| 880 | case ImmTyInstOffset: OS << "InstOffset" ; break; |
| 881 | case ImmTyOffset0: OS << "Offset0" ; break; |
| 882 | case ImmTyOffset1: OS << "Offset1" ; break; |
| 883 | case ImmTySMEMOffsetMod: OS << "SMEMOffsetMod" ; break; |
| 884 | case ImmTyCPol: OS << "CPol" ; break; |
| 885 | case ImmTyIndexKey8bit: OS << "index_key" ; break; |
| 886 | case ImmTyIndexKey16bit: OS << "index_key" ; break; |
| 887 | case ImmTyIndexKey32bit: OS << "index_key" ; break; |
| 888 | case ImmTyTFE: OS << "TFE" ; break; |
| 889 | case ImmTyIsAsync: OS << "IsAsync" ; break; |
| 890 | case ImmTyD16: OS << "D16" ; break; |
| 891 | case ImmTyFormat: OS << "Format" ; break; |
| 892 | case ImmTyFormatOp: OS << "FormatOp" ; break; |
| 893 | case ImmTyClamp: OS << "Clamp" ; break; |
| 894 | case ImmTyOModSI: OS << "OModSI" ; break; |
| 895 | case ImmTyDPP8: OS << "DPP8" ; break; |
| 896 | case ImmTyDppCtrl: OS << "DppCtrl" ; break; |
| 897 | case ImmTyDppRowMask: OS << "DppRowMask" ; break; |
| 898 | case ImmTyDppBankMask: OS << "DppBankMask" ; break; |
| 899 | case ImmTyDppBoundCtrl: OS << "DppBoundCtrl" ; break; |
| 900 | case ImmTyDppFI: OS << "DppFI" ; break; |
| 901 | case ImmTySDWADstSel: OS << "SDWADstSel" ; break; |
| 902 | case ImmTySDWASrc0Sel: OS << "SDWASrc0Sel" ; break; |
| 903 | case ImmTySDWASrc1Sel: OS << "SDWASrc1Sel" ; break; |
| 904 | case ImmTySDWADstUnused: OS << "SDWADstUnused" ; break; |
| 905 | case ImmTyDMask: OS << "DMask" ; break; |
| 906 | case ImmTyDim: OS << "Dim" ; break; |
| 907 | case ImmTyUNorm: OS << "UNorm" ; break; |
| 908 | case ImmTyDA: OS << "DA" ; break; |
| 909 | case ImmTyR128A16: OS << "R128A16" ; break; |
| 910 | case ImmTyA16: OS << "A16" ; break; |
| 911 | case ImmTyLWE: OS << "LWE" ; break; |
| 912 | case ImmTyOff: OS << "Off" ; break; |
| 913 | case ImmTyExpTgt: OS << "ExpTgt" ; break; |
| 914 | case ImmTyExpCompr: OS << "ExpCompr" ; break; |
| 915 | case ImmTyExpVM: OS << "ExpVM" ; break; |
| 916 | case ImmTyDone: OS << "Done" ; break; |
| 917 | case ImmTyRowEn: OS << "RowEn" ; break; |
| 918 | case ImmTyHwreg: OS << "Hwreg" ; break; |
| 919 | case ImmTySendMsg: OS << "SendMsg" ; break; |
| 920 | case ImmTyWaitEvent: OS << "WaitEvent" ; break; |
| 921 | case ImmTyInterpSlot: OS << "InterpSlot" ; break; |
| 922 | case ImmTyInterpAttr: OS << "InterpAttr" ; break; |
| 923 | case ImmTyInterpAttrChan: OS << "InterpAttrChan" ; break; |
| 924 | case ImmTyOpSel: OS << "OpSel" ; break; |
| 925 | case ImmTyOpSelHi: OS << "OpSelHi" ; break; |
| 926 | case ImmTyNegLo: OS << "NegLo" ; break; |
| 927 | case ImmTyNegHi: OS << "NegHi" ; break; |
| 928 | case ImmTySwizzle: OS << "Swizzle" ; break; |
| 929 | case ImmTyGprIdxMode: OS << "GprIdxMode" ; break; |
| 930 | case ImmTyHigh: OS << "High" ; break; |
| 931 | case ImmTyBLGP: OS << "BLGP" ; break; |
| 932 | case ImmTyCBSZ: OS << "CBSZ" ; break; |
| 933 | case ImmTyABID: OS << "ABID" ; break; |
| 934 | case ImmTyEndpgm: OS << "Endpgm" ; break; |
| 935 | case ImmTyWaitVDST: OS << "WaitVDST" ; break; |
| 936 | case ImmTyWaitEXP: OS << "WaitEXP" ; break; |
| 937 | case ImmTyWaitVAVDst: OS << "WaitVAVDst" ; break; |
| 938 | case ImmTyWaitVMVSrc: OS << "WaitVMVSrc" ; break; |
| 939 | case ImmTyBitOp3: OS << "BitOp3" ; break; |
| 940 | case ImmTyMatrixAFMT: OS << "ImmTyMatrixAFMT" ; break; |
| 941 | case ImmTyMatrixBFMT: OS << "ImmTyMatrixBFMT" ; break; |
| 942 | case ImmTyMatrixAScale: OS << "ImmTyMatrixAScale" ; break; |
| 943 | case ImmTyMatrixBScale: OS << "ImmTyMatrixBScale" ; break; |
| 944 | case ImmTyMatrixAScaleFmt: OS << "ImmTyMatrixAScaleFmt" ; break; |
| 945 | case ImmTyMatrixBScaleFmt: OS << "ImmTyMatrixBScaleFmt" ; break; |
| 946 | case ImmTyMatrixAReuse: OS << "ImmTyMatrixAReuse" ; break; |
| 947 | case ImmTyMatrixBReuse: OS << "ImmTyMatrixBReuse" ; break; |
| 948 | case ImmTyScaleSel: OS << "ScaleSel" ; break; |
| 949 | case ImmTyByteSel: OS << "ByteSel" ; break; |
| 950 | } |
| 951 | // clang-format on |
| 952 | } |
| 953 | |
| 954 | void print(raw_ostream &OS, const MCAsmInfo &MAI) const override { |
| 955 | switch (Kind) { |
| 956 | case Register: |
| 957 | OS << "<register " << AMDGPUInstPrinter::getRegisterName(Reg: getReg()) |
| 958 | << " mods: " << Reg.Mods << '>'; |
| 959 | break; |
| 960 | case Immediate: |
| 961 | OS << '<' << getImm(); |
| 962 | if (getImmTy() != ImmTyNone) { |
| 963 | OS << " type: " ; |
| 964 | printImmTy(OS, Type: getImmTy()); |
| 965 | } |
| 966 | OS << " mods: " << Imm.Mods << '>'; |
| 967 | break; |
| 968 | case Token: |
| 969 | OS << '\'' << getToken() << '\''; |
| 970 | break; |
| 971 | case Expression: |
| 972 | OS << "<expr " ; |
| 973 | MAI.printExpr(OS, *Expr); |
| 974 | OS << '>'; |
| 975 | break; |
| 976 | } |
| 977 | } |
| 978 | |
| 979 | static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser, |
| 980 | int64_t Val, SMLoc Loc, |
| 981 | ImmTy Type = ImmTyNone, |
| 982 | bool IsFPImm = false) { |
| 983 | auto Op = std::make_unique<AMDGPUOperand>(args: Immediate, args&: AsmParser); |
| 984 | Op->Imm.Val = Val; |
| 985 | Op->Imm.IsFPImm = IsFPImm; |
| 986 | Op->Imm.Type = Type; |
| 987 | Op->Imm.Mods = Modifiers(); |
| 988 | Op->StartLoc = Loc; |
| 989 | Op->EndLoc = Loc; |
| 990 | return Op; |
| 991 | } |
| 992 | |
| 993 | static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser, |
| 994 | StringRef Str, SMLoc Loc, |
| 995 | bool HasExplicitEncodingSize = true) { |
| 996 | auto Res = std::make_unique<AMDGPUOperand>(args: Token, args&: AsmParser); |
| 997 | Res->Tok.Data = Str.data(); |
| 998 | Res->Tok.Length = Str.size(); |
| 999 | Res->StartLoc = Loc; |
| 1000 | Res->EndLoc = Loc; |
| 1001 | return Res; |
| 1002 | } |
| 1003 | |
| 1004 | static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser, |
| 1005 | MCRegister Reg, SMLoc S, SMLoc E) { |
| 1006 | auto Op = std::make_unique<AMDGPUOperand>(args: Register, args&: AsmParser); |
| 1007 | Op->Reg.RegNo = Reg; |
| 1008 | Op->Reg.Mods = Modifiers(); |
| 1009 | Op->StartLoc = S; |
| 1010 | Op->EndLoc = E; |
| 1011 | return Op; |
| 1012 | } |
| 1013 | |
| 1014 | static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser, |
| 1015 | const class MCExpr *Expr, SMLoc S) { |
| 1016 | auto Op = std::make_unique<AMDGPUOperand>(args: Expression, args&: AsmParser); |
| 1017 | Op->Expr = Expr; |
| 1018 | Op->StartLoc = S; |
| 1019 | Op->EndLoc = S; |
| 1020 | return Op; |
| 1021 | } |
| 1022 | }; |
| 1023 | |
| 1024 | raw_ostream &operator<<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) { |
| 1025 | OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext; |
| 1026 | return OS; |
| 1027 | } |
| 1028 | |
| 1029 | //===----------------------------------------------------------------------===// |
| 1030 | // AsmParser |
| 1031 | //===----------------------------------------------------------------------===// |
| 1032 | |
| 1033 | // TODO: define GET_SUBTARGET_FEATURE_NAME |
| 1034 | #define GET_REGISTER_MATCHER |
| 1035 | #include "AMDGPUGenAsmMatcher.inc" |
| 1036 | #undef GET_REGISTER_MATCHER |
| 1037 | #undef GET_SUBTARGET_FEATURE_NAME |
| 1038 | |
| 1039 | // Holds info related to the current kernel, e.g. count of SGPRs used. |
| 1040 | // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next |
| 1041 | // .amdgpu_hsa_kernel or at EOF. |
| 1042 | class KernelScopeInfo { |
| 1043 | int SgprIndexUnusedMin = -1; |
| 1044 | int VgprIndexUnusedMin = -1; |
| 1045 | int AgprIndexUnusedMin = -1; |
| 1046 | MCContext *Ctx = nullptr; |
| 1047 | MCSubtargetInfo const *MSTI = nullptr; |
| 1048 | |
| 1049 | void usesSgprAt(int i) { |
| 1050 | if (i >= SgprIndexUnusedMin) { |
| 1051 | SgprIndexUnusedMin = ++i; |
| 1052 | if (Ctx) { |
| 1053 | MCSymbol *const Sym = |
| 1054 | Ctx->getOrCreateSymbol(Name: Twine(".kernel.sgpr_count" )); |
| 1055 | Sym->setVariableValue(MCConstantExpr::create(Value: SgprIndexUnusedMin, Ctx&: *Ctx)); |
| 1056 | } |
| 1057 | } |
| 1058 | } |
| 1059 | |
| 1060 | void usesVgprAt(int i) { |
| 1061 | if (i >= VgprIndexUnusedMin) { |
| 1062 | VgprIndexUnusedMin = ++i; |
| 1063 | if (Ctx) { |
| 1064 | MCSymbol *const Sym = |
| 1065 | Ctx->getOrCreateSymbol(Name: Twine(".kernel.vgpr_count" )); |
| 1066 | int totalVGPR = |
| 1067 | getTotalNumVGPRs(has90AInsts: isGFX90A(STI: *MSTI), ArgNumAGPR: std::max(a: AgprIndexUnusedMin, b: 0), |
| 1068 | ArgNumVGPR: VgprIndexUnusedMin); |
| 1069 | Sym->setVariableValue(MCConstantExpr::create(Value: totalVGPR, Ctx&: *Ctx)); |
| 1070 | } |
| 1071 | } |
| 1072 | } |
| 1073 | |
| 1074 | void usesAgprAt(int i) { |
| 1075 | // Instruction will error in AMDGPUAsmParser::matchAndEmitInstruction |
| 1076 | if (!hasMAIInsts(STI: *MSTI)) |
| 1077 | return; |
| 1078 | |
| 1079 | if (i >= AgprIndexUnusedMin) { |
| 1080 | AgprIndexUnusedMin = ++i; |
| 1081 | if (Ctx) { |
| 1082 | MCSymbol *const Sym = |
| 1083 | Ctx->getOrCreateSymbol(Name: Twine(".kernel.agpr_count" )); |
| 1084 | Sym->setVariableValue(MCConstantExpr::create(Value: AgprIndexUnusedMin, Ctx&: *Ctx)); |
| 1085 | |
| 1086 | // Also update vgpr_count (dependent on agpr_count for gfx908/gfx90a) |
| 1087 | MCSymbol *const vSym = |
| 1088 | Ctx->getOrCreateSymbol(Name: Twine(".kernel.vgpr_count" )); |
| 1089 | int totalVGPR = |
| 1090 | getTotalNumVGPRs(has90AInsts: isGFX90A(STI: *MSTI), ArgNumAGPR: std::max(a: AgprIndexUnusedMin, b: 0), |
| 1091 | ArgNumVGPR: VgprIndexUnusedMin); |
| 1092 | vSym->setVariableValue(MCConstantExpr::create(Value: totalVGPR, Ctx&: *Ctx)); |
| 1093 | } |
| 1094 | } |
| 1095 | } |
| 1096 | |
| 1097 | public: |
| 1098 | KernelScopeInfo() = default; |
| 1099 | |
| 1100 | void initialize(MCContext &Context) { |
| 1101 | Ctx = &Context; |
| 1102 | MSTI = Ctx->getSubtargetInfo(); |
| 1103 | |
| 1104 | usesSgprAt(i: SgprIndexUnusedMin = -1); |
| 1105 | usesVgprAt(i: VgprIndexUnusedMin = -1); |
| 1106 | if (hasMAIInsts(STI: *MSTI)) { |
| 1107 | usesAgprAt(i: AgprIndexUnusedMin = -1); |
| 1108 | } |
| 1109 | } |
| 1110 | |
| 1111 | void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, |
| 1112 | unsigned RegWidth) { |
| 1113 | switch (RegKind) { |
| 1114 | case IS_SGPR: |
| 1115 | usesSgprAt(i: DwordRegIndex + divideCeil(Numerator: RegWidth, Denominator: 32) - 1); |
| 1116 | break; |
| 1117 | case IS_AGPR: |
| 1118 | usesAgprAt(i: DwordRegIndex + divideCeil(Numerator: RegWidth, Denominator: 32) - 1); |
| 1119 | break; |
| 1120 | case IS_VGPR: |
| 1121 | usesVgprAt(i: DwordRegIndex + divideCeil(Numerator: RegWidth, Denominator: 32) - 1); |
| 1122 | break; |
| 1123 | default: |
| 1124 | break; |
| 1125 | } |
| 1126 | } |
| 1127 | }; |
| 1128 | |
| 1129 | class AMDGPUAsmParser : public MCTargetAsmParser { |
| 1130 | MCAsmParser &Parser; |
| 1131 | |
| 1132 | unsigned ForcedEncodingSize = 0; |
| 1133 | bool ForcedDPP = false; |
| 1134 | bool ForcedSDWA = false; |
| 1135 | KernelScopeInfo KernelScope; |
| 1136 | const unsigned HwMode; |
| 1137 | const AMDGPU::GPUKind Gfx; |
| 1138 | const AMDGPU::IsaVersion ISA; |
| 1139 | |
| 1140 | /// @name Auto-generated Match Functions |
| 1141 | /// { |
| 1142 | |
| 1143 | #define |
| 1144 | #include "AMDGPUGenAsmMatcher.inc" |
| 1145 | |
| 1146 | /// } |
| 1147 | |
| 1148 | /// Get size of register operand |
| 1149 | unsigned getRegOperandSize(const MCInstrDesc &Desc, unsigned OpNo) const { |
| 1150 | assert(OpNo < Desc.NumOperands); |
| 1151 | int16_t RCID = MII.getOpRegClassID(OpInfo: Desc.operands()[OpNo], HwModeId: HwMode); |
| 1152 | return getRegBitWidth(RCID) / 8; |
| 1153 | } |
| 1154 | |
| 1155 | std::optional<AMDGPU::InfoSectionData> InfoData; |
| 1156 | |
| 1157 | /// State for checking that every kernel named in a .amdhsa_kernel directive |
| 1158 | /// begins with the required prologue instruction sequence. Because the |
| 1159 | /// directive may appear either before or after the kernel's label (it is |
| 1160 | /// normally emitted after the function body, in .rodata), validation is |
| 1161 | /// deferred to onEndOfFile(). We record an order-independent timeline of |
| 1162 | /// parsed labels and emitted instruction opcodes, plus the set of symbols |
| 1163 | /// named by .amdhsa_kernel directives, and match them up at end of file. |
| 1164 | SmallVector<unsigned> OpcodeStream; |
| 1165 | SmallVector<std::tuple<const MCSymbol *, SMLoc, unsigned>> |
| 1166 | OpcodeStreamSymbols; |
| 1167 | SmallPtrSet<const MCSymbol *, 8> AMDHSAKernelSymbols; |
| 1168 | |
| 1169 | /// Verify recorded kernel prologues. |
| 1170 | void checkKernelPrologues(); |
| 1171 | |
| 1172 | private: |
| 1173 | void createConstantSymbol(StringRef Id, int64_t Val); |
| 1174 | |
| 1175 | bool ParseAsAbsoluteExpression(uint32_t &Ret); |
| 1176 | bool OutOfRangeError(SMRange Range); |
| 1177 | /// Calculate VGPR/SGPR blocks required for given target, reserved |
| 1178 | /// registers, and user-specified NextFreeXGPR values. |
| 1179 | /// |
| 1180 | /// \param Features [in] Target features, used for bug corrections. |
| 1181 | /// \param VCCUsed [in] Whether VCC special SGPR is reserved. |
| 1182 | /// \param FlatScrUsed [in] Whether FLAT_SCRATCH special SGPR is reserved. |
| 1183 | /// \param XNACKUsed [in] Whether XNACK_MASK special SGPR is reserved. |
| 1184 | /// \param EnableWavefrontSize32 [in] Value of ENABLE_WAVEFRONT_SIZE32 kernel |
| 1185 | /// descriptor field, if valid. |
| 1186 | /// \param NextFreeVGPR [in] Max VGPR number referenced, plus one. |
| 1187 | /// \param VGPRRange [in] Token range, used for VGPR diagnostics. |
| 1188 | /// \param NextFreeSGPR [in] Max SGPR number referenced, plus one. |
| 1189 | /// \param SGPRRange [in] Token range, used for SGPR diagnostics. |
| 1190 | /// \param VGPRBlocks [out] Result VGPR block count. |
| 1191 | /// \param SGPRBlocks [out] Result SGPR block count. |
| 1192 | bool calculateGPRBlocks(const FeatureBitset &Features, const MCExpr *VCCUsed, |
| 1193 | const MCExpr *FlatScrUsed, bool XNACKUsed, |
| 1194 | std::optional<bool> EnableWavefrontSize32, |
| 1195 | const MCExpr *NextFreeVGPR, SMRange VGPRRange, |
| 1196 | const MCExpr *NextFreeSGPR, SMRange SGPRRange, |
| 1197 | const MCExpr *&VGPRBlocks, const MCExpr *&SGPRBlocks); |
| 1198 | bool ParseDirectiveAMDGCNTarget(); |
| 1199 | bool ParseDirectiveAMDHSACodeObjectVersion(); |
| 1200 | bool ParseDirectiveAMDHSAKernel(); |
| 1201 | bool ParseAMDKernelCodeTValue(StringRef ID, AMDGPUMCKernelCodeT &); |
| 1202 | bool ParseDirectiveAMDKernelCodeT(); |
| 1203 | // TODO: Possibly make subtargetHasRegister const. |
| 1204 | bool subtargetHasRegister(const MCRegisterInfo &MRI, MCRegister Reg); |
| 1205 | bool ParseDirectiveAMDGPUHsaKernel(); |
| 1206 | |
| 1207 | bool ParseDirectiveISAVersion(); |
| 1208 | bool ParseDirectiveHSAMetadata(); |
| 1209 | bool ParseDirectivePALMetadataBegin(); |
| 1210 | bool ParseDirectivePALMetadata(); |
| 1211 | bool ParseDirectiveAMDGPULDS(); |
| 1212 | bool ParseDirectiveAMDGPUInfo(); |
| 1213 | |
| 1214 | /// Common code to parse out a block of text (typically YAML) between start |
| 1215 | /// and end directives. |
| 1216 | bool ParseToEndDirective(const char *AssemblerDirectiveBegin, |
| 1217 | const char *AssemblerDirectiveEnd, |
| 1218 | std::string &CollectString); |
| 1219 | |
| 1220 | bool AddNextRegisterToList(MCRegister &Reg, unsigned &RegWidth, |
| 1221 | RegisterKind RegKind, MCRegister Reg1, |
| 1222 | RegisterKind RegKind1, SMLoc Loc); |
| 1223 | bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &Reg, |
| 1224 | unsigned &RegNum, unsigned &RegWidth, |
| 1225 | bool RestoreOnFailure = false); |
| 1226 | bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &Reg, |
| 1227 | unsigned &RegNum, unsigned &RegWidth, |
| 1228 | SmallVectorImpl<AsmToken> &Tokens); |
| 1229 | MCRegister ParseRegularReg(RegisterKind &RegKind, unsigned &RegNum, |
| 1230 | unsigned &RegWidth, |
| 1231 | SmallVectorImpl<AsmToken> &Tokens); |
| 1232 | MCRegister ParseSpecialReg(RegisterKind &RegKind, unsigned &RegNum, |
| 1233 | unsigned &RegWidth, |
| 1234 | SmallVectorImpl<AsmToken> &Tokens); |
| 1235 | MCRegister ParseRegList(RegisterKind &RegKind, unsigned &RegNum, |
| 1236 | unsigned &RegWidth, |
| 1237 | SmallVectorImpl<AsmToken> &Tokens); |
| 1238 | bool ParseRegRange(unsigned &Num, unsigned &Width, unsigned &SubReg); |
| 1239 | MCRegister getRegularReg(RegisterKind RegKind, unsigned RegNum, |
| 1240 | unsigned SubReg, unsigned RegWidth, SMLoc Loc); |
| 1241 | |
| 1242 | bool isRegister(); |
| 1243 | bool isRegister(const AsmToken &Token, const AsmToken &NextToken) const; |
| 1244 | std::optional<StringRef> getGprCountSymbolName(RegisterKind RegKind); |
| 1245 | void initializeGprCountSymbol(RegisterKind RegKind); |
| 1246 | bool updateGprCountSymbols(RegisterKind RegKind, unsigned DwordRegIndex, |
| 1247 | unsigned RegWidth); |
| 1248 | void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic); |
| 1249 | |
| 1250 | public: |
| 1251 | enum OperandMode { |
| 1252 | OperandMode_Default, |
| 1253 | OperandMode_NSA, |
| 1254 | }; |
| 1255 | |
| 1256 | using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>; |
| 1257 | |
| 1258 | AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser, |
| 1259 | const MCInstrInfo &MII) |
| 1260 | : MCTargetAsmParser(STI, MII), Parser(_Parser), |
| 1261 | HwMode(STI.getHwMode(type: MCSubtargetInfo::HwMode_RegInfo)), |
| 1262 | Gfx(AMDGPU::parseArchAMDGCN(CPU: STI.getCPU())), |
| 1263 | ISA(AMDGPU::getIsaVersion(GPU: STI.getCPU())) { |
| 1264 | MCAsmParserExtension::Initialize(Parser); |
| 1265 | |
| 1266 | setAvailableFeatures(ComputeAvailableFeatures(FB: getFeatureBits())); |
| 1267 | |
| 1268 | if (ISA.Major >= 6 && isHsaAbi(STI: getSTI())) { |
| 1269 | createConstantSymbol(Id: ".amdgcn.gfx_generation_number" , Val: ISA.Major); |
| 1270 | createConstantSymbol(Id: ".amdgcn.gfx_generation_minor" , Val: ISA.Minor); |
| 1271 | createConstantSymbol(Id: ".amdgcn.gfx_generation_stepping" , Val: ISA.Stepping); |
| 1272 | } else { |
| 1273 | createConstantSymbol(Id: ".option.machine_version_major" , Val: ISA.Major); |
| 1274 | createConstantSymbol(Id: ".option.machine_version_minor" , Val: ISA.Minor); |
| 1275 | createConstantSymbol(Id: ".option.machine_version_stepping" , Val: ISA.Stepping); |
| 1276 | } |
| 1277 | if (ISA.Major >= 6 && isHsaAbi(STI: getSTI())) { |
| 1278 | initializeGprCountSymbol(RegKind: IS_VGPR); |
| 1279 | initializeGprCountSymbol(RegKind: IS_SGPR); |
| 1280 | } else |
| 1281 | KernelScope.initialize(Context&: getContext()); |
| 1282 | |
| 1283 | for (auto [Symbol, Code] : AMDGPU::UCVersion::getGFXVersions()) |
| 1284 | createConstantSymbol(Id: Symbol, Val: Code); |
| 1285 | |
| 1286 | createConstantSymbol(Id: "UC_VERSION_W64_BIT" , Val: 0x2000); |
| 1287 | createConstantSymbol(Id: "UC_VERSION_W32_BIT" , Val: 0x4000); |
| 1288 | createConstantSymbol(Id: "UC_VERSION_MDP_BIT" , Val: 0x8000); |
| 1289 | } |
| 1290 | |
| 1291 | bool hasMIMG_R128() const { return AMDGPU::hasMIMG_R128(STI: getSTI()); } |
| 1292 | |
| 1293 | bool hasPackedD16() const { return AMDGPU::hasPackedD16(STI: getSTI()); } |
| 1294 | |
| 1295 | bool hasA16() const { return AMDGPU::hasA16(STI: getSTI()); } |
| 1296 | |
| 1297 | bool hasG16() const { return AMDGPU::hasG16(STI: getSTI()); } |
| 1298 | |
| 1299 | bool hasGDS() const { return AMDGPU::hasGDS(STI: getSTI()); } |
| 1300 | |
| 1301 | bool isSI() const { return AMDGPU::isSI(STI: getSTI()); } |
| 1302 | |
| 1303 | bool isCI() const { return AMDGPU::isCI(STI: getSTI()); } |
| 1304 | |
| 1305 | bool isVI() const { return AMDGPU::isVI(STI: getSTI()); } |
| 1306 | |
| 1307 | bool isGFX9() const { return AMDGPU::isGFX9(STI: getSTI()); } |
| 1308 | |
| 1309 | // TODO: isGFX90A is also true for GFX940. We need to clean it. |
| 1310 | bool isGFX90A() const { return AMDGPU::isGFX90A(STI: getSTI()); } |
| 1311 | |
| 1312 | bool isGFX940() const { return AMDGPU::isGFX940(STI: getSTI()); } |
| 1313 | |
| 1314 | bool isGFX9Plus() const { return AMDGPU::isGFX9Plus(STI: getSTI()); } |
| 1315 | |
| 1316 | bool isGFX10Plus() const { return AMDGPU::isGFX10Plus(STI: getSTI()); } |
| 1317 | |
| 1318 | bool isGFX11() const { return AMDGPU::isGFX11(STI: getSTI()); } |
| 1319 | |
| 1320 | bool isGFX11Plus() const { return AMDGPU::isGFX11Plus(STI: getSTI()); } |
| 1321 | |
| 1322 | bool isGFX12() const { return AMDGPU::isGFX12(STI: getSTI()); } |
| 1323 | |
| 1324 | bool isGFX12Plus() const { return AMDGPU::isGFX12Plus(STI: getSTI()); } |
| 1325 | |
| 1326 | bool isGFX1250Plus() const { return AMDGPU::isGFX1250Plus(STI: getSTI()); } |
| 1327 | |
| 1328 | bool hasBVHRayTracingInsts() const { |
| 1329 | return getFeatureBits()[AMDGPU::FeatureBVHRayTracingInsts]; |
| 1330 | } |
| 1331 | |
| 1332 | bool isWave32() const { return getAvailableFeatures()[Feature_isWave32Bit]; } |
| 1333 | |
| 1334 | bool isWave64() const { return getAvailableFeatures()[Feature_isWave64Bit]; } |
| 1335 | |
| 1336 | bool hasInv2PiInlineImm() const { |
| 1337 | return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm]; |
| 1338 | } |
| 1339 | |
| 1340 | bool has64BitLiterals() const { |
| 1341 | return getFeatureBits()[AMDGPU::Feature64BitLiterals]; |
| 1342 | } |
| 1343 | |
| 1344 | bool hasFlatOffsets() const { |
| 1345 | return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets]; |
| 1346 | } |
| 1347 | |
| 1348 | bool hasTrue16Insts() const { |
| 1349 | return getFeatureBits()[AMDGPU::FeatureTrue16BitInsts]; |
| 1350 | } |
| 1351 | |
| 1352 | bool hasArchitectedFlatScratch() const { |
| 1353 | return getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch]; |
| 1354 | } |
| 1355 | |
| 1356 | bool hasSGPR102_SGPR103() const { return !isVI() && !isGFX9(); } |
| 1357 | |
| 1358 | bool hasSGPR104_SGPR105() const { return isGFX10Plus(); } |
| 1359 | |
| 1360 | bool hasIntClamp() const { return getFeatureBits()[AMDGPU::FeatureIntClamp]; } |
| 1361 | |
| 1362 | bool hasPartialNSAEncoding() const { |
| 1363 | return getFeatureBits()[AMDGPU::FeaturePartialNSAEncoding]; |
| 1364 | } |
| 1365 | |
| 1366 | bool hasGloballyAddressableScratch() const { |
| 1367 | return getFeatureBits()[AMDGPU::FeatureGloballyAddressableScratch]; |
| 1368 | } |
| 1369 | |
| 1370 | bool hasScaleOffset() const { |
| 1371 | return getFeatureBits()[AMDGPU::FeatureScaleOffset]; |
| 1372 | } |
| 1373 | |
| 1374 | bool hasCacheFillSize() const { |
| 1375 | return getFeatureBits()[AMDGPU::FeatureCacheFillSize]; |
| 1376 | } |
| 1377 | |
| 1378 | unsigned getNSAMaxSize(bool HasSampler = false) const { |
| 1379 | return AMDGPU::getNSAMaxSize(STI: getSTI(), HasSampler); |
| 1380 | } |
| 1381 | |
| 1382 | unsigned getMaxNumUserSGPRs() const { |
| 1383 | return AMDGPU::getMaxNumUserSGPRs(STI: getSTI()); |
| 1384 | } |
| 1385 | |
| 1386 | bool hasKernargPreload() const { return AMDGPU::hasKernargPreload(STI: getSTI()); } |
| 1387 | |
| 1388 | AMDGPUTargetStreamer &getTargetStreamer() { |
| 1389 | MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer(); |
| 1390 | return static_cast<AMDGPUTargetStreamer &>(TS); |
| 1391 | } |
| 1392 | |
| 1393 | MCContext &getContext() const { |
| 1394 | // We need this const_cast because for some reason getContext() is not const |
| 1395 | // in MCAsmParser. |
| 1396 | return const_cast<AMDGPUAsmParser *>(this)->MCTargetAsmParser::getContext(); |
| 1397 | } |
| 1398 | |
| 1399 | const MCRegisterInfo *getMRI() const { |
| 1400 | return getContext().getRegisterInfo(); |
| 1401 | } |
| 1402 | |
| 1403 | const MCInstrInfo *getMII() const { return &MII; } |
| 1404 | |
| 1405 | // Resolve a RegClassByHwModeUses index to a register class id for the active |
| 1406 | // HwMode; -1 if the mode has no entry. |
| 1407 | int16_t getTargetRegClass(unsigned TargetRCIdx) const { |
| 1408 | return MII.getRegClassByHwModeTable(ModeId: HwMode)[TargetRCIdx]; |
| 1409 | } |
| 1410 | |
| 1411 | // FIXME: This should not be used. Instead, should use queries derived from |
| 1412 | // getAvailableFeatures(). |
| 1413 | const FeatureBitset &getFeatureBits() const { |
| 1414 | return getSTI().getFeatureBits(); |
| 1415 | } |
| 1416 | |
| 1417 | void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; } |
| 1418 | void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; } |
| 1419 | void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; } |
| 1420 | |
| 1421 | unsigned getForcedEncodingSize() const { return ForcedEncodingSize; } |
| 1422 | bool isForcedVOP3() const { return ForcedEncodingSize == 64; } |
| 1423 | bool isForcedDPP() const { return ForcedDPP; } |
| 1424 | bool isForcedSDWA() const { return ForcedSDWA; } |
| 1425 | ArrayRef<unsigned> getMatchedVariants() const; |
| 1426 | StringRef getMatchedVariantName() const; |
| 1427 | |
| 1428 | std::unique_ptr<AMDGPUOperand> parseRegister(bool RestoreOnFailure = false); |
| 1429 | bool ParseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc, |
| 1430 | bool RestoreOnFailure); |
| 1431 | bool parseRegister(MCRegister &Reg, SMLoc &StartLoc, SMLoc &EndLoc) override; |
| 1432 | ParseStatus tryParseRegister(MCRegister &Reg, SMLoc &StartLoc, |
| 1433 | SMLoc &EndLoc) override; |
| 1434 | unsigned checkTargetMatchPredicate(MCInst &Inst) override; |
| 1435 | unsigned validateTargetOperandClass(MCParsedAsmOperand &Op, |
| 1436 | unsigned Kind) override; |
| 1437 | bool matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode, |
| 1438 | OperandVector &Operands, MCStreamer &Out, |
| 1439 | uint64_t &ErrorInfo, |
| 1440 | bool MatchingInlineAsm) override; |
| 1441 | bool ParseDirective(AsmToken DirectiveID) override; |
| 1442 | void doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) override; |
| 1443 | void onEndOfFile() override; |
| 1444 | ParseStatus parseOperand(OperandVector &Operands, StringRef Mnemonic, |
| 1445 | OperandMode Mode = OperandMode_Default); |
| 1446 | StringRef parseMnemonicSuffix(StringRef Name); |
| 1447 | bool parseInstruction(ParseInstructionInfo &Info, StringRef Name, |
| 1448 | SMLoc NameLoc, OperandVector &Operands) override; |
| 1449 | // bool ProcessInstruction(MCInst &Inst); |
| 1450 | |
| 1451 | ParseStatus parseTokenOp(StringRef Name, OperandVector &Operands); |
| 1452 | |
| 1453 | ParseStatus parseIntWithPrefix(const char *Prefix, int64_t &Int); |
| 1454 | |
| 1455 | ParseStatus |
| 1456 | parseIntWithPrefix(const char *Prefix, OperandVector &Operands, |
| 1457 | AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone, |
| 1458 | std::function<bool(int64_t &)> ConvertResult = nullptr); |
| 1459 | |
| 1460 | ParseStatus parseOperandArrayWithPrefix( |
| 1461 | const char *Prefix, OperandVector &Operands, |
| 1462 | AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone, |
| 1463 | bool (*ConvertResult)(int64_t &) = nullptr); |
| 1464 | |
| 1465 | ParseStatus |
| 1466 | parseNamedBit(StringRef Name, OperandVector &Operands, |
| 1467 | AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone, |
| 1468 | bool IgnoreNegative = false); |
| 1469 | unsigned getCPolKind(StringRef Id, StringRef Mnemo, bool &Disabling) const; |
| 1470 | ParseStatus parseCPol(OperandVector &Operands); |
| 1471 | ParseStatus parseScope(OperandVector &Operands, int64_t &Scope); |
| 1472 | ParseStatus parseTH(OperandVector &Operands, int64_t &TH); |
| 1473 | ParseStatus parseCFS(OperandVector &Operands, int64_t &CFS); |
| 1474 | ParseStatus parseStringWithPrefix(StringRef Prefix, StringRef &Value, |
| 1475 | SMLoc &StringLoc); |
| 1476 | ParseStatus parseStringOrIntWithPrefix(OperandVector &Operands, |
| 1477 | StringRef Name, |
| 1478 | ArrayRef<const char *> Ids, |
| 1479 | int64_t &IntVal); |
| 1480 | ParseStatus parseStringOrIntWithPrefix(OperandVector &Operands, |
| 1481 | StringRef Name, |
| 1482 | ArrayRef<const char *> Ids, |
| 1483 | AMDGPUOperand::ImmTy Type); |
| 1484 | |
| 1485 | bool isModifier(); |
| 1486 | bool isOperandModifier(const AsmToken &Token, |
| 1487 | const AsmToken &NextToken) const; |
| 1488 | bool isRegOrOperandModifier(const AsmToken &Token, |
| 1489 | const AsmToken &NextToken) const; |
| 1490 | bool isNamedOperandModifier(const AsmToken &Token, |
| 1491 | const AsmToken &NextToken) const; |
| 1492 | bool isOpcodeModifierWithVal(const AsmToken &Token, |
| 1493 | const AsmToken &NextToken) const; |
| 1494 | bool parseSP3NegModifier(); |
| 1495 | ParseStatus parseImm(OperandVector &Operands, bool HasSP3AbsModifier = false, |
| 1496 | LitModifier Lit = LitModifier::None); |
| 1497 | ParseStatus parseReg(OperandVector &Operands); |
| 1498 | ParseStatus parseRegOrImm(OperandVector &Operands, bool HasSP3AbsMod = false, |
| 1499 | LitModifier Lit = LitModifier::None); |
| 1500 | ParseStatus parseRegOrImmWithFPInputMods(OperandVector &Operands, |
| 1501 | bool AllowImm = true); |
| 1502 | ParseStatus parseRegOrImmWithIntInputMods(OperandVector &Operands, |
| 1503 | bool AllowImm = true); |
| 1504 | ParseStatus parseRegWithFPInputMods(OperandVector &Operands); |
| 1505 | ParseStatus parseRegWithIntInputMods(OperandVector &Operands); |
| 1506 | ParseStatus parseRsrcReg(OperandVector &Operands); |
| 1507 | ParseStatus parseVReg32OrOff(OperandVector &Operands); |
| 1508 | ParseStatus tryParseIndexKey(OperandVector &Operands, |
| 1509 | AMDGPUOperand::ImmTy ImmTy); |
| 1510 | ParseStatus parseIndexKey8bit(OperandVector &Operands); |
| 1511 | ParseStatus parseIndexKey16bit(OperandVector &Operands); |
| 1512 | ParseStatus parseIndexKey32bit(OperandVector &Operands); |
| 1513 | ParseStatus tryParseMatrixFMT(OperandVector &Operands, StringRef Name, |
| 1514 | AMDGPUOperand::ImmTy Type); |
| 1515 | ParseStatus parseMatrixAFMT(OperandVector &Operands); |
| 1516 | ParseStatus parseMatrixBFMT(OperandVector &Operands); |
| 1517 | ParseStatus tryParseMatrixScale(OperandVector &Operands, StringRef Name, |
| 1518 | AMDGPUOperand::ImmTy Type); |
| 1519 | ParseStatus parseMatrixAScale(OperandVector &Operands); |
| 1520 | ParseStatus parseMatrixBScale(OperandVector &Operands); |
| 1521 | ParseStatus tryParseMatrixScaleFmt(OperandVector &Operands, StringRef Name, |
| 1522 | AMDGPUOperand::ImmTy Type); |
| 1523 | ParseStatus parseMatrixAScaleFmt(OperandVector &Operands); |
| 1524 | ParseStatus parseMatrixBScaleFmt(OperandVector &Operands); |
| 1525 | |
| 1526 | ParseStatus parseDfmtNfmt(int64_t &Format); |
| 1527 | ParseStatus parseUfmt(int64_t &Format); |
| 1528 | ParseStatus parseSymbolicSplitFormat(StringRef FormatStr, SMLoc Loc, |
| 1529 | int64_t &Format); |
| 1530 | ParseStatus parseSymbolicUnifiedFormat(StringRef FormatStr, SMLoc Loc, |
| 1531 | int64_t &Format); |
| 1532 | ParseStatus parseFormat(OperandVector &Operands); |
| 1533 | ParseStatus parseFormatOp(OperandVector &Operands); |
| 1534 | ParseStatus parseSymbolicOrNumericFormat(int64_t &Format); |
| 1535 | ParseStatus parseNumericFormat(int64_t &Format); |
| 1536 | ParseStatus parseFlatOffset(OperandVector &Operands); |
| 1537 | ParseStatus parseR128A16(OperandVector &Operands); |
| 1538 | ParseStatus parseBLGP(OperandVector &Operands); |
| 1539 | ParseStatus tryParseFmt(const char *Pref, int64_t MaxVal, int64_t &Fmt); |
| 1540 | bool matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt, StringRef FormatStr, |
| 1541 | SMLoc Loc); |
| 1542 | |
| 1543 | void cvtExp(MCInst &Inst, const OperandVector &Operands); |
| 1544 | |
| 1545 | bool parseCnt(int64_t &IntVal); |
| 1546 | ParseStatus parseSWaitCnt(OperandVector &Operands); |
| 1547 | |
| 1548 | bool parseDepCtr(int64_t &IntVal, unsigned &Mask); |
| 1549 | void depCtrError(SMLoc Loc, int ErrorId, StringRef DepCtrName); |
| 1550 | ParseStatus parseDepCtr(OperandVector &Operands); |
| 1551 | |
| 1552 | bool parseDelay(int64_t &Delay); |
| 1553 | ParseStatus parseSDelayALU(OperandVector &Operands); |
| 1554 | |
| 1555 | ParseStatus parseHwreg(OperandVector &Operands); |
| 1556 | |
| 1557 | private: |
| 1558 | struct OperandInfoTy { |
| 1559 | SMLoc Loc; |
| 1560 | int64_t Val; |
| 1561 | bool IsSymbolic = false; |
| 1562 | bool IsDefined = false; |
| 1563 | |
| 1564 | constexpr OperandInfoTy(int64_t Val) : Val(Val) {} |
| 1565 | }; |
| 1566 | |
| 1567 | struct StructuredOpField : OperandInfoTy { |
| 1568 | StringLiteral Id; |
| 1569 | StringLiteral Desc; |
| 1570 | unsigned Width; |
| 1571 | bool IsDefined = false; |
| 1572 | |
| 1573 | constexpr StructuredOpField(StringLiteral Id, StringLiteral Desc, |
| 1574 | unsigned Width, int64_t Default) |
| 1575 | : OperandInfoTy(Default), Id(Id), Desc(Desc), Width(Width) {} |
| 1576 | virtual ~StructuredOpField() = default; |
| 1577 | |
| 1578 | bool Error(AMDGPUAsmParser &Parser, const Twine &Err) const { |
| 1579 | Parser.Error(L: Loc, Msg: "invalid " + Desc + ": " + Err); |
| 1580 | return false; |
| 1581 | } |
| 1582 | |
| 1583 | virtual bool validate(AMDGPUAsmParser &Parser) const { |
| 1584 | if (IsSymbolic && Val == OPR_ID_UNSUPPORTED) |
| 1585 | return Error(Parser, Err: "not supported on this GPU" ); |
| 1586 | if (!isUIntN(N: Width, x: Val)) |
| 1587 | return Error(Parser, Err: "only " + Twine(Width) + "-bit values are legal" ); |
| 1588 | return true; |
| 1589 | } |
| 1590 | }; |
| 1591 | |
| 1592 | ParseStatus parseStructuredOpFields(ArrayRef<StructuredOpField *> Fields); |
| 1593 | bool validateStructuredOpFields(ArrayRef<const StructuredOpField *> Fields); |
| 1594 | |
| 1595 | bool parseSendMsgBody(OperandInfoTy &Msg, OperandInfoTy &Op, |
| 1596 | OperandInfoTy &Stream); |
| 1597 | bool validateSendMsg(const OperandInfoTy &Msg, const OperandInfoTy &Op, |
| 1598 | const OperandInfoTy &Stream); |
| 1599 | |
| 1600 | ParseStatus parseHwregFunc(OperandInfoTy &HwReg, OperandInfoTy &Offset, |
| 1601 | OperandInfoTy &Width); |
| 1602 | |
| 1603 | const AMDGPUOperand &findMCOperand(const OperandVector &Operands, |
| 1604 | int MCOpIdx) const; |
| 1605 | |
| 1606 | static SMLoc getLaterLoc(SMLoc a, SMLoc b); |
| 1607 | |
| 1608 | SMLoc getFlatOffsetLoc(const OperandVector &Operands) const; |
| 1609 | SMLoc getSMEMOffsetLoc(const OperandVector &Operands) const; |
| 1610 | SMLoc getBLGPLoc(const OperandVector &Operands) const; |
| 1611 | |
| 1612 | SMLoc getOperandLoc(const OperandVector &Operands, int MCOpIdx) const; |
| 1613 | SMLoc getOperandLoc(std::function<bool(const AMDGPUOperand &)> Test, |
| 1614 | const OperandVector &Operands) const; |
| 1615 | SMLoc getImmLoc(AMDGPUOperand::ImmTy Type, |
| 1616 | const OperandVector &Operands) const; |
| 1617 | SMLoc getInstLoc(const OperandVector &Operands) const; |
| 1618 | |
| 1619 | bool validateInstruction(const MCInst &Inst, SMLoc IDLoc, |
| 1620 | const OperandVector &Operands); |
| 1621 | bool validateOffset(const MCInst &Inst, const OperandVector &Operands); |
| 1622 | bool validateFlatOffset(const MCInst &Inst, const OperandVector &Operands); |
| 1623 | bool validateSMEMOffset(const MCInst &Inst, const OperandVector &Operands); |
| 1624 | bool validateSMEMSData(const MCInst &Inst, const OperandVector &Operands); |
| 1625 | bool validateBF16InlineConst(const MCInst &Inst, |
| 1626 | const OperandVector &Operands); |
| 1627 | bool validateSOPLiteral(const MCInst &Inst, const OperandVector &Operands); |
| 1628 | bool validateConstantBusLimitations(const MCInst &Inst, |
| 1629 | const OperandVector &Operands); |
| 1630 | std::optional<unsigned> checkVOPDRegBankConstraints(const MCInst &Inst, |
| 1631 | bool AsVOPD3); |
| 1632 | bool validateVOPD(const MCInst &Inst, const OperandVector &Operands); |
| 1633 | bool tryVOPD(const MCInst &Inst); |
| 1634 | bool tryVOPD3(const MCInst &Inst); |
| 1635 | bool tryAnotherVOPDEncoding(const MCInst &Inst); |
| 1636 | |
| 1637 | bool validateIntClampSupported(const MCInst &Inst); |
| 1638 | bool validateMIMGAtomicDMask(const MCInst &Inst); |
| 1639 | bool validateMIMGGatherDMask(const MCInst &Inst); |
| 1640 | bool validateMovrels(const MCInst &Inst, const OperandVector &Operands); |
| 1641 | bool validateMIMGDataSize(const MCInst &Inst, SMLoc IDLoc); |
| 1642 | bool validateMIMGAddrSize(const MCInst &Inst, SMLoc IDLoc); |
| 1643 | bool validateMIMGD16(const MCInst &Inst); |
| 1644 | bool validateMIMGDim(const MCInst &Inst, const OperandVector &Operands); |
| 1645 | bool validateTensorR128(const MCInst &Inst); |
| 1646 | bool validateMIMGMSAA(const MCInst &Inst); |
| 1647 | bool validateOpSel(const MCInst &Inst); |
| 1648 | bool validateTrue16OpSel(const MCInst &Inst); |
| 1649 | bool validateNeg(const MCInst &Inst, AMDGPU::OpName OpName); |
| 1650 | bool validateDPP(const MCInst &Inst, const OperandVector &Operands); |
| 1651 | bool validateVccOperand(MCRegister Reg) const; |
| 1652 | bool validateVOPLiteral(const MCInst &Inst, const OperandVector &Operands); |
| 1653 | bool validateMAIAccWrite(const MCInst &Inst, const OperandVector &Operands); |
| 1654 | bool validateMAISrc2(const MCInst &Inst, const OperandVector &Operands); |
| 1655 | bool validateMFMA(const MCInst &Inst, const OperandVector &Operands); |
| 1656 | bool validateAGPRLdSt(const MCInst &Inst) const; |
| 1657 | bool validateVGPRAlign(const MCInst &Inst) const; |
| 1658 | bool validateBLGP(const MCInst &Inst, const OperandVector &Operands); |
| 1659 | bool validateDS(const MCInst &Inst, const OperandVector &Operands); |
| 1660 | bool validateGWS(const MCInst &Inst, const OperandVector &Operands); |
| 1661 | bool validateDivScale(const MCInst &Inst); |
| 1662 | bool validateWaitCnt(const MCInst &Inst, const OperandVector &Operands); |
| 1663 | bool validateCoherencyBits(const MCInst &Inst, const OperandVector &Operands, |
| 1664 | SMLoc IDLoc); |
| 1665 | bool validateTHAndScopeBits(const MCInst &Inst, const OperandVector &Operands, |
| 1666 | const unsigned CPol); |
| 1667 | bool validateCFSBits(const MCInst &Inst, const OperandVector &Operands); |
| 1668 | bool validateTFE(const MCInst &Inst, const OperandVector &Operands); |
| 1669 | bool validateLdsDirect(const MCInst &Inst, const OperandVector &Operands); |
| 1670 | bool validateWMMA(const MCInst &Inst, const OperandVector &Operands); |
| 1671 | bool validateMonitorSleep(const MCInst &Inst, const OperandVector &Operands); |
| 1672 | bool validateClusterBarrierIsFirst(const MCInst &Inst, |
| 1673 | const OperandVector &Operands); |
| 1674 | bool validateScaleSel(const MCInst &Inst, const OperandVector &Operands); |
| 1675 | unsigned getConstantBusLimit(unsigned Opcode) const; |
| 1676 | bool usesConstantBus(const MCInst &Inst, unsigned OpIdx); |
| 1677 | bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const; |
| 1678 | MCRegister findImplicitSGPRReadInVOP(const MCInst &Inst) const; |
| 1679 | |
| 1680 | bool isSupportedMnemo(StringRef Mnemo, const FeatureBitset &FBS); |
| 1681 | bool isSupportedMnemo(StringRef Mnemo, const FeatureBitset &FBS, |
| 1682 | ArrayRef<unsigned> Variants); |
| 1683 | bool checkUnsupportedInstruction(StringRef Name, SMLoc IDLoc); |
| 1684 | |
| 1685 | bool isId(const StringRef Id) const; |
| 1686 | bool isId(const AsmToken &Token, const StringRef Id) const; |
| 1687 | bool isToken(const AsmToken::TokenKind Kind) const; |
| 1688 | StringRef getId() const; |
| 1689 | bool trySkipId(const StringRef Id); |
| 1690 | bool trySkipId(const StringRef Pref, const StringRef Id); |
| 1691 | bool trySkipId(const StringRef Id, const AsmToken::TokenKind Kind); |
| 1692 | bool trySkipToken(const AsmToken::TokenKind Kind); |
| 1693 | bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg); |
| 1694 | bool parseString(StringRef &Val, |
| 1695 | const StringRef ErrMsg = "expected a string" ); |
| 1696 | bool parseId(StringRef &Val, const StringRef ErrMsg = "" ); |
| 1697 | |
| 1698 | void peekTokens(MutableArrayRef<AsmToken> Tokens); |
| 1699 | AsmToken::TokenKind getTokenKind() const; |
| 1700 | bool parseExpr(int64_t &Imm, StringRef Expected = "" ); |
| 1701 | bool parseExpr(OperandVector &Operands); |
| 1702 | StringRef getTokenStr() const; |
| 1703 | AsmToken peekToken(bool ShouldSkipSpace = true); |
| 1704 | AsmToken getToken() const; |
| 1705 | SMLoc getLoc() const; |
| 1706 | void lex(); |
| 1707 | |
| 1708 | public: |
| 1709 | void onBeginOfFile() override; |
| 1710 | /// Emit the deferred leading .amdgcn_target directive if it has not been |
| 1711 | /// emitted yet. Called before emitting the first instruction or kernel |
| 1712 | /// descriptor, so that any leading .amdgcn_target/.amd_amdgpu_isa directive |
| 1713 | /// in the source has had a chance to update the target ID first. |
| 1714 | void emitTargetDirective(); |
| 1715 | bool parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) override; |
| 1716 | |
| 1717 | ParseStatus parseCustomOperand(OperandVector &Operands, unsigned MCK); |
| 1718 | |
| 1719 | ParseStatus parseExpTgt(OperandVector &Operands); |
| 1720 | ParseStatus parseSendMsg(OperandVector &Operands); |
| 1721 | ParseStatus parseWaitEvent(OperandVector &Operands); |
| 1722 | ParseStatus parseInterpSlot(OperandVector &Operands); |
| 1723 | ParseStatus parseInterpAttr(OperandVector &Operands); |
| 1724 | ParseStatus parseSOPPBrTarget(OperandVector &Operands); |
| 1725 | ParseStatus parseBoolReg(OperandVector &Operands); |
| 1726 | |
| 1727 | bool parseSwizzleOperand(int64_t &Op, const unsigned MinVal, |
| 1728 | const unsigned MaxVal, const Twine &ErrMsg, |
| 1729 | SMLoc &Loc); |
| 1730 | bool parseSwizzleOperands(const unsigned OpNum, int64_t *Op, |
| 1731 | const unsigned MinVal, const unsigned MaxVal, |
| 1732 | const StringRef ErrMsg); |
| 1733 | ParseStatus parseSwizzle(OperandVector &Operands); |
| 1734 | bool parseSwizzleOffset(int64_t &Imm); |
| 1735 | bool parseSwizzleMacro(int64_t &Imm); |
| 1736 | bool parseSwizzleQuadPerm(int64_t &Imm); |
| 1737 | bool parseSwizzleBitmaskPerm(int64_t &Imm); |
| 1738 | bool parseSwizzleBroadcast(int64_t &Imm); |
| 1739 | bool parseSwizzleSwap(int64_t &Imm); |
| 1740 | bool parseSwizzleReverse(int64_t &Imm); |
| 1741 | bool parseSwizzleFFT(int64_t &Imm); |
| 1742 | bool parseSwizzleRotate(int64_t &Imm); |
| 1743 | |
| 1744 | ParseStatus parseGPRIdxMode(OperandVector &Operands); |
| 1745 | int64_t parseGPRIdxMacro(); |
| 1746 | |
| 1747 | void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { |
| 1748 | cvtMubufImpl(Inst, Operands, IsAtomic: false); |
| 1749 | } |
| 1750 | void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { |
| 1751 | cvtMubufImpl(Inst, Operands, IsAtomic: true); |
| 1752 | } |
| 1753 | |
| 1754 | ParseStatus parseOModSI(OperandVector &Operands); |
| 1755 | |
| 1756 | void cvtVOP3(MCInst &Inst, const OperandVector &Operands, |
| 1757 | OptionalImmIndexMap &OptionalIdx); |
| 1758 | void cvtScaledMFMA(MCInst &Inst, const OperandVector &Operands); |
| 1759 | void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands); |
| 1760 | void cvtVOP3(MCInst &Inst, const OperandVector &Operands); |
| 1761 | void cvtVOP3P(MCInst &Inst, const OperandVector &Operands); |
| 1762 | void cvtSWMMAC(MCInst &Inst, const OperandVector &Operands); |
| 1763 | |
| 1764 | void cvtVOPD(MCInst &Inst, const OperandVector &Operands); |
| 1765 | void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands, |
| 1766 | OptionalImmIndexMap &OptionalIdx); |
| 1767 | void cvtVOP3P(MCInst &Inst, const OperandVector &Operands, |
| 1768 | OptionalImmIndexMap &OptionalIdx); |
| 1769 | |
| 1770 | void cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands); |
| 1771 | void cvtVINTERP(MCInst &Inst, const OperandVector &Operands); |
| 1772 | void cvtOpSelHelper(MCInst &Inst, unsigned OpSel); |
| 1773 | |
| 1774 | bool parseDimId(unsigned &Encoding); |
| 1775 | ParseStatus parseDim(OperandVector &Operands); |
| 1776 | bool convertDppBoundCtrl(int64_t &BoundCtrl); |
| 1777 | ParseStatus parseDPP8(OperandVector &Operands); |
| 1778 | ParseStatus parseDPPCtrl(OperandVector &Operands); |
| 1779 | bool isSupportedDPPCtrl(StringRef Ctrl, const OperandVector &Operands); |
| 1780 | int64_t parseDPPCtrlSel(StringRef Ctrl); |
| 1781 | int64_t parseDPPCtrlPerm(); |
| 1782 | void cvtDPP(MCInst &Inst, const OperandVector &Operands, bool IsDPP8 = false); |
| 1783 | void cvtDPP8(MCInst &Inst, const OperandVector &Operands) { |
| 1784 | cvtDPP(Inst, Operands, IsDPP8: true); |
| 1785 | } |
| 1786 | void cvtVOP3DPP(MCInst &Inst, const OperandVector &Operands, |
| 1787 | bool IsDPP8 = false); |
| 1788 | void cvtVOP3DPP8(MCInst &Inst, const OperandVector &Operands) { |
| 1789 | cvtVOP3DPP(Inst, Operands, IsDPP8: true); |
| 1790 | } |
| 1791 | |
| 1792 | ParseStatus parseSDWASel(OperandVector &Operands, StringRef Prefix, |
| 1793 | AMDGPUOperand::ImmTy Type); |
| 1794 | ParseStatus parseSDWADstUnused(OperandVector &Operands); |
| 1795 | void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands); |
| 1796 | void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands); |
| 1797 | void cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands); |
| 1798 | void cvtSdwaVOP2e(MCInst &Inst, const OperandVector &Operands); |
| 1799 | void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands); |
| 1800 | |
| 1801 | enum class SDWAInstType : unsigned { VOP1 = 0, VOP2 = 1, VOPC = 2 }; |
| 1802 | |
| 1803 | void cvtSDWA(MCInst &Inst, const OperandVector &Operands, |
| 1804 | SDWAInstType BasicInstType, bool SkipDstVcc = false, |
| 1805 | bool SkipSrcVcc = false); |
| 1806 | |
| 1807 | ParseStatus parseEndpgm(OperandVector &Operands); |
| 1808 | |
| 1809 | ParseStatus parseVOPD(OperandVector &Operands); |
| 1810 | }; |
| 1811 | |
| 1812 | } // end anonymous namespace |
| 1813 | |
| 1814 | // May be called with integer type with equivalent bitwidth. |
| 1815 | static const fltSemantics *getFltSemantics(unsigned Size) { |
| 1816 | switch (Size) { |
| 1817 | case 4: |
| 1818 | return &APFloat::IEEEsingle(); |
| 1819 | case 8: |
| 1820 | return &APFloat::IEEEdouble(); |
| 1821 | case 2: |
| 1822 | return &APFloat::IEEEhalf(); |
| 1823 | default: |
| 1824 | llvm_unreachable("unsupported fp type" ); |
| 1825 | } |
| 1826 | } |
| 1827 | |
| 1828 | static const fltSemantics *getFltSemantics(MVT VT) { |
| 1829 | return getFltSemantics(Size: VT.getScalarSizeInBits() / 8); |
| 1830 | } |
| 1831 | |
| 1832 | static const fltSemantics *getOpFltSemantics(uint8_t OperandType) { |
| 1833 | switch (OperandType) { |
| 1834 | // When floating-point immediate is used as operand of type i16, the 32-bit |
| 1835 | // representation of the constant truncated to the 16 LSBs should be used. |
| 1836 | case AMDGPU::OPERAND_REG_IMM_INT16: |
| 1837 | case AMDGPU::OPERAND_REG_INLINE_C_INT16: |
| 1838 | case AMDGPU::OPERAND_REG_IMM_INT32: |
| 1839 | case AMDGPU::OPERAND_REG_IMM_FP32: |
| 1840 | case AMDGPU::OPERAND_REG_INLINE_C_INT32: |
| 1841 | case AMDGPU::OPERAND_REG_INLINE_C_FP32: |
| 1842 | case AMDGPU::OPERAND_REG_INLINE_AC_INT32: |
| 1843 | case AMDGPU::OPERAND_REG_INLINE_AC_FP32: |
| 1844 | case AMDGPU::OPERAND_REG_IMM_V2FP32: |
| 1845 | case AMDGPU::OPERAND_REG_IMM_V2INT32: |
| 1846 | case AMDGPU::OPERAND_REG_IMM_V2INT16: |
| 1847 | case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: |
| 1848 | case AMDGPU::OPERAND_KIMM32: |
| 1849 | case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: |
| 1850 | return &APFloat::IEEEsingle(); |
| 1851 | case AMDGPU::OPERAND_REG_IMM_INT64: |
| 1852 | case AMDGPU::OPERAND_REG_IMM_FP64: |
| 1853 | case AMDGPU::OPERAND_REG_INLINE_C_INT64: |
| 1854 | case AMDGPU::OPERAND_REG_INLINE_C_FP64: |
| 1855 | case AMDGPU::OPERAND_REG_INLINE_AC_FP64: |
| 1856 | case AMDGPU::OPERAND_REG_IMM_V2FP64: |
| 1857 | case AMDGPU::OPERAND_REG_IMM_V2INT64: |
| 1858 | case AMDGPU::OPERAND_KIMM64: |
| 1859 | return &APFloat::IEEEdouble(); |
| 1860 | case AMDGPU::OPERAND_REG_IMM_FP16: |
| 1861 | case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16: |
| 1862 | case AMDGPU::OPERAND_REG_INLINE_C_FP16: |
| 1863 | case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: |
| 1864 | case AMDGPU::OPERAND_REG_IMM_V2FP16: |
| 1865 | case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT: |
| 1866 | case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16: |
| 1867 | case AMDGPU::OPERAND_KIMM16: |
| 1868 | return &APFloat::IEEEhalf(); |
| 1869 | case AMDGPU::OPERAND_REG_IMM_BF16: |
| 1870 | case AMDGPU::OPERAND_REG_INLINE_C_BF16: |
| 1871 | case AMDGPU::OPERAND_REG_INLINE_C_V2BF16: |
| 1872 | case AMDGPU::OPERAND_REG_IMM_V2BF16: |
| 1873 | return &APFloat::BFloat(); |
| 1874 | default: |
| 1875 | llvm_unreachable("unsupported fp type" ); |
| 1876 | } |
| 1877 | } |
| 1878 | |
| 1879 | //===----------------------------------------------------------------------===// |
| 1880 | // Operand |
| 1881 | //===----------------------------------------------------------------------===// |
| 1882 | |
| 1883 | static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) { |
| 1884 | bool Lost; |
| 1885 | |
| 1886 | // Convert literal to single precision |
| 1887 | APFloat::opStatus Status = FPLiteral.convert( |
| 1888 | ToSemantics: *getFltSemantics(VT), RM: APFloat::rmNearestTiesToEven, losesInfo: &Lost); |
| 1889 | // We allow precision lost but not overflow or underflow |
| 1890 | if (Status != APFloat::opOK && Lost && |
| 1891 | ((Status & APFloat::opOverflow) != 0 || |
| 1892 | (Status & APFloat::opUnderflow) != 0)) { |
| 1893 | return false; |
| 1894 | } |
| 1895 | |
| 1896 | return true; |
| 1897 | } |
| 1898 | |
| 1899 | static bool isSafeTruncation(int64_t Val, unsigned Size) { |
| 1900 | return isUIntN(N: Size, x: Val) || isIntN(N: Size, x: Val); |
| 1901 | } |
| 1902 | |
| 1903 | static bool isInlineableLiteralOp16(int64_t Val, MVT VT, bool HasInv2Pi) { |
| 1904 | if (VT.getScalarType() == MVT::i16) |
| 1905 | return isInlinableLiteral32(Literal: Val, HasInv2Pi); |
| 1906 | |
| 1907 | if (VT.getScalarType() == MVT::f16) |
| 1908 | return AMDGPU::isInlinableLiteralFP16(Literal: Val, HasInv2Pi); |
| 1909 | |
| 1910 | assert(VT.getScalarType() == MVT::bf16); |
| 1911 | |
| 1912 | return AMDGPU::isInlinableLiteralBF16(Literal: Val, HasInv2Pi); |
| 1913 | } |
| 1914 | |
| 1915 | bool AMDGPUOperand::isInlinableImm(MVT type) const { |
| 1916 | |
| 1917 | // This is a hack to enable named inline values like |
| 1918 | // shared_base with both 32-bit and 64-bit operands. |
| 1919 | // Note that these values are defined as |
| 1920 | // 32-bit operands only. |
| 1921 | if (isInlineValue()) { |
| 1922 | return true; |
| 1923 | } |
| 1924 | |
| 1925 | if (!isImmTy(ImmT: ImmTyNone)) { |
| 1926 | // Only plain immediates are inlinable (e.g. "clamp" attribute is not) |
| 1927 | return false; |
| 1928 | } |
| 1929 | |
| 1930 | if (getModifiers().Lit != LitModifier::None) |
| 1931 | return false; |
| 1932 | |
| 1933 | // TODO: We should avoid using host float here. It would be better to |
| 1934 | // check the float bit values which is what a few other places do. |
| 1935 | // We've had bot failures before due to weird NaN support on mips hosts. |
| 1936 | |
| 1937 | APInt Literal(64, Imm.Val); |
| 1938 | |
| 1939 | if (Imm.IsFPImm) { // We got fp literal token |
| 1940 | if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand |
| 1941 | return AMDGPU::isInlinableLiteral64(Literal: Imm.Val, |
| 1942 | HasInv2Pi: AsmParser->hasInv2PiInlineImm()); |
| 1943 | } |
| 1944 | |
| 1945 | APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val)); |
| 1946 | if (!canLosslesslyConvertToFPType(FPLiteral, VT: type)) |
| 1947 | return false; |
| 1948 | |
| 1949 | if (type.getScalarSizeInBits() == 16) { |
| 1950 | bool Lost = false; |
| 1951 | switch (type.getScalarType().SimpleTy) { |
| 1952 | default: |
| 1953 | llvm_unreachable("unknown 16-bit type" ); |
| 1954 | case MVT::bf16: |
| 1955 | FPLiteral.convert(ToSemantics: APFloatBase::BFloat(), RM: APFloat::rmNearestTiesToEven, |
| 1956 | losesInfo: &Lost); |
| 1957 | break; |
| 1958 | case MVT::f16: |
| 1959 | FPLiteral.convert(ToSemantics: APFloatBase::IEEEhalf(), RM: APFloat::rmNearestTiesToEven, |
| 1960 | losesInfo: &Lost); |
| 1961 | break; |
| 1962 | case MVT::i16: |
| 1963 | FPLiteral.convert(ToSemantics: APFloatBase::IEEEsingle(), |
| 1964 | RM: APFloat::rmNearestTiesToEven, losesInfo: &Lost); |
| 1965 | break; |
| 1966 | } |
| 1967 | // We need to use 32-bit representation here because when a floating-point |
| 1968 | // inline constant is used as an i16 operand, its 32-bit representation |
| 1969 | // representation will be used. We will need the 32-bit value to check if |
| 1970 | // it is FP inline constant. |
| 1971 | uint32_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue(); |
| 1972 | return isInlineableLiteralOp16(Val: ImmVal, VT: type, |
| 1973 | HasInv2Pi: AsmParser->hasInv2PiInlineImm()); |
| 1974 | } |
| 1975 | |
| 1976 | // Check if single precision literal is inlinable |
| 1977 | return AMDGPU::isInlinableLiteral32( |
| 1978 | Literal: static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()), |
| 1979 | HasInv2Pi: AsmParser->hasInv2PiInlineImm()); |
| 1980 | } |
| 1981 | |
| 1982 | // We got int literal token. |
| 1983 | if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand |
| 1984 | return AMDGPU::isInlinableLiteral64(Literal: Imm.Val, |
| 1985 | HasInv2Pi: AsmParser->hasInv2PiInlineImm()); |
| 1986 | } |
| 1987 | |
| 1988 | if (!isSafeTruncation(Val: Imm.Val, Size: type.getScalarSizeInBits())) { |
| 1989 | return false; |
| 1990 | } |
| 1991 | |
| 1992 | if (type.getScalarSizeInBits() == 16) { |
| 1993 | return isInlineableLiteralOp16( |
| 1994 | Val: static_cast<int16_t>(Literal.getLoBits(numBits: 16).getSExtValue()), VT: type, |
| 1995 | HasInv2Pi: AsmParser->hasInv2PiInlineImm()); |
| 1996 | } |
| 1997 | |
| 1998 | return AMDGPU::isInlinableLiteral32( |
| 1999 | Literal: static_cast<int32_t>(Literal.getLoBits(numBits: 32).getZExtValue()), |
| 2000 | HasInv2Pi: AsmParser->hasInv2PiInlineImm()); |
| 2001 | } |
| 2002 | |
| 2003 | bool AMDGPUOperand::isLiteralImm(MVT type) const { |
| 2004 | // Check that this immediate can be added as literal |
| 2005 | if (!isImmTy(ImmT: ImmTyNone)) { |
| 2006 | return false; |
| 2007 | } |
| 2008 | |
| 2009 | bool Allow64Bit = |
| 2010 | (type == MVT::i64 || type == MVT::f64) && AsmParser->has64BitLiterals(); |
| 2011 | |
| 2012 | if (!Imm.IsFPImm) { |
| 2013 | // We got int literal token. |
| 2014 | |
| 2015 | if (type == MVT::f64 && hasFPModifiers()) { |
| 2016 | // Cannot apply fp modifiers to int literals preserving the same semantics |
| 2017 | // for VOP1/2/C and VOP3 because of integer truncation. To avoid |
| 2018 | // ambiguity, disable these cases. |
| 2019 | return false; |
| 2020 | } |
| 2021 | |
| 2022 | unsigned Size = type.getSizeInBits(); |
| 2023 | if (Size == 64) { |
| 2024 | if (Allow64Bit && !AMDGPU::isValid32BitLiteral(Val: Imm.Val, IsFP64: false)) |
| 2025 | return true; |
| 2026 | Size = 32; |
| 2027 | } |
| 2028 | |
| 2029 | // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP |
| 2030 | // types. |
| 2031 | return isSafeTruncation(Val: Imm.Val, Size); |
| 2032 | } |
| 2033 | |
| 2034 | // We got fp literal token |
| 2035 | if (type == MVT::f64) { // Expected 64-bit fp operand |
| 2036 | // We would set low 64-bits of literal to zeroes but we accept this literals |
| 2037 | return true; |
| 2038 | } |
| 2039 | |
| 2040 | if (type == MVT::i64) { // Expected 64-bit int operand |
| 2041 | // We don't allow fp literals in 64-bit integer instructions. It is |
| 2042 | // unclear how we should encode them. |
| 2043 | return false; |
| 2044 | } |
| 2045 | |
| 2046 | // We allow fp literals with f16x2 operands assuming that the specified |
| 2047 | // literal goes into the lower half and the upper half is zero. We also |
| 2048 | // require that the literal may be losslessly converted to f16. |
| 2049 | // |
| 2050 | // For i16x2 operands, we assume that the specified literal is encoded as a |
| 2051 | // single-precision float. This is pretty odd, but it matches SP3 and what |
| 2052 | // happens in hardware. |
| 2053 | MVT ExpectedType = (type == MVT::v2f16) ? MVT::f16 |
| 2054 | : (type == MVT::v2i16) ? MVT::f32 |
| 2055 | : (type == MVT::v2f32) ? MVT::f32 |
| 2056 | : type; |
| 2057 | |
| 2058 | APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val)); |
| 2059 | return canLosslesslyConvertToFPType(FPLiteral, VT: ExpectedType); |
| 2060 | } |
| 2061 | |
| 2062 | bool AMDGPUOperand::isRegClassTarget(unsigned TargetRCIdx) const { |
| 2063 | if (!isRegKind()) |
| 2064 | return false; |
| 2065 | int16_t RCID = AsmParser->getTargetRegClass(TargetRCIdx); |
| 2066 | return RCID >= 0 && isRegClass(RCID); |
| 2067 | } |
| 2068 | |
| 2069 | bool AMDGPUOperand::isRegClass(unsigned RCID) const { |
| 2070 | return isRegKind() && |
| 2071 | AsmParser->getMRI()->getRegClass(i: RCID).contains(Reg: getReg()); |
| 2072 | } |
| 2073 | |
| 2074 | bool AMDGPUOperand::isVRegWithInputMods() const { |
| 2075 | return isRegClass(RCID: AMDGPU::VGPR_32RegClassID) || |
| 2076 | // GFX90A allows DPP on 64-bit operands. |
| 2077 | (AsmParser->getFeatureBits()[AMDGPU::FeatureDPALU_DPP] && |
| 2078 | isRegClassTarget(TargetRCIdx: AMDGPU::VReg_64_AlignTarget)); |
| 2079 | } |
| 2080 | |
| 2081 | template <bool IsFake16> |
| 2082 | bool AMDGPUOperand::isT16_Lo128VRegWithInputMods() const { |
| 2083 | return isRegClass(RCID: IsFake16 ? AMDGPU::VGPR_32_Lo128RegClassID |
| 2084 | : AMDGPU::VGPR_16_Lo128RegClassID); |
| 2085 | } |
| 2086 | |
| 2087 | template <bool IsFake16> bool AMDGPUOperand::isT16VRegWithInputMods() const { |
| 2088 | return isRegClass(RCID: IsFake16 ? AMDGPU::VGPR_32RegClassID |
| 2089 | : AMDGPU::VGPR_16RegClassID); |
| 2090 | } |
| 2091 | |
| 2092 | bool AMDGPUOperand::isT16_LO16VRegWithInputMods() const { |
| 2093 | return isRegClass(RCID: AMDGPU::VGPR_16_LO16RegClassID); |
| 2094 | } |
| 2095 | |
| 2096 | bool AMDGPUOperand::isSDWAOperand(MVT type) const { |
| 2097 | if (AsmParser->isVI()) |
| 2098 | return isVReg32(); |
| 2099 | if (AsmParser->isGFX9Plus()) |
| 2100 | return isRegClass(RCID: AMDGPU::VS_32RegClassID) || isInlinableImm(type); |
| 2101 | return false; |
| 2102 | } |
| 2103 | |
| 2104 | bool AMDGPUOperand::isSDWAFP16Operand() const { |
| 2105 | return isSDWAOperand(type: MVT::f16); |
| 2106 | } |
| 2107 | |
| 2108 | bool AMDGPUOperand::isSDWAFP32Operand() const { |
| 2109 | return isSDWAOperand(type: MVT::f32); |
| 2110 | } |
| 2111 | |
| 2112 | bool AMDGPUOperand::isSDWAInt16Operand() const { |
| 2113 | return isSDWAOperand(type: MVT::i16); |
| 2114 | } |
| 2115 | |
| 2116 | bool AMDGPUOperand::isSDWAInt32Operand() const { |
| 2117 | return isSDWAOperand(type: MVT::i32); |
| 2118 | } |
| 2119 | |
| 2120 | bool AMDGPUOperand::isBoolReg() const { |
| 2121 | return isReg() && ((AsmParser->isWave64() && isSCSrc_b64()) || |
| 2122 | (AsmParser->isWave32() && isSCSrc_b32())); |
| 2123 | } |
| 2124 | |
| 2125 | uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val, |
| 2126 | unsigned Size) const { |
| 2127 | assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers()); |
| 2128 | assert(Size == 2 || Size == 4 || Size == 8); |
| 2129 | |
| 2130 | const uint64_t FpSignMask = (1ULL << (Size * 8 - 1)); |
| 2131 | |
| 2132 | if (Imm.Mods.Abs) { |
| 2133 | Val &= ~FpSignMask; |
| 2134 | } |
| 2135 | if (Imm.Mods.Neg) { |
| 2136 | Val ^= FpSignMask; |
| 2137 | } |
| 2138 | |
| 2139 | return Val; |
| 2140 | } |
| 2141 | |
| 2142 | void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, |
| 2143 | bool ApplyModifiers) const { |
| 2144 | MCOpIdx = Inst.getNumOperands(); |
| 2145 | |
| 2146 | if (isExpr()) { |
| 2147 | Inst.addOperand(Op: MCOperand::createExpr(Val: Expr)); |
| 2148 | return; |
| 2149 | } |
| 2150 | |
| 2151 | if (AMDGPU::isSISrcOperand(Desc: AsmParser->getMII()->get(Opcode: Inst.getOpcode()), |
| 2152 | OpNo: Inst.getNumOperands())) { |
| 2153 | addLiteralImmOperand(Inst, Val: Imm.Val, |
| 2154 | ApplyModifiers: ApplyModifiers & isImmTy(ImmT: ImmTyNone) && |
| 2155 | Imm.Mods.hasFPModifiers()); |
| 2156 | } else { |
| 2157 | assert(!isImmTy(ImmTyNone) || !hasModifiers()); |
| 2158 | Inst.addOperand(Op: MCOperand::createImm(Val: Imm.Val)); |
| 2159 | } |
| 2160 | } |
| 2161 | |
| 2162 | void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, |
| 2163 | bool ApplyModifiers) const { |
| 2164 | const auto &InstDesc = AsmParser->getMII()->get(Opcode: Inst.getOpcode()); |
| 2165 | auto OpNum = Inst.getNumOperands(); |
| 2166 | // Check that this operand accepts literals |
| 2167 | assert(AMDGPU::isSISrcOperand(InstDesc, OpNum)); |
| 2168 | |
| 2169 | if (ApplyModifiers) { |
| 2170 | assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum)); |
| 2171 | const unsigned Size = |
| 2172 | Imm.IsFPImm ? sizeof(double) : getOperandSize(Desc: InstDesc, OpNo: OpNum); |
| 2173 | Val = applyInputFPModifiers(Val, Size); |
| 2174 | } |
| 2175 | |
| 2176 | APInt Literal(64, Val); |
| 2177 | uint8_t OpTy = InstDesc.operands()[OpNum].OperandType; |
| 2178 | |
| 2179 | bool CanUse64BitLiterals = |
| 2180 | AsmParser->has64BitLiterals() && !SIInstrFlags::isVOP3Like(O: InstDesc); |
| 2181 | LitModifier Lit = getModifiers().Lit; |
| 2182 | MCContext &Ctx = AsmParser->getContext(); |
| 2183 | |
| 2184 | if (Imm.IsFPImm) { // We got fp literal token |
| 2185 | switch (OpTy) { |
| 2186 | case AMDGPU::OPERAND_REG_IMM_INT64: |
| 2187 | case AMDGPU::OPERAND_REG_IMM_FP64: |
| 2188 | case AMDGPU::OPERAND_REG_INLINE_C_INT64: |
| 2189 | case AMDGPU::OPERAND_REG_INLINE_C_FP64: |
| 2190 | case AMDGPU::OPERAND_REG_INLINE_AC_FP64: |
| 2191 | case AMDGPU::OPERAND_REG_IMM_V2FP64: |
| 2192 | case AMDGPU::OPERAND_REG_IMM_V2INT64: |
| 2193 | if (Lit == LitModifier::None && |
| 2194 | AMDGPU::isInlinableLiteral64(Literal: Literal.getZExtValue(), |
| 2195 | HasInv2Pi: AsmParser->hasInv2PiInlineImm())) { |
| 2196 | Inst.addOperand(Op: MCOperand::createImm(Val: Literal.getZExtValue())); |
| 2197 | return; |
| 2198 | } |
| 2199 | |
| 2200 | // Non-inlineable |
| 2201 | if (AMDGPU::isSISrcFPOperand(Desc: InstDesc, |
| 2202 | OpNo: OpNum)) { // Expected 64-bit fp operand |
| 2203 | bool HasMandatoryLiteral = |
| 2204 | AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::imm); |
| 2205 | // For fp operands we check if low 32 bits are zeros |
| 2206 | if (Literal.getLoBits(numBits: 32) != 0 && |
| 2207 | (InstDesc.getSize() != 4 || !AsmParser->has64BitLiterals()) && |
| 2208 | !HasMandatoryLiteral) { |
| 2209 | const_cast<AMDGPUAsmParser *>(AsmParser)->Warning( |
| 2210 | L: Inst.getLoc(), |
| 2211 | Msg: "Can't encode literal as exact 64-bit floating-point operand. " |
| 2212 | "Low 32-bits will be set to zero" ); |
| 2213 | Val &= 0xffffffff00000000u; |
| 2214 | } |
| 2215 | |
| 2216 | if ((OpTy == AMDGPU::OPERAND_REG_IMM_FP64 || |
| 2217 | OpTy == AMDGPU::OPERAND_REG_INLINE_C_FP64 || |
| 2218 | OpTy == AMDGPU::OPERAND_REG_INLINE_AC_FP64)) { |
| 2219 | if (CanUse64BitLiterals && Lit == LitModifier::None && |
| 2220 | (isInt<32>(x: Val) || isUInt<32>(x: Val))) { |
| 2221 | // The floating-point operand will be verbalized as an |
| 2222 | // integer one. If that integer happens to fit 32 bits, on |
| 2223 | // re-assembling it will be intepreted as the high half of |
| 2224 | // the actual value, so we have to wrap it into lit64(). |
| 2225 | Lit = LitModifier::Lit64; |
| 2226 | } else if (Lit == LitModifier::Lit) { |
| 2227 | // For FP64 operands lit() specifies the high half of the value. |
| 2228 | Val = Hi_32(Value: Val); |
| 2229 | } |
| 2230 | } |
| 2231 | break; |
| 2232 | } |
| 2233 | |
| 2234 | // We don't allow fp literals in 64-bit integer instructions. It is |
| 2235 | // unclear how we should encode them. This case should be checked earlier |
| 2236 | // in predicate methods (isLiteralImm()) |
| 2237 | llvm_unreachable("fp literal in 64-bit integer instruction." ); |
| 2238 | |
| 2239 | case AMDGPU::OPERAND_KIMM64: |
| 2240 | if (CanUse64BitLiterals && Lit == LitModifier::None && |
| 2241 | (isInt<32>(x: Val) || isUInt<32>(x: Val))) |
| 2242 | Lit = LitModifier::Lit64; |
| 2243 | break; |
| 2244 | |
| 2245 | case AMDGPU::OPERAND_REG_IMM_BF16: |
| 2246 | case AMDGPU::OPERAND_REG_INLINE_C_BF16: |
| 2247 | case AMDGPU::OPERAND_REG_INLINE_C_V2BF16: |
| 2248 | case AMDGPU::OPERAND_REG_IMM_V2BF16: |
| 2249 | if (Lit == LitModifier::None && AsmParser->hasInv2PiInlineImm() && |
| 2250 | Literal == 0x3fc45f306725feed) { |
| 2251 | // This is the 1/(2*pi) which is going to be truncated to bf16 with the |
| 2252 | // loss of precision. The constant represents idiomatic fp32 value of |
| 2253 | // 1/(2*pi) = 0.15915494 since bf16 is in fact fp32 with cleared low 16 |
| 2254 | // bits. Prevent rounding below. |
| 2255 | Inst.addOperand(Op: MCOperand::createImm(Val: 0x3e22)); |
| 2256 | return; |
| 2257 | } |
| 2258 | [[fallthrough]]; |
| 2259 | |
| 2260 | case AMDGPU::OPERAND_REG_IMM_INT32: |
| 2261 | case AMDGPU::OPERAND_REG_IMM_FP32: |
| 2262 | case AMDGPU::OPERAND_REG_INLINE_C_INT32: |
| 2263 | case AMDGPU::OPERAND_REG_INLINE_C_FP32: |
| 2264 | case AMDGPU::OPERAND_REG_INLINE_AC_INT32: |
| 2265 | case AMDGPU::OPERAND_REG_INLINE_AC_FP32: |
| 2266 | case AMDGPU::OPERAND_REG_IMM_INT16: |
| 2267 | case AMDGPU::OPERAND_REG_IMM_FP16: |
| 2268 | case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16: |
| 2269 | case AMDGPU::OPERAND_REG_INLINE_C_INT16: |
| 2270 | case AMDGPU::OPERAND_REG_INLINE_C_FP16: |
| 2271 | case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: |
| 2272 | case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: |
| 2273 | case AMDGPU::OPERAND_REG_IMM_V2INT16: |
| 2274 | case AMDGPU::OPERAND_REG_IMM_V2FP16: |
| 2275 | case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT: |
| 2276 | case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16: |
| 2277 | case AMDGPU::OPERAND_REG_IMM_V2FP32: |
| 2278 | case AMDGPU::OPERAND_REG_IMM_V2INT32: |
| 2279 | case AMDGPU::OPERAND_KIMM32: |
| 2280 | case AMDGPU::OPERAND_KIMM16: |
| 2281 | case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: { |
| 2282 | bool lost; |
| 2283 | APFloat FPLiteral(APFloat::IEEEdouble(), Literal); |
| 2284 | // Convert literal to single precision |
| 2285 | FPLiteral.convert(ToSemantics: *getOpFltSemantics(OperandType: OpTy), RM: APFloat::rmNearestTiesToEven, |
| 2286 | losesInfo: &lost); |
| 2287 | // We allow precision lost but not overflow or underflow. This should be |
| 2288 | // checked earlier in isLiteralImm() |
| 2289 | |
| 2290 | Val = FPLiteral.bitcastToAPInt().getZExtValue(); |
| 2291 | break; |
| 2292 | } |
| 2293 | default: |
| 2294 | llvm_unreachable("invalid operand size" ); |
| 2295 | } |
| 2296 | |
| 2297 | if (Lit != LitModifier::None) { |
| 2298 | Inst.addOperand( |
| 2299 | Op: MCOperand::createExpr(Val: AMDGPUMCExpr::createLit(Lit, Value: Val, Ctx))); |
| 2300 | } else { |
| 2301 | Inst.addOperand(Op: MCOperand::createImm(Val)); |
| 2302 | } |
| 2303 | return; |
| 2304 | } |
| 2305 | |
| 2306 | // We got int literal token. |
| 2307 | // Only sign extend inline immediates. |
| 2308 | switch (OpTy) { |
| 2309 | case AMDGPU::OPERAND_REG_IMM_INT32: |
| 2310 | case AMDGPU::OPERAND_REG_IMM_FP32: |
| 2311 | case AMDGPU::OPERAND_REG_INLINE_C_INT32: |
| 2312 | case AMDGPU::OPERAND_REG_INLINE_C_FP32: |
| 2313 | case AMDGPU::OPERAND_REG_INLINE_AC_INT32: |
| 2314 | case AMDGPU::OPERAND_REG_INLINE_AC_FP32: |
| 2315 | case AMDGPU::OPERAND_REG_IMM_V2INT16: |
| 2316 | case AMDGPU::OPERAND_REG_IMM_V2BF16: |
| 2317 | case AMDGPU::OPERAND_REG_IMM_V2FP16: |
| 2318 | case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT: |
| 2319 | case AMDGPU::OPERAND_REG_IMM_V2FP32: |
| 2320 | case AMDGPU::OPERAND_REG_IMM_V2INT32: |
| 2321 | case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: |
| 2322 | case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16: |
| 2323 | break; |
| 2324 | |
| 2325 | case AMDGPU::OPERAND_REG_IMM_INT64: |
| 2326 | case AMDGPU::OPERAND_REG_INLINE_C_INT64: |
| 2327 | case AMDGPU::OPERAND_REG_IMM_V2INT64: |
| 2328 | if (Lit == LitModifier::None && |
| 2329 | AMDGPU::isInlinableLiteral64(Literal: Val, HasInv2Pi: AsmParser->hasInv2PiInlineImm())) { |
| 2330 | Inst.addOperand(Op: MCOperand::createImm(Val)); |
| 2331 | return; |
| 2332 | } |
| 2333 | |
| 2334 | // When the 32 MSBs are not zero (effectively means it can't be safely |
| 2335 | // truncated to uint32_t), if the target doesn't support 64-bit literals, or |
| 2336 | // the lit modifier is explicitly used, we need to truncate it to the 32 |
| 2337 | // LSBs. |
| 2338 | if (!AsmParser->has64BitLiterals() || Lit == LitModifier::Lit) |
| 2339 | Val = Lo_32(Value: Val); |
| 2340 | break; |
| 2341 | |
| 2342 | case AMDGPU::OPERAND_REG_IMM_FP64: |
| 2343 | case AMDGPU::OPERAND_REG_INLINE_C_FP64: |
| 2344 | case AMDGPU::OPERAND_REG_INLINE_AC_FP64: |
| 2345 | case AMDGPU::OPERAND_REG_IMM_V2FP64: |
| 2346 | if (Lit == LitModifier::None && |
| 2347 | AMDGPU::isInlinableLiteral64(Literal: Val, HasInv2Pi: AsmParser->hasInv2PiInlineImm())) { |
| 2348 | Inst.addOperand(Op: MCOperand::createImm(Val)); |
| 2349 | return; |
| 2350 | } |
| 2351 | |
| 2352 | // If the target doesn't support 64-bit literals, we need to use the |
| 2353 | // constant as the high 32 MSBs of a double-precision floating point value. |
| 2354 | if (!AsmParser->has64BitLiterals()) { |
| 2355 | Val = static_cast<uint64_t>(Val) << 32; |
| 2356 | } else { |
| 2357 | // Now the target does support 64-bit literals, there are two cases |
| 2358 | // where we still want to use src_literal encoding: |
| 2359 | // 1) explicitly forced by using lit modifier; |
| 2360 | // 2) the value is a valid 32-bit representation (signed or unsigned), |
| 2361 | // meanwhile not forced by lit64 modifier. |
| 2362 | if (Lit == LitModifier::Lit || |
| 2363 | (Lit != LitModifier::Lit64 && (isInt<32>(x: Val) || isUInt<32>(x: Val)))) |
| 2364 | Val = static_cast<uint64_t>(Val) << 32; |
| 2365 | } |
| 2366 | |
| 2367 | // For FP64 operands lit() specifies the high half of the value. |
| 2368 | if (Lit == LitModifier::Lit) |
| 2369 | Val = Hi_32(Value: Val); |
| 2370 | break; |
| 2371 | |
| 2372 | case AMDGPU::OPERAND_REG_IMM_INT16: |
| 2373 | case AMDGPU::OPERAND_REG_INLINE_C_INT16: |
| 2374 | case AMDGPU::OPERAND_REG_INLINE_C_FP16: |
| 2375 | case AMDGPU::OPERAND_REG_IMM_FP16: |
| 2376 | case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16: |
| 2377 | case AMDGPU::OPERAND_REG_IMM_BF16: |
| 2378 | case AMDGPU::OPERAND_REG_INLINE_C_BF16: |
| 2379 | case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: |
| 2380 | case AMDGPU::OPERAND_REG_INLINE_C_V2FP16: |
| 2381 | case AMDGPU::OPERAND_REG_INLINE_C_V2BF16: |
| 2382 | case AMDGPU::OPERAND_KIMM32: |
| 2383 | case AMDGPU::OPERAND_KIMM16: |
| 2384 | break; |
| 2385 | |
| 2386 | case AMDGPU::OPERAND_KIMM64: |
| 2387 | if ((isInt<32>(x: Val) || isUInt<32>(x: Val)) && Lit != LitModifier::Lit64) |
| 2388 | Val <<= 32; |
| 2389 | break; |
| 2390 | |
| 2391 | default: |
| 2392 | llvm_unreachable("invalid operand type" ); |
| 2393 | } |
| 2394 | |
| 2395 | if (Lit != LitModifier::None) { |
| 2396 | Inst.addOperand( |
| 2397 | Op: MCOperand::createExpr(Val: AMDGPUMCExpr::createLit(Lit, Value: Val, Ctx))); |
| 2398 | } else { |
| 2399 | Inst.addOperand(Op: MCOperand::createImm(Val)); |
| 2400 | } |
| 2401 | } |
| 2402 | |
| 2403 | void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const { |
| 2404 | MCOpIdx = Inst.getNumOperands(); |
| 2405 | Inst.addOperand( |
| 2406 | Op: MCOperand::createReg(Reg: AMDGPU::getMCReg(Reg: getReg(), STI: AsmParser->getSTI()))); |
| 2407 | } |
| 2408 | |
| 2409 | bool AMDGPUOperand::isInlineValue() const { |
| 2410 | return isRegKind() && ::isInlineValue(Reg: getReg()); |
| 2411 | } |
| 2412 | |
| 2413 | //===----------------------------------------------------------------------===// |
| 2414 | // AsmParser |
| 2415 | //===----------------------------------------------------------------------===// |
| 2416 | |
| 2417 | void AMDGPUAsmParser::createConstantSymbol(StringRef Id, int64_t Val) { |
| 2418 | // TODO: make those pre-defined variables read-only. |
| 2419 | // Currently there is none suitable machinery in the core llvm-mc for this. |
| 2420 | // MCSymbol::isRedefinable is intended for another purpose, and |
| 2421 | // AsmParser::parseDirectiveSet() cannot be specialized for specific target. |
| 2422 | MCContext &Ctx = getContext(); |
| 2423 | MCSymbol *Sym = Ctx.getOrCreateSymbol(Name: Id); |
| 2424 | Sym->setVariableValue(MCConstantExpr::create(Value: Val, Ctx)); |
| 2425 | } |
| 2426 | |
| 2427 | static int getRegClass(RegisterKind Is, unsigned RegWidth) { |
| 2428 | if (Is == IS_VGPR) { |
| 2429 | switch (RegWidth) { |
| 2430 | default: |
| 2431 | return -1; |
| 2432 | case 32: |
| 2433 | return AMDGPU::VGPR_32RegClassID; |
| 2434 | case 64: |
| 2435 | return AMDGPU::VReg_64RegClassID; |
| 2436 | case 96: |
| 2437 | return AMDGPU::VReg_96RegClassID; |
| 2438 | case 128: |
| 2439 | return AMDGPU::VReg_128RegClassID; |
| 2440 | case 160: |
| 2441 | return AMDGPU::VReg_160RegClassID; |
| 2442 | case 192: |
| 2443 | return AMDGPU::VReg_192RegClassID; |
| 2444 | case 224: |
| 2445 | return AMDGPU::VReg_224RegClassID; |
| 2446 | case 256: |
| 2447 | return AMDGPU::VReg_256RegClassID; |
| 2448 | case 288: |
| 2449 | return AMDGPU::VReg_288RegClassID; |
| 2450 | case 320: |
| 2451 | return AMDGPU::VReg_320RegClassID; |
| 2452 | case 352: |
| 2453 | return AMDGPU::VReg_352RegClassID; |
| 2454 | case 384: |
| 2455 | return AMDGPU::VReg_384RegClassID; |
| 2456 | case 512: |
| 2457 | return AMDGPU::VReg_512RegClassID; |
| 2458 | case 1024: |
| 2459 | return AMDGPU::VReg_1024RegClassID; |
| 2460 | } |
| 2461 | } else if (Is == IS_TTMP) { |
| 2462 | switch (RegWidth) { |
| 2463 | default: |
| 2464 | return -1; |
| 2465 | case 32: |
| 2466 | return AMDGPU::TTMP_32RegClassID; |
| 2467 | case 64: |
| 2468 | return AMDGPU::TTMP_64RegClassID; |
| 2469 | case 128: |
| 2470 | return AMDGPU::TTMP_128RegClassID; |
| 2471 | case 256: |
| 2472 | return AMDGPU::TTMP_256RegClassID; |
| 2473 | case 512: |
| 2474 | return AMDGPU::TTMP_512RegClassID; |
| 2475 | } |
| 2476 | } else if (Is == IS_SGPR) { |
| 2477 | switch (RegWidth) { |
| 2478 | default: |
| 2479 | return -1; |
| 2480 | case 32: |
| 2481 | return AMDGPU::SGPR_32RegClassID; |
| 2482 | case 64: |
| 2483 | return AMDGPU::SGPR_64RegClassID; |
| 2484 | case 96: |
| 2485 | return AMDGPU::SGPR_96RegClassID; |
| 2486 | case 128: |
| 2487 | return AMDGPU::SGPR_128RegClassID; |
| 2488 | case 160: |
| 2489 | return AMDGPU::SGPR_160RegClassID; |
| 2490 | case 192: |
| 2491 | return AMDGPU::SGPR_192RegClassID; |
| 2492 | case 224: |
| 2493 | return AMDGPU::SGPR_224RegClassID; |
| 2494 | case 256: |
| 2495 | return AMDGPU::SGPR_256RegClassID; |
| 2496 | case 288: |
| 2497 | return AMDGPU::SGPR_288RegClassID; |
| 2498 | case 320: |
| 2499 | return AMDGPU::SGPR_320RegClassID; |
| 2500 | case 352: |
| 2501 | return AMDGPU::SGPR_352RegClassID; |
| 2502 | case 384: |
| 2503 | return AMDGPU::SGPR_384RegClassID; |
| 2504 | case 512: |
| 2505 | return AMDGPU::SGPR_512RegClassID; |
| 2506 | } |
| 2507 | } else if (Is == IS_AGPR) { |
| 2508 | switch (RegWidth) { |
| 2509 | default: |
| 2510 | return -1; |
| 2511 | case 32: |
| 2512 | return AMDGPU::AGPR_32RegClassID; |
| 2513 | case 64: |
| 2514 | return AMDGPU::AReg_64RegClassID; |
| 2515 | case 96: |
| 2516 | return AMDGPU::AReg_96RegClassID; |
| 2517 | case 128: |
| 2518 | return AMDGPU::AReg_128RegClassID; |
| 2519 | case 160: |
| 2520 | return AMDGPU::AReg_160RegClassID; |
| 2521 | case 192: |
| 2522 | return AMDGPU::AReg_192RegClassID; |
| 2523 | case 224: |
| 2524 | return AMDGPU::AReg_224RegClassID; |
| 2525 | case 256: |
| 2526 | return AMDGPU::AReg_256RegClassID; |
| 2527 | case 288: |
| 2528 | return AMDGPU::AReg_288RegClassID; |
| 2529 | case 320: |
| 2530 | return AMDGPU::AReg_320RegClassID; |
| 2531 | case 352: |
| 2532 | return AMDGPU::AReg_352RegClassID; |
| 2533 | case 384: |
| 2534 | return AMDGPU::AReg_384RegClassID; |
| 2535 | case 512: |
| 2536 | return AMDGPU::AReg_512RegClassID; |
| 2537 | case 1024: |
| 2538 | return AMDGPU::AReg_1024RegClassID; |
| 2539 | } |
| 2540 | } |
| 2541 | return -1; |
| 2542 | } |
| 2543 | |
| 2544 | static MCRegister getSpecialRegForName(StringRef RegName) { |
| 2545 | return StringSwitch<unsigned>(RegName) |
| 2546 | .Case(S: "exec" , Value: AMDGPU::EXEC) |
| 2547 | .Case(S: "vcc" , Value: AMDGPU::VCC) |
| 2548 | .Case(S: "flat_scratch" , Value: AMDGPU::FLAT_SCR) |
| 2549 | .Case(S: "xnack_mask" , Value: AMDGPU::XNACK_MASK) |
| 2550 | .Case(S: "shared_base" , Value: AMDGPU::SRC_SHARED_BASE) |
| 2551 | .Case(S: "src_shared_base" , Value: AMDGPU::SRC_SHARED_BASE) |
| 2552 | .Case(S: "shared_limit" , Value: AMDGPU::SRC_SHARED_LIMIT) |
| 2553 | .Case(S: "src_shared_limit" , Value: AMDGPU::SRC_SHARED_LIMIT) |
| 2554 | .Case(S: "private_base" , Value: AMDGPU::SRC_PRIVATE_BASE) |
| 2555 | .Case(S: "src_private_base" , Value: AMDGPU::SRC_PRIVATE_BASE) |
| 2556 | .Case(S: "private_limit" , Value: AMDGPU::SRC_PRIVATE_LIMIT) |
| 2557 | .Case(S: "src_private_limit" , Value: AMDGPU::SRC_PRIVATE_LIMIT) |
| 2558 | .Case(S: "src_flat_scratch_base_lo" , Value: AMDGPU::SRC_FLAT_SCRATCH_BASE_LO) |
| 2559 | .Case(S: "src_flat_scratch_base_hi" , Value: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI) |
| 2560 | .Case(S: "pops_exiting_wave_id" , Value: AMDGPU::SRC_POPS_EXITING_WAVE_ID) |
| 2561 | .Case(S: "src_pops_exiting_wave_id" , Value: AMDGPU::SRC_POPS_EXITING_WAVE_ID) |
| 2562 | .Case(S: "lds_direct" , Value: AMDGPU::LDS_DIRECT) |
| 2563 | .Case(S: "src_lds_direct" , Value: AMDGPU::LDS_DIRECT) |
| 2564 | .Case(S: "m0" , Value: AMDGPU::M0) |
| 2565 | .Case(S: "vccz" , Value: AMDGPU::SRC_VCCZ) |
| 2566 | .Case(S: "src_vccz" , Value: AMDGPU::SRC_VCCZ) |
| 2567 | .Case(S: "execz" , Value: AMDGPU::SRC_EXECZ) |
| 2568 | .Case(S: "src_execz" , Value: AMDGPU::SRC_EXECZ) |
| 2569 | .Case(S: "scc" , Value: AMDGPU::SRC_SCC) |
| 2570 | .Case(S: "src_scc" , Value: AMDGPU::SRC_SCC) |
| 2571 | .Case(S: "tba" , Value: AMDGPU::TBA) |
| 2572 | .Case(S: "tma" , Value: AMDGPU::TMA) |
| 2573 | .Case(S: "flat_scratch_lo" , Value: AMDGPU::FLAT_SCR_LO) |
| 2574 | .Case(S: "flat_scratch_hi" , Value: AMDGPU::FLAT_SCR_HI) |
| 2575 | .Case(S: "xnack_mask_lo" , Value: AMDGPU::XNACK_MASK_LO) |
| 2576 | .Case(S: "xnack_mask_hi" , Value: AMDGPU::XNACK_MASK_HI) |
| 2577 | .Case(S: "vcc_lo" , Value: AMDGPU::VCC_LO) |
| 2578 | .Case(S: "vcc_hi" , Value: AMDGPU::VCC_HI) |
| 2579 | .Case(S: "exec_lo" , Value: AMDGPU::EXEC_LO) |
| 2580 | .Case(S: "exec_hi" , Value: AMDGPU::EXEC_HI) |
| 2581 | .Case(S: "tma_lo" , Value: AMDGPU::TMA_LO) |
| 2582 | .Case(S: "tma_hi" , Value: AMDGPU::TMA_HI) |
| 2583 | .Case(S: "tba_lo" , Value: AMDGPU::TBA_LO) |
| 2584 | .Case(S: "tba_hi" , Value: AMDGPU::TBA_HI) |
| 2585 | .Case(S: "pc" , Value: AMDGPU::PC_REG) |
| 2586 | .Case(S: "null" , Value: AMDGPU::SGPR_NULL) |
| 2587 | .Default(Value: AMDGPU::NoRegister); |
| 2588 | } |
| 2589 | |
| 2590 | bool AMDGPUAsmParser::ParseRegister(MCRegister &RegNo, SMLoc &StartLoc, |
| 2591 | SMLoc &EndLoc, bool RestoreOnFailure) { |
| 2592 | auto R = parseRegister(); |
| 2593 | if (!R) |
| 2594 | return true; |
| 2595 | assert(R->isReg()); |
| 2596 | RegNo = R->getReg(); |
| 2597 | StartLoc = R->getStartLoc(); |
| 2598 | EndLoc = R->getEndLoc(); |
| 2599 | return false; |
| 2600 | } |
| 2601 | |
| 2602 | bool AMDGPUAsmParser::parseRegister(MCRegister &Reg, SMLoc &StartLoc, |
| 2603 | SMLoc &EndLoc) { |
| 2604 | return ParseRegister(RegNo&: Reg, StartLoc, EndLoc, /*RestoreOnFailure=*/false); |
| 2605 | } |
| 2606 | |
| 2607 | ParseStatus AMDGPUAsmParser::tryParseRegister(MCRegister &Reg, SMLoc &StartLoc, |
| 2608 | SMLoc &EndLoc) { |
| 2609 | bool Result = ParseRegister(RegNo&: Reg, StartLoc, EndLoc, /*RestoreOnFailure=*/true); |
| 2610 | bool PendingErrors = getParser().hasPendingError(); |
| 2611 | getParser().clearPendingErrors(); |
| 2612 | if (PendingErrors) |
| 2613 | return ParseStatus::Failure; |
| 2614 | if (Result) |
| 2615 | return ParseStatus::NoMatch; |
| 2616 | return ParseStatus::Success; |
| 2617 | } |
| 2618 | |
| 2619 | bool AMDGPUAsmParser::AddNextRegisterToList(MCRegister &Reg, unsigned &RegWidth, |
| 2620 | RegisterKind RegKind, |
| 2621 | MCRegister Reg1, |
| 2622 | RegisterKind RegKind1, SMLoc Loc) { |
| 2623 | // Allow VCC_LO/HI at the end of SGPR lists. |
| 2624 | if (RegKind == IS_SGPR) { |
| 2625 | unsigned RegIdx = (Reg - AMDGPU::SGPR0) + RegWidth / 32; |
| 2626 | if ((RegIdx == 106 && Reg1 == AMDGPU::VCC_LO) || |
| 2627 | (RegIdx == 107 && Reg1 == AMDGPU::VCC_HI)) { |
| 2628 | RegWidth += 32; |
| 2629 | return true; |
| 2630 | } |
| 2631 | } |
| 2632 | |
| 2633 | if (RegKind != RegKind1) { |
| 2634 | Error(L: Loc, Msg: "registers in a list must be of the same kind" ); |
| 2635 | return false; |
| 2636 | } |
| 2637 | |
| 2638 | switch (RegKind) { |
| 2639 | case IS_SPECIAL: |
| 2640 | if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) { |
| 2641 | Reg = AMDGPU::EXEC; |
| 2642 | RegWidth = 64; |
| 2643 | return true; |
| 2644 | } |
| 2645 | if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) { |
| 2646 | Reg = AMDGPU::FLAT_SCR; |
| 2647 | RegWidth = 64; |
| 2648 | return true; |
| 2649 | } |
| 2650 | if (Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) { |
| 2651 | Reg = AMDGPU::XNACK_MASK; |
| 2652 | RegWidth = 64; |
| 2653 | return true; |
| 2654 | } |
| 2655 | if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) { |
| 2656 | Reg = AMDGPU::VCC; |
| 2657 | RegWidth = 64; |
| 2658 | return true; |
| 2659 | } |
| 2660 | if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) { |
| 2661 | Reg = AMDGPU::TBA; |
| 2662 | RegWidth = 64; |
| 2663 | return true; |
| 2664 | } |
| 2665 | if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) { |
| 2666 | Reg = AMDGPU::TMA; |
| 2667 | RegWidth = 64; |
| 2668 | return true; |
| 2669 | } |
| 2670 | Error(L: Loc, Msg: "register does not fit in the list" ); |
| 2671 | return false; |
| 2672 | case IS_VGPR: |
| 2673 | case IS_SGPR: |
| 2674 | case IS_AGPR: |
| 2675 | case IS_TTMP: |
| 2676 | if (Reg1 != Reg + RegWidth / 32) { |
| 2677 | Error(L: Loc, Msg: "registers in a list must have consecutive indices" ); |
| 2678 | return false; |
| 2679 | } |
| 2680 | RegWidth += 32; |
| 2681 | return true; |
| 2682 | default: |
| 2683 | llvm_unreachable("unexpected register kind" ); |
| 2684 | } |
| 2685 | } |
| 2686 | |
| 2687 | struct RegInfo { |
| 2688 | StringLiteral Name; |
| 2689 | RegisterKind Kind; |
| 2690 | }; |
| 2691 | |
| 2692 | static constexpr RegInfo RegularRegisters[] = { |
| 2693 | {.Name: {"v" }, .Kind: IS_VGPR}, {.Name: {"s" }, .Kind: IS_SGPR}, {.Name: {"ttmp" }, .Kind: IS_TTMP}, |
| 2694 | {.Name: {"acc" }, .Kind: IS_AGPR}, {.Name: {"a" }, .Kind: IS_AGPR}, |
| 2695 | }; |
| 2696 | |
| 2697 | static bool isRegularReg(RegisterKind Kind) { |
| 2698 | return Kind == IS_VGPR || Kind == IS_SGPR || Kind == IS_TTMP || |
| 2699 | Kind == IS_AGPR; |
| 2700 | } |
| 2701 | |
| 2702 | static const RegInfo *getRegularRegInfo(StringRef Str) { |
| 2703 | for (const RegInfo &Reg : RegularRegisters) |
| 2704 | if (Str.starts_with(Prefix: Reg.Name)) |
| 2705 | return &Reg; |
| 2706 | return nullptr; |
| 2707 | } |
| 2708 | |
| 2709 | static bool getRegNum(StringRef Str, unsigned &Num) { |
| 2710 | return !Str.getAsInteger(Radix: 10, Result&: Num); |
| 2711 | } |
| 2712 | |
| 2713 | bool AMDGPUAsmParser::isRegister(const AsmToken &Token, |
| 2714 | const AsmToken &NextToken) const { |
| 2715 | |
| 2716 | // A list of consecutive registers: [s0,s1,s2,s3] |
| 2717 | if (Token.is(K: AsmToken::LBrac)) |
| 2718 | return true; |
| 2719 | |
| 2720 | if (!Token.is(K: AsmToken::Identifier)) |
| 2721 | return false; |
| 2722 | |
| 2723 | // A single register like s0 or a range of registers like s[0:1] |
| 2724 | |
| 2725 | StringRef Str = Token.getString(); |
| 2726 | const RegInfo *Reg = getRegularRegInfo(Str); |
| 2727 | if (Reg) { |
| 2728 | StringRef RegName = Reg->Name; |
| 2729 | StringRef RegSuffix = Str.substr(Start: RegName.size()); |
| 2730 | if (!RegSuffix.empty()) { |
| 2731 | RegSuffix.consume_back(Suffix: ".l" ); |
| 2732 | RegSuffix.consume_back(Suffix: ".h" ); |
| 2733 | unsigned Num; |
| 2734 | // A single register with an index: rXX |
| 2735 | if (getRegNum(Str: RegSuffix, Num)) |
| 2736 | return true; |
| 2737 | } else { |
| 2738 | // A range of registers: r[XX:YY]. |
| 2739 | if (NextToken.is(K: AsmToken::LBrac)) |
| 2740 | return true; |
| 2741 | } |
| 2742 | } |
| 2743 | |
| 2744 | return getSpecialRegForName(RegName: Str).isValid(); |
| 2745 | } |
| 2746 | |
| 2747 | bool AMDGPUAsmParser::isRegister() { |
| 2748 | return isRegister(Token: getToken(), NextToken: peekToken()); |
| 2749 | } |
| 2750 | |
| 2751 | MCRegister AMDGPUAsmParser::getRegularReg(RegisterKind RegKind, unsigned RegNum, |
| 2752 | unsigned SubReg, unsigned RegWidth, |
| 2753 | SMLoc Loc) { |
| 2754 | assert(isRegularReg(RegKind)); |
| 2755 | |
| 2756 | unsigned AlignSize = 1; |
| 2757 | if (RegKind == IS_SGPR || RegKind == IS_TTMP) { |
| 2758 | // SGPR and TTMP registers must be aligned. |
| 2759 | // Max required alignment is 4 dwords. |
| 2760 | AlignSize = std::min(a: llvm::bit_ceil(Value: RegWidth / 32), b: 4u); |
| 2761 | } |
| 2762 | |
| 2763 | if (RegNum % AlignSize != 0) { |
| 2764 | Error(L: Loc, Msg: "invalid register alignment" ); |
| 2765 | return MCRegister(); |
| 2766 | } |
| 2767 | |
| 2768 | unsigned RegIdx = RegNum / AlignSize; |
| 2769 | int RCID = getRegClass(Is: RegKind, RegWidth); |
| 2770 | if (RCID == -1) { |
| 2771 | Error(L: Loc, Msg: "invalid or unsupported register size" ); |
| 2772 | return MCRegister(); |
| 2773 | } |
| 2774 | |
| 2775 | const MCRegisterInfo *TRI = getContext().getRegisterInfo(); |
| 2776 | const MCRegisterClass &RC = TRI->getRegClass(i: RCID); |
| 2777 | if (RegIdx >= RC.getNumRegs() || (RegKind == IS_VGPR && RegIdx > 255)) { |
| 2778 | Error(L: Loc, Msg: "register index is out of range" ); |
| 2779 | return AMDGPU::NoRegister; |
| 2780 | } |
| 2781 | |
| 2782 | if (RegKind == IS_VGPR && !isGFX1250Plus() && RegIdx + RegWidth / 32 > 256) { |
| 2783 | Error(L: Loc, Msg: "register index is out of range" ); |
| 2784 | return MCRegister(); |
| 2785 | } |
| 2786 | |
| 2787 | MCRegister Reg = RC.getRegister(i: RegIdx); |
| 2788 | |
| 2789 | if (SubReg) { |
| 2790 | Reg = TRI->getSubReg(Reg, Idx: SubReg); |
| 2791 | |
| 2792 | if (!Reg) |
| 2793 | Error(L: Loc, Msg: "invalid subregister" ); |
| 2794 | } |
| 2795 | |
| 2796 | return Reg; |
| 2797 | } |
| 2798 | |
| 2799 | bool AMDGPUAsmParser::ParseRegRange(unsigned &Num, unsigned &RegWidth, |
| 2800 | unsigned &SubReg) { |
| 2801 | int64_t RegLo, RegHi; |
| 2802 | if (!skipToken(Kind: AsmToken::LBrac, ErrMsg: "missing register index" )) |
| 2803 | return false; |
| 2804 | |
| 2805 | SMLoc FirstIdxLoc = getLoc(); |
| 2806 | SMLoc SecondIdxLoc; |
| 2807 | |
| 2808 | if (!parseExpr(Imm&: RegLo)) |
| 2809 | return false; |
| 2810 | |
| 2811 | if (trySkipToken(Kind: AsmToken::Colon)) { |
| 2812 | SecondIdxLoc = getLoc(); |
| 2813 | if (!parseExpr(Imm&: RegHi)) |
| 2814 | return false; |
| 2815 | } else { |
| 2816 | RegHi = RegLo; |
| 2817 | } |
| 2818 | |
| 2819 | if (!skipToken(Kind: AsmToken::RBrac, ErrMsg: "expected a closing square bracket" )) |
| 2820 | return false; |
| 2821 | |
| 2822 | if (!isUInt<32>(x: RegLo)) { |
| 2823 | Error(L: FirstIdxLoc, Msg: "invalid register index" ); |
| 2824 | return false; |
| 2825 | } |
| 2826 | |
| 2827 | if (!isUInt<32>(x: RegHi)) { |
| 2828 | Error(L: SecondIdxLoc, Msg: "invalid register index" ); |
| 2829 | return false; |
| 2830 | } |
| 2831 | |
| 2832 | if (RegLo > RegHi) { |
| 2833 | Error(L: FirstIdxLoc, Msg: "first register index should not exceed second index" ); |
| 2834 | return false; |
| 2835 | } |
| 2836 | |
| 2837 | if (RegHi == RegLo) { |
| 2838 | StringRef RegSuffix = getTokenStr(); |
| 2839 | if (RegSuffix == ".l" ) { |
| 2840 | SubReg = AMDGPU::lo16; |
| 2841 | lex(); |
| 2842 | } else if (RegSuffix == ".h" ) { |
| 2843 | SubReg = AMDGPU::hi16; |
| 2844 | lex(); |
| 2845 | } |
| 2846 | } |
| 2847 | |
| 2848 | Num = static_cast<unsigned>(RegLo); |
| 2849 | RegWidth = 32 * ((RegHi - RegLo) + 1); |
| 2850 | |
| 2851 | return true; |
| 2852 | } |
| 2853 | |
| 2854 | MCRegister AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind, |
| 2855 | unsigned &RegNum, |
| 2856 | unsigned &RegWidth, |
| 2857 | SmallVectorImpl<AsmToken> &Tokens) { |
| 2858 | assert(isToken(AsmToken::Identifier)); |
| 2859 | MCRegister Reg = getSpecialRegForName(RegName: getTokenStr()); |
| 2860 | if (Reg) { |
| 2861 | RegNum = 0; |
| 2862 | RegWidth = 32; |
| 2863 | RegKind = IS_SPECIAL; |
| 2864 | Tokens.push_back(Elt: getToken()); |
| 2865 | lex(); // skip register name |
| 2866 | } |
| 2867 | return Reg; |
| 2868 | } |
| 2869 | |
| 2870 | MCRegister AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind, |
| 2871 | unsigned &RegNum, |
| 2872 | unsigned &RegWidth, |
| 2873 | SmallVectorImpl<AsmToken> &Tokens) { |
| 2874 | assert(isToken(AsmToken::Identifier)); |
| 2875 | StringRef RegName = getTokenStr(); |
| 2876 | auto Loc = getLoc(); |
| 2877 | |
| 2878 | const RegInfo *RI = getRegularRegInfo(Str: RegName); |
| 2879 | if (!RI) { |
| 2880 | Error(L: Loc, Msg: "invalid register name" ); |
| 2881 | return MCRegister(); |
| 2882 | } |
| 2883 | |
| 2884 | Tokens.push_back(Elt: getToken()); |
| 2885 | lex(); // skip register name |
| 2886 | |
| 2887 | RegKind = RI->Kind; |
| 2888 | StringRef RegSuffix = RegName.substr(Start: RI->Name.size()); |
| 2889 | unsigned SubReg = NoSubRegister; |
| 2890 | bool IsRange = false; |
| 2891 | if (!RegSuffix.empty()) { |
| 2892 | if (RegSuffix.consume_back(Suffix: ".l" )) |
| 2893 | SubReg = AMDGPU::lo16; |
| 2894 | else if (RegSuffix.consume_back(Suffix: ".h" )) |
| 2895 | SubReg = AMDGPU::hi16; |
| 2896 | |
| 2897 | // Single 32-bit register: vXX. |
| 2898 | if (!getRegNum(Str: RegSuffix, Num&: RegNum)) { |
| 2899 | Error(L: Loc, Msg: "invalid register index" ); |
| 2900 | return MCRegister(); |
| 2901 | } |
| 2902 | RegWidth = 32; |
| 2903 | } else { |
| 2904 | // Range of registers: v[XX:YY]. ":YY" is optional. |
| 2905 | IsRange = true; |
| 2906 | if (!ParseRegRange(Num&: RegNum, RegWidth, SubReg)) |
| 2907 | return MCRegister(); |
| 2908 | } |
| 2909 | |
| 2910 | // Do not allow vcc_lo/hi be referred as s106/107. |
| 2911 | MCRegister Reg = getRegularReg(RegKind, RegNum, SubReg, RegWidth, Loc); |
| 2912 | const MCRegisterInfo &TRI = *getContext().getRegisterInfo(); |
| 2913 | if (RegKind == IS_SGPR && IsRange |
| 2914 | ? (TRI.isSubRegister(RegA: Reg, RegB: VCC_LO) || TRI.isSubRegister(RegA: Reg, RegB: VCC_HI)) |
| 2915 | : (Reg == VCC_LO || Reg == VCC_HI)) { |
| 2916 | Error(L: Loc, Msg: "register index is out of range" ); |
| 2917 | return MCRegister(); |
| 2918 | } |
| 2919 | |
| 2920 | return Reg; |
| 2921 | } |
| 2922 | |
| 2923 | MCRegister AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind, |
| 2924 | unsigned &RegNum, unsigned &RegWidth, |
| 2925 | SmallVectorImpl<AsmToken> &Tokens) { |
| 2926 | MCRegister Reg; |
| 2927 | auto ListLoc = getLoc(); |
| 2928 | |
| 2929 | if (!skipToken(Kind: AsmToken::LBrac, |
| 2930 | ErrMsg: "expected a register or a list of registers" )) { |
| 2931 | return MCRegister(); |
| 2932 | } |
| 2933 | |
| 2934 | // List of consecutive registers, e.g.: [s0,s1,s2,s3] |
| 2935 | |
| 2936 | auto Loc = getLoc(); |
| 2937 | if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) |
| 2938 | return MCRegister(); |
| 2939 | if (RegWidth != 32) { |
| 2940 | Error(L: Loc, Msg: "expected a single 32-bit register" ); |
| 2941 | return MCRegister(); |
| 2942 | } |
| 2943 | |
| 2944 | for (; trySkipToken(Kind: AsmToken::Comma);) { |
| 2945 | RegisterKind NextRegKind; |
| 2946 | MCRegister NextReg; |
| 2947 | unsigned NextRegNum, NextRegWidth; |
| 2948 | Loc = getLoc(); |
| 2949 | |
| 2950 | if (!ParseAMDGPURegister(RegKind&: NextRegKind, Reg&: NextReg, RegNum&: NextRegNum, RegWidth&: NextRegWidth, |
| 2951 | Tokens)) { |
| 2952 | return MCRegister(); |
| 2953 | } |
| 2954 | if (NextRegWidth != 32) { |
| 2955 | Error(L: Loc, Msg: "expected a single 32-bit register" ); |
| 2956 | return MCRegister(); |
| 2957 | } |
| 2958 | if (!AddNextRegisterToList(Reg, RegWidth, RegKind, Reg1: NextReg, RegKind1: NextRegKind, |
| 2959 | Loc)) |
| 2960 | return MCRegister(); |
| 2961 | } |
| 2962 | |
| 2963 | if (!skipToken(Kind: AsmToken::RBrac, |
| 2964 | ErrMsg: "expected a comma or a closing square bracket" )) { |
| 2965 | return MCRegister(); |
| 2966 | } |
| 2967 | |
| 2968 | if (isRegularReg(Kind: RegKind)) |
| 2969 | Reg = getRegularReg(RegKind, RegNum, SubReg: NoSubRegister, RegWidth, Loc: ListLoc); |
| 2970 | |
| 2971 | return Reg; |
| 2972 | } |
| 2973 | |
| 2974 | bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, |
| 2975 | MCRegister &Reg, unsigned &RegNum, |
| 2976 | unsigned &RegWidth, |
| 2977 | SmallVectorImpl<AsmToken> &Tokens) { |
| 2978 | auto Loc = getLoc(); |
| 2979 | Reg = MCRegister(); |
| 2980 | |
| 2981 | if (isToken(Kind: AsmToken::Identifier)) { |
| 2982 | Reg = ParseSpecialReg(RegKind, RegNum, RegWidth, Tokens); |
| 2983 | if (!Reg) |
| 2984 | Reg = ParseRegularReg(RegKind, RegNum, RegWidth, Tokens); |
| 2985 | } else { |
| 2986 | Reg = ParseRegList(RegKind, RegNum, RegWidth, Tokens); |
| 2987 | } |
| 2988 | |
| 2989 | const MCRegisterInfo *TRI = getContext().getRegisterInfo(); |
| 2990 | if (!Reg) { |
| 2991 | assert(Parser.hasPendingError()); |
| 2992 | return false; |
| 2993 | } |
| 2994 | |
| 2995 | if (!subtargetHasRegister(MRI: *TRI, Reg)) { |
| 2996 | if (Reg == AMDGPU::SGPR_NULL) { |
| 2997 | Error(L: Loc, Msg: "'null' operand is not supported on this GPU" ); |
| 2998 | } else { |
| 2999 | Error(L: Loc, Msg: Twine(AMDGPUInstPrinter::getRegisterName(Reg)) + |
| 3000 | " register not available on this GPU" ); |
| 3001 | } |
| 3002 | return false; |
| 3003 | } |
| 3004 | |
| 3005 | return true; |
| 3006 | } |
| 3007 | |
| 3008 | bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, |
| 3009 | MCRegister &Reg, unsigned &RegNum, |
| 3010 | unsigned &RegWidth, |
| 3011 | bool RestoreOnFailure /*=false*/) { |
| 3012 | Reg = MCRegister(); |
| 3013 | |
| 3014 | SmallVector<AsmToken, 1> Tokens; |
| 3015 | if (ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, Tokens)) { |
| 3016 | if (RestoreOnFailure) { |
| 3017 | while (!Tokens.empty()) { |
| 3018 | getLexer().UnLex(Token: Tokens.pop_back_val()); |
| 3019 | } |
| 3020 | } |
| 3021 | return true; |
| 3022 | } |
| 3023 | return false; |
| 3024 | } |
| 3025 | |
| 3026 | std::optional<StringRef> |
| 3027 | AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) { |
| 3028 | switch (RegKind) { |
| 3029 | case IS_VGPR: |
| 3030 | return StringRef(".amdgcn.next_free_vgpr" ); |
| 3031 | case IS_SGPR: |
| 3032 | return StringRef(".amdgcn.next_free_sgpr" ); |
| 3033 | default: |
| 3034 | return std::nullopt; |
| 3035 | } |
| 3036 | } |
| 3037 | |
| 3038 | void AMDGPUAsmParser::initializeGprCountSymbol(RegisterKind RegKind) { |
| 3039 | auto SymbolName = getGprCountSymbolName(RegKind); |
| 3040 | assert(SymbolName && "initializing invalid register kind" ); |
| 3041 | MCSymbol *Sym = getContext().getOrCreateSymbol(Name: *SymbolName); |
| 3042 | Sym->setVariableValue(MCConstantExpr::create(Value: 0, Ctx&: getContext())); |
| 3043 | Sym->setRedefinable(true); |
| 3044 | } |
| 3045 | |
| 3046 | bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind, |
| 3047 | unsigned DwordRegIndex, |
| 3048 | unsigned RegWidth) { |
| 3049 | // Symbols are only defined for GCN targets |
| 3050 | if (ISA.Major < 6) |
| 3051 | return true; |
| 3052 | |
| 3053 | auto SymbolName = getGprCountSymbolName(RegKind); |
| 3054 | if (!SymbolName) |
| 3055 | return true; |
| 3056 | MCSymbol *Sym = getContext().getOrCreateSymbol(Name: *SymbolName); |
| 3057 | |
| 3058 | int64_t NewMax = DwordRegIndex + divideCeil(Numerator: RegWidth, Denominator: 32) - 1; |
| 3059 | int64_t OldCount; |
| 3060 | |
| 3061 | if (!Sym->isVariable()) |
| 3062 | return !Error(L: getLoc(), |
| 3063 | Msg: ".amdgcn.next_free_{v,s}gpr symbols must be variable" ); |
| 3064 | if (!Sym->getVariableValue()->evaluateAsAbsolute(Res&: OldCount)) |
| 3065 | return !Error( |
| 3066 | L: getLoc(), |
| 3067 | Msg: ".amdgcn.next_free_{v,s}gpr symbols must be absolute expressions" ); |
| 3068 | |
| 3069 | if (OldCount <= NewMax) |
| 3070 | Sym->setVariableValue(MCConstantExpr::create(Value: NewMax + 1, Ctx&: getContext())); |
| 3071 | |
| 3072 | return true; |
| 3073 | } |
| 3074 | |
| 3075 | std::unique_ptr<AMDGPUOperand> |
| 3076 | AMDGPUAsmParser::parseRegister(bool RestoreOnFailure) { |
| 3077 | const auto &Tok = getToken(); |
| 3078 | SMLoc StartLoc = Tok.getLoc(); |
| 3079 | SMLoc EndLoc = Tok.getEndLoc(); |
| 3080 | RegisterKind RegKind; |
| 3081 | MCRegister Reg; |
| 3082 | unsigned RegNum, RegWidth; |
| 3083 | |
| 3084 | if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) { |
| 3085 | return nullptr; |
| 3086 | } |
| 3087 | if (isHsaAbi(STI: getSTI())) { |
| 3088 | if (!updateGprCountSymbols(RegKind, DwordRegIndex: RegNum, RegWidth)) |
| 3089 | return nullptr; |
| 3090 | } else |
| 3091 | KernelScope.usesRegister(RegKind, DwordRegIndex: RegNum, RegWidth); |
| 3092 | return AMDGPUOperand::CreateReg(AsmParser: this, Reg, S: StartLoc, E: EndLoc); |
| 3093 | } |
| 3094 | |
| 3095 | ParseStatus AMDGPUAsmParser::parseImm(OperandVector &Operands, |
| 3096 | bool HasSP3AbsModifier, LitModifier Lit) { |
| 3097 | // TODO: add syntactic sugar for 1/(2*PI) |
| 3098 | |
| 3099 | if (isRegister() || isModifier()) |
| 3100 | return ParseStatus::NoMatch; |
| 3101 | |
| 3102 | if (Lit == LitModifier::None) { |
| 3103 | if (trySkipId(Id: "lit" )) |
| 3104 | Lit = LitModifier::Lit; |
| 3105 | else if (trySkipId(Id: "lit64" )) |
| 3106 | Lit = LitModifier::Lit64; |
| 3107 | |
| 3108 | if (Lit != LitModifier::None) { |
| 3109 | if (!skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after lit" )) |
| 3110 | return ParseStatus::Failure; |
| 3111 | ParseStatus S = parseImm(Operands, HasSP3AbsModifier, Lit); |
| 3112 | if (S.isSuccess() && |
| 3113 | !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected closing parentheses" )) |
| 3114 | return ParseStatus::Failure; |
| 3115 | return S; |
| 3116 | } |
| 3117 | } |
| 3118 | |
| 3119 | const auto &Tok = getToken(); |
| 3120 | const auto &NextTok = peekToken(); |
| 3121 | bool IsReal = Tok.is(K: AsmToken::Real); |
| 3122 | SMLoc S = getLoc(); |
| 3123 | bool Negate = false; |
| 3124 | |
| 3125 | if (!IsReal && Tok.is(K: AsmToken::Minus) && NextTok.is(K: AsmToken::Real)) { |
| 3126 | lex(); |
| 3127 | IsReal = true; |
| 3128 | Negate = true; |
| 3129 | } |
| 3130 | |
| 3131 | AMDGPUOperand::Modifiers Mods; |
| 3132 | Mods.Lit = Lit; |
| 3133 | |
| 3134 | if (IsReal) { |
| 3135 | // Floating-point expressions are not supported. |
| 3136 | // Can only allow floating-point literals with an |
| 3137 | // optional sign. |
| 3138 | |
| 3139 | StringRef Num = getTokenStr(); |
| 3140 | lex(); |
| 3141 | |
| 3142 | APFloat RealVal(APFloat::IEEEdouble()); |
| 3143 | auto roundMode = APFloat::rmNearestTiesToEven; |
| 3144 | if (errorToBool(Err: RealVal.convertFromString(Num, roundMode).takeError())) |
| 3145 | return ParseStatus::Failure; |
| 3146 | if (Negate) |
| 3147 | RealVal.changeSign(); |
| 3148 | |
| 3149 | Operands.push_back( |
| 3150 | Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: RealVal.bitcastToAPInt().getZExtValue(), |
| 3151 | Loc: S, Type: AMDGPUOperand::ImmTyNone, IsFPImm: true)); |
| 3152 | AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back()); |
| 3153 | Op.setModifiers(Mods); |
| 3154 | |
| 3155 | return ParseStatus::Success; |
| 3156 | |
| 3157 | } else { |
| 3158 | int64_t IntVal; |
| 3159 | const MCExpr *Expr; |
| 3160 | SMLoc S = getLoc(); |
| 3161 | |
| 3162 | if (HasSP3AbsModifier) { |
| 3163 | // This is a workaround for handling expressions |
| 3164 | // as arguments of SP3 'abs' modifier, for example: |
| 3165 | // |1.0| |
| 3166 | // |-1| |
| 3167 | // |1+x| |
| 3168 | // This syntax is not compatible with syntax of standard |
| 3169 | // MC expressions (due to the trailing '|'). |
| 3170 | SMLoc EndLoc; |
| 3171 | if (getParser().parsePrimaryExpr(Res&: Expr, EndLoc, TypeInfo: nullptr)) |
| 3172 | return ParseStatus::Failure; |
| 3173 | } else { |
| 3174 | if (Parser.parseExpression(Res&: Expr)) |
| 3175 | return ParseStatus::Failure; |
| 3176 | } |
| 3177 | |
| 3178 | if (Expr->evaluateAsAbsolute(Res&: IntVal)) { |
| 3179 | if (Lit == LitModifier::Lit && !isInt<32>(x: IntVal) && !isUInt<32>(x: IntVal)) |
| 3180 | return Error(L: S, Msg: "literal value out of range" ); |
| 3181 | Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: IntVal, Loc: S)); |
| 3182 | AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back()); |
| 3183 | Op.setModifiers(Mods); |
| 3184 | } else { |
| 3185 | if (Lit != LitModifier::None) |
| 3186 | return ParseStatus::NoMatch; |
| 3187 | Operands.push_back(Elt: AMDGPUOperand::CreateExpr(AsmParser: this, Expr, S)); |
| 3188 | } |
| 3189 | |
| 3190 | return ParseStatus::Success; |
| 3191 | } |
| 3192 | |
| 3193 | return ParseStatus::NoMatch; |
| 3194 | } |
| 3195 | |
| 3196 | ParseStatus AMDGPUAsmParser::parseReg(OperandVector &Operands) { |
| 3197 | if (!isRegister()) |
| 3198 | return ParseStatus::NoMatch; |
| 3199 | |
| 3200 | if (auto R = parseRegister()) { |
| 3201 | assert(R->isReg()); |
| 3202 | Operands.push_back(Elt: std::move(R)); |
| 3203 | return ParseStatus::Success; |
| 3204 | } |
| 3205 | return ParseStatus::Failure; |
| 3206 | } |
| 3207 | |
| 3208 | ParseStatus AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands, |
| 3209 | bool HasSP3AbsMod, LitModifier Lit) { |
| 3210 | ParseStatus Res = parseReg(Operands); |
| 3211 | if (!Res.isNoMatch()) |
| 3212 | return Res; |
| 3213 | if (isModifier()) |
| 3214 | return ParseStatus::NoMatch; |
| 3215 | return parseImm(Operands, HasSP3AbsModifier: HasSP3AbsMod, Lit); |
| 3216 | } |
| 3217 | |
| 3218 | bool AMDGPUAsmParser::isNamedOperandModifier(const AsmToken &Token, |
| 3219 | const AsmToken &NextToken) const { |
| 3220 | if (Token.is(K: AsmToken::Identifier) && NextToken.is(K: AsmToken::LParen)) { |
| 3221 | const auto &str = Token.getString(); |
| 3222 | return str == "abs" || str == "neg" || str == "sext" ; |
| 3223 | } |
| 3224 | return false; |
| 3225 | } |
| 3226 | |
| 3227 | bool AMDGPUAsmParser::isOpcodeModifierWithVal(const AsmToken &Token, |
| 3228 | const AsmToken &NextToken) const { |
| 3229 | return Token.is(K: AsmToken::Identifier) && NextToken.is(K: AsmToken::Colon); |
| 3230 | } |
| 3231 | |
| 3232 | bool AMDGPUAsmParser::isOperandModifier(const AsmToken &Token, |
| 3233 | const AsmToken &NextToken) const { |
| 3234 | return isNamedOperandModifier(Token, NextToken) || Token.is(K: AsmToken::Pipe); |
| 3235 | } |
| 3236 | |
| 3237 | bool AMDGPUAsmParser::isRegOrOperandModifier(const AsmToken &Token, |
| 3238 | const AsmToken &NextToken) const { |
| 3239 | return isRegister(Token, NextToken) || isOperandModifier(Token, NextToken); |
| 3240 | } |
| 3241 | |
| 3242 | // Check if this is an operand modifier or an opcode modifier |
| 3243 | // which may look like an expression but it is not. We should |
| 3244 | // avoid parsing these modifiers as expressions. Currently |
| 3245 | // recognized sequences are: |
| 3246 | // |...| |
| 3247 | // abs(...) |
| 3248 | // neg(...) |
| 3249 | // sext(...) |
| 3250 | // -reg |
| 3251 | // -|...| |
| 3252 | // -abs(...) |
| 3253 | // name:... |
| 3254 | // "name ::" is the VOPD separator, not an opcode modifier. |
| 3255 | // |
| 3256 | bool AMDGPUAsmParser::isModifier() { |
| 3257 | |
| 3258 | AsmToken Tok = getToken(); |
| 3259 | AsmToken NextToken[2]; |
| 3260 | peekTokens(Tokens: NextToken); |
| 3261 | |
| 3262 | // "name:value" is an opcode modifier. The second colon of "::" is the |
| 3263 | // VOPD separator, so a symbol written immediately before "::" is a literal. |
| 3264 | bool IsOpcodeModifier = isOpcodeModifierWithVal(Token: Tok, NextToken: NextToken[0]) && |
| 3265 | !NextToken[1].is(K: AsmToken::Colon); |
| 3266 | |
| 3267 | return isOperandModifier(Token: Tok, NextToken: NextToken[0]) || |
| 3268 | (Tok.is(K: AsmToken::Minus) && |
| 3269 | isRegOrOperandModifier(Token: NextToken[0], NextToken: NextToken[1])) || |
| 3270 | IsOpcodeModifier; |
| 3271 | } |
| 3272 | |
| 3273 | // Check if the current token is an SP3 'neg' modifier. |
| 3274 | // Currently this modifier is allowed in the following context: |
| 3275 | // |
| 3276 | // 1. Before a register, e.g. "-v0", "-v[...]" or "-[v0,v1]". |
| 3277 | // 2. Before an 'abs' modifier: -abs(...) |
| 3278 | // 3. Before an SP3 'abs' modifier: -|...| |
| 3279 | // |
| 3280 | // In all other cases "-" is handled as a part |
| 3281 | // of an expression that follows the sign. |
| 3282 | // |
| 3283 | // Note: When "-" is followed by an integer literal, |
| 3284 | // this is interpreted as integer negation rather |
| 3285 | // than a floating-point NEG modifier applied to N. |
| 3286 | // Beside being contr-intuitive, such use of floating-point |
| 3287 | // NEG modifier would have resulted in different meaning |
| 3288 | // of integer literals used with VOP1/2/C and VOP3, |
| 3289 | // for example: |
| 3290 | // v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF |
| 3291 | // v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001 |
| 3292 | // Negative fp literals with preceding "-" are |
| 3293 | // handled likewise for uniformity |
| 3294 | // |
| 3295 | bool AMDGPUAsmParser::parseSP3NegModifier() { |
| 3296 | |
| 3297 | AsmToken NextToken[2]; |
| 3298 | peekTokens(Tokens: NextToken); |
| 3299 | |
| 3300 | if (isToken(Kind: AsmToken::Minus) && |
| 3301 | (isRegister(Token: NextToken[0], NextToken: NextToken[1]) || |
| 3302 | NextToken[0].is(K: AsmToken::Pipe) || isId(Token: NextToken[0], Id: "abs" ))) { |
| 3303 | lex(); |
| 3304 | return true; |
| 3305 | } |
| 3306 | |
| 3307 | return false; |
| 3308 | } |
| 3309 | |
| 3310 | ParseStatus |
| 3311 | AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands, |
| 3312 | bool AllowImm) { |
| 3313 | bool Neg, SP3Neg; |
| 3314 | bool Abs, SP3Abs; |
| 3315 | SMLoc Loc; |
| 3316 | |
| 3317 | // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead. |
| 3318 | if (isToken(Kind: AsmToken::Minus) && peekToken().is(K: AsmToken::Minus)) |
| 3319 | return Error(L: getLoc(), Msg: "invalid syntax, expected 'neg' modifier" ); |
| 3320 | |
| 3321 | SP3Neg = parseSP3NegModifier(); |
| 3322 | |
| 3323 | Loc = getLoc(); |
| 3324 | Neg = trySkipId(Id: "neg" ); |
| 3325 | if (Neg && SP3Neg) |
| 3326 | return Error(L: Loc, Msg: "expected register or immediate" ); |
| 3327 | if (Neg && !skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after neg" )) |
| 3328 | return ParseStatus::Failure; |
| 3329 | |
| 3330 | Abs = trySkipId(Id: "abs" ); |
| 3331 | if (Abs && !skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after abs" )) |
| 3332 | return ParseStatus::Failure; |
| 3333 | |
| 3334 | LitModifier Lit = LitModifier::None; |
| 3335 | if (trySkipId(Id: "lit" )) { |
| 3336 | Lit = LitModifier::Lit; |
| 3337 | if (!skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after lit" )) |
| 3338 | return ParseStatus::Failure; |
| 3339 | } else if (trySkipId(Id: "lit64" )) { |
| 3340 | Lit = LitModifier::Lit64; |
| 3341 | if (!skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after lit64" )) |
| 3342 | return ParseStatus::Failure; |
| 3343 | if (!has64BitLiterals()) |
| 3344 | return Error(L: Loc, Msg: "lit64 is not supported on this GPU" ); |
| 3345 | } |
| 3346 | |
| 3347 | Loc = getLoc(); |
| 3348 | SP3Abs = trySkipToken(Kind: AsmToken::Pipe); |
| 3349 | if (Abs && SP3Abs) |
| 3350 | return Error(L: Loc, Msg: "expected register or immediate" ); |
| 3351 | |
| 3352 | ParseStatus Res; |
| 3353 | if (AllowImm) { |
| 3354 | Res = parseRegOrImm(Operands, HasSP3AbsMod: SP3Abs, Lit); |
| 3355 | } else { |
| 3356 | Res = parseReg(Operands); |
| 3357 | } |
| 3358 | if (!Res.isSuccess()) |
| 3359 | return (SP3Neg || Neg || SP3Abs || Abs || Lit != LitModifier::None) |
| 3360 | ? ParseStatus::Failure |
| 3361 | : Res; |
| 3362 | |
| 3363 | if (Lit != LitModifier::None && !Operands.back()->isImm()) |
| 3364 | Error(L: Loc, Msg: "expected immediate with lit modifier" ); |
| 3365 | |
| 3366 | if (SP3Abs && !skipToken(Kind: AsmToken::Pipe, ErrMsg: "expected vertical bar" )) |
| 3367 | return ParseStatus::Failure; |
| 3368 | if (Abs && !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected closing parentheses" )) |
| 3369 | return ParseStatus::Failure; |
| 3370 | if (Neg && !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected closing parentheses" )) |
| 3371 | return ParseStatus::Failure; |
| 3372 | if (Lit != LitModifier::None && |
| 3373 | !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected closing parentheses" )) |
| 3374 | return ParseStatus::Failure; |
| 3375 | |
| 3376 | AMDGPUOperand::Modifiers Mods; |
| 3377 | Mods.Abs = Abs || SP3Abs; |
| 3378 | Mods.Neg = Neg || SP3Neg; |
| 3379 | Mods.Lit = Lit; |
| 3380 | |
| 3381 | if (Mods.hasFPModifiers() || Lit != LitModifier::None) { |
| 3382 | AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back()); |
| 3383 | if (Op.isExpr()) |
| 3384 | return Error(L: Op.getStartLoc(), Msg: "expected an absolute expression" ); |
| 3385 | Op.setModifiers(Mods); |
| 3386 | } |
| 3387 | return ParseStatus::Success; |
| 3388 | } |
| 3389 | |
| 3390 | ParseStatus |
| 3391 | AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands, |
| 3392 | bool AllowImm) { |
| 3393 | bool Sext = trySkipId(Id: "sext" ); |
| 3394 | if (Sext && !skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after sext" )) |
| 3395 | return ParseStatus::Failure; |
| 3396 | |
| 3397 | ParseStatus Res; |
| 3398 | if (AllowImm) { |
| 3399 | Res = parseRegOrImm(Operands); |
| 3400 | } else { |
| 3401 | Res = parseReg(Operands); |
| 3402 | } |
| 3403 | if (!Res.isSuccess()) |
| 3404 | return Sext ? ParseStatus::Failure : Res; |
| 3405 | |
| 3406 | if (Sext && !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected closing parentheses" )) |
| 3407 | return ParseStatus::Failure; |
| 3408 | |
| 3409 | AMDGPUOperand::Modifiers Mods; |
| 3410 | Mods.Sext = Sext; |
| 3411 | |
| 3412 | if (Mods.hasIntModifiers()) { |
| 3413 | AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back()); |
| 3414 | if (Op.isExpr()) |
| 3415 | return Error(L: Op.getStartLoc(), Msg: "expected an absolute expression" ); |
| 3416 | Op.setModifiers(Mods); |
| 3417 | } |
| 3418 | |
| 3419 | return ParseStatus::Success; |
| 3420 | } |
| 3421 | |
| 3422 | ParseStatus AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) { |
| 3423 | return parseRegOrImmWithFPInputMods(Operands, AllowImm: false); |
| 3424 | } |
| 3425 | |
| 3426 | ParseStatus AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) { |
| 3427 | return parseRegOrImmWithIntInputMods(Operands, AllowImm: false); |
| 3428 | } |
| 3429 | |
| 3430 | ParseStatus AMDGPUAsmParser::parseRsrcReg(OperandVector &Operands) { |
| 3431 | // Without the marker, fall back to plain register parsing so the legacy |
| 3432 | // bare-register form (e.g. `s8`, `v8`) still assembles for indexed |
| 3433 | // buffer/image instructions. |
| 3434 | if (!trySkipId(Id: "rsrcidx" )) |
| 3435 | return parseReg(Operands); |
| 3436 | |
| 3437 | if (!skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after rsrcidx" )) |
| 3438 | return ParseStatus::Failure; |
| 3439 | |
| 3440 | SMLoc RegLoc = getLoc(); |
| 3441 | std::unique_ptr<AMDGPUOperand> Reg = parseRegister(); |
| 3442 | if (!Reg) |
| 3443 | return ParseStatus::Failure; |
| 3444 | |
| 3445 | // Enforce that the inner register is a valid index register. The matcher |
| 3446 | // predicate alone is not sufficient: if it fails, the matcher will fall back |
| 3447 | // to a non-indexed instruction variant whose resource operand happens to |
| 3448 | // accept the same register, silently dropping the `rsrcidx` intent. |
| 3449 | if (!Reg->isRsrcReg32()) |
| 3450 | return Error(L: RegLoc, Msg: "rsrcidx operand must be a 32-bit SGPR or VGPR" ); |
| 3451 | |
| 3452 | if (!skipToken(Kind: AsmToken::RParen, ErrMsg: "expected closing parenthesis" )) |
| 3453 | return ParseStatus::Failure; |
| 3454 | |
| 3455 | Operands.push_back(Elt: std::move(Reg)); |
| 3456 | return ParseStatus::Success; |
| 3457 | } |
| 3458 | |
| 3459 | ParseStatus AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) { |
| 3460 | auto Loc = getLoc(); |
| 3461 | if (trySkipId(Id: "off" )) { |
| 3462 | Operands.push_back( |
| 3463 | Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: 0, Loc, Type: AMDGPUOperand::ImmTyOff, IsFPImm: false)); |
| 3464 | return ParseStatus::Success; |
| 3465 | } |
| 3466 | |
| 3467 | if (!isRegister()) |
| 3468 | return ParseStatus::NoMatch; |
| 3469 | |
| 3470 | std::unique_ptr<AMDGPUOperand> Reg = parseRegister(); |
| 3471 | if (Reg) { |
| 3472 | Operands.push_back(Elt: std::move(Reg)); |
| 3473 | return ParseStatus::Success; |
| 3474 | } |
| 3475 | |
| 3476 | return ParseStatus::Failure; |
| 3477 | } |
| 3478 | |
| 3479 | unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) { |
| 3480 | if ((getForcedEncodingSize() == 32 && SIInstrFlags::isVOP3(O: MII, O: Inst)) || |
| 3481 | (getForcedEncodingSize() == 64 && !SIInstrFlags::isVOP3(O: MII, O: Inst)) || |
| 3482 | (isForcedDPP() && !SIInstrFlags::isDPP(O: MII, O: Inst)) || |
| 3483 | (isForcedSDWA() && !SIInstrFlags::isSDWA(O: MII, O: Inst))) |
| 3484 | return Match_InvalidOperand; |
| 3485 | |
| 3486 | if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi || |
| 3487 | Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) { |
| 3488 | // v_mac_f32/16 allow only dst_sel == DWORD; |
| 3489 | auto OpNum = |
| 3490 | AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::dst_sel); |
| 3491 | const auto &Op = Inst.getOperand(i: OpNum); |
| 3492 | if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) { |
| 3493 | return Match_InvalidOperand; |
| 3494 | } |
| 3495 | } |
| 3496 | |
| 3497 | // Asm can first try to match VOPD or VOPD3. By failing early here with |
| 3498 | // Match_InvalidOperand, the parser will retry parsing as VOPD3 or VOPD. |
| 3499 | // Checking later during validateInstruction does not give a chance to retry |
| 3500 | // parsing as a different encoding. |
| 3501 | if (tryAnotherVOPDEncoding(Inst)) |
| 3502 | return Match_InvalidOperand; |
| 3503 | |
| 3504 | return Match_Success; |
| 3505 | } |
| 3506 | |
| 3507 | static ArrayRef<unsigned> getAllVariants() { |
| 3508 | static const unsigned Variants[] = { |
| 3509 | AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3, |
| 3510 | AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::SDWA9, |
| 3511 | AMDGPUAsmVariants::DPP, AMDGPUAsmVariants::VOP3_DPP}; |
| 3512 | |
| 3513 | return ArrayRef(Variants); |
| 3514 | } |
| 3515 | |
| 3516 | // What asm variants we should check |
| 3517 | ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const { |
| 3518 | if (isForcedDPP() && isForcedVOP3()) { |
| 3519 | static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3_DPP}; |
| 3520 | return ArrayRef(Variants); |
| 3521 | } |
| 3522 | if (getForcedEncodingSize() == 32) { |
| 3523 | static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT}; |
| 3524 | return ArrayRef(Variants); |
| 3525 | } |
| 3526 | |
| 3527 | if (isForcedVOP3()) { |
| 3528 | static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3}; |
| 3529 | return ArrayRef(Variants); |
| 3530 | } |
| 3531 | |
| 3532 | if (isForcedSDWA()) { |
| 3533 | static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA, |
| 3534 | AMDGPUAsmVariants::SDWA9}; |
| 3535 | return ArrayRef(Variants); |
| 3536 | } |
| 3537 | |
| 3538 | if (isForcedDPP()) { |
| 3539 | static const unsigned Variants[] = {AMDGPUAsmVariants::DPP}; |
| 3540 | return ArrayRef(Variants); |
| 3541 | } |
| 3542 | |
| 3543 | return getAllVariants(); |
| 3544 | } |
| 3545 | |
| 3546 | StringRef AMDGPUAsmParser::getMatchedVariantName() const { |
| 3547 | if (isForcedDPP() && isForcedVOP3()) |
| 3548 | return "e64_dpp" ; |
| 3549 | |
| 3550 | if (getForcedEncodingSize() == 32) |
| 3551 | return "e32" ; |
| 3552 | |
| 3553 | if (isForcedVOP3()) |
| 3554 | return "e64" ; |
| 3555 | |
| 3556 | if (isForcedSDWA()) |
| 3557 | return "sdwa" ; |
| 3558 | |
| 3559 | if (isForcedDPP()) |
| 3560 | return "dpp" ; |
| 3561 | |
| 3562 | return "" ; |
| 3563 | } |
| 3564 | |
| 3565 | MCRegister |
| 3566 | AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const { |
| 3567 | const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode()); |
| 3568 | for (MCPhysReg Reg : Desc.implicit_uses()) { |
| 3569 | switch (Reg) { |
| 3570 | case AMDGPU::FLAT_SCR: |
| 3571 | case AMDGPU::VCC: |
| 3572 | case AMDGPU::VCC_LO: |
| 3573 | case AMDGPU::VCC_HI: |
| 3574 | case AMDGPU::M0: |
| 3575 | return Reg; |
| 3576 | default: |
| 3577 | break; |
| 3578 | } |
| 3579 | } |
| 3580 | return MCRegister(); |
| 3581 | } |
| 3582 | |
| 3583 | // NB: This code is correct only when used to check constant |
| 3584 | // bus limitations because GFX7 support no f16 inline constants. |
| 3585 | // Note that there are no cases when a GFX7 opcode violates |
| 3586 | // constant bus limitations due to the use of an f16 constant. |
| 3587 | bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst, |
| 3588 | unsigned OpIdx) const { |
| 3589 | const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode()); |
| 3590 | |
| 3591 | if (!AMDGPU::isSISrcOperand(Desc, OpNo: OpIdx) || |
| 3592 | AMDGPU::isKImmOperand(Desc, OpNo: OpIdx)) { |
| 3593 | return false; |
| 3594 | } |
| 3595 | |
| 3596 | const MCOperand &MO = Inst.getOperand(i: OpIdx); |
| 3597 | |
| 3598 | int64_t Val = MO.isImm() ? MO.getImm() : getLitValue(Expr: MO.getExpr()); |
| 3599 | auto OpSize = AMDGPU::getOperandSize(Desc, OpNo: OpIdx); |
| 3600 | |
| 3601 | switch (OpSize) { // expected operand size |
| 3602 | case 8: |
| 3603 | return AMDGPU::isInlinableLiteral64(Literal: Val, HasInv2Pi: hasInv2PiInlineImm()); |
| 3604 | case 4: |
| 3605 | return AMDGPU::isInlinableLiteral32(Literal: Val, HasInv2Pi: hasInv2PiInlineImm()); |
| 3606 | case 2: { |
| 3607 | const unsigned OperandType = Desc.operands()[OpIdx].OperandType; |
| 3608 | if (OperandType == AMDGPU::OPERAND_REG_IMM_INT16 || |
| 3609 | OperandType == AMDGPU::OPERAND_REG_INLINE_C_INT16) |
| 3610 | return AMDGPU::isInlinableLiteralI16(Literal: Val, HasInv2Pi: hasInv2PiInlineImm()); |
| 3611 | |
| 3612 | if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 || |
| 3613 | OperandType == AMDGPU::OPERAND_REG_IMM_V2INT16) |
| 3614 | return AMDGPU::isInlinableLiteralV2I16(Literal: Val); |
| 3615 | |
| 3616 | if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16 || |
| 3617 | OperandType == AMDGPU::OPERAND_REG_IMM_V2FP16) |
| 3618 | return AMDGPU::isInlinableLiteralV2F16(Literal: Val); |
| 3619 | |
| 3620 | if (OperandType == AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT) |
| 3621 | return AMDGPU::isPKFMACF16InlineConstant(Literal: Val, IsGFX11Plus: isGFX11Plus()); |
| 3622 | |
| 3623 | if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2BF16 || |
| 3624 | OperandType == AMDGPU::OPERAND_REG_IMM_V2BF16) |
| 3625 | return AMDGPU::isInlinableLiteralV2BF16(Literal: Val); |
| 3626 | |
| 3627 | if (OperandType == AMDGPU::OPERAND_REG_IMM_FP16 || |
| 3628 | OperandType == AMDGPU::OPERAND_REG_INLINE_C_FP16) |
| 3629 | return AMDGPU::isInlinableLiteralFP16(Literal: Val, HasInv2Pi: hasInv2PiInlineImm()); |
| 3630 | |
| 3631 | if (OperandType == AMDGPU::OPERAND_REG_IMM_BF16 || |
| 3632 | OperandType == AMDGPU::OPERAND_REG_INLINE_C_BF16) |
| 3633 | return AMDGPU::isInlinableLiteralBF16(Literal: Val, HasInv2Pi: hasInv2PiInlineImm()); |
| 3634 | |
| 3635 | if (OperandType == AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16 || |
| 3636 | OperandType == AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16) |
| 3637 | return false; |
| 3638 | |
| 3639 | llvm_unreachable("invalid operand type" ); |
| 3640 | } |
| 3641 | default: |
| 3642 | llvm_unreachable("invalid operand size" ); |
| 3643 | } |
| 3644 | } |
| 3645 | |
| 3646 | unsigned AMDGPUAsmParser::getConstantBusLimit(unsigned Opcode) const { |
| 3647 | if (!isGFX10Plus()) |
| 3648 | return 1; |
| 3649 | |
| 3650 | switch (Opcode) { |
| 3651 | // 64-bit shift instructions can use only one scalar value input |
| 3652 | case AMDGPU::V_LSHLREV_B64_e64: |
| 3653 | case AMDGPU::V_LSHLREV_B64_gfx10: |
| 3654 | case AMDGPU::V_LSHLREV_B64_e64_gfx11: |
| 3655 | case AMDGPU::V_LSHLREV_B64_e32_gfx12: |
| 3656 | case AMDGPU::V_LSHLREV_B64_e64_gfx12: |
| 3657 | case AMDGPU::V_LSHRREV_B64_e64: |
| 3658 | case AMDGPU::V_LSHRREV_B64_gfx10: |
| 3659 | case AMDGPU::V_LSHRREV_B64_e64_gfx11: |
| 3660 | case AMDGPU::V_LSHRREV_B64_e64_gfx12: |
| 3661 | case AMDGPU::V_ASHRREV_I64_e64: |
| 3662 | case AMDGPU::V_ASHRREV_I64_gfx10: |
| 3663 | case AMDGPU::V_ASHRREV_I64_e64_gfx11: |
| 3664 | case AMDGPU::V_ASHRREV_I64_e64_gfx12: |
| 3665 | case AMDGPU::V_LSHL_B64_e64: |
| 3666 | case AMDGPU::V_LSHR_B64_e64: |
| 3667 | case AMDGPU::V_ASHR_I64_e64: |
| 3668 | return 1; |
| 3669 | default: |
| 3670 | return 2; |
| 3671 | } |
| 3672 | } |
| 3673 | |
| 3674 | constexpr unsigned MAX_SRC_OPERANDS_NUM = 6; |
| 3675 | using OperandIndices = SmallVector<int16_t, MAX_SRC_OPERANDS_NUM>; |
| 3676 | |
| 3677 | // Get regular operand indices in the same order as specified |
| 3678 | // in the instruction (but append mandatory literals to the end). |
| 3679 | static OperandIndices getSrcOperandIndices(unsigned Opcode, |
| 3680 | bool AddMandatoryLiterals = false) { |
| 3681 | |
| 3682 | int16_t ImmIdx = |
| 3683 | AddMandatoryLiterals ? getNamedOperandIdx(Opcode, Name: OpName::imm) : -1; |
| 3684 | |
| 3685 | if (isVOPD(Opc: Opcode)) { |
| 3686 | int16_t ImmXIdx = |
| 3687 | AddMandatoryLiterals ? getNamedOperandIdx(Opcode, Name: OpName::immX) : -1; |
| 3688 | |
| 3689 | return {getNamedOperandIdx(Opcode, Name: OpName::src0X), |
| 3690 | getNamedOperandIdx(Opcode, Name: OpName::vsrc1X), |
| 3691 | getNamedOperandIdx(Opcode, Name: OpName::vsrc2X), |
| 3692 | getNamedOperandIdx(Opcode, Name: OpName::src0Y), |
| 3693 | getNamedOperandIdx(Opcode, Name: OpName::vsrc1Y), |
| 3694 | getNamedOperandIdx(Opcode, Name: OpName::vsrc2Y), |
| 3695 | ImmXIdx, |
| 3696 | ImmIdx}; |
| 3697 | } |
| 3698 | |
| 3699 | return {getNamedOperandIdx(Opcode, Name: OpName::src0), |
| 3700 | getNamedOperandIdx(Opcode, Name: OpName::src1), |
| 3701 | getNamedOperandIdx(Opcode, Name: OpName::src2), ImmIdx}; |
| 3702 | } |
| 3703 | |
| 3704 | bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) { |
| 3705 | const MCOperand &MO = Inst.getOperand(i: OpIdx); |
| 3706 | if (MO.isImm()) |
| 3707 | return !isInlineConstant(Inst, OpIdx); |
| 3708 | if (MO.isReg()) { |
| 3709 | auto Reg = MO.getReg(); |
| 3710 | if (!Reg) |
| 3711 | return false; |
| 3712 | const MCRegisterInfo *TRI = getContext().getRegisterInfo(); |
| 3713 | auto PReg = mc2PseudoReg(Reg); |
| 3714 | return isSGPR(Reg: PReg, TRI) && PReg != SGPR_NULL; |
| 3715 | } |
| 3716 | return true; |
| 3717 | } |
| 3718 | |
| 3719 | // Based on the comment for `AMDGPUInstructionSelector::selectWritelane`: |
| 3720 | // Writelane is special in that it can use SGPR and M0 (which would normally |
| 3721 | // count as using the constant bus twice - but in this case it is allowed since |
| 3722 | // the lane selector doesn't count as a use of the constant bus). However, it is |
| 3723 | // still required to abide by the 1 SGPR rule. |
| 3724 | static bool checkWriteLane(const MCInst &Inst) { |
| 3725 | const unsigned Opcode = Inst.getOpcode(); |
| 3726 | if (Opcode != V_WRITELANE_B32_gfx6_gfx7 && Opcode != V_WRITELANE_B32_vi) |
| 3727 | return false; |
| 3728 | const MCOperand &LaneSelOp = Inst.getOperand(i: 2); |
| 3729 | if (!LaneSelOp.isReg()) |
| 3730 | return false; |
| 3731 | auto LaneSelReg = mc2PseudoReg(Reg: LaneSelOp.getReg()); |
| 3732 | return LaneSelReg == M0 || LaneSelReg == M0_gfxpre11; |
| 3733 | } |
| 3734 | |
| 3735 | bool AMDGPUAsmParser::validateConstantBusLimitations( |
| 3736 | const MCInst &Inst, const OperandVector &Operands) { |
| 3737 | const unsigned Opcode = Inst.getOpcode(); |
| 3738 | const MCInstrDesc &Desc = MII.get(Opcode); |
| 3739 | MCRegister LastSGPR; |
| 3740 | unsigned ConstantBusUseCount = 0; |
| 3741 | unsigned NumLiterals = 0; |
| 3742 | unsigned LiteralSize; |
| 3743 | |
| 3744 | if (!SIInstrFlags::isVOPC(O: Desc) && !SIInstrFlags::isVOP1(O: Desc) && |
| 3745 | !SIInstrFlags::isVOP2(O: Desc) && !SIInstrFlags::isVOP3Like(O: Desc) && |
| 3746 | !SIInstrFlags::isSDWA(O: Desc) && !isVOPD(Opc: Opcode)) |
| 3747 | return true; |
| 3748 | |
| 3749 | if (checkWriteLane(Inst)) |
| 3750 | return true; |
| 3751 | |
| 3752 | // Check special imm operands (used by madmk, etc) |
| 3753 | if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::imm)) { |
| 3754 | ++NumLiterals; |
| 3755 | LiteralSize = 4; |
| 3756 | } |
| 3757 | |
| 3758 | SmallDenseSet<MCRegister> SGPRsUsed; |
| 3759 | MCRegister SGPRUsed = findImplicitSGPRReadInVOP(Inst); |
| 3760 | if (SGPRUsed) { |
| 3761 | SGPRsUsed.insert(V: SGPRUsed); |
| 3762 | ++ConstantBusUseCount; |
| 3763 | } |
| 3764 | |
| 3765 | OperandIndices OpIndices = getSrcOperandIndices(Opcode); |
| 3766 | |
| 3767 | unsigned ConstantBusLimit = getConstantBusLimit(Opcode); |
| 3768 | |
| 3769 | for (int OpIdx : OpIndices) { |
| 3770 | if (OpIdx == -1) |
| 3771 | continue; |
| 3772 | |
| 3773 | const MCOperand &MO = Inst.getOperand(i: OpIdx); |
| 3774 | if (usesConstantBus(Inst, OpIdx)) { |
| 3775 | if (MO.isReg()) { |
| 3776 | LastSGPR = mc2PseudoReg(Reg: MO.getReg()); |
| 3777 | // Pairs of registers with a partial intersections like these |
| 3778 | // s0, s[0:1] |
| 3779 | // flat_scratch_lo, flat_scratch |
| 3780 | // flat_scratch_lo, flat_scratch_hi |
| 3781 | // are theoretically valid but they are disabled anyway. |
| 3782 | // Note that this code mimics SIInstrInfo::verifyInstruction |
| 3783 | if (SGPRsUsed.insert(V: LastSGPR).second) { |
| 3784 | ++ConstantBusUseCount; |
| 3785 | } |
| 3786 | } else { // Expression or a literal |
| 3787 | |
| 3788 | if (Desc.operands()[OpIdx].OperandType == MCOI::OPERAND_IMMEDIATE) |
| 3789 | continue; // special operand like VINTERP attr_chan |
| 3790 | |
| 3791 | // An instruction may use only one literal. |
| 3792 | // This has been validated on the previous step. |
| 3793 | // See validateVOPLiteral. |
| 3794 | // This literal may be used as more than one operand. |
| 3795 | // If all these operands are of the same size, |
| 3796 | // this literal counts as one scalar value. |
| 3797 | // Otherwise it counts as 2 scalar values. |
| 3798 | // See "GFX10 Shader Programming", section 3.6.2.3. |
| 3799 | |
| 3800 | unsigned Size = AMDGPU::getOperandSize(Desc, OpNo: OpIdx); |
| 3801 | if (Size < 4) |
| 3802 | Size = 4; |
| 3803 | |
| 3804 | if (NumLiterals == 0) { |
| 3805 | NumLiterals = 1; |
| 3806 | LiteralSize = Size; |
| 3807 | } else if (LiteralSize != Size) { |
| 3808 | NumLiterals = 2; |
| 3809 | } |
| 3810 | } |
| 3811 | } |
| 3812 | |
| 3813 | if (ConstantBusUseCount + NumLiterals > ConstantBusLimit) { |
| 3814 | Error(L: getOperandLoc(Operands, MCOpIdx: OpIdx), |
| 3815 | Msg: "invalid operand (violates constant bus restrictions)" ); |
| 3816 | return false; |
| 3817 | } |
| 3818 | } |
| 3819 | return true; |
| 3820 | } |
| 3821 | |
| 3822 | std::optional<unsigned> |
| 3823 | AMDGPUAsmParser::checkVOPDRegBankConstraints(const MCInst &Inst, bool AsVOPD3) { |
| 3824 | |
| 3825 | const unsigned Opcode = Inst.getOpcode(); |
| 3826 | if (!isVOPD(Opc: Opcode)) |
| 3827 | return {}; |
| 3828 | |
| 3829 | const MCRegisterInfo *TRI = getContext().getRegisterInfo(); |
| 3830 | |
| 3831 | auto getVRegIdx = [&](unsigned, unsigned OperandIdx) { |
| 3832 | const MCOperand &Opr = Inst.getOperand(i: OperandIdx); |
| 3833 | return (Opr.isReg() && !isSGPR(Reg: mc2PseudoReg(Reg: Opr.getReg()), TRI)) |
| 3834 | ? Opr.getReg() |
| 3835 | : MCRegister(); |
| 3836 | }; |
| 3837 | |
| 3838 | // On GFX1170+ if both OpX and OpY are V_MOV_B32 then OPY uses SRC2 |
| 3839 | // source-cache. |
| 3840 | bool SkipSrc = |
| 3841 | Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1170 || |
| 3842 | Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 || |
| 3843 | Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 || |
| 3844 | Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx13 || |
| 3845 | Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 || |
| 3846 | Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx13; |
| 3847 | bool AllowSameVGPR = isGFX12Plus(); |
| 3848 | |
| 3849 | if (AsVOPD3) { // Literal constants are not allowed with VOPD3. |
| 3850 | for (auto OpName : {OpName::src0X, OpName::src0Y}) { |
| 3851 | int I = getNamedOperandIdx(Opcode, Name: OpName); |
| 3852 | const MCOperand &Op = Inst.getOperand(i: I); |
| 3853 | if (!Op.isImm()) |
| 3854 | continue; |
| 3855 | int64_t Imm = Op.getImm(); |
| 3856 | if (!AMDGPU::isInlinableLiteral32(Literal: Imm, HasInv2Pi: hasInv2PiInlineImm()) && |
| 3857 | !AMDGPU::isInlinableLiteral64(Literal: Imm, HasInv2Pi: hasInv2PiInlineImm())) |
| 3858 | return (unsigned)I; |
| 3859 | } |
| 3860 | |
| 3861 | for (auto OpName : {OpName::vsrc1X, OpName::vsrc1Y, OpName::vsrc2X, |
| 3862 | OpName::vsrc2Y, OpName::imm}) { |
| 3863 | int I = getNamedOperandIdx(Opcode, Name: OpName); |
| 3864 | if (I == -1) |
| 3865 | continue; |
| 3866 | const MCOperand &Op = Inst.getOperand(i: I); |
| 3867 | if (Op.isImm()) |
| 3868 | return (unsigned)I; |
| 3869 | } |
| 3870 | } |
| 3871 | |
| 3872 | const auto &InstInfo = getVOPDInstInfo(VOPDOpcode: Opcode, InstrInfo: &MII); |
| 3873 | auto InvalidCompOprIdx = InstInfo.getInvalidCompOperandIndex( |
| 3874 | GetRegIdx: getVRegIdx, MRI: *TRI, SkipSrc, AllowSameVGPR, VOPD3: AsVOPD3); |
| 3875 | |
| 3876 | return InvalidCompOprIdx; |
| 3877 | } |
| 3878 | |
| 3879 | bool AMDGPUAsmParser::validateVOPD(const MCInst &Inst, |
| 3880 | const OperandVector &Operands) { |
| 3881 | |
| 3882 | unsigned Opcode = Inst.getOpcode(); |
| 3883 | bool AsVOPD3 = SIInstrFlags::isVOPD3(O: MII, O: Inst); |
| 3884 | |
| 3885 | if (AsVOPD3) { |
| 3886 | for (const std::unique_ptr<MCParsedAsmOperand> &Operand : Operands) { |
| 3887 | AMDGPUOperand &Op = (AMDGPUOperand &)*Operand; |
| 3888 | if ((Op.isRegKind() || Op.isImmTy(ImmT: AMDGPUOperand::ImmTyNone)) && |
| 3889 | (Op.getModifiers().getFPModifiersOperand() & SISrcMods::ABS)) |
| 3890 | Error(L: Op.getStartLoc(), Msg: "ABS not allowed in VOPD3 instructions" ); |
| 3891 | } |
| 3892 | } |
| 3893 | |
| 3894 | auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3); |
| 3895 | if (!InvalidCompOprIdx.has_value()) |
| 3896 | return true; |
| 3897 | |
| 3898 | auto CompOprIdx = *InvalidCompOprIdx; |
| 3899 | const auto &InstInfo = getVOPDInstInfo(VOPDOpcode: Opcode, InstrInfo: &MII); |
| 3900 | auto ParsedIdx = |
| 3901 | std::max(a: InstInfo[VOPD::X].getIndexInParsedOperands(CompOprIdx), |
| 3902 | b: InstInfo[VOPD::Y].getIndexInParsedOperands(CompOprIdx)); |
| 3903 | assert(ParsedIdx > 0 && ParsedIdx < Operands.size()); |
| 3904 | |
| 3905 | auto Loc = ((AMDGPUOperand &)*Operands[ParsedIdx]).getStartLoc(); |
| 3906 | if (CompOprIdx == VOPD::Component::DST) { |
| 3907 | if (AsVOPD3) |
| 3908 | Error(L: Loc, Msg: "dst registers must be distinct" ); |
| 3909 | else |
| 3910 | Error(L: Loc, Msg: "one dst register must be even and the other odd" ); |
| 3911 | } else { |
| 3912 | auto CompSrcIdx = CompOprIdx - VOPD::Component::DST_NUM; |
| 3913 | Error(L: Loc, Msg: Twine("src" ) + Twine(CompSrcIdx) + |
| 3914 | " operands must use different VGPR banks" ); |
| 3915 | } |
| 3916 | |
| 3917 | return false; |
| 3918 | } |
| 3919 | |
| 3920 | // \returns true if \p Inst does not satisfy VOPD constraints, but can be |
| 3921 | // potentially used as VOPD3 with the same operands. |
| 3922 | bool AMDGPUAsmParser::tryVOPD3(const MCInst &Inst) { |
| 3923 | // First check if it fits VOPD |
| 3924 | auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3: false); |
| 3925 | if (!InvalidCompOprIdx.has_value()) |
| 3926 | return false; |
| 3927 | |
| 3928 | // Then if it fits VOPD3 |
| 3929 | InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3: true); |
| 3930 | if (InvalidCompOprIdx.has_value()) { |
| 3931 | // If failed operand is dst it is better to show error about VOPD3 |
| 3932 | // instruction as it has more capabilities and error message will be |
| 3933 | // more informative. If the dst is not legal for VOPD3, then it is not |
| 3934 | // legal for VOPD either. |
| 3935 | if (*InvalidCompOprIdx == VOPD::Component::DST) |
| 3936 | return true; |
| 3937 | |
| 3938 | // Otherwise prefer VOPD as we may find ourselves in an awkward situation |
| 3939 | // with a conflict in tied implicit src2 of fmac and no asm operand to |
| 3940 | // to point to. |
| 3941 | return false; |
| 3942 | } |
| 3943 | return true; |
| 3944 | } |
| 3945 | |
| 3946 | // \returns true is a VOPD3 instruction can be also represented as a shorter |
| 3947 | // VOPD encoding. |
| 3948 | bool AMDGPUAsmParser::tryVOPD(const MCInst &Inst) { |
| 3949 | const unsigned Opcode = Inst.getOpcode(); |
| 3950 | const auto &II = getVOPDInstInfo(VOPDOpcode: Opcode, InstrInfo: &MII); |
| 3951 | unsigned EncodingFamily = AMDGPU::getVOPDEncodingFamily(ST: getSTI()); |
| 3952 | if (!getCanBeVOPD(Opc: II[VOPD::X].getOpcode(), EncodingFamily, VOPD3: false).X || |
| 3953 | !getCanBeVOPD(Opc: II[VOPD::Y].getOpcode(), EncodingFamily, VOPD3: false).Y) |
| 3954 | return false; |
| 3955 | |
| 3956 | // This is an awkward exception, VOPD3 variant of V_DUAL_CNDMASK_B32 has |
| 3957 | // explicit src2 even if it is vcc_lo. If it was parsed as VOPD3 it cannot |
| 3958 | // be parsed as VOPD which does not accept src2. |
| 3959 | if (II[VOPD::X].getOpcode() == AMDGPU::V_CNDMASK_B32_e32 || |
| 3960 | II[VOPD::Y].getOpcode() == AMDGPU::V_CNDMASK_B32_e32) |
| 3961 | return false; |
| 3962 | |
| 3963 | // If any modifiers are set this cannot be VOPD. |
| 3964 | for (auto OpName : {OpName::src0X_modifiers, OpName::src0Y_modifiers, |
| 3965 | OpName::vsrc1X_modifiers, OpName::vsrc1Y_modifiers, |
| 3966 | OpName::vsrc2X_modifiers, OpName::vsrc2Y_modifiers}) { |
| 3967 | int I = getNamedOperandIdx(Opcode, Name: OpName); |
| 3968 | if (I == -1) |
| 3969 | continue; |
| 3970 | if (Inst.getOperand(i: I).getImm()) |
| 3971 | return false; |
| 3972 | } |
| 3973 | |
| 3974 | return !tryVOPD3(Inst); |
| 3975 | } |
| 3976 | |
| 3977 | // VOPD3 has more relaxed register constraints than VOPD. We prefer shorter VOPD |
| 3978 | // form but switch to VOPD3 otherwise. |
| 3979 | bool AMDGPUAsmParser::tryAnotherVOPDEncoding(const MCInst &Inst) { |
| 3980 | if (!isGFX1250Plus() || !isVOPD(Opc: Inst.getOpcode())) |
| 3981 | return false; |
| 3982 | |
| 3983 | if (SIInstrFlags::isVOPD3(O: MII, O: Inst)) |
| 3984 | return tryVOPD(Inst); |
| 3985 | return tryVOPD3(Inst); |
| 3986 | } |
| 3987 | |
| 3988 | bool AMDGPUAsmParser::validateIntClampSupported(const MCInst &Inst) { |
| 3989 | |
| 3990 | const unsigned Opc = Inst.getOpcode(); |
| 3991 | |
| 3992 | if (SIInstrFlags::hasIntClamp(O: MII, O: Inst) && !hasIntClamp()) { |
| 3993 | int ClampIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::clamp); |
| 3994 | assert(ClampIdx != -1); |
| 3995 | return Inst.getOperand(i: ClampIdx).getImm() == 0; |
| 3996 | } |
| 3997 | |
| 3998 | return true; |
| 3999 | } |
| 4000 | |
| 4001 | bool AMDGPUAsmParser::validateMIMGDataSize(const MCInst &Inst, SMLoc IDLoc) { |
| 4002 | |
| 4003 | const unsigned Opc = Inst.getOpcode(); |
| 4004 | const MCInstrDesc &Desc = MII.get(Opcode: Opc); |
| 4005 | |
| 4006 | if ((SIInstrFlags::isImage(O: Desc)) == 0) |
| 4007 | return true; |
| 4008 | |
| 4009 | int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdata); |
| 4010 | int DMaskIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dmask); |
| 4011 | int TFEIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::tfe); |
| 4012 | |
| 4013 | if (VDataIdx == -1 && isGFX10Plus()) // no return image_sample |
| 4014 | return true; |
| 4015 | |
| 4016 | if ((DMaskIdx == -1 || TFEIdx == -1) && |
| 4017 | hasBVHRayTracingInsts()) // intersect_ray |
| 4018 | return true; |
| 4019 | |
| 4020 | unsigned VDataSize = getRegOperandSize(Desc, OpNo: VDataIdx); |
| 4021 | unsigned TFESize = (TFEIdx != -1 && Inst.getOperand(i: TFEIdx).getImm()) ? 1 : 0; |
| 4022 | unsigned DMask = Inst.getOperand(i: DMaskIdx).getImm() & 0xf; |
| 4023 | if (DMask == 0) |
| 4024 | DMask = 1; |
| 4025 | |
| 4026 | bool IsPackedD16 = false; |
| 4027 | unsigned DataSize = SIInstrFlags::isGather4(O: Desc) ? 4 : llvm::popcount(Value: DMask); |
| 4028 | if (hasPackedD16()) { |
| 4029 | int D16Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::d16); |
| 4030 | IsPackedD16 = D16Idx >= 0; |
| 4031 | if (IsPackedD16 && Inst.getOperand(i: D16Idx).getImm()) |
| 4032 | DataSize = (DataSize + 1) / 2; |
| 4033 | } |
| 4034 | |
| 4035 | if ((VDataSize / 4) == DataSize + TFESize) |
| 4036 | return true; |
| 4037 | |
| 4038 | StringRef Modifiers; |
| 4039 | if (isGFX90A()) |
| 4040 | Modifiers = IsPackedD16 ? "dmask and d16" : "dmask" ; |
| 4041 | else |
| 4042 | Modifiers = IsPackedD16 ? "dmask, d16 and tfe" : "dmask and tfe" ; |
| 4043 | |
| 4044 | Error(L: IDLoc, Msg: Twine("image data size does not match " ) + Modifiers); |
| 4045 | return false; |
| 4046 | } |
| 4047 | |
| 4048 | bool AMDGPUAsmParser::validateMIMGAddrSize(const MCInst &Inst, SMLoc IDLoc) { |
| 4049 | const unsigned Opc = Inst.getOpcode(); |
| 4050 | const MCInstrDesc &Desc = MII.get(Opcode: Opc); |
| 4051 | |
| 4052 | if (!SIInstrFlags::isImage(O: Desc) || !isGFX10Plus()) |
| 4053 | return true; |
| 4054 | |
| 4055 | const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc); |
| 4056 | |
| 4057 | const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = |
| 4058 | AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode); |
| 4059 | int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr0); |
| 4060 | AMDGPU::OpName RSrcOpName = |
| 4061 | SIInstrFlags::isMIMG(O: Desc) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc; |
| 4062 | int SrsrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: RSrcOpName); |
| 4063 | int DimIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dim); |
| 4064 | int A16Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::a16); |
| 4065 | |
| 4066 | assert(VAddr0Idx != -1); |
| 4067 | assert(SrsrcIdx != -1); |
| 4068 | assert(SrsrcIdx > VAddr0Idx); |
| 4069 | |
| 4070 | bool IsA16 = (A16Idx != -1 && Inst.getOperand(i: A16Idx).getImm()); |
| 4071 | if (BaseOpcode->BVH) { |
| 4072 | if (IsA16 == BaseOpcode->A16) |
| 4073 | return true; |
| 4074 | Error(L: IDLoc, Msg: "image address size does not match a16" ); |
| 4075 | return false; |
| 4076 | } |
| 4077 | |
| 4078 | unsigned Dim = Inst.getOperand(i: DimIdx).getImm(); |
| 4079 | const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfoByEncoding(DimEnc: Dim); |
| 4080 | bool IsNSA = SrsrcIdx - VAddr0Idx > 1; |
| 4081 | unsigned ActualAddrSize = |
| 4082 | IsNSA ? SrsrcIdx - VAddr0Idx : getRegOperandSize(Desc, OpNo: VAddr0Idx) / 4; |
| 4083 | |
| 4084 | unsigned ExpectedAddrSize = |
| 4085 | AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim: DimInfo, IsA16, IsG16Supported: hasG16()); |
| 4086 | |
| 4087 | if (IsNSA) { |
| 4088 | if (hasPartialNSAEncoding() && |
| 4089 | ExpectedAddrSize > getNSAMaxSize(HasSampler: SIInstrFlags::isVSAMPLE(O: Desc))) { |
| 4090 | int VAddrLastIdx = SrsrcIdx - 1; |
| 4091 | unsigned VAddrLastSize = getRegOperandSize(Desc, OpNo: VAddrLastIdx) / 4; |
| 4092 | |
| 4093 | ActualAddrSize = VAddrLastIdx - VAddr0Idx + VAddrLastSize; |
| 4094 | } |
| 4095 | } else { |
| 4096 | if (ExpectedAddrSize > 12) |
| 4097 | ExpectedAddrSize = 16; |
| 4098 | |
| 4099 | // Allow oversized 8 VGPR vaddr when only 5/6/7 VGPRs are required. |
| 4100 | // This provides backward compatibility for assembly created |
| 4101 | // before 160b/192b/224b types were directly supported. |
| 4102 | if (ActualAddrSize == 8 && (ExpectedAddrSize >= 5 && ExpectedAddrSize <= 7)) |
| 4103 | return true; |
| 4104 | } |
| 4105 | |
| 4106 | if (ActualAddrSize == ExpectedAddrSize) |
| 4107 | return true; |
| 4108 | |
| 4109 | Error(L: IDLoc, Msg: "image address size does not match dim and a16" ); |
| 4110 | return false; |
| 4111 | } |
| 4112 | |
| 4113 | bool AMDGPUAsmParser::validateMIMGAtomicDMask(const MCInst &Inst) { |
| 4114 | |
| 4115 | const unsigned Opc = Inst.getOpcode(); |
| 4116 | const MCInstrDesc &Desc = MII.get(Opcode: Opc); |
| 4117 | |
| 4118 | if ((SIInstrFlags::isImage(O: Desc)) == 0) |
| 4119 | return true; |
| 4120 | if (!Desc.mayLoad() || !Desc.mayStore()) |
| 4121 | return true; // Not atomic |
| 4122 | |
| 4123 | int DMaskIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dmask); |
| 4124 | unsigned DMask = Inst.getOperand(i: DMaskIdx).getImm() & 0xf; |
| 4125 | |
| 4126 | // This is an incomplete check because image_atomic_cmpswap |
| 4127 | // may only use 0x3 and 0xf while other atomic operations |
| 4128 | // may use 0x1 and 0x3. However these limitations are |
| 4129 | // verified when we check that dmask matches dst size. |
| 4130 | return DMask == 0x1 || DMask == 0x3 || DMask == 0xf; |
| 4131 | } |
| 4132 | |
| 4133 | bool AMDGPUAsmParser::validateMIMGGatherDMask(const MCInst &Inst) { |
| 4134 | |
| 4135 | const unsigned Opc = Inst.getOpcode(); |
| 4136 | |
| 4137 | if (!SIInstrFlags::isGather4(O: MII, O: Inst)) |
| 4138 | return true; |
| 4139 | |
| 4140 | int DMaskIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dmask); |
| 4141 | unsigned DMask = Inst.getOperand(i: DMaskIdx).getImm() & 0xf; |
| 4142 | |
| 4143 | // GATHER4 instructions use dmask in a different fashion compared to |
| 4144 | // other MIMG instructions. The only useful DMASK values are |
| 4145 | // 1=red, 2=green, 4=blue, 8=alpha. (e.g. 1 returns |
| 4146 | // (red,red,red,red) etc.) The ISA document doesn't mention |
| 4147 | // this. |
| 4148 | return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8; |
| 4149 | } |
| 4150 | |
| 4151 | bool AMDGPUAsmParser::validateMIMGDim(const MCInst &Inst, |
| 4152 | const OperandVector &Operands) { |
| 4153 | if (!isGFX10Plus()) |
| 4154 | return true; |
| 4155 | |
| 4156 | const unsigned Opc = Inst.getOpcode(); |
| 4157 | |
| 4158 | if ((SIInstrFlags::isImage(O: MII, O: Inst)) == 0) |
| 4159 | return true; |
| 4160 | |
| 4161 | // image_bvh_intersect_ray instructions do not have dim |
| 4162 | if (AMDGPU::getMIMGBaseOpcode(Opc)->BVH) |
| 4163 | return true; |
| 4164 | |
| 4165 | for (unsigned i = 1, e = Operands.size(); i != e; ++i) { |
| 4166 | AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]); |
| 4167 | if (Op.isDim()) |
| 4168 | return true; |
| 4169 | } |
| 4170 | return false; |
| 4171 | } |
| 4172 | |
| 4173 | bool AMDGPUAsmParser::validateMIMGMSAA(const MCInst &Inst) { |
| 4174 | const unsigned Opc = Inst.getOpcode(); |
| 4175 | |
| 4176 | if ((SIInstrFlags::isImage(O: MII, O: Inst)) == 0) |
| 4177 | return true; |
| 4178 | |
| 4179 | const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc); |
| 4180 | const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = |
| 4181 | AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode); |
| 4182 | |
| 4183 | if (!BaseOpcode->MSAA) |
| 4184 | return true; |
| 4185 | |
| 4186 | int DimIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dim); |
| 4187 | assert(DimIdx != -1); |
| 4188 | |
| 4189 | unsigned Dim = Inst.getOperand(i: DimIdx).getImm(); |
| 4190 | const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfoByEncoding(DimEnc: Dim); |
| 4191 | |
| 4192 | return DimInfo->MSAA; |
| 4193 | } |
| 4194 | |
| 4195 | static bool IsMovrelsSDWAOpcode(const unsigned Opcode) { |
| 4196 | switch (Opcode) { |
| 4197 | case AMDGPU::V_MOVRELS_B32_sdwa_gfx10: |
| 4198 | case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10: |
| 4199 | case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10: |
| 4200 | return true; |
| 4201 | default: |
| 4202 | return false; |
| 4203 | } |
| 4204 | } |
| 4205 | |
| 4206 | // movrels* opcodes should only allow VGPRS as src0. |
| 4207 | // This is specified in .td description for vop1/vop3, |
| 4208 | // but sdwa is handled differently. See isSDWAOperand. |
| 4209 | bool AMDGPUAsmParser::validateMovrels(const MCInst &Inst, |
| 4210 | const OperandVector &Operands) { |
| 4211 | |
| 4212 | const unsigned Opc = Inst.getOpcode(); |
| 4213 | |
| 4214 | if (!SIInstrFlags::isSDWA(O: MII, O: Inst) || !IsMovrelsSDWAOpcode(Opcode: Opc)) |
| 4215 | return true; |
| 4216 | |
| 4217 | const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0); |
| 4218 | assert(Src0Idx != -1); |
| 4219 | |
| 4220 | const MCOperand &Src0 = Inst.getOperand(i: Src0Idx); |
| 4221 | if (Src0.isReg()) { |
| 4222 | auto Reg = mc2PseudoReg(Reg: Src0.getReg()); |
| 4223 | const MCRegisterInfo *TRI = getContext().getRegisterInfo(); |
| 4224 | if (!isSGPR(Reg, TRI)) |
| 4225 | return true; |
| 4226 | } |
| 4227 | |
| 4228 | Error(L: getOperandLoc(Operands, MCOpIdx: Src0Idx), Msg: "source operand must be a VGPR" ); |
| 4229 | return false; |
| 4230 | } |
| 4231 | |
| 4232 | bool AMDGPUAsmParser::validateMAIAccWrite(const MCInst &Inst, |
| 4233 | const OperandVector &Operands) { |
| 4234 | |
| 4235 | const unsigned Opc = Inst.getOpcode(); |
| 4236 | |
| 4237 | if (Opc != AMDGPU::V_ACCVGPR_WRITE_B32_vi) |
| 4238 | return true; |
| 4239 | |
| 4240 | const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0); |
| 4241 | assert(Src0Idx != -1); |
| 4242 | |
| 4243 | const MCOperand &Src0 = Inst.getOperand(i: Src0Idx); |
| 4244 | if (!Src0.isReg()) |
| 4245 | return true; |
| 4246 | |
| 4247 | auto Reg = mc2PseudoReg(Reg: Src0.getReg()); |
| 4248 | const MCRegisterInfo *TRI = getContext().getRegisterInfo(); |
| 4249 | if (!isGFX90A() && isSGPR(Reg, TRI)) { |
| 4250 | Error(L: getOperandLoc(Operands, MCOpIdx: Src0Idx), |
| 4251 | Msg: "source operand must be either a VGPR or an inline constant" ); |
| 4252 | return false; |
| 4253 | } |
| 4254 | |
| 4255 | return true; |
| 4256 | } |
| 4257 | |
| 4258 | bool AMDGPUAsmParser::validateMAISrc2(const MCInst &Inst, |
| 4259 | const OperandVector &Operands) { |
| 4260 | unsigned Opcode = Inst.getOpcode(); |
| 4261 | |
| 4262 | if (!SIInstrFlags::isMAI(O: MII, O: Inst) || |
| 4263 | !getFeatureBits()[FeatureMFMAInlineLiteralBug]) |
| 4264 | return true; |
| 4265 | |
| 4266 | const int Src2Idx = getNamedOperandIdx(Opcode, Name: OpName::src2); |
| 4267 | if (Src2Idx == -1) |
| 4268 | return true; |
| 4269 | |
| 4270 | if (Inst.getOperand(i: Src2Idx).isImm() && isInlineConstant(Inst, OpIdx: Src2Idx)) { |
| 4271 | Error(L: getOperandLoc(Operands, MCOpIdx: Src2Idx), |
| 4272 | Msg: "inline constants are not allowed for this operand" ); |
| 4273 | return false; |
| 4274 | } |
| 4275 | |
| 4276 | return true; |
| 4277 | } |
| 4278 | |
| 4279 | bool AMDGPUAsmParser::validateMFMA(const MCInst &Inst, |
| 4280 | const OperandVector &Operands) { |
| 4281 | const unsigned Opc = Inst.getOpcode(); |
| 4282 | const MCInstrDesc &Desc = MII.get(Opcode: Opc); |
| 4283 | |
| 4284 | if (!SIInstrFlags::isMAI(O: Desc)) |
| 4285 | return true; |
| 4286 | |
| 4287 | int BlgpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::blgp); |
| 4288 | if (BlgpIdx != -1) { |
| 4289 | if (const MFMA_F8F6F4_Info *Info = AMDGPU::isMFMA_F8F6F4(Opcode: Opc)) { |
| 4290 | int CbszIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::cbsz); |
| 4291 | |
| 4292 | unsigned CBSZ = Inst.getOperand(i: CbszIdx).getImm(); |
| 4293 | unsigned BLGP = Inst.getOperand(i: BlgpIdx).getImm(); |
| 4294 | |
| 4295 | // Validate the correct register size was used for the floating point |
| 4296 | // format operands |
| 4297 | |
| 4298 | bool Success = true; |
| 4299 | if (Info->NumRegsSrcA != mfmaScaleF8F6F4FormatToNumRegs(EncodingVal: CBSZ)) { |
| 4300 | int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0); |
| 4301 | Error(L: getOperandLoc(Operands, MCOpIdx: Src0Idx), |
| 4302 | Msg: "wrong register tuple size for cbsz value " + Twine(CBSZ)); |
| 4303 | Success = false; |
| 4304 | } |
| 4305 | |
| 4306 | if (Info->NumRegsSrcB != mfmaScaleF8F6F4FormatToNumRegs(EncodingVal: BLGP)) { |
| 4307 | int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1); |
| 4308 | Error(L: getOperandLoc(Operands, MCOpIdx: Src1Idx), |
| 4309 | Msg: "wrong register tuple size for blgp value " + Twine(BLGP)); |
| 4310 | Success = false; |
| 4311 | } |
| 4312 | |
| 4313 | return Success; |
| 4314 | } |
| 4315 | } |
| 4316 | |
| 4317 | const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2); |
| 4318 | if (Src2Idx == -1) |
| 4319 | return true; |
| 4320 | |
| 4321 | const MCOperand &Src2 = Inst.getOperand(i: Src2Idx); |
| 4322 | if (!Src2.isReg()) |
| 4323 | return true; |
| 4324 | |
| 4325 | MCRegister Src2Reg = Src2.getReg(); |
| 4326 | MCRegister DstReg = Inst.getOperand(i: 0).getReg(); |
| 4327 | if (Src2Reg == DstReg) |
| 4328 | return true; |
| 4329 | |
| 4330 | const MCRegisterInfo *TRI = getContext().getRegisterInfo(); |
| 4331 | if (TRI->getRegClass(i: MII.getOpRegClassID(OpInfo: Desc.operands()[0], HwModeId: HwMode)) |
| 4332 | .getSizeInBits() <= 128) |
| 4333 | return true; |
| 4334 | |
| 4335 | if (TRI->regsOverlap(RegA: Src2Reg, RegB: DstReg)) { |
| 4336 | Error(L: getOperandLoc(Operands, MCOpIdx: Src2Idx), |
| 4337 | Msg: "source 2 operand must not partially overlap with dst" ); |
| 4338 | return false; |
| 4339 | } |
| 4340 | |
| 4341 | return true; |
| 4342 | } |
| 4343 | |
| 4344 | bool AMDGPUAsmParser::validateDivScale(const MCInst &Inst) { |
| 4345 | switch (Inst.getOpcode()) { |
| 4346 | default: |
| 4347 | return true; |
| 4348 | case V_DIV_SCALE_F32_gfx6_gfx7: |
| 4349 | case V_DIV_SCALE_F32_vi: |
| 4350 | case V_DIV_SCALE_F32_gfx10: |
| 4351 | case V_DIV_SCALE_F64_gfx6_gfx7: |
| 4352 | case V_DIV_SCALE_F64_vi: |
| 4353 | case V_DIV_SCALE_F64_gfx10: |
| 4354 | break; |
| 4355 | } |
| 4356 | |
| 4357 | // TODO: Check that src0 = src1 or src2. |
| 4358 | |
| 4359 | for (auto Name : |
| 4360 | {AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src2_modifiers, |
| 4361 | AMDGPU::OpName::src2_modifiers}) { |
| 4362 | if (Inst.getOperand(i: AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name)) |
| 4363 | .getImm() & |
| 4364 | SISrcMods::ABS) { |
| 4365 | return false; |
| 4366 | } |
| 4367 | } |
| 4368 | |
| 4369 | return true; |
| 4370 | } |
| 4371 | |
| 4372 | bool AMDGPUAsmParser::validateMIMGD16(const MCInst &Inst) { |
| 4373 | |
| 4374 | const unsigned Opc = Inst.getOpcode(); |
| 4375 | |
| 4376 | if ((SIInstrFlags::isImage(O: MII, O: Inst)) == 0) |
| 4377 | return true; |
| 4378 | |
| 4379 | int D16Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::d16); |
| 4380 | if (D16Idx >= 0 && Inst.getOperand(i: D16Idx).getImm()) { |
| 4381 | if (isCI() || isSI()) |
| 4382 | return false; |
| 4383 | } |
| 4384 | |
| 4385 | return true; |
| 4386 | } |
| 4387 | |
| 4388 | bool AMDGPUAsmParser::validateTensorR128(const MCInst &Inst) { |
| 4389 | const unsigned Opc = Inst.getOpcode(); |
| 4390 | |
| 4391 | if (!SIInstrFlags::usesTENSOR_CNT(O: MII, O: Inst)) |
| 4392 | return true; |
| 4393 | |
| 4394 | int R128Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::r128); |
| 4395 | |
| 4396 | return R128Idx < 0 || !Inst.getOperand(i: R128Idx).getImm(); |
| 4397 | } |
| 4398 | |
| 4399 | static bool IsRevOpcode(const unsigned Opcode) { |
| 4400 | switch (Opcode) { |
| 4401 | case AMDGPU::V_SUBREV_F32_e32: |
| 4402 | case AMDGPU::V_SUBREV_F32_e64: |
| 4403 | case AMDGPU::V_SUBREV_F32_e32_gfx10: |
| 4404 | case AMDGPU::V_SUBREV_F32_e32_gfx6_gfx7: |
| 4405 | case AMDGPU::V_SUBREV_F32_e32_vi: |
| 4406 | case AMDGPU::V_SUBREV_F32_e64_gfx10: |
| 4407 | case AMDGPU::V_SUBREV_F32_e64_gfx6_gfx7: |
| 4408 | case AMDGPU::V_SUBREV_F32_e64_vi: |
| 4409 | |
| 4410 | case AMDGPU::V_SUBREV_CO_U32_e32: |
| 4411 | case AMDGPU::V_SUBREV_CO_U32_e64: |
| 4412 | case AMDGPU::V_SUBREV_I32_e32_gfx6_gfx7: |
| 4413 | case AMDGPU::V_SUBREV_I32_e64_gfx6_gfx7: |
| 4414 | |
| 4415 | case AMDGPU::V_SUBBREV_U32_e32: |
| 4416 | case AMDGPU::V_SUBBREV_U32_e64: |
| 4417 | case AMDGPU::V_SUBBREV_U32_e32_gfx6_gfx7: |
| 4418 | case AMDGPU::V_SUBBREV_U32_e32_vi: |
| 4419 | case AMDGPU::V_SUBBREV_U32_e64_gfx6_gfx7: |
| 4420 | case AMDGPU::V_SUBBREV_U32_e64_vi: |
| 4421 | |
| 4422 | case AMDGPU::V_SUBREV_U32_e32: |
| 4423 | case AMDGPU::V_SUBREV_U32_e64: |
| 4424 | case AMDGPU::V_SUBREV_U32_e32_gfx9: |
| 4425 | case AMDGPU::V_SUBREV_U32_e32_vi: |
| 4426 | case AMDGPU::V_SUBREV_U32_e64_gfx9: |
| 4427 | case AMDGPU::V_SUBREV_U32_e64_vi: |
| 4428 | |
| 4429 | case AMDGPU::V_SUBREV_F16_e32: |
| 4430 | case AMDGPU::V_SUBREV_F16_e64: |
| 4431 | case AMDGPU::V_SUBREV_F16_e32_gfx10: |
| 4432 | case AMDGPU::V_SUBREV_F16_e32_vi: |
| 4433 | case AMDGPU::V_SUBREV_F16_e64_gfx10: |
| 4434 | case AMDGPU::V_SUBREV_F16_e64_vi: |
| 4435 | |
| 4436 | case AMDGPU::V_SUBREV_U16_e32: |
| 4437 | case AMDGPU::V_SUBREV_U16_e64: |
| 4438 | case AMDGPU::V_SUBREV_U16_e32_vi: |
| 4439 | case AMDGPU::V_SUBREV_U16_e64_vi: |
| 4440 | |
| 4441 | case AMDGPU::V_SUBREV_CO_U32_e32_gfx9: |
| 4442 | case AMDGPU::V_SUBREV_CO_U32_e64_gfx10: |
| 4443 | case AMDGPU::V_SUBREV_CO_U32_e64_gfx9: |
| 4444 | |
| 4445 | case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9: |
| 4446 | case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9: |
| 4447 | |
| 4448 | case AMDGPU::V_SUBREV_NC_U32_e32_gfx10: |
| 4449 | case AMDGPU::V_SUBREV_NC_U32_e64_gfx10: |
| 4450 | |
| 4451 | case AMDGPU::V_SUBREV_CO_CI_U32_e32_gfx10: |
| 4452 | case AMDGPU::V_SUBREV_CO_CI_U32_e64_gfx10: |
| 4453 | |
| 4454 | case AMDGPU::V_LSHRREV_B32_e32: |
| 4455 | case AMDGPU::V_LSHRREV_B32_e64: |
| 4456 | case AMDGPU::V_LSHRREV_B32_e32_gfx6_gfx7: |
| 4457 | case AMDGPU::V_LSHRREV_B32_e64_gfx6_gfx7: |
| 4458 | case AMDGPU::V_LSHRREV_B32_e32_vi: |
| 4459 | case AMDGPU::V_LSHRREV_B32_e64_vi: |
| 4460 | case AMDGPU::V_LSHRREV_B32_e32_gfx10: |
| 4461 | case AMDGPU::V_LSHRREV_B32_e64_gfx10: |
| 4462 | |
| 4463 | case AMDGPU::V_ASHRREV_I32_e32: |
| 4464 | case AMDGPU::V_ASHRREV_I32_e64: |
| 4465 | case AMDGPU::V_ASHRREV_I32_e32_gfx10: |
| 4466 | case AMDGPU::V_ASHRREV_I32_e32_gfx6_gfx7: |
| 4467 | case AMDGPU::V_ASHRREV_I32_e32_vi: |
| 4468 | case AMDGPU::V_ASHRREV_I32_e64_gfx10: |
| 4469 | case AMDGPU::V_ASHRREV_I32_e64_gfx6_gfx7: |
| 4470 | case AMDGPU::V_ASHRREV_I32_e64_vi: |
| 4471 | |
| 4472 | case AMDGPU::V_LSHLREV_B32_e32: |
| 4473 | case AMDGPU::V_LSHLREV_B32_e64: |
| 4474 | case AMDGPU::V_LSHLREV_B32_e32_gfx10: |
| 4475 | case AMDGPU::V_LSHLREV_B32_e32_gfx6_gfx7: |
| 4476 | case AMDGPU::V_LSHLREV_B32_e32_vi: |
| 4477 | case AMDGPU::V_LSHLREV_B32_e64_gfx10: |
| 4478 | case AMDGPU::V_LSHLREV_B32_e64_gfx6_gfx7: |
| 4479 | case AMDGPU::V_LSHLREV_B32_e64_vi: |
| 4480 | |
| 4481 | case AMDGPU::V_LSHLREV_B16_e32: |
| 4482 | case AMDGPU::V_LSHLREV_B16_e64: |
| 4483 | case AMDGPU::V_LSHLREV_B16_e32_vi: |
| 4484 | case AMDGPU::V_LSHLREV_B16_e64_vi: |
| 4485 | case AMDGPU::V_LSHLREV_B16_gfx10: |
| 4486 | |
| 4487 | case AMDGPU::V_LSHRREV_B16_e32: |
| 4488 | case AMDGPU::V_LSHRREV_B16_e64: |
| 4489 | case AMDGPU::V_LSHRREV_B16_e32_vi: |
| 4490 | case AMDGPU::V_LSHRREV_B16_e64_vi: |
| 4491 | case AMDGPU::V_LSHRREV_B16_gfx10: |
| 4492 | |
| 4493 | case AMDGPU::V_ASHRREV_I16_e32: |
| 4494 | case AMDGPU::V_ASHRREV_I16_e64: |
| 4495 | case AMDGPU::V_ASHRREV_I16_e32_vi: |
| 4496 | case AMDGPU::V_ASHRREV_I16_e64_vi: |
| 4497 | case AMDGPU::V_ASHRREV_I16_gfx10: |
| 4498 | |
| 4499 | case AMDGPU::V_LSHLREV_B64_e64: |
| 4500 | case AMDGPU::V_LSHLREV_B64_gfx10: |
| 4501 | case AMDGPU::V_LSHLREV_B64_vi: |
| 4502 | |
| 4503 | case AMDGPU::V_LSHRREV_B64_e64: |
| 4504 | case AMDGPU::V_LSHRREV_B64_gfx10: |
| 4505 | case AMDGPU::V_LSHRREV_B64_vi: |
| 4506 | |
| 4507 | case AMDGPU::V_ASHRREV_I64_e64: |
| 4508 | case AMDGPU::V_ASHRREV_I64_gfx10: |
| 4509 | case AMDGPU::V_ASHRREV_I64_vi: |
| 4510 | |
| 4511 | case AMDGPU::V_PK_LSHLREV_B16: |
| 4512 | case AMDGPU::V_PK_LSHLREV_B16_gfx10: |
| 4513 | case AMDGPU::V_PK_LSHLREV_B16_vi: |
| 4514 | |
| 4515 | case AMDGPU::V_PK_LSHRREV_B16: |
| 4516 | case AMDGPU::V_PK_LSHRREV_B16_gfx10: |
| 4517 | case AMDGPU::V_PK_LSHRREV_B16_vi: |
| 4518 | case AMDGPU::V_PK_ASHRREV_I16: |
| 4519 | case AMDGPU::V_PK_ASHRREV_I16_gfx10: |
| 4520 | case AMDGPU::V_PK_ASHRREV_I16_vi: |
| 4521 | return true; |
| 4522 | default: |
| 4523 | return false; |
| 4524 | } |
| 4525 | } |
| 4526 | |
| 4527 | bool AMDGPUAsmParser::validateLdsDirect(const MCInst &Inst, |
| 4528 | const OperandVector &Operands) { |
| 4529 | const unsigned Opcode = Inst.getOpcode(); |
| 4530 | |
| 4531 | // lds_direct register is defined so that it can be used |
| 4532 | // with 9-bit operands only. Ignore encodings which do not accept these. |
| 4533 | if (!SIInstrFlags::isVOP1(O: MII, O: Inst) && !SIInstrFlags::isVOP2(O: MII, O: Inst) && |
| 4534 | !SIInstrFlags::isVOP3Like(O: MII, O: Inst) && |
| 4535 | !SIInstrFlags::isVOPC(O: MII, O: Inst) && !SIInstrFlags::isSDWA(O: MII, O: Inst)) |
| 4536 | return true; |
| 4537 | |
| 4538 | for (auto SrcName : {OpName::src0, OpName::src1, OpName::src2}) { |
| 4539 | auto SrcIdx = getNamedOperandIdx(Opcode, Name: SrcName); |
| 4540 | if (SrcIdx == -1) |
| 4541 | break; |
| 4542 | const auto &Src = Inst.getOperand(i: SrcIdx); |
| 4543 | if (Src.isReg() && Src.getReg() == LDS_DIRECT) { |
| 4544 | |
| 4545 | if (isGFX90A() || isGFX11Plus()) { |
| 4546 | Error(L: getOperandLoc(Operands, MCOpIdx: SrcIdx), |
| 4547 | Msg: "lds_direct is not supported on this GPU" ); |
| 4548 | return false; |
| 4549 | } |
| 4550 | |
| 4551 | if (IsRevOpcode(Opcode) || SIInstrFlags::isSDWA(O: MII, O: Inst)) { |
| 4552 | Error(L: getOperandLoc(Operands, MCOpIdx: SrcIdx), |
| 4553 | Msg: "lds_direct cannot be used with this instruction" ); |
| 4554 | return false; |
| 4555 | } |
| 4556 | |
| 4557 | if (SrcName != OpName::src0) { |
| 4558 | Error(L: getOperandLoc(Operands, MCOpIdx: SrcIdx), |
| 4559 | Msg: "lds_direct may be used as src0 only" ); |
| 4560 | return false; |
| 4561 | } |
| 4562 | } |
| 4563 | } |
| 4564 | |
| 4565 | return true; |
| 4566 | } |
| 4567 | |
| 4568 | SMLoc AMDGPUAsmParser::getFlatOffsetLoc(const OperandVector &Operands) const { |
| 4569 | for (unsigned i = 1, e = Operands.size(); i != e; ++i) { |
| 4570 | AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]); |
| 4571 | if (Op.isFlatOffset()) |
| 4572 | return Op.getStartLoc(); |
| 4573 | } |
| 4574 | return getLoc(); |
| 4575 | } |
| 4576 | |
| 4577 | bool AMDGPUAsmParser::validateOffset(const MCInst &Inst, |
| 4578 | const OperandVector &Operands) { |
| 4579 | auto Opcode = Inst.getOpcode(); |
| 4580 | auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::offset); |
| 4581 | if (OpNum == -1) |
| 4582 | return true; |
| 4583 | |
| 4584 | if (SIInstrFlags::isFLAT(O: MII, O: Inst)) |
| 4585 | return validateFlatOffset(Inst, Operands); |
| 4586 | |
| 4587 | if (SIInstrFlags::isSMRD(O: MII, O: Inst)) |
| 4588 | return validateSMEMOffset(Inst, Operands); |
| 4589 | |
| 4590 | const auto &Op = Inst.getOperand(i: OpNum); |
| 4591 | // GFX12+ buffer ops: InstOffset is signed 24, but must not be a negative. |
| 4592 | if (isGFX12Plus() && SIInstrFlags::isBuffer(O: MII, O: Inst)) { |
| 4593 | const unsigned OffsetSize = 24; |
| 4594 | if (!isUIntN(N: OffsetSize - 1, x: Op.getImm())) { |
| 4595 | Error(L: getFlatOffsetLoc(Operands), |
| 4596 | Msg: Twine("expected a " ) + Twine(OffsetSize - 1) + |
| 4597 | "-bit unsigned offset for buffer ops" ); |
| 4598 | return false; |
| 4599 | } |
| 4600 | } else { |
| 4601 | const unsigned OffsetSize = 16; |
| 4602 | if (!isUIntN(N: OffsetSize, x: Op.getImm())) { |
| 4603 | Error(L: getFlatOffsetLoc(Operands), |
| 4604 | Msg: Twine("expected a " ) + Twine(OffsetSize) + "-bit unsigned offset" ); |
| 4605 | return false; |
| 4606 | } |
| 4607 | } |
| 4608 | return true; |
| 4609 | } |
| 4610 | |
| 4611 | bool AMDGPUAsmParser::validateFlatOffset(const MCInst &Inst, |
| 4612 | const OperandVector &Operands) { |
| 4613 | if (!SIInstrFlags::isFLAT(O: MII, O: Inst)) |
| 4614 | return true; |
| 4615 | |
| 4616 | auto Opcode = Inst.getOpcode(); |
| 4617 | auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::offset); |
| 4618 | assert(OpNum != -1); |
| 4619 | |
| 4620 | const auto &Op = Inst.getOperand(i: OpNum); |
| 4621 | if (!hasFlatOffsets() && Op.getImm() != 0) { |
| 4622 | Error(L: getFlatOffsetLoc(Operands), |
| 4623 | Msg: "flat offset modifier is not supported on this GPU" ); |
| 4624 | return false; |
| 4625 | } |
| 4626 | |
| 4627 | // For pre-GFX12 FLAT instructions the offset must be positive; |
| 4628 | // MSB is ignored and forced to zero. |
| 4629 | unsigned OffsetSize = AMDGPU::getNumFlatOffsetBits(ST: getSTI()); |
| 4630 | bool AllowNegative = |
| 4631 | SIInstrFlags::isSegmentSpecificFLAT(O: MII, O: Inst) || isGFX12Plus(); |
| 4632 | if (!isIntN(N: OffsetSize, x: Op.getImm()) || (!AllowNegative && Op.getImm() < 0)) { |
| 4633 | Error(L: getFlatOffsetLoc(Operands), |
| 4634 | Msg: Twine("expected a " ) + |
| 4635 | (AllowNegative ? Twine(OffsetSize) + "-bit signed offset" |
| 4636 | : Twine(OffsetSize - 1) + "-bit unsigned offset" )); |
| 4637 | return false; |
| 4638 | } |
| 4639 | |
| 4640 | return true; |
| 4641 | } |
| 4642 | |
| 4643 | SMLoc AMDGPUAsmParser::getSMEMOffsetLoc(const OperandVector &Operands) const { |
| 4644 | // Start with second operand because SMEM Offset cannot be dst or src0. |
| 4645 | for (unsigned i = 2, e = Operands.size(); i != e; ++i) { |
| 4646 | AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]); |
| 4647 | if (Op.isSMEMOffset() || Op.isSMEMOffsetMod()) |
| 4648 | return Op.getStartLoc(); |
| 4649 | } |
| 4650 | return getLoc(); |
| 4651 | } |
| 4652 | |
| 4653 | bool AMDGPUAsmParser::validateSMEMOffset(const MCInst &Inst, |
| 4654 | const OperandVector &Operands) { |
| 4655 | if (isCI() || isSI()) |
| 4656 | return true; |
| 4657 | |
| 4658 | if (!SIInstrFlags::isSMRD(O: MII, O: Inst)) |
| 4659 | return true; |
| 4660 | |
| 4661 | auto Opcode = Inst.getOpcode(); |
| 4662 | auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::offset); |
| 4663 | if (OpNum == -1) |
| 4664 | return true; |
| 4665 | |
| 4666 | const auto &Op = Inst.getOperand(i: OpNum); |
| 4667 | if (!Op.isImm()) |
| 4668 | return true; |
| 4669 | |
| 4670 | uint64_t Offset = Op.getImm(); |
| 4671 | bool IsBuffer = AMDGPU::getSMEMIsBuffer(Opc: Opcode); |
| 4672 | if (AMDGPU::isLegalSMRDEncodedUnsignedOffset(ST: getSTI(), EncodedOffset: Offset) || |
| 4673 | AMDGPU::isLegalSMRDEncodedSignedOffset(ST: getSTI(), EncodedOffset: Offset, IsBuffer)) |
| 4674 | return true; |
| 4675 | |
| 4676 | Error(L: getSMEMOffsetLoc(Operands), |
| 4677 | Msg: isGFX12Plus() && IsBuffer |
| 4678 | ? "expected a 23-bit unsigned offset for buffer ops" |
| 4679 | : isGFX12Plus() ? "expected a 24-bit signed offset" |
| 4680 | : (isVI() || IsBuffer) ? "expected a 20-bit unsigned offset" |
| 4681 | : "expected a 21-bit signed offset" ); |
| 4682 | |
| 4683 | return false; |
| 4684 | } |
| 4685 | |
| 4686 | // GFX12+ SMEM prefetches and probes encode an immediate sdata in 5 bits. |
| 4687 | bool AMDGPUAsmParser::validateSMEMSData(const MCInst &Inst, |
| 4688 | const OperandVector &Operands) { |
| 4689 | if (!isGFX12Plus() || !SIInstrFlags::isSMRD(O: MII, O: Inst)) |
| 4690 | return true; |
| 4691 | |
| 4692 | int OpNum = |
| 4693 | AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::sdata); |
| 4694 | if (OpNum == -1 || !Inst.getOperand(i: OpNum).isImm() || |
| 4695 | isUInt<5>(x: Inst.getOperand(i: OpNum).getImm())) |
| 4696 | return true; |
| 4697 | |
| 4698 | Error(L: getOperandLoc(Operands, MCOpIdx: OpNum), Msg: "expected a 5-bit unsigned immediate" ); |
| 4699 | return false; |
| 4700 | } |
| 4701 | |
| 4702 | // On subtargets with FeatureBF16InlineConstFromUpperFP32 the hardware generates |
| 4703 | // a bf16 inline constant in the high half of the corresponding fp32 inline |
| 4704 | // constant. A VOP1 bf16 opcode (v_cvt_f32_bf16 and the bf16 transcendentals) |
| 4705 | // reads the low half of its source, so it must use the VOP3 encoding with |
| 4706 | // op_sel[0] set in order to see the constant at all. |
| 4707 | bool AMDGPUAsmParser::validateBF16InlineConst(const MCInst &Inst, |
| 4708 | const OperandVector &Operands) { |
| 4709 | if (!getFeatureBits()[AMDGPU::FeatureBF16InlineConstFromUpperFP32]) |
| 4710 | return true; |
| 4711 | |
| 4712 | const unsigned Opc = Inst.getOpcode(); |
| 4713 | const MCInstrDesc &Desc = MII.get(Opcode: Opc); |
| 4714 | const bool IsVOP3 = |
| 4715 | SIInstrFlags::isVOP3(O: Desc) && !SIInstrFlags::isVOP3P(O: Desc); |
| 4716 | if (!SIInstrFlags::isVOP1(O: Desc) && !IsVOP3) |
| 4717 | return true; |
| 4718 | |
| 4719 | // Only the single-source VOP1 bf16 opcodes are affected. Multi-source and |
| 4720 | // packed bf16 instructions such as v_fma_mix*_bf16 are not. |
| 4721 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::src1)) |
| 4722 | return true; |
| 4723 | |
| 4724 | const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0); |
| 4725 | if (Src0Idx == -1) |
| 4726 | return true; |
| 4727 | |
| 4728 | const MCOperandInfo &Src0Info = Desc.operands()[Src0Idx]; |
| 4729 | if (!AMDGPU::isBF16SrcOperand(OpInfo: Src0Info)) |
| 4730 | return true; |
| 4731 | |
| 4732 | const MCOperand &Src0 = Inst.getOperand(i: Src0Idx); |
| 4733 | if (!Src0.isImm() || |
| 4734 | !AMDGPU::isInlinableLiteralBF16(Literal: static_cast<int16_t>(Src0.getImm()), |
| 4735 | HasInv2Pi: hasInv2PiInlineImm())) |
| 4736 | return true; |
| 4737 | |
| 4738 | if (IsVOP3) { |
| 4739 | const int ModsIdx = |
| 4740 | AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0_modifiers); |
| 4741 | if (ModsIdx != -1 && |
| 4742 | (Inst.getOperand(i: ModsIdx).getImm() & SISrcMods::OP_SEL_0)) |
| 4743 | return true; |
| 4744 | } |
| 4745 | |
| 4746 | Error(L: getOperandLoc(Operands, MCOpIdx: Src0Idx), |
| 4747 | Msg: "bf16 inline constant is read from the high half of the fp32 inline " |
| 4748 | "constant on this GPU; use the e64 encoding with op_sel:[1,0]" ); |
| 4749 | return false; |
| 4750 | } |
| 4751 | |
| 4752 | bool AMDGPUAsmParser::validateSOPLiteral(const MCInst &Inst, |
| 4753 | const OperandVector &Operands) { |
| 4754 | unsigned Opcode = Inst.getOpcode(); |
| 4755 | const MCInstrDesc &Desc = MII.get(Opcode); |
| 4756 | if (!SIInstrFlags::isSOP2(O: Desc) && !SIInstrFlags::isSOPC(O: Desc)) |
| 4757 | return true; |
| 4758 | |
| 4759 | const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0); |
| 4760 | const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src1); |
| 4761 | |
| 4762 | const int OpIndices[] = {Src0Idx, Src1Idx}; |
| 4763 | |
| 4764 | unsigned NumExprs = 0; |
| 4765 | unsigned NumLiterals = 0; |
| 4766 | int64_t LiteralValue; |
| 4767 | |
| 4768 | for (int OpIdx : OpIndices) { |
| 4769 | if (OpIdx == -1) |
| 4770 | break; |
| 4771 | |
| 4772 | const MCOperand &MO = Inst.getOperand(i: OpIdx); |
| 4773 | // Exclude special imm operands (like that used by s_set_gpr_idx_on) |
| 4774 | if (AMDGPU::isSISrcOperand(Desc, OpNo: OpIdx)) { |
| 4775 | bool IsLit = false; |
| 4776 | std::optional<int64_t> Imm; |
| 4777 | if (MO.isImm()) { |
| 4778 | Imm = MO.getImm(); |
| 4779 | } else if (MO.isExpr()) { |
| 4780 | if (isLitExpr(Expr: MO.getExpr())) { |
| 4781 | IsLit = true; |
| 4782 | Imm = getLitValue(Expr: MO.getExpr()); |
| 4783 | } |
| 4784 | } else { |
| 4785 | continue; |
| 4786 | } |
| 4787 | |
| 4788 | if (!Imm.has_value()) { |
| 4789 | ++NumExprs; |
| 4790 | } else if (!isInlineConstant(Inst, OpIdx)) { |
| 4791 | auto OpType = static_cast<AMDGPU::OperandType>( |
| 4792 | Desc.operands()[OpIdx].OperandType); |
| 4793 | int64_t Value = encode32BitLiteral(Imm: *Imm, Type: OpType, IsLit); |
| 4794 | if (NumLiterals == 0 || LiteralValue != Value) { |
| 4795 | LiteralValue = Value; |
| 4796 | ++NumLiterals; |
| 4797 | } |
| 4798 | } |
| 4799 | } |
| 4800 | } |
| 4801 | |
| 4802 | if (NumLiterals + NumExprs <= 1) |
| 4803 | return true; |
| 4804 | |
| 4805 | Error(L: getOperandLoc(Operands, MCOpIdx: Src1Idx), |
| 4806 | Msg: "only one unique literal operand is allowed" ); |
| 4807 | return false; |
| 4808 | } |
| 4809 | |
| 4810 | bool AMDGPUAsmParser::validateOpSel(const MCInst &Inst) { |
| 4811 | const unsigned Opc = Inst.getOpcode(); |
| 4812 | if (isPermlane16(Opc)) { |
| 4813 | int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel); |
| 4814 | unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm(); |
| 4815 | |
| 4816 | if (OpSel & ~3) |
| 4817 | return false; |
| 4818 | } |
| 4819 | |
| 4820 | if (isGFX940() && SIInstrFlags::isDOT(O: MII, O: Inst)) { |
| 4821 | int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel); |
| 4822 | if (OpSelIdx != -1) { |
| 4823 | if (Inst.getOperand(i: OpSelIdx).getImm() != 0) |
| 4824 | return false; |
| 4825 | } |
| 4826 | int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel_hi); |
| 4827 | if (OpSelHiIdx != -1) { |
| 4828 | if (Inst.getOperand(i: OpSelHiIdx).getImm() != -1) |
| 4829 | return false; |
| 4830 | } |
| 4831 | } |
| 4832 | |
| 4833 | // op_sel[0:1] must be 0 for v_dot2_bf16_bf16 and v_dot2_f16_f16 (VOP3 Dot). |
| 4834 | if (isGFX11Plus() && SIInstrFlags::isDOT(O: MII, O: Inst) && |
| 4835 | SIInstrFlags::isVOP3(O: MII, O: Inst) && !SIInstrFlags::isVOP3P(O: MII, O: Inst)) { |
| 4836 | int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel); |
| 4837 | unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm(); |
| 4838 | if (OpSel & 3) |
| 4839 | return false; |
| 4840 | } |
| 4841 | |
| 4842 | // gfx950 v_pk_add/mul/fma_f32 have several invalid op_sel encodings |
| 4843 | if (getFeatureBits()[AMDGPU::FeatureGFX950Insts] && |
| 4844 | (Opc == AMDGPU::V_PK_ADD_F32_vi || Opc == AMDGPU::V_PK_MUL_F32_vi || |
| 4845 | Opc == AMDGPU::V_PK_FMA_F32_vi)) { |
| 4846 | int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel); |
| 4847 | unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm(); |
| 4848 | |
| 4849 | // op_sel[2:1] == 2'b01, 2'b10, or 2'b11 for v_pk_fma. |
| 4850 | if (Opc == AMDGPU::V_PK_FMA_F32_vi && |
| 4851 | (((OpSel >> 1) & 1) || ((OpSel >> 2) & 1))) |
| 4852 | return false; |
| 4853 | // op_sel[1:0] == 2'b10 for v_pk_add/mul_f32. |
| 4854 | else if ((OpSel & 3) == 2) |
| 4855 | return false; |
| 4856 | } |
| 4857 | |
| 4858 | // Packed math FP32 instructions typically accept SGPRs or VGPRs as source |
| 4859 | // operands. On gfx12+, if a source operand uses SGPRs, the HW can only read |
| 4860 | // the first SGPR and use it for both the low and high operations. |
| 4861 | if (isPackedSingleSGPRFP32Inst(Opc)) { |
| 4862 | int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0); |
| 4863 | int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1); |
| 4864 | int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel); |
| 4865 | int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel_hi); |
| 4866 | |
| 4867 | const MCOperand &Src0 = Inst.getOperand(i: Src0Idx); |
| 4868 | const MCOperand &Src1 = Inst.getOperand(i: Src1Idx); |
| 4869 | unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm(); |
| 4870 | unsigned OpSelHi = Inst.getOperand(i: OpSelHiIdx).getImm(); |
| 4871 | |
| 4872 | const MCRegisterInfo *TRI = getContext().getRegisterInfo(); |
| 4873 | |
| 4874 | auto VerifyOneSGPR = [OpSel, OpSelHi](unsigned Index) -> bool { |
| 4875 | unsigned Mask = 1U << Index; |
| 4876 | return ((OpSel & Mask) == 0) && ((OpSelHi & Mask) == 0); |
| 4877 | }; |
| 4878 | |
| 4879 | if (Src0.isReg() && isSGPR(Reg: Src0.getReg(), TRI) && |
| 4880 | !VerifyOneSGPR(/*Index=*/0)) |
| 4881 | return false; |
| 4882 | if (Src1.isReg() && isSGPR(Reg: Src1.getReg(), TRI) && |
| 4883 | !VerifyOneSGPR(/*Index=*/1)) |
| 4884 | return false; |
| 4885 | |
| 4886 | int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2); |
| 4887 | if (Src2Idx != -1) { |
| 4888 | const MCOperand &Src2 = Inst.getOperand(i: Src2Idx); |
| 4889 | if (Src2.isReg() && isSGPR(Reg: Src2.getReg(), TRI) && |
| 4890 | !VerifyOneSGPR(/*Index=*/2)) |
| 4891 | return false; |
| 4892 | } |
| 4893 | } |
| 4894 | |
| 4895 | return true; |
| 4896 | } |
| 4897 | |
| 4898 | bool AMDGPUAsmParser::validateTrue16OpSel(const MCInst &Inst) { |
| 4899 | if (!hasTrue16Insts()) |
| 4900 | return true; |
| 4901 | const MCRegisterInfo *MRI = getMRI(); |
| 4902 | const unsigned Opc = Inst.getOpcode(); |
| 4903 | int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel); |
| 4904 | if (OpSelIdx == -1) |
| 4905 | return true; |
| 4906 | unsigned OpSelOpValue = Inst.getOperand(i: OpSelIdx).getImm(); |
| 4907 | // If the value is 0 we could have a default OpSel Operand, so conservatively |
| 4908 | // allow it. |
| 4909 | if (OpSelOpValue == 0) |
| 4910 | return true; |
| 4911 | unsigned OpCount = 0; |
| 4912 | for (AMDGPU::OpName OpName : {AMDGPU::OpName::src0, AMDGPU::OpName::src1, |
| 4913 | AMDGPU::OpName::src2, AMDGPU::OpName::vdst}) { |
| 4914 | int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: OpName); |
| 4915 | if (OpIdx == -1) |
| 4916 | continue; |
| 4917 | const MCOperand &Op = Inst.getOperand(i: OpIdx); |
| 4918 | if (Op.isReg() && |
| 4919 | MRI->getRegClass(i: AMDGPU::VGPR_16RegClassID).contains(Reg: Op.getReg())) { |
| 4920 | bool VGPRSuffixIsHi = AMDGPU::isHi16Reg(Reg: Op.getReg(), MRI: *MRI); |
| 4921 | bool OpSelOpIsHi = ((OpSelOpValue & (1 << OpCount)) != 0); |
| 4922 | if (OpSelOpIsHi != VGPRSuffixIsHi) |
| 4923 | return false; |
| 4924 | } |
| 4925 | ++OpCount; |
| 4926 | } |
| 4927 | |
| 4928 | return true; |
| 4929 | } |
| 4930 | |
| 4931 | bool AMDGPUAsmParser::validateNeg(const MCInst &Inst, AMDGPU::OpName OpName) { |
| 4932 | assert(OpName == AMDGPU::OpName::neg_lo || OpName == AMDGPU::OpName::neg_hi); |
| 4933 | |
| 4934 | const unsigned Opc = Inst.getOpcode(); |
| 4935 | |
| 4936 | // v_dot4 fp8/bf8 neg_lo/neg_hi not allowed on src0 and src1 (allowed on src2) |
| 4937 | // v_wmma iu4/iu8 neg_lo not allowed on src2 (allowed on src0, src1) |
| 4938 | // v_swmmac f16/bf16 neg_lo/neg_hi not allowed on src2 (allowed on src0, src1) |
| 4939 | // other wmma/swmmac instructions don't have neg_lo/neg_hi operand. |
| 4940 | if (!SIInstrFlags::isDOT(O: MII, O: Inst) && !SIInstrFlags::isWMMA(O: MII, O: Inst) && |
| 4941 | !SIInstrFlags::isSWMMAC(O: MII, O: Inst)) |
| 4942 | return true; |
| 4943 | |
| 4944 | int NegIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: OpName); |
| 4945 | if (NegIdx == -1) |
| 4946 | return true; |
| 4947 | |
| 4948 | unsigned Neg = Inst.getOperand(i: NegIdx).getImm(); |
| 4949 | |
| 4950 | // Instructions that have neg_lo or neg_hi operand but neg modifier is allowed |
| 4951 | // on some src operands but not allowed on other. |
| 4952 | // It is convenient that such instructions don't have src_modifiers operand |
| 4953 | // for src operands that don't allow neg because they also don't allow opsel. |
| 4954 | |
| 4955 | const AMDGPU::OpName SrcMods[3] = {AMDGPU::OpName::src0_modifiers, |
| 4956 | AMDGPU::OpName::src1_modifiers, |
| 4957 | AMDGPU::OpName::src2_modifiers}; |
| 4958 | |
| 4959 | for (unsigned i = 0; i < 3; ++i) { |
| 4960 | if (!AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: SrcMods[i])) { |
| 4961 | if (Neg & (1 << i)) |
| 4962 | return false; |
| 4963 | } |
| 4964 | } |
| 4965 | |
| 4966 | return true; |
| 4967 | } |
| 4968 | |
| 4969 | bool AMDGPUAsmParser::validateDPP(const MCInst &Inst, |
| 4970 | const OperandVector &Operands) { |
| 4971 | const unsigned Opc = Inst.getOpcode(); |
| 4972 | int DppCtrlIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dpp_ctrl); |
| 4973 | if (DppCtrlIdx >= 0) { |
| 4974 | unsigned DppCtrl = Inst.getOperand(i: DppCtrlIdx).getImm(); |
| 4975 | |
| 4976 | if (!AMDGPU::isLegalDPALU_DPPControl(ST: getSTI(), DC: DppCtrl) && |
| 4977 | getSTI().hasFeature(Feature: AMDGPU::FeatureDPALU_DPP) && |
| 4978 | AMDGPU::isDPALU_DPP(OpDesc: MII.get(Opcode: Opc), MII, ST: getSTI())) { |
| 4979 | // DP ALU DPP is supported for row_newbcast only on GFX9* and row_share |
| 4980 | // only on GFX12. |
| 4981 | SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyDppCtrl, Operands); |
| 4982 | Error(L: S, Msg: isGFX12() ? "DP ALU dpp only supports row_share" |
| 4983 | : "DP ALU dpp only supports row_newbcast" ); |
| 4984 | return false; |
| 4985 | } |
| 4986 | } |
| 4987 | |
| 4988 | int Dpp8Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dpp8); |
| 4989 | bool IsDPP = DppCtrlIdx >= 0 || Dpp8Idx >= 0; |
| 4990 | |
| 4991 | if (IsDPP && !hasDPPSrc1SGPR(STI: getSTI())) { |
| 4992 | int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1); |
| 4993 | if (Src1Idx >= 0) { |
| 4994 | const MCOperand &Src1 = Inst.getOperand(i: Src1Idx); |
| 4995 | const MCRegisterInfo *TRI = getContext().getRegisterInfo(); |
| 4996 | if (Src1.isReg() && isSGPR(Reg: mc2PseudoReg(Reg: Src1.getReg()), TRI)) { |
| 4997 | Error(L: getOperandLoc(Operands, MCOpIdx: Src1Idx), |
| 4998 | Msg: "invalid operand for instruction" ); |
| 4999 | return false; |
| 5000 | } |
| 5001 | if (Src1.isImm()) { |
| 5002 | Error(L: getInstLoc(Operands), |
| 5003 | Msg: "src1 immediate operand invalid for instruction" ); |
| 5004 | return false; |
| 5005 | } |
| 5006 | } |
| 5007 | } |
| 5008 | |
| 5009 | return true; |
| 5010 | } |
| 5011 | |
| 5012 | // Check if VCC register matches wavefront size |
| 5013 | bool AMDGPUAsmParser::validateVccOperand(MCRegister Reg) const { |
| 5014 | return (Reg == AMDGPU::VCC && isWave64()) || |
| 5015 | (Reg == AMDGPU::VCC_LO && isWave32()); |
| 5016 | } |
| 5017 | |
| 5018 | // One unique literal can be used. VOP3 literal is only allowed in GFX10+ |
| 5019 | bool AMDGPUAsmParser::validateVOPLiteral(const MCInst &Inst, |
| 5020 | const OperandVector &Operands) { |
| 5021 | unsigned Opcode = Inst.getOpcode(); |
| 5022 | const MCInstrDesc &Desc = MII.get(Opcode); |
| 5023 | bool HasMandatoryLiteral = getNamedOperandIdx(Opcode, Name: OpName::imm) != -1; |
| 5024 | if (!SIInstrFlags::isVOP3Like(O: Desc) && !HasMandatoryLiteral && |
| 5025 | !isVOPD(Opc: Opcode)) |
| 5026 | return true; |
| 5027 | |
| 5028 | OperandIndices OpIndices = getSrcOperandIndices(Opcode, AddMandatoryLiterals: HasMandatoryLiteral); |
| 5029 | |
| 5030 | std::optional<unsigned> LiteralOpIdx; |
| 5031 | std::optional<uint64_t> LiteralValue; |
| 5032 | |
| 5033 | for (int OpIdx : OpIndices) { |
| 5034 | if (OpIdx == -1) |
| 5035 | continue; |
| 5036 | |
| 5037 | const MCOperand &MO = Inst.getOperand(i: OpIdx); |
| 5038 | if (!MO.isImm() && !MO.isExpr()) |
| 5039 | continue; |
| 5040 | if (!isSISrcOperand(Desc, OpNo: OpIdx)) |
| 5041 | continue; |
| 5042 | |
| 5043 | std::optional<int64_t> Imm; |
| 5044 | if (MO.isImm()) |
| 5045 | Imm = MO.getImm(); |
| 5046 | else if (MO.isExpr() && isLitExpr(Expr: MO.getExpr())) |
| 5047 | Imm = getLitValue(Expr: MO.getExpr()); |
| 5048 | |
| 5049 | bool IsAnotherLiteral = false; |
| 5050 | bool IsForcedLit = findMCOperand(Operands, MCOpIdx: OpIdx).isForcedLit(); |
| 5051 | bool IsForcedLit64 = findMCOperand(Operands, MCOpIdx: OpIdx).isForcedLit64(); |
| 5052 | if (!Imm.has_value()) { |
| 5053 | // Literal value not known, so we conservately assume it's different. |
| 5054 | IsAnotherLiteral = true; |
| 5055 | } else if (IsForcedLit || IsForcedLit64 || !isInlineConstant(Inst, OpIdx)) { |
| 5056 | uint64_t Value = *Imm; |
| 5057 | bool IsForcedFP64 = |
| 5058 | Desc.operands()[OpIdx].OperandType == AMDGPU::OPERAND_KIMM64 || |
| 5059 | (Desc.operands()[OpIdx].OperandType == AMDGPU::OPERAND_REG_IMM_FP64 && |
| 5060 | HasMandatoryLiteral); |
| 5061 | AMDGPU::OperandType OpTy = |
| 5062 | static_cast<AMDGPU::OperandType>(Desc.operands()[OpIdx].OperandType); |
| 5063 | bool IsFP64 = |
| 5064 | (IsForcedFP64 || (AMDGPU::isSISrcFPOperand(Desc, OpNo: OpIdx) && |
| 5065 | OpTy != AMDGPU::OPERAND_REG_IMM_V2INT64)) && |
| 5066 | AMDGPU::getOperandSize(OpInfo: Desc.operands()[OpIdx]) == 8; |
| 5067 | bool IsValid32Op = |
| 5068 | IsForcedLit || AMDGPU::isValid32BitLiteral(Val: Value, IsFP64); |
| 5069 | |
| 5070 | if (((!IsValid32Op && !isInt<32>(x: Value) && !isUInt<32>(x: Value) && |
| 5071 | !IsForcedFP64) || |
| 5072 | (IsForcedLit64 && !HasMandatoryLiteral)) && |
| 5073 | (!has64BitLiterals() || Desc.getSize() != 4)) { |
| 5074 | Error(L: getOperandLoc(Operands, MCOpIdx: OpIdx), |
| 5075 | Msg: "invalid operand for instruction" ); |
| 5076 | return false; |
| 5077 | } |
| 5078 | |
| 5079 | // Only src0 can use lit64 in VOP* encoding. |
| 5080 | if (!IsForcedFP64 && (IsForcedLit64 || !IsValid32Op) && |
| 5081 | OpIdx != getNamedOperandIdx(Opcode, Name: OpName::src0)) { |
| 5082 | Error(L: getOperandLoc(Operands, MCOpIdx: OpIdx), |
| 5083 | Msg: "invalid operand for instruction" ); |
| 5084 | return false; |
| 5085 | } |
| 5086 | |
| 5087 | // Compare values using the word encoded by a 32-bit literal. |
| 5088 | if (IsValid32Op && !IsForcedFP64 && !IsForcedLit64) { |
| 5089 | Value = static_cast<uint32_t>( |
| 5090 | AMDGPU::encode32BitLiteral(Imm: Value, Type: OpTy, IsLit: IsForcedLit)); |
| 5091 | } |
| 5092 | |
| 5093 | IsAnotherLiteral = !LiteralValue || *LiteralValue != Value; |
| 5094 | LiteralValue = Value; |
| 5095 | } |
| 5096 | |
| 5097 | if (IsAnotherLiteral && !HasMandatoryLiteral && |
| 5098 | !getFeatureBits()[FeatureVOP3Literal]) { |
| 5099 | Error(L: getOperandLoc(Operands, MCOpIdx: OpIdx), |
| 5100 | Msg: "literal operands are not supported" ); |
| 5101 | return false; |
| 5102 | } |
| 5103 | |
| 5104 | if (LiteralOpIdx && IsAnotherLiteral) { |
| 5105 | Error(L: getLaterLoc(a: getOperandLoc(Operands, MCOpIdx: OpIdx), |
| 5106 | b: getOperandLoc(Operands, MCOpIdx: *LiteralOpIdx)), |
| 5107 | Msg: "only one unique literal operand is allowed" ); |
| 5108 | return false; |
| 5109 | } |
| 5110 | |
| 5111 | if (IsAnotherLiteral) |
| 5112 | LiteralOpIdx = OpIdx; |
| 5113 | } |
| 5114 | |
| 5115 | return true; |
| 5116 | } |
| 5117 | |
| 5118 | // Returns -1 if not a register, 0 if VGPR and 1 if AGPR. |
| 5119 | static int IsAGPROperand(const MCInst &Inst, AMDGPU::OpName Name, |
| 5120 | const MCRegisterInfo *MRI) { |
| 5121 | int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name); |
| 5122 | if (OpIdx < 0) |
| 5123 | return -1; |
| 5124 | |
| 5125 | const MCOperand &Op = Inst.getOperand(i: OpIdx); |
| 5126 | if (!Op.isReg()) |
| 5127 | return -1; |
| 5128 | |
| 5129 | MCRegister Sub = MRI->getSubReg(Reg: Op.getReg(), Idx: AMDGPU::sub0); |
| 5130 | auto Reg = Sub ? Sub : Op.getReg(); |
| 5131 | const MCRegisterClass &AGPR32 = MRI->getRegClass(i: AMDGPU::AGPR_32RegClassID); |
| 5132 | return AGPR32.contains(Reg) ? 1 : 0; |
| 5133 | } |
| 5134 | |
| 5135 | bool AMDGPUAsmParser::validateAGPRLdSt(const MCInst &Inst) const { |
| 5136 | if (!SIInstrFlags::isFLAT(O: MII, O: Inst) && !SIInstrFlags::isBuffer(O: MII, O: Inst) && |
| 5137 | !SIInstrFlags::isMIMG(O: MII, O: Inst) && !SIInstrFlags::isDS(O: MII, O: Inst)) |
| 5138 | return true; |
| 5139 | |
| 5140 | AMDGPU::OpName DataName = SIInstrFlags::isDS(O: MII, O: Inst) |
| 5141 | ? AMDGPU::OpName::data0 |
| 5142 | : AMDGPU::OpName::vdata; |
| 5143 | |
| 5144 | const MCRegisterInfo *MRI = getMRI(); |
| 5145 | int DstAreg = IsAGPROperand(Inst, Name: AMDGPU::OpName::vdst, MRI); |
| 5146 | int DataAreg = IsAGPROperand(Inst, Name: DataName, MRI); |
| 5147 | |
| 5148 | if (SIInstrFlags::isDS(O: MII, O: Inst) && DataAreg >= 0) { |
| 5149 | int Data2Areg = IsAGPROperand(Inst, Name: AMDGPU::OpName::data1, MRI); |
| 5150 | if (Data2Areg >= 0 && Data2Areg != DataAreg) |
| 5151 | return false; |
| 5152 | } |
| 5153 | |
| 5154 | auto FB = getFeatureBits(); |
| 5155 | if (FB[AMDGPU::FeatureGFX90AInsts]) { |
| 5156 | if (DataAreg < 0 || DstAreg < 0) |
| 5157 | return true; |
| 5158 | return DstAreg == DataAreg; |
| 5159 | } |
| 5160 | |
| 5161 | return DstAreg < 1 && DataAreg < 1; |
| 5162 | } |
| 5163 | |
| 5164 | bool AMDGPUAsmParser::validateVGPRAlign(const MCInst &Inst) const { |
| 5165 | auto FB = getFeatureBits(); |
| 5166 | if (!FB[AMDGPU::FeatureRequiresAlignedVGPRs]) |
| 5167 | return true; |
| 5168 | |
| 5169 | unsigned Opc = Inst.getOpcode(); |
| 5170 | const MCRegisterInfo *MRI = getMRI(); |
| 5171 | // DS_READ_B96_TR_B6 is the only DS instruction in GFX950, that allows |
| 5172 | // unaligned VGPR. All others only allow even aligned VGPRs. |
| 5173 | if (FB[AMDGPU::FeatureGFX90AInsts] && Opc == AMDGPU::DS_READ_B96_TR_B6_vi) |
| 5174 | return true; |
| 5175 | |
| 5176 | if (FB[AMDGPU::FeatureGFX1250Insts]) { |
| 5177 | switch (Opc) { |
| 5178 | default: |
| 5179 | break; |
| 5180 | case AMDGPU::DS_LOAD_TR6_B96: |
| 5181 | case AMDGPU::DS_LOAD_TR6_B96_gfx12: |
| 5182 | // DS_LOAD_TR6_B96 is the only DS instruction in GFX1250, that |
| 5183 | // allows unaligned VGPR. All others only allow even aligned VGPRs. |
| 5184 | return true; |
| 5185 | case AMDGPU::GLOBAL_LOAD_TR6_B96: |
| 5186 | case AMDGPU::GLOBAL_LOAD_TR6_B96_gfx1250: { |
| 5187 | // GLOBAL_LOAD_TR6_B96 is the only GLOBAL instruction in GFX1250, that |
| 5188 | // allows unaligned VGPR for vdst, but other operands still only allow |
| 5189 | // even aligned VGPRs. |
| 5190 | int VAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr); |
| 5191 | if (VAddrIdx != -1) { |
| 5192 | const MCOperand &Op = Inst.getOperand(i: VAddrIdx); |
| 5193 | MCRegister Sub = MRI->getSubReg(Reg: Op.getReg(), Idx: AMDGPU::sub0); |
| 5194 | if ((Sub - AMDGPU::VGPR0) & 1) |
| 5195 | return false; |
| 5196 | } |
| 5197 | return true; |
| 5198 | } |
| 5199 | case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR: |
| 5200 | case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR_gfx1250: |
| 5201 | return true; |
| 5202 | } |
| 5203 | } |
| 5204 | |
| 5205 | const MCRegisterClass &VGPR32 = MRI->getRegClass(i: AMDGPU::VGPR_32RegClassID); |
| 5206 | const MCRegisterClass &AGPR32 = MRI->getRegClass(i: AMDGPU::AGPR_32RegClassID); |
| 5207 | for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) { |
| 5208 | const MCOperand &Op = Inst.getOperand(i: I); |
| 5209 | if (!Op.isReg()) |
| 5210 | continue; |
| 5211 | |
| 5212 | MCRegister Sub = MRI->getSubReg(Reg: Op.getReg(), Idx: AMDGPU::sub0); |
| 5213 | if (!Sub) |
| 5214 | continue; |
| 5215 | |
| 5216 | if (VGPR32.contains(Reg: Sub) && ((Sub - AMDGPU::VGPR0) & 1)) |
| 5217 | return false; |
| 5218 | if (AGPR32.contains(Reg: Sub) && ((Sub - AMDGPU::AGPR0) & 1)) |
| 5219 | return false; |
| 5220 | } |
| 5221 | |
| 5222 | return true; |
| 5223 | } |
| 5224 | |
| 5225 | SMLoc AMDGPUAsmParser::getBLGPLoc(const OperandVector &Operands) const { |
| 5226 | for (unsigned i = 1, e = Operands.size(); i != e; ++i) { |
| 5227 | AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]); |
| 5228 | if (Op.isBLGP()) |
| 5229 | return Op.getStartLoc(); |
| 5230 | } |
| 5231 | return SMLoc(); |
| 5232 | } |
| 5233 | |
| 5234 | bool AMDGPUAsmParser::validateBLGP(const MCInst &Inst, |
| 5235 | const OperandVector &Operands) { |
| 5236 | unsigned Opc = Inst.getOpcode(); |
| 5237 | int BlgpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::blgp); |
| 5238 | if (BlgpIdx == -1) |
| 5239 | return true; |
| 5240 | SMLoc BLGPLoc = getBLGPLoc(Operands); |
| 5241 | if (!BLGPLoc.isValid()) |
| 5242 | return true; |
| 5243 | bool IsNeg = StringRef(BLGPLoc.getPointer()).starts_with(Prefix: "neg:" ); |
| 5244 | auto FB = getFeatureBits(); |
| 5245 | bool UsesNeg = false; |
| 5246 | if (FB[AMDGPU::FeatureGFX940Insts]) { |
| 5247 | switch (Opc) { |
| 5248 | case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_acd: |
| 5249 | case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_vcd: |
| 5250 | case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_acd: |
| 5251 | case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_vcd: |
| 5252 | UsesNeg = true; |
| 5253 | } |
| 5254 | } |
| 5255 | |
| 5256 | if (IsNeg == UsesNeg) |
| 5257 | return true; |
| 5258 | |
| 5259 | Error(L: BLGPLoc, Msg: UsesNeg ? "invalid modifier: blgp is not supported" |
| 5260 | : "invalid modifier: neg is not supported" ); |
| 5261 | |
| 5262 | return false; |
| 5263 | } |
| 5264 | |
| 5265 | bool AMDGPUAsmParser::validateWaitCnt(const MCInst &Inst, |
| 5266 | const OperandVector &Operands) { |
| 5267 | if (!isGFX11Plus()) |
| 5268 | return true; |
| 5269 | |
| 5270 | unsigned Opc = Inst.getOpcode(); |
| 5271 | if (Opc != AMDGPU::S_WAITCNT_EXPCNT_gfx11 && |
| 5272 | Opc != AMDGPU::S_WAITCNT_LGKMCNT_gfx11 && |
| 5273 | Opc != AMDGPU::S_WAITCNT_VMCNT_gfx11 && |
| 5274 | Opc != AMDGPU::S_WAITCNT_VSCNT_gfx11) |
| 5275 | return true; |
| 5276 | |
| 5277 | int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::sdst); |
| 5278 | assert(Src0Idx >= 0 && Inst.getOperand(Src0Idx).isReg()); |
| 5279 | auto Reg = mc2PseudoReg(Reg: Inst.getOperand(i: Src0Idx).getReg()); |
| 5280 | if (Reg == AMDGPU::SGPR_NULL) |
| 5281 | return true; |
| 5282 | |
| 5283 | Error(L: getOperandLoc(Operands, MCOpIdx: Src0Idx), Msg: "src0 must be null" ); |
| 5284 | return false; |
| 5285 | } |
| 5286 | |
| 5287 | bool AMDGPUAsmParser::validateDS(const MCInst &Inst, |
| 5288 | const OperandVector &Operands) { |
| 5289 | if (!SIInstrFlags::isDS(O: MII, O: Inst)) |
| 5290 | return true; |
| 5291 | if (SIInstrFlags::isGWS(O: MII, O: Inst)) |
| 5292 | return validateGWS(Inst, Operands); |
| 5293 | // Only validate GDS for non-GWS instructions. |
| 5294 | if (hasGDS()) |
| 5295 | return true; |
| 5296 | int GDSIdx = |
| 5297 | AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::gds); |
| 5298 | if (GDSIdx < 0) |
| 5299 | return true; |
| 5300 | unsigned GDS = Inst.getOperand(i: GDSIdx).getImm(); |
| 5301 | if (GDS) { |
| 5302 | SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyGDS, Operands); |
| 5303 | Error(L: S, Msg: "gds modifier is not supported on this GPU" ); |
| 5304 | return false; |
| 5305 | } |
| 5306 | return true; |
| 5307 | } |
| 5308 | |
| 5309 | // gfx90a has an undocumented limitation: |
| 5310 | // DS_GWS opcodes must use even aligned registers. |
| 5311 | bool AMDGPUAsmParser::validateGWS(const MCInst &Inst, |
| 5312 | const OperandVector &Operands) { |
| 5313 | if (!getFeatureBits()[AMDGPU::FeatureGFX90AInsts]) |
| 5314 | return true; |
| 5315 | |
| 5316 | int Opc = Inst.getOpcode(); |
| 5317 | if (Opc != AMDGPU::DS_GWS_INIT_vi && Opc != AMDGPU::DS_GWS_BARRIER_vi && |
| 5318 | Opc != AMDGPU::DS_GWS_SEMA_BR_vi) |
| 5319 | return true; |
| 5320 | |
| 5321 | const MCRegisterInfo *MRI = getMRI(); |
| 5322 | const MCRegisterClass &VGPR32 = MRI->getRegClass(i: AMDGPU::VGPR_32RegClassID); |
| 5323 | int Data0Pos = |
| 5324 | AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::data0); |
| 5325 | assert(Data0Pos != -1); |
| 5326 | auto Reg = Inst.getOperand(i: Data0Pos).getReg(); |
| 5327 | auto RegIdx = Reg - (VGPR32.contains(Reg) ? AMDGPU::VGPR0 : AMDGPU::AGPR0); |
| 5328 | if (RegIdx & 1) { |
| 5329 | Error(L: getOperandLoc(Operands, MCOpIdx: Data0Pos), Msg: "vgpr must be even aligned" ); |
| 5330 | return false; |
| 5331 | } |
| 5332 | |
| 5333 | return true; |
| 5334 | } |
| 5335 | |
| 5336 | bool AMDGPUAsmParser::validateCoherencyBits(const MCInst &Inst, |
| 5337 | const OperandVector &Operands, |
| 5338 | SMLoc IDLoc) { |
| 5339 | int CPolPos = |
| 5340 | AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::cpol); |
| 5341 | if (CPolPos == -1) |
| 5342 | return true; |
| 5343 | |
| 5344 | unsigned CPol = Inst.getOperand(i: CPolPos).getImm(); |
| 5345 | |
| 5346 | if (CPol & CPol::CFS) { |
| 5347 | if (!hasCacheFillSize()) { |
| 5348 | SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands); |
| 5349 | StringRef CStr(S.getPointer()); |
| 5350 | S = SMLoc::getFromPointer(Ptr: &CStr.data()[CStr.find(Str: "cfs" )]); |
| 5351 | Error(L: S, Msg: "Cache fill size is not supported on this GPU" ); |
| 5352 | } else if (!validateCFSBits(Inst, Operands)) { |
| 5353 | return false; |
| 5354 | } |
| 5355 | } |
| 5356 | |
| 5357 | if (!hasScaleOffset() && (CPol & CPol::SCAL)) { |
| 5358 | SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands); |
| 5359 | StringRef CStr(S.getPointer()); |
| 5360 | S = SMLoc::getFromPointer(Ptr: &CStr.data()[CStr.find(Str: "scale_offset" )]); |
| 5361 | Error(L: S, Msg: "scale_offset is not supported on this GPU" ); |
| 5362 | } |
| 5363 | |
| 5364 | if (!isGFX1250Plus() && (CPol & CPol::NV)) { |
| 5365 | SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands); |
| 5366 | StringRef CStr(S.getPointer()); |
| 5367 | S = SMLoc::getFromPointer(Ptr: &CStr.data()[CStr.find(Str: "nv" )]); |
| 5368 | Error(L: S, Msg: "nv is not supported on this GPU" ); |
| 5369 | } |
| 5370 | |
| 5371 | if ((CPol & CPol::SCAL) && !supportsScaleOffset(MII, Opcode: Inst.getOpcode())) { |
| 5372 | SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands); |
| 5373 | StringRef CStr(S.getPointer()); |
| 5374 | S = SMLoc::getFromPointer(Ptr: &CStr.data()[CStr.find(Str: "scale_offset" )]); |
| 5375 | Error(L: S, Msg: "scale_offset is not supported for this instruction" ); |
| 5376 | } |
| 5377 | |
| 5378 | if (isGFX12Plus()) |
| 5379 | return validateTHAndScopeBits(Inst, Operands, CPol); |
| 5380 | |
| 5381 | if (SIInstrFlags::isSMRD(O: MII, O: Inst)) { |
| 5382 | if (CPol && (isSI() || isCI())) { |
| 5383 | SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands); |
| 5384 | Error(L: S, Msg: "cache policy is not supported for SMRD instructions" ); |
| 5385 | return false; |
| 5386 | } |
| 5387 | if (CPol & ~(AMDGPU::CPol::GLC | AMDGPU::CPol::DLC)) { |
| 5388 | Error(L: IDLoc, Msg: "invalid cache policy for SMEM instruction" ); |
| 5389 | return false; |
| 5390 | } |
| 5391 | } |
| 5392 | |
| 5393 | if (isGFX90A() && !isGFX940() && (CPol & CPol::SCC)) { |
| 5394 | if (!SIInstrFlags::isVMEM(O: MII, O: Inst)) { |
| 5395 | SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands); |
| 5396 | StringRef CStr(S.getPointer()); |
| 5397 | S = SMLoc::getFromPointer(Ptr: &CStr.data()[CStr.find(Str: "scc" )]); |
| 5398 | Error(L: S, |
| 5399 | Msg: "scc modifier is not supported for this instruction on this GPU" ); |
| 5400 | return false; |
| 5401 | } |
| 5402 | } |
| 5403 | |
| 5404 | if (!SIInstrFlags::isAtomic(O: MII, O: Inst)) |
| 5405 | return true; |
| 5406 | |
| 5407 | if (SIInstrFlags::isAtomicRet(O: MII, O: Inst)) { |
| 5408 | if (!SIInstrFlags::isMIMG(O: MII, O: Inst) && !(CPol & CPol::GLC)) { |
| 5409 | Error(L: IDLoc, Msg: isGFX940() ? "instruction must use sc0" |
| 5410 | : "instruction must use glc" ); |
| 5411 | return false; |
| 5412 | } |
| 5413 | } else { |
| 5414 | if (CPol & CPol::GLC) { |
| 5415 | SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands); |
| 5416 | StringRef CStr(S.getPointer()); |
| 5417 | S = SMLoc::getFromPointer( |
| 5418 | Ptr: &CStr.data()[CStr.find(Str: isGFX940() ? "sc0" : "glc" )]); |
| 5419 | Error(L: S, Msg: isGFX940() ? "instruction must not use sc0" |
| 5420 | : "instruction must not use glc" ); |
| 5421 | return false; |
| 5422 | } |
| 5423 | } |
| 5424 | |
| 5425 | return true; |
| 5426 | } |
| 5427 | |
| 5428 | bool AMDGPUAsmParser::validateTHAndScopeBits(const MCInst &Inst, |
| 5429 | const OperandVector &Operands, |
| 5430 | const unsigned CPol) { |
| 5431 | const unsigned TH = CPol & AMDGPU::CPol::TH; |
| 5432 | const unsigned Scope = CPol & AMDGPU::CPol::SCOPE; |
| 5433 | |
| 5434 | auto PrintError = [&](StringRef Msg) { |
| 5435 | SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands); |
| 5436 | Error(L: S, Msg); |
| 5437 | return false; |
| 5438 | }; |
| 5439 | |
| 5440 | if ((TH & AMDGPU::CPol::TH_ATOMIC_RETURN) && |
| 5441 | SIInstrFlags::isAtomicNoRet(O: MII, O: Inst)) |
| 5442 | return PrintError("th:TH_ATOMIC_RETURN requires a destination operand" ); |
| 5443 | |
| 5444 | if (SIInstrFlags::isAtomicRet(O: MII, O: Inst) && |
| 5445 | (SIInstrFlags::isFLAT(O: MII, O: Inst) || SIInstrFlags::isMUBUF(O: MII, O: Inst)) && |
| 5446 | (!(TH & AMDGPU::CPol::TH_ATOMIC_RETURN))) |
| 5447 | return PrintError("instruction must use th:TH_ATOMIC_RETURN" ); |
| 5448 | |
| 5449 | if (TH == 0) |
| 5450 | return true; |
| 5451 | |
| 5452 | if (SIInstrFlags::isSMRD(O: MII, O: Inst) && |
| 5453 | ((TH == AMDGPU::CPol::TH_NT_RT) || (TH == AMDGPU::CPol::TH_RT_NT) || |
| 5454 | (TH == AMDGPU::CPol::TH_NT_HT))) |
| 5455 | return PrintError("invalid th value for SMEM instruction" ); |
| 5456 | |
| 5457 | if (TH == AMDGPU::CPol::TH_BYPASS) { |
| 5458 | if ((Scope != AMDGPU::CPol::SCOPE_SYS && |
| 5459 | CPol & AMDGPU::CPol::TH_REAL_BYPASS) || |
| 5460 | (Scope == AMDGPU::CPol::SCOPE_SYS && |
| 5461 | !(CPol & AMDGPU::CPol::TH_REAL_BYPASS))) |
| 5462 | return PrintError("scope and th combination is not valid" ); |
| 5463 | } |
| 5464 | |
| 5465 | unsigned THType = AMDGPU::getTemporalHintType(TID: MII.get(Opcode: Inst.getOpcode())); |
| 5466 | if (THType == AMDGPU::CPol::TH_TYPE_ATOMIC) { |
| 5467 | if (!(CPol & AMDGPU::CPol::TH_TYPE_ATOMIC)) |
| 5468 | return PrintError("invalid th value for atomic instructions" ); |
| 5469 | } else if (THType == AMDGPU::CPol::TH_TYPE_STORE) { |
| 5470 | if (!(CPol & AMDGPU::CPol::TH_TYPE_STORE)) |
| 5471 | return PrintError("invalid th value for store instructions" ); |
| 5472 | } else { |
| 5473 | if (!(CPol & AMDGPU::CPol::TH_TYPE_LOAD)) |
| 5474 | return PrintError("invalid th value for load instructions" ); |
| 5475 | } |
| 5476 | |
| 5477 | return true; |
| 5478 | } |
| 5479 | |
| 5480 | bool AMDGPUAsmParser::validateCFSBits(const MCInst &Inst, |
| 5481 | const OperandVector &Operands) { |
| 5482 | const unsigned Opcode = Inst.getOpcode(); |
| 5483 | const MCInstrDesc &Desc = MII.get(Opcode); |
| 5484 | |
| 5485 | auto PrintError = [&](StringRef Msg) { |
| 5486 | SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands); |
| 5487 | Error(L: S, Msg); |
| 5488 | return false; |
| 5489 | }; |
| 5490 | |
| 5491 | if (SIInstrFlags::isVMEM(O: Desc)) |
| 5492 | return true; |
| 5493 | |
| 5494 | return PrintError("Cache fill size is only supported for VC operations" ); |
| 5495 | } |
| 5496 | |
| 5497 | bool AMDGPUAsmParser::validateTFE(const MCInst &Inst, |
| 5498 | const OperandVector &Operands) { |
| 5499 | const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode()); |
| 5500 | if (Desc.mayStore() && SIInstrFlags::isBuffer(O: Desc)) { |
| 5501 | SMLoc Loc = getImmLoc(Type: AMDGPUOperand::ImmTyTFE, Operands); |
| 5502 | if (Loc != getInstLoc(Operands)) { |
| 5503 | Error(L: Loc, Msg: "TFE modifier has no meaning for store instructions" ); |
| 5504 | return false; |
| 5505 | } |
| 5506 | } |
| 5507 | |
| 5508 | return true; |
| 5509 | } |
| 5510 | |
| 5511 | bool AMDGPUAsmParser::validateWMMA(const MCInst &Inst, |
| 5512 | const OperandVector &Operands) { |
| 5513 | unsigned Opc = Inst.getOpcode(); |
| 5514 | const MCRegisterInfo *TRI = getContext().getRegisterInfo(); |
| 5515 | const MCInstrDesc &Desc = MII.get(Opcode: Opc); |
| 5516 | |
| 5517 | int AFmtIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_a_fmt); |
| 5518 | if (AFmtIdx == -1) |
| 5519 | return true; |
| 5520 | unsigned AFmt = Inst.getOperand(i: AFmtIdx).getImm(); |
| 5521 | int BFmtIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_b_fmt); |
| 5522 | unsigned BFmt = Inst.getOperand(i: BFmtIdx).getImm(); |
| 5523 | |
| 5524 | auto validateFmt = [&](unsigned Fmt, AMDGPU::OpName SrcOp) -> bool { |
| 5525 | int SrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: SrcOp); |
| 5526 | unsigned RegSize = |
| 5527 | TRI->getRegClass(i: MII.getOpRegClassID(OpInfo: Desc.operands()[SrcIdx], HwModeId: HwMode)) |
| 5528 | .getSizeInBits(); |
| 5529 | |
| 5530 | if (RegSize == AMDGPU::wmmaScaleF8F6F4FormatToNumRegs(Fmt) * 32) |
| 5531 | return true; |
| 5532 | |
| 5533 | Error(L: getOperandLoc(Operands, MCOpIdx: SrcIdx), |
| 5534 | Msg: "wrong register tuple size for " + |
| 5535 | Twine(WMMAMods::ModMatrixFmt[Fmt])); |
| 5536 | return false; |
| 5537 | }; |
| 5538 | |
| 5539 | if (!validateFmt(AFmt, AMDGPU::OpName::src0) || |
| 5540 | !validateFmt(BFmt, AMDGPU::OpName::src1)) |
| 5541 | return false; |
| 5542 | |
| 5543 | int AScaleIdx = |
| 5544 | AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_a_scale_fmt); |
| 5545 | if (AScaleIdx == -1) |
| 5546 | return true; |
| 5547 | unsigned AScale = Inst.getOperand(i: AScaleIdx).getImm(); |
| 5548 | int BScaleIdx = |
| 5549 | AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_b_scale_fmt); |
| 5550 | unsigned BScale = Inst.getOperand(i: BScaleIdx).getImm(); |
| 5551 | if (!isValidWMMAScaleFmtCombination(AFmt, AScale, BFmt, BScale)) { |
| 5552 | Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyMatrixAFMT, Operands), |
| 5553 | Msg: "invalid matrix and scale format combination" ); |
| 5554 | return false; |
| 5555 | } |
| 5556 | |
| 5557 | return true; |
| 5558 | } |
| 5559 | |
| 5560 | bool AMDGPUAsmParser::validateMonitorSleep(const MCInst &Inst, |
| 5561 | const OperandVector &Operands) { |
| 5562 | unsigned Opc = Inst.getOpcode(); |
| 5563 | if (Opc != AMDGPU::S_MONITOR_SLEEP_gfx12 || |
| 5564 | !getSTI().hasFeature(Feature: AMDGPU::FeatureNoSleepForever)) |
| 5565 | return true; |
| 5566 | |
| 5567 | int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::simm16); |
| 5568 | if (Inst.getOperand(i: ImmIdx).getImm() & 0x8000) { |
| 5569 | Error(L: getOperandLoc(Operands, MCOpIdx: ImmIdx), |
| 5570 | Msg: "sleep forever is unsupported on the target" ); |
| 5571 | return false; |
| 5572 | } |
| 5573 | |
| 5574 | return true; |
| 5575 | } |
| 5576 | |
| 5577 | bool AMDGPUAsmParser::validateClusterBarrierIsFirst( |
| 5578 | const MCInst &Inst, const OperandVector &Operands) { |
| 5579 | unsigned Opc = Inst.getOpcode(); |
| 5580 | if (Opc != AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM_gfx12 && |
| 5581 | Opc != AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM_gfx13) |
| 5582 | return true; |
| 5583 | |
| 5584 | int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0); |
| 5585 | int BarrierID = Inst.getOperand(i: Src0Idx).getImm(); |
| 5586 | if (BarrierID != AMDGPU::Barrier::CLUSTER) |
| 5587 | return true; |
| 5588 | |
| 5589 | Error( |
| 5590 | L: getOperandLoc(Operands, MCOpIdx: Src0Idx), |
| 5591 | Msg: "s_barrier_signal_isfirst does not support user_cluster_barrier_id (-3)" ); |
| 5592 | return false; |
| 5593 | } |
| 5594 | |
| 5595 | bool AMDGPUAsmParser::validateScaleSel(const MCInst &Inst, |
| 5596 | const OperandVector &Operands) { |
| 5597 | unsigned Opc = Inst.getOpcode(); |
| 5598 | int ScaleSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::scale_sel); |
| 5599 | if (ScaleSelIdx == -1) |
| 5600 | return true; |
| 5601 | int MaxSel = 0; |
| 5602 | switch (Opc) { |
| 5603 | case AMDGPU::V_CVT_SCALE_PK16_F16_FP6_e64_gfx1250: |
| 5604 | case AMDGPU::V_CVT_SCALE_PK16_BF16_FP6_e64_gfx1250: |
| 5605 | case AMDGPU::V_CVT_SCALE_PK16_F16_BF6_e64_gfx1250: |
| 5606 | case AMDGPU::V_CVT_SCALE_PK16_BF16_BF6_e64_gfx1250: |
| 5607 | case AMDGPU::V_CVT_SCALE_PK16_F32_FP6_e64_gfx1250: |
| 5608 | case AMDGPU::V_CVT_SCALE_PK16_F32_BF6_e64_gfx1250: |
| 5609 | case AMDGPU::V_CVT_SCALE_PK8_F16_FP4_e64_gfx1250: |
| 5610 | case AMDGPU::V_CVT_SCALE_PK8_BF16_FP4_e64_gfx1250: |
| 5611 | case AMDGPU::V_CVT_SCALE_PK8_F32_FP4_e64_gfx1250: |
| 5612 | MaxSel = 4; |
| 5613 | break; |
| 5614 | case AMDGPU::V_CVT_SCALE_PK8_F16_FP8_e64_gfx1250: |
| 5615 | case AMDGPU::V_CVT_SCALE_PK8_BF16_FP8_e64_gfx1250: |
| 5616 | case AMDGPU::V_CVT_SCALE_PK8_F16_BF8_e64_gfx1250: |
| 5617 | case AMDGPU::V_CVT_SCALE_PK8_BF16_BF8_e64_gfx1250: |
| 5618 | case AMDGPU::V_CVT_SCALE_PK8_F32_FP8_e64_gfx1250: |
| 5619 | case AMDGPU::V_CVT_SCALE_PK8_F32_BF8_e64_gfx1250: |
| 5620 | MaxSel = 8; |
| 5621 | break; |
| 5622 | default: |
| 5623 | return true; |
| 5624 | } |
| 5625 | |
| 5626 | if (getSTI().hasFeature(Feature: AMDGPU::FeatureBlock16ConversionScaleInsts)) |
| 5627 | MaxSel *= 2; |
| 5628 | |
| 5629 | int ScaleSel = Inst.getOperand(i: ScaleSelIdx).getImm(); |
| 5630 | if (ScaleSel < MaxSel) |
| 5631 | return true; |
| 5632 | |
| 5633 | Error(L: getOperandLoc(Operands, MCOpIdx: ScaleSelIdx), |
| 5634 | Msg: "scale_sel maximum supported value is " + Twine(MaxSel - 1)); |
| 5635 | return false; |
| 5636 | } |
| 5637 | |
| 5638 | bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst, SMLoc IDLoc, |
| 5639 | const OperandVector &Operands) { |
| 5640 | if (!validateLdsDirect(Inst, Operands)) |
| 5641 | return false; |
| 5642 | if (!validateTrue16OpSel(Inst)) { |
| 5643 | Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyOpSel, Operands), |
| 5644 | Msg: "op_sel operand conflicts with 16-bit operand suffix" ); |
| 5645 | return false; |
| 5646 | } |
| 5647 | if (!validateSOPLiteral(Inst, Operands)) |
| 5648 | return false; |
| 5649 | if (!validateVOPLiteral(Inst, Operands)) { |
| 5650 | return false; |
| 5651 | } |
| 5652 | if (!validateConstantBusLimitations(Inst, Operands)) { |
| 5653 | return false; |
| 5654 | } |
| 5655 | if (!validateVOPD(Inst, Operands)) { |
| 5656 | return false; |
| 5657 | } |
| 5658 | if (!validateIntClampSupported(Inst)) { |
| 5659 | Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyClamp, Operands), |
| 5660 | Msg: "integer clamping is not supported on this GPU" ); |
| 5661 | return false; |
| 5662 | } |
| 5663 | if (!validateOpSel(Inst)) { |
| 5664 | Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyOpSel, Operands), |
| 5665 | Msg: "invalid op_sel operand" ); |
| 5666 | return false; |
| 5667 | } |
| 5668 | if (!validateNeg(Inst, OpName: AMDGPU::OpName::neg_lo)) { |
| 5669 | Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyNegLo, Operands), |
| 5670 | Msg: "invalid neg_lo operand" ); |
| 5671 | return false; |
| 5672 | } |
| 5673 | if (!validateNeg(Inst, OpName: AMDGPU::OpName::neg_hi)) { |
| 5674 | Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyNegHi, Operands), |
| 5675 | Msg: "invalid neg_hi operand" ); |
| 5676 | return false; |
| 5677 | } |
| 5678 | if (!validateDPP(Inst, Operands)) { |
| 5679 | return false; |
| 5680 | } |
| 5681 | // For MUBUF/MTBUF d16 is a part of opcode, so there is nothing to validate. |
| 5682 | if (!validateMIMGD16(Inst)) { |
| 5683 | Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyD16, Operands), |
| 5684 | Msg: "d16 modifier is not supported on this GPU" ); |
| 5685 | return false; |
| 5686 | } |
| 5687 | if (!validateMIMGDim(Inst, Operands)) { |
| 5688 | Error(L: IDLoc, Msg: "missing dim operand" ); |
| 5689 | return false; |
| 5690 | } |
| 5691 | if (!validateTensorR128(Inst)) { |
| 5692 | Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyD16, Operands), |
| 5693 | Msg: "instruction must set modifier r128=0" ); |
| 5694 | return false; |
| 5695 | } |
| 5696 | if (!validateMIMGMSAA(Inst)) { |
| 5697 | Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyDim, Operands), |
| 5698 | Msg: "invalid dim; must be MSAA type" ); |
| 5699 | return false; |
| 5700 | } |
| 5701 | if (!validateMIMGDataSize(Inst, IDLoc)) { |
| 5702 | return false; |
| 5703 | } |
| 5704 | if (!validateMIMGAddrSize(Inst, IDLoc)) |
| 5705 | return false; |
| 5706 | if (!validateMIMGAtomicDMask(Inst)) { |
| 5707 | Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyDMask, Operands), |
| 5708 | Msg: "invalid atomic image dmask" ); |
| 5709 | return false; |
| 5710 | } |
| 5711 | if (!validateMIMGGatherDMask(Inst)) { |
| 5712 | Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyDMask, Operands), |
| 5713 | Msg: "invalid image_gather dmask: only one bit must be set" ); |
| 5714 | return false; |
| 5715 | } |
| 5716 | if (!validateMovrels(Inst, Operands)) { |
| 5717 | return false; |
| 5718 | } |
| 5719 | if (!validateOffset(Inst, Operands)) { |
| 5720 | return false; |
| 5721 | } |
| 5722 | if (!validateSMEMSData(Inst, Operands)) |
| 5723 | return false; |
| 5724 | if (!validateBF16InlineConst(Inst, Operands)) { |
| 5725 | return false; |
| 5726 | } |
| 5727 | if (!validateMAIAccWrite(Inst, Operands)) { |
| 5728 | return false; |
| 5729 | } |
| 5730 | if (!validateMAISrc2(Inst, Operands)) { |
| 5731 | return false; |
| 5732 | } |
| 5733 | if (!validateMFMA(Inst, Operands)) { |
| 5734 | return false; |
| 5735 | } |
| 5736 | if (!validateCoherencyBits(Inst, Operands, IDLoc)) { |
| 5737 | return false; |
| 5738 | } |
| 5739 | |
| 5740 | if (!validateAGPRLdSt(Inst)) { |
| 5741 | Error( |
| 5742 | L: IDLoc, |
| 5743 | Msg: getFeatureBits()[AMDGPU::FeatureGFX90AInsts] |
| 5744 | ? "invalid register class: data and dst should be all VGPR or AGPR" |
| 5745 | : "invalid register class: agpr loads and stores not supported on " |
| 5746 | "this GPU" ); |
| 5747 | return false; |
| 5748 | } |
| 5749 | if (!validateVGPRAlign(Inst)) { |
| 5750 | Error(L: IDLoc, Msg: "invalid register class: vgpr tuples must be 64 bit aligned" ); |
| 5751 | return false; |
| 5752 | } |
| 5753 | if (!validateDS(Inst, Operands)) { |
| 5754 | return false; |
| 5755 | } |
| 5756 | |
| 5757 | if (!validateBLGP(Inst, Operands)) { |
| 5758 | return false; |
| 5759 | } |
| 5760 | |
| 5761 | if (!validateDivScale(Inst)) { |
| 5762 | Error(L: IDLoc, Msg: "ABS not allowed in VOP3B instructions" ); |
| 5763 | return false; |
| 5764 | } |
| 5765 | if (!validateWaitCnt(Inst, Operands)) { |
| 5766 | return false; |
| 5767 | } |
| 5768 | if (!validateTFE(Inst, Operands)) { |
| 5769 | return false; |
| 5770 | } |
| 5771 | if (!validateWMMA(Inst, Operands)) { |
| 5772 | return false; |
| 5773 | } |
| 5774 | if (!validateMonitorSleep(Inst, Operands)) { |
| 5775 | return false; |
| 5776 | } |
| 5777 | if (!validateClusterBarrierIsFirst(Inst, Operands)) { |
| 5778 | return false; |
| 5779 | } |
| 5780 | if (!validateScaleSel(Inst, Operands)) { |
| 5781 | return false; |
| 5782 | } |
| 5783 | |
| 5784 | return true; |
| 5785 | } |
| 5786 | |
| 5787 | static std::string AMDGPUMnemonicSpellCheck(StringRef S, |
| 5788 | const FeatureBitset &FBS, |
| 5789 | unsigned VariantID = 0); |
| 5790 | |
| 5791 | static bool AMDGPUCheckMnemonic(StringRef Mnemonic, |
| 5792 | const FeatureBitset &AvailableFeatures, |
| 5793 | unsigned VariantID); |
| 5794 | |
| 5795 | bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo, |
| 5796 | const FeatureBitset &FBS) { |
| 5797 | return isSupportedMnemo(Mnemo, FBS, Variants: getAllVariants()); |
| 5798 | } |
| 5799 | |
| 5800 | bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo, |
| 5801 | const FeatureBitset &FBS, |
| 5802 | ArrayRef<unsigned> Variants) { |
| 5803 | for (auto Variant : Variants) { |
| 5804 | if (AMDGPUCheckMnemonic(Mnemonic: Mnemo, AvailableFeatures: FBS, VariantID: Variant)) |
| 5805 | return true; |
| 5806 | } |
| 5807 | |
| 5808 | return false; |
| 5809 | } |
| 5810 | |
| 5811 | bool AMDGPUAsmParser::checkUnsupportedInstruction(StringRef Mnemo, |
| 5812 | SMLoc IDLoc) { |
| 5813 | FeatureBitset FBS = ComputeAvailableFeatures(FB: getFeatureBits()); |
| 5814 | |
| 5815 | // Check if requested instruction variant is supported. |
| 5816 | if (isSupportedMnemo(Mnemo, FBS, Variants: getMatchedVariants())) |
| 5817 | return false; |
| 5818 | |
| 5819 | // This instruction is not supported. |
| 5820 | // Clear any other pending errors because they are no longer relevant. |
| 5821 | getParser().clearPendingErrors(); |
| 5822 | |
| 5823 | // Requested instruction variant is not supported. |
| 5824 | // Check if any other variants are supported. |
| 5825 | StringRef VariantName = getMatchedVariantName(); |
| 5826 | if (!VariantName.empty() && isSupportedMnemo(Mnemo, FBS)) { |
| 5827 | return Error(L: IDLoc, Msg: Twine(VariantName, |
| 5828 | " variant of this instruction is not supported" )); |
| 5829 | } |
| 5830 | |
| 5831 | // Check if this instruction may be used with a different wavesize. |
| 5832 | if (isGFX10Plus() && getFeatureBits()[AMDGPU::FeatureWavefrontSize64] && |
| 5833 | !getFeatureBits()[AMDGPU::FeatureWavefrontSize32]) { |
| 5834 | // FIXME: Use getAvailableFeatures, and do not manually recompute |
| 5835 | FeatureBitset FeaturesWS32 = getFeatureBits(); |
| 5836 | FeaturesWS32.flip(I: AMDGPU::FeatureWavefrontSize64) |
| 5837 | .flip(I: AMDGPU::FeatureWavefrontSize32); |
| 5838 | FeatureBitset AvailableFeaturesWS32 = |
| 5839 | ComputeAvailableFeatures(FB: FeaturesWS32); |
| 5840 | |
| 5841 | if (isSupportedMnemo(Mnemo, FBS: AvailableFeaturesWS32, Variants: getMatchedVariants())) |
| 5842 | return Error(L: IDLoc, Msg: "instruction requires wavesize=32" ); |
| 5843 | } |
| 5844 | |
| 5845 | // Finally check if this instruction is supported on any other GPU. |
| 5846 | if (isSupportedMnemo(Mnemo, FBS: FeatureBitset().set())) { |
| 5847 | return Error(L: IDLoc, Msg: "instruction not supported on this GPU (" + |
| 5848 | getSTI().getCPU() + ")" + ": " + Mnemo); |
| 5849 | } |
| 5850 | |
| 5851 | // Instruction not supported on any GPU. Probably a typo. |
| 5852 | std::string Suggestion = AMDGPUMnemonicSpellCheck(S: Mnemo, FBS); |
| 5853 | return Error(L: IDLoc, Msg: "invalid instruction" + Suggestion); |
| 5854 | } |
| 5855 | |
| 5856 | static bool isInvalidVOPDY(const OperandVector &Operands, |
| 5857 | uint64_t InvalidOprIdx) { |
| 5858 | assert(InvalidOprIdx < Operands.size()); |
| 5859 | const auto &Op = ((AMDGPUOperand &)*Operands[InvalidOprIdx]); |
| 5860 | if (Op.isToken() && InvalidOprIdx > 1) { |
| 5861 | const auto &PrevOp = ((AMDGPUOperand &)*Operands[InvalidOprIdx - 1]); |
| 5862 | return PrevOp.isToken() && PrevOp.getToken() == "::" ; |
| 5863 | } |
| 5864 | return false; |
| 5865 | } |
| 5866 | |
| 5867 | bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode, |
| 5868 | OperandVector &Operands, |
| 5869 | MCStreamer &Out, |
| 5870 | uint64_t &ErrorInfo, |
| 5871 | bool MatchingInlineAsm) { |
| 5872 | MCInst Inst; |
| 5873 | Inst.setLoc(IDLoc); |
| 5874 | unsigned Result = Match_Success; |
| 5875 | |
| 5876 | // Order match statuses from least to most specific and keep the most |
| 5877 | // specific one: |
| 5878 | // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature |
| 5879 | auto atLeastAsSpecific = [](unsigned New, unsigned Cur) { |
| 5880 | auto rank = [](unsigned M) { |
| 5881 | return M == Match_MnemonicFail ? 1 |
| 5882 | : M == Match_InvalidOperand ? 2 |
| 5883 | : M == Match_MissingFeature ? 3 |
| 5884 | : 0; // Match_Success sentinel |
| 5885 | }; |
| 5886 | return rank(New) >= rank(Cur); |
| 5887 | }; |
| 5888 | |
| 5889 | for (auto Variant : getMatchedVariants()) { |
| 5890 | uint64_t EI; |
| 5891 | auto R = |
| 5892 | MatchInstructionImpl(Operands, Inst, ErrorInfo&: EI, matchingInlineAsm: MatchingInlineAsm, VariantID: Variant); |
| 5893 | if (R == Match_Success || atLeastAsSpecific(R, Result)) { |
| 5894 | Result = R; |
| 5895 | ErrorInfo = EI; |
| 5896 | } |
| 5897 | if (R == Match_Success) |
| 5898 | break; |
| 5899 | } |
| 5900 | |
| 5901 | if (Result == Match_Success) { |
| 5902 | if (!validateInstruction(Inst, IDLoc, Operands)) { |
| 5903 | return true; |
| 5904 | } |
| 5905 | emitTargetDirective(); |
| 5906 | Out.emitInstruction(Inst, STI: getSTI()); |
| 5907 | // Record for kernel prologue checking. |
| 5908 | OpcodeStream.push_back(Elt: Inst.getOpcode()); |
| 5909 | return false; |
| 5910 | } |
| 5911 | |
| 5912 | StringRef Mnemo = ((AMDGPUOperand &)*Operands[0]).getToken(); |
| 5913 | if (checkUnsupportedInstruction(Mnemo, IDLoc)) { |
| 5914 | return true; |
| 5915 | } |
| 5916 | |
| 5917 | switch (Result) { |
| 5918 | default: |
| 5919 | break; |
| 5920 | case Match_MissingFeature: |
| 5921 | // It has been verified that the specified instruction |
| 5922 | // mnemonic is valid. A match was found but it requires |
| 5923 | // features which are not supported on this GPU. |
| 5924 | return Error(L: IDLoc, Msg: "operands are not valid for this GPU or mode" ); |
| 5925 | |
| 5926 | case Match_InvalidOperand: { |
| 5927 | SMLoc ErrorLoc = IDLoc; |
| 5928 | if (ErrorInfo != ~0ULL) { |
| 5929 | if (ErrorInfo >= Operands.size()) { |
| 5930 | return Error(L: IDLoc, Msg: "too few operands for instruction" ); |
| 5931 | } |
| 5932 | AMDGPUOperand &ErrorOp = (AMDGPUOperand &)*Operands[ErrorInfo]; |
| 5933 | ErrorLoc = ErrorOp.getStartLoc(); |
| 5934 | if (ErrorLoc == SMLoc()) |
| 5935 | ErrorLoc = IDLoc; |
| 5936 | |
| 5937 | if (isInvalidVOPDY(Operands, InvalidOprIdx: ErrorInfo)) |
| 5938 | return Error(L: ErrorLoc, Msg: "invalid VOPDY instruction" ); |
| 5939 | } |
| 5940 | return Error(L: ErrorLoc, Msg: "invalid operand for instruction" ); |
| 5941 | } |
| 5942 | |
| 5943 | case Match_MnemonicFail: |
| 5944 | llvm_unreachable("Invalid instructions should have been handled already" ); |
| 5945 | } |
| 5946 | llvm_unreachable("Implement any new match types added!" ); |
| 5947 | } |
| 5948 | |
| 5949 | bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) { |
| 5950 | int64_t Tmp = -1; |
| 5951 | if (!isToken(Kind: AsmToken::Integer) && !isToken(Kind: AsmToken::Identifier)) { |
| 5952 | return true; |
| 5953 | } |
| 5954 | if (getParser().parseAbsoluteExpression(Res&: Tmp)) { |
| 5955 | return true; |
| 5956 | } |
| 5957 | Ret = static_cast<uint32_t>(Tmp); |
| 5958 | return false; |
| 5959 | } |
| 5960 | |
| 5961 | bool AMDGPUAsmParser::ParseDirectiveAMDGCNTarget() { |
| 5962 | if (!getSTI().getTargetTriple().isAMDGCN()) |
| 5963 | return TokError(Msg: "directive only supported for amdgcn architecture" ); |
| 5964 | |
| 5965 | std::string TargetIDDirective; |
| 5966 | SMLoc TargetStart = getTok().getLoc(); |
| 5967 | if (getParser().parseEscapedString(Data&: TargetIDDirective)) |
| 5968 | return true; |
| 5969 | |
| 5970 | std::optional<AMDGPU::TargetID> MaybeParsed = |
| 5971 | AMDGPU::TargetID::parseTargetIDString(TargetIDDirective); |
| 5972 | if (!MaybeParsed) |
| 5973 | return getParser().Error(L: TargetStart, |
| 5974 | Msg: "malformed target id '" + TargetIDDirective + "'" ); |
| 5975 | |
| 5976 | const AMDGPU::TargetID &ParsedTargetID = *MaybeParsed; |
| 5977 | const Triple &TT = getSTI().getTargetTriple(); |
| 5978 | |
| 5979 | // The processor named in the target id must be covered by the triple's |
| 5980 | // subarch. |
| 5981 | if (!AMDGPU::isCPUValidForSubArch(SubArch: TT.getSubArch(), |
| 5982 | AK: ParsedTargetID.getGPUKind())) { |
| 5983 | return getParser().Error( |
| 5984 | L: TargetStart, Msg: "target id '" + TargetIDDirective + |
| 5985 | "' specifies a processor that is not valid for " |
| 5986 | "subarch '" + |
| 5987 | TT.getArchName() + "'" ); |
| 5988 | } |
| 5989 | |
| 5990 | const std::optional<AMDGPU::TargetID> &CurrentTargetID = |
| 5991 | getTargetStreamer().getTargetID(); |
| 5992 | |
| 5993 | Triple DirectiveTriple(ParsedTargetID.getTargetTripleString()); |
| 5994 | const Triple &STITriple = getSTI().getTargetTriple(); |
| 5995 | if (!DirectiveTriple.isCompatibleWith(Other: STITriple)) { |
| 5996 | return getParser().Error( |
| 5997 | L: TargetStart, Msg: ".amdgcn_target " + Twine(ParsedTargetID.toString()) + |
| 5998 | " is incompatible with " + |
| 5999 | Twine(CurrentTargetID->toString())); |
| 6000 | } |
| 6001 | |
| 6002 | // Error if the ISA version doesn't match |
| 6003 | StringRef DirectiveProcessor = |
| 6004 | AMDGPU::getArchNameAMDGCN(AK: ParsedTargetID.getGPUKind()); |
| 6005 | AMDGPU::IsaVersion DirectiveISA = AMDGPU::getIsaVersion(GPU: DirectiveProcessor); |
| 6006 | if (DirectiveISA != ISA) { |
| 6007 | return getParser().Error(L: TargetStart, |
| 6008 | Msg: ".amdgcn_target directive processor " + |
| 6009 | Twine(DirectiveProcessor) + |
| 6010 | " does not match the specified processor " + |
| 6011 | Twine(getSTI().getCPU())); |
| 6012 | } |
| 6013 | |
| 6014 | // Warn if sramecc or xnack mismatch. These do not change the encoding. |
| 6015 | if (AMDGPU::IsaInfo::targetIDSettingsConflict( |
| 6016 | Lhs: ParsedTargetID.getXnackSetting(), |
| 6017 | Rhs: CurrentTargetID->getXnackSetting())) { |
| 6018 | Warning(L: TargetStart, |
| 6019 | Msg: ".amdgcn_target directive has conflicting xnack settings" ); |
| 6020 | } |
| 6021 | if (AMDGPU::IsaInfo::targetIDSettingsConflict( |
| 6022 | Lhs: ParsedTargetID.getSramEccSetting(), |
| 6023 | Rhs: CurrentTargetID->getSramEccSetting())) { |
| 6024 | Warning(L: TargetStart, |
| 6025 | Msg: ".amdgcn_target directive has conflicting sramecc settings" ); |
| 6026 | } |
| 6027 | |
| 6028 | // Update the target streamer's TargetID with settings from the directive. |
| 6029 | // We don't update the MCSubtargetInfo because we've already validated |
| 6030 | // that the directive matches the command-line CPU. |
| 6031 | getTargetStreamer().getTargetID()->setXnackSetting( |
| 6032 | ParsedTargetID.getXnackSetting()); |
| 6033 | getTargetStreamer().getTargetID()->setSramEccSetting( |
| 6034 | ParsedTargetID.getSramEccSetting()); |
| 6035 | |
| 6036 | return false; |
| 6037 | } |
| 6038 | |
| 6039 | bool AMDGPUAsmParser::OutOfRangeError(SMRange Range) { |
| 6040 | return Error(L: Range.Start, Msg: "value out of range" , Range); |
| 6041 | } |
| 6042 | |
| 6043 | bool AMDGPUAsmParser::calculateGPRBlocks( |
| 6044 | const FeatureBitset &Features, const MCExpr *VCCUsed, |
| 6045 | const MCExpr *FlatScrUsed, bool XNACKUsed, |
| 6046 | std::optional<bool> EnableWavefrontSize32, const MCExpr *NextFreeVGPR, |
| 6047 | SMRange VGPRRange, const MCExpr *NextFreeSGPR, SMRange SGPRRange, |
| 6048 | const MCExpr *&VGPRBlocks, const MCExpr *&SGPRBlocks) { |
| 6049 | // TODO(scott.linder): These calculations are duplicated from |
| 6050 | // AMDGPUAsmPrinter::getSIProgramInfo and could be unified. |
| 6051 | MCContext &Ctx = getContext(); |
| 6052 | |
| 6053 | const MCExpr *NumSGPRs = NextFreeSGPR; |
| 6054 | int64_t EvaluatedSGPRs; |
| 6055 | |
| 6056 | if (ISA.Major >= 10) |
| 6057 | NumSGPRs = MCConstantExpr::create(Value: 0, Ctx); |
| 6058 | else { |
| 6059 | unsigned MaxAddressableNumSGPRs = AMDGPU::getAddressableNumSGPRs(AK: Gfx); |
| 6060 | |
| 6061 | if (NumSGPRs->evaluateAsAbsolute(Res&: EvaluatedSGPRs) && ISA.Major >= 8 && |
| 6062 | !Features.test(I: FeatureSGPRInitBug) && |
| 6063 | static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs) |
| 6064 | return OutOfRangeError(Range: SGPRRange); |
| 6065 | |
| 6066 | const MCExpr * = |
| 6067 | AMDGPUMCExpr::createExtraSGPRs(VCCUsed, FlatScrUsed, XNACKUsed, Ctx); |
| 6068 | NumSGPRs = MCBinaryExpr::createAdd(LHS: NumSGPRs, RHS: ExtraSGPRs, Ctx); |
| 6069 | |
| 6070 | if (NumSGPRs->evaluateAsAbsolute(Res&: EvaluatedSGPRs) && |
| 6071 | (ISA.Major <= 7 || Features.test(I: FeatureSGPRInitBug)) && |
| 6072 | static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs) |
| 6073 | return OutOfRangeError(Range: SGPRRange); |
| 6074 | |
| 6075 | if (Features.test(I: FeatureSGPRInitBug)) |
| 6076 | NumSGPRs = |
| 6077 | MCConstantExpr::create(Value: IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG, Ctx); |
| 6078 | } |
| 6079 | |
| 6080 | // The MCExpr equivalent of getNumSGPRBlocks/getNumVGPRBlocks: |
| 6081 | // (alignTo(max(1u, NumGPR), GPREncodingGranule) / GPREncodingGranule) - 1 |
| 6082 | auto GetNumGPRBlocks = [&Ctx](const MCExpr *NumGPR, |
| 6083 | unsigned Granule) -> const MCExpr * { |
| 6084 | const MCExpr *OneConst = MCConstantExpr::create(Value: 1ul, Ctx); |
| 6085 | const MCExpr *GranuleConst = MCConstantExpr::create(Value: Granule, Ctx); |
| 6086 | const MCExpr *MaxNumGPR = AMDGPUMCExpr::createMax(Args: {NumGPR, OneConst}, Ctx); |
| 6087 | const MCExpr *AlignToGPR = |
| 6088 | AMDGPUMCExpr::createAlignTo(Value: MaxNumGPR, Align: GranuleConst, Ctx); |
| 6089 | const MCExpr *DivGPR = |
| 6090 | MCBinaryExpr::createDiv(LHS: AlignToGPR, RHS: GranuleConst, Ctx); |
| 6091 | const MCExpr *SubGPR = MCBinaryExpr::createSub(LHS: DivGPR, RHS: OneConst, Ctx); |
| 6092 | return SubGPR; |
| 6093 | }; |
| 6094 | |
| 6095 | VGPRBlocks = GetNumGPRBlocks( |
| 6096 | NextFreeVGPR, |
| 6097 | IsaInfo::getVGPREncodingGranule(STI: getSTI(), EnableWavefrontSize32)); |
| 6098 | SGPRBlocks = |
| 6099 | GetNumGPRBlocks(NumSGPRs, IsaInfo::getSGPREncodingGranule(STI: getSTI())); |
| 6100 | |
| 6101 | return false; |
| 6102 | } |
| 6103 | |
| 6104 | bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() { |
| 6105 | if (!getSTI().getTargetTriple().isAMDGCN()) |
| 6106 | return TokError(Msg: "directive only supported for amdgcn architecture" ); |
| 6107 | |
| 6108 | if (!isHsaAbi(STI: getSTI())) |
| 6109 | return TokError(Msg: "directive only supported for amdhsa OS" ); |
| 6110 | |
| 6111 | StringRef KernelName; |
| 6112 | if (getParser().parseIdentifier(Res&: KernelName)) |
| 6113 | return true; |
| 6114 | |
| 6115 | // Remember the kernel name so its prologue can be checked at end of file. |
| 6116 | // The matching label may have been parsed already or may follow later. |
| 6117 | AMDHSAKernelSymbols.insert(Ptr: getContext().getOrCreateSymbol(Name: KernelName)); |
| 6118 | |
| 6119 | AMDGPU::MCKernelDescriptor KD = |
| 6120 | AMDGPU::MCKernelDescriptor::getDefaultAmdhsaKernelDescriptor( |
| 6121 | STI: &getSTI(), Ctx&: getContext()); |
| 6122 | |
| 6123 | StringSet<> Seen; |
| 6124 | |
| 6125 | const MCExpr *ZeroExpr = MCConstantExpr::create(Value: 0, Ctx&: getContext()); |
| 6126 | const MCExpr *OneExpr = MCConstantExpr::create(Value: 1, Ctx&: getContext()); |
| 6127 | |
| 6128 | SMRange VGPRRange; |
| 6129 | const MCExpr *NextFreeVGPR = ZeroExpr; |
| 6130 | const MCExpr *AccumOffset = MCConstantExpr::create(Value: 0, Ctx&: getContext()); |
| 6131 | const MCExpr *NamedBarCnt = ZeroExpr; |
| 6132 | uint64_t SharedVGPRCount = 0; |
| 6133 | uint64_t PreloadLength = 0; |
| 6134 | uint64_t PreloadOffset = 0; |
| 6135 | SMRange SGPRRange; |
| 6136 | const MCExpr *NextFreeSGPR = ZeroExpr; |
| 6137 | |
| 6138 | // Count the number of user SGPRs implied from the enabled feature bits. |
| 6139 | unsigned ImpliedUserSGPRCount = 0; |
| 6140 | |
| 6141 | // Track if the asm explicitly contains the directive for the user SGPR |
| 6142 | // count. |
| 6143 | std::optional<unsigned> ExplicitUserSGPRCount; |
| 6144 | const MCExpr *ReserveVCC = OneExpr; |
| 6145 | const MCExpr *ReserveFlatScr = OneExpr; |
| 6146 | std::optional<bool> EnableWavefrontSize32; |
| 6147 | |
| 6148 | while (true) { |
| 6149 | while (trySkipToken(Kind: AsmToken::EndOfStatement)) |
| 6150 | ; |
| 6151 | |
| 6152 | StringRef ID; |
| 6153 | SMRange IDRange = getTok().getLocRange(); |
| 6154 | if (!parseId(Val&: ID, ErrMsg: "expected .amdhsa_ directive or .end_amdhsa_kernel" )) |
| 6155 | return true; |
| 6156 | |
| 6157 | if (ID == ".end_amdhsa_kernel" ) |
| 6158 | break; |
| 6159 | |
| 6160 | if (!Seen.insert(key: ID).second) |
| 6161 | return TokError(Msg: ".amdhsa_ directives cannot be repeated" ); |
| 6162 | |
| 6163 | SMLoc ValStart = getLoc(); |
| 6164 | const MCExpr *ExprVal; |
| 6165 | if (getParser().parseExpression(Res&: ExprVal)) |
| 6166 | return true; |
| 6167 | SMLoc ValEnd = getLoc(); |
| 6168 | SMRange ValRange = SMRange(ValStart, ValEnd); |
| 6169 | |
| 6170 | int64_t IVal = 0; |
| 6171 | uint64_t Val = IVal; |
| 6172 | bool EvaluatableExpr; |
| 6173 | if ((EvaluatableExpr = ExprVal->evaluateAsAbsolute(Res&: IVal))) { |
| 6174 | if (IVal < 0) |
| 6175 | return OutOfRangeError(Range: ValRange); |
| 6176 | Val = IVal; |
| 6177 | } |
| 6178 | |
| 6179 | #define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE) \ |
| 6180 | if (!isUInt<ENTRY##_WIDTH>(Val)) \ |
| 6181 | return OutOfRangeError(RANGE); \ |
| 6182 | AMDGPU::MCKernelDescriptor::bits_set(FIELD, VALUE, ENTRY##_SHIFT, ENTRY, \ |
| 6183 | getContext()); |
| 6184 | |
| 6185 | // Some fields use the parsed value immediately which requires the expression to |
| 6186 | // be solvable. |
| 6187 | #define EXPR_RESOLVE_OR_ERROR(RESOLVED) \ |
| 6188 | if (!(RESOLVED)) \ |
| 6189 | return Error(IDRange.Start, "directive should have resolvable expression", \ |
| 6190 | IDRange); |
| 6191 | |
| 6192 | if (ID == ".amdhsa_group_segment_fixed_size" ) { |
| 6193 | if (!isUInt<sizeof(kernel_descriptor_t::group_segment_fixed_size) * |
| 6194 | CHAR_BIT>(x: Val)) |
| 6195 | return OutOfRangeError(Range: ValRange); |
| 6196 | KD.group_segment_fixed_size = ExprVal; |
| 6197 | } else if (ID == ".amdhsa_private_segment_fixed_size" ) { |
| 6198 | if (!isUInt<sizeof(kernel_descriptor_t::private_segment_fixed_size) * |
| 6199 | CHAR_BIT>(x: Val)) |
| 6200 | return OutOfRangeError(Range: ValRange); |
| 6201 | KD.private_segment_fixed_size = ExprVal; |
| 6202 | } else if (ID == ".amdhsa_kernarg_size" ) { |
| 6203 | if (!isUInt<sizeof(kernel_descriptor_t::kernarg_size) * CHAR_BIT>(x: Val)) |
| 6204 | return OutOfRangeError(Range: ValRange); |
| 6205 | KD.kernarg_size = ExprVal; |
| 6206 | } else if (ID == ".amdhsa_user_sgpr_count" ) { |
| 6207 | EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); |
| 6208 | ExplicitUserSGPRCount = Val; |
| 6209 | } else if (ID == ".amdhsa_user_sgpr_private_segment_buffer" ) { |
| 6210 | EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); |
| 6211 | if (hasArchitectedFlatScratch()) |
| 6212 | return Error(L: IDRange.Start, |
| 6213 | Msg: "directive is not supported with architected flat scratch" , |
| 6214 | Range: IDRange); |
| 6215 | PARSE_BITS_ENTRY(KD.kernel_code_properties, |
| 6216 | KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER, |
| 6217 | ExprVal, ValRange); |
| 6218 | if (Val) |
| 6219 | ImpliedUserSGPRCount += 4; |
| 6220 | } else if (ID == ".amdhsa_user_sgpr_kernarg_preload_length" ) { |
| 6221 | EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); |
| 6222 | if (!hasKernargPreload()) |
| 6223 | return Error(L: IDRange.Start, Msg: "directive requires gfx90a+" , Range: IDRange); |
| 6224 | |
| 6225 | if (Val > getMaxNumUserSGPRs()) |
| 6226 | return OutOfRangeError(Range: ValRange); |
| 6227 | PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_LENGTH, ExprVal, |
| 6228 | ValRange); |
| 6229 | if (Val) { |
| 6230 | ImpliedUserSGPRCount += Val; |
| 6231 | PreloadLength = Val; |
| 6232 | } |
| 6233 | } else if (ID == ".amdhsa_user_sgpr_kernarg_preload_offset" ) { |
| 6234 | EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); |
| 6235 | if (!hasKernargPreload()) |
| 6236 | return Error(L: IDRange.Start, Msg: "directive requires gfx90a+" , Range: IDRange); |
| 6237 | |
| 6238 | if (Val >= 1024) |
| 6239 | return OutOfRangeError(Range: ValRange); |
| 6240 | PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_OFFSET, ExprVal, |
| 6241 | ValRange); |
| 6242 | if (Val) |
| 6243 | PreloadOffset = Val; |
| 6244 | } else if (ID == ".amdhsa_user_sgpr_dispatch_ptr" ) { |
| 6245 | EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); |
| 6246 | PARSE_BITS_ENTRY(KD.kernel_code_properties, |
| 6247 | KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, ExprVal, |
| 6248 | ValRange); |
| 6249 | if (Val) |
| 6250 | ImpliedUserSGPRCount += 2; |
| 6251 | } else if (ID == ".amdhsa_user_sgpr_queue_ptr" ) { |
| 6252 | EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); |
| 6253 | PARSE_BITS_ENTRY(KD.kernel_code_properties, |
| 6254 | KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, ExprVal, |
| 6255 | ValRange); |
| 6256 | if (Val) |
| 6257 | ImpliedUserSGPRCount += 2; |
| 6258 | } else if (ID == ".amdhsa_user_sgpr_kernarg_segment_ptr" ) { |
| 6259 | EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); |
| 6260 | PARSE_BITS_ENTRY(KD.kernel_code_properties, |
| 6261 | KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR, |
| 6262 | ExprVal, ValRange); |
| 6263 | if (Val) |
| 6264 | ImpliedUserSGPRCount += 2; |
| 6265 | } else if (ID == ".amdhsa_user_sgpr_dispatch_id" ) { |
| 6266 | EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); |
| 6267 | PARSE_BITS_ENTRY(KD.kernel_code_properties, |
| 6268 | KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, ExprVal, |
| 6269 | ValRange); |
| 6270 | if (Val) |
| 6271 | ImpliedUserSGPRCount += 2; |
| 6272 | } else if (ID == ".amdhsa_user_sgpr_flat_scratch_init" ) { |
| 6273 | if (hasArchitectedFlatScratch()) |
| 6274 | return Error(L: IDRange.Start, |
| 6275 | Msg: "directive is not supported with architected flat scratch" , |
| 6276 | Range: IDRange); |
| 6277 | EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); |
| 6278 | PARSE_BITS_ENTRY(KD.kernel_code_properties, |
| 6279 | KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT, |
| 6280 | ExprVal, ValRange); |
| 6281 | if (Val) |
| 6282 | ImpliedUserSGPRCount += 2; |
| 6283 | } else if (ID == ".amdhsa_user_sgpr_private_segment_size" ) { |
| 6284 | EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); |
| 6285 | PARSE_BITS_ENTRY(KD.kernel_code_properties, |
| 6286 | KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE, |
| 6287 | ExprVal, ValRange); |
| 6288 | if (Val) |
| 6289 | ImpliedUserSGPRCount += 1; |
| 6290 | } else if (ID == ".amdhsa_wavefront_size32" ) { |
| 6291 | EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); |
| 6292 | if (ISA.Major < 10) |
| 6293 | return Error(L: IDRange.Start, Msg: "directive requires gfx10+" , Range: IDRange); |
| 6294 | EnableWavefrontSize32 = Val; |
| 6295 | PARSE_BITS_ENTRY(KD.kernel_code_properties, |
| 6296 | KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, ExprVal, |
| 6297 | ValRange); |
| 6298 | } else if (ID == ".amdhsa_uses_dynamic_stack" ) { |
| 6299 | PARSE_BITS_ENTRY(KD.kernel_code_properties, |
| 6300 | KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, ExprVal, |
| 6301 | ValRange); |
| 6302 | } else if (ID == ".amdhsa_system_sgpr_private_segment_wavefront_offset" ) { |
| 6303 | if (hasArchitectedFlatScratch()) |
| 6304 | return Error(L: IDRange.Start, |
| 6305 | Msg: "directive is not supported with architected flat scratch" , |
| 6306 | Range: IDRange); |
| 6307 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, |
| 6308 | COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal, |
| 6309 | ValRange); |
| 6310 | } else if (ID == ".amdhsa_enable_private_segment" ) { |
| 6311 | if (!hasArchitectedFlatScratch()) |
| 6312 | return Error( |
| 6313 | L: IDRange.Start, |
| 6314 | Msg: "directive is not supported without architected flat scratch" , |
| 6315 | Range: IDRange); |
| 6316 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, |
| 6317 | COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal, |
| 6318 | ValRange); |
| 6319 | } else if (ID == ".amdhsa_system_sgpr_workgroup_id_x" ) { |
| 6320 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, |
| 6321 | COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, ExprVal, |
| 6322 | ValRange); |
| 6323 | } else if (ID == ".amdhsa_system_sgpr_workgroup_id_y" ) { |
| 6324 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, |
| 6325 | COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, ExprVal, |
| 6326 | ValRange); |
| 6327 | } else if (ID == ".amdhsa_system_sgpr_workgroup_id_z" ) { |
| 6328 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, |
| 6329 | COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, ExprVal, |
| 6330 | ValRange); |
| 6331 | } else if (ID == ".amdhsa_system_sgpr_workgroup_info" ) { |
| 6332 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, |
| 6333 | COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, ExprVal, |
| 6334 | ValRange); |
| 6335 | } else if (ID == ".amdhsa_system_vgpr_workitem_id" ) { |
| 6336 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, |
| 6337 | COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, ExprVal, |
| 6338 | ValRange); |
| 6339 | } else if (ID == ".amdhsa_next_free_vgpr" ) { |
| 6340 | VGPRRange = ValRange; |
| 6341 | NextFreeVGPR = ExprVal; |
| 6342 | } else if (ID == ".amdhsa_next_free_sgpr" ) { |
| 6343 | SGPRRange = ValRange; |
| 6344 | NextFreeSGPR = ExprVal; |
| 6345 | } else if (ID == ".amdhsa_accum_offset" ) { |
| 6346 | if (!isGFX90A()) |
| 6347 | return Error(L: IDRange.Start, Msg: "directive requires gfx90a+" , Range: IDRange); |
| 6348 | AccumOffset = ExprVal; |
| 6349 | } else if (ID == ".amdhsa_named_barrier_count" ) { |
| 6350 | if (!isGFX1250Plus()) |
| 6351 | return Error(L: IDRange.Start, Msg: "directive requires gfx1250+" , Range: IDRange); |
| 6352 | NamedBarCnt = ExprVal; |
| 6353 | } else if (ID == ".amdhsa_reserve_vcc" ) { |
| 6354 | if (EvaluatableExpr && !isUInt<1>(x: Val)) |
| 6355 | return OutOfRangeError(Range: ValRange); |
| 6356 | ReserveVCC = ExprVal; |
| 6357 | } else if (ID == ".amdhsa_reserve_flat_scratch" ) { |
| 6358 | if (ISA.Major < 7) |
| 6359 | return Error(L: IDRange.Start, Msg: "directive requires gfx7+" , Range: IDRange); |
| 6360 | if (hasArchitectedFlatScratch()) |
| 6361 | return Error(L: IDRange.Start, |
| 6362 | Msg: "directive is not supported with architected flat scratch" , |
| 6363 | Range: IDRange); |
| 6364 | if (EvaluatableExpr && !isUInt<1>(x: Val)) |
| 6365 | return OutOfRangeError(Range: ValRange); |
| 6366 | ReserveFlatScr = ExprVal; |
| 6367 | } else if (ID == ".amdhsa_reserve_xnack_mask" ) { |
| 6368 | if (ISA.Major < 8) |
| 6369 | return Error(L: IDRange.Start, Msg: "directive requires gfx8+" , Range: IDRange); |
| 6370 | if (!isUInt<1>(x: Val)) |
| 6371 | return OutOfRangeError(Range: ValRange); |
| 6372 | bool XnackOn = getTargetStreamer().getTargetID()->isXnackOnOrAny(); |
| 6373 | if (Val != XnackOn) { |
| 6374 | return getParser().Error( |
| 6375 | L: IDRange.Start, |
| 6376 | Msg: ".amdhsa_reserve_xnack_mask does not match target id" , Range: IDRange); |
| 6377 | } |
| 6378 | } else if (ID == ".amdhsa_float_round_mode_32" ) { |
| 6379 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, |
| 6380 | COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, ExprVal, |
| 6381 | ValRange); |
| 6382 | } else if (ID == ".amdhsa_float_round_mode_16_64" ) { |
| 6383 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, |
| 6384 | COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, ExprVal, |
| 6385 | ValRange); |
| 6386 | } else if (ID == ".amdhsa_float_denorm_mode_32" ) { |
| 6387 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, |
| 6388 | COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, ExprVal, |
| 6389 | ValRange); |
| 6390 | } else if (ID == ".amdhsa_float_denorm_mode_16_64" ) { |
| 6391 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, |
| 6392 | COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, ExprVal, |
| 6393 | ValRange); |
| 6394 | } else if (ID == ".amdhsa_dx10_clamp" ) { |
| 6395 | if (!getSTI().hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode)) |
| 6396 | return Error(L: IDRange.Start, Msg: "directive unsupported on gfx1170+" , |
| 6397 | Range: IDRange); |
| 6398 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, |
| 6399 | COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, ExprVal, |
| 6400 | ValRange); |
| 6401 | } else if (ID == ".amdhsa_ieee_mode" ) { |
| 6402 | if (!getSTI().hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode)) |
| 6403 | return Error(L: IDRange.Start, Msg: "directive unsupported on gfx1170+" , |
| 6404 | Range: IDRange); |
| 6405 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, |
| 6406 | COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, ExprVal, |
| 6407 | ValRange); |
| 6408 | } else if (ID == ".amdhsa_fp16_overflow" ) { |
| 6409 | if (ISA.Major < 9) |
| 6410 | return Error(L: IDRange.Start, Msg: "directive requires gfx9+" , Range: IDRange); |
| 6411 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, |
| 6412 | COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, ExprVal, |
| 6413 | ValRange); |
| 6414 | } else if (ID == ".amdhsa_tg_split" ) { |
| 6415 | if (!isGFX90A()) |
| 6416 | return Error(L: IDRange.Start, Msg: "directive requires gfx90a+" , Range: IDRange); |
| 6417 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, |
| 6418 | ExprVal, ValRange); |
| 6419 | } else if (ID == ".amdhsa_workgroup_processor_mode" ) { |
| 6420 | if (!supportsWGP(STI: getSTI())) |
| 6421 | return Error(L: IDRange.Start, |
| 6422 | Msg: "directive unsupported on " + getSTI().getCPU(), Range: IDRange); |
| 6423 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, |
| 6424 | COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal, |
| 6425 | ValRange); |
| 6426 | } else if (ID == ".amdhsa_memory_ordered" ) { |
| 6427 | if (ISA.Major < 10) |
| 6428 | return Error(L: IDRange.Start, Msg: "directive requires gfx10+" , Range: IDRange); |
| 6429 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, |
| 6430 | COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, ExprVal, |
| 6431 | ValRange); |
| 6432 | } else if (ID == ".amdhsa_forward_progress" ) { |
| 6433 | if (ISA.Major < 10) |
| 6434 | return Error(L: IDRange.Start, Msg: "directive requires gfx10+" , Range: IDRange); |
| 6435 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, |
| 6436 | COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, ExprVal, |
| 6437 | ValRange); |
| 6438 | } else if (ID == ".amdhsa_shared_vgpr_count" ) { |
| 6439 | EXPR_RESOLVE_OR_ERROR(EvaluatableExpr); |
| 6440 | if (ISA.Major < 10 || ISA.Major >= 12) |
| 6441 | return Error(L: IDRange.Start, Msg: "directive requires gfx10 or gfx11" , |
| 6442 | Range: IDRange); |
| 6443 | SharedVGPRCount = Val; |
| 6444 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, |
| 6445 | COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal, |
| 6446 | ValRange); |
| 6447 | } else if (ID == ".amdhsa_inst_pref_size" ) { |
| 6448 | if (ISA.Major < 11) |
| 6449 | return Error(L: IDRange.Start, Msg: "directive requires gfx11+" , Range: IDRange); |
| 6450 | if (ISA.Major == 11) { |
| 6451 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, |
| 6452 | COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE, ExprVal, |
| 6453 | ValRange); |
| 6454 | } else { |
| 6455 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, |
| 6456 | COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE, ExprVal, |
| 6457 | ValRange); |
| 6458 | } |
| 6459 | } else if (ID == ".amdhsa_exception_fp_ieee_invalid_op" ) { |
| 6460 | PARSE_BITS_ENTRY( |
| 6461 | KD.compute_pgm_rsrc2, |
| 6462 | COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION, |
| 6463 | ExprVal, ValRange); |
| 6464 | } else if (ID == ".amdhsa_exception_fp_denorm_src" ) { |
| 6465 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, |
| 6466 | COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE, |
| 6467 | ExprVal, ValRange); |
| 6468 | } else if (ID == ".amdhsa_exception_fp_ieee_div_zero" ) { |
| 6469 | PARSE_BITS_ENTRY( |
| 6470 | KD.compute_pgm_rsrc2, |
| 6471 | COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO, |
| 6472 | ExprVal, ValRange); |
| 6473 | } else if (ID == ".amdhsa_exception_fp_ieee_overflow" ) { |
| 6474 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, |
| 6475 | COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW, |
| 6476 | ExprVal, ValRange); |
| 6477 | } else if (ID == ".amdhsa_exception_fp_ieee_underflow" ) { |
| 6478 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, |
| 6479 | COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW, |
| 6480 | ExprVal, ValRange); |
| 6481 | } else if (ID == ".amdhsa_exception_fp_ieee_inexact" ) { |
| 6482 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, |
| 6483 | COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT, |
| 6484 | ExprVal, ValRange); |
| 6485 | } else if (ID == ".amdhsa_exception_int_div_zero" ) { |
| 6486 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2, |
| 6487 | COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO, |
| 6488 | ExprVal, ValRange); |
| 6489 | } else if (ID == ".amdhsa_round_robin_scheduling" ) { |
| 6490 | if (ISA.Major < 12) |
| 6491 | return Error(L: IDRange.Start, Msg: "directive requires gfx12+" , Range: IDRange); |
| 6492 | PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, |
| 6493 | COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, ExprVal, |
| 6494 | ValRange); |
| 6495 | } else { |
| 6496 | return Error(L: IDRange.Start, Msg: "unknown .amdhsa_kernel directive" , Range: IDRange); |
| 6497 | } |
| 6498 | |
| 6499 | #undef PARSE_BITS_ENTRY |
| 6500 | } |
| 6501 | |
| 6502 | if (!Seen.contains(key: ".amdhsa_next_free_vgpr" )) |
| 6503 | return TokError(Msg: ".amdhsa_next_free_vgpr directive is required" ); |
| 6504 | |
| 6505 | if (!Seen.contains(key: ".amdhsa_next_free_sgpr" )) |
| 6506 | return TokError(Msg: ".amdhsa_next_free_sgpr directive is required" ); |
| 6507 | |
| 6508 | unsigned UserSGPRCount = ExplicitUserSGPRCount.value_or(u&: ImpliedUserSGPRCount); |
| 6509 | if (UserSGPRCount > getMaxNumUserSGPRs()) |
| 6510 | return TokError(Msg: "too many user SGPRs enabled, found " + |
| 6511 | Twine(UserSGPRCount) + ", but only " + |
| 6512 | Twine(getMaxNumUserSGPRs()) + " are supported." ); |
| 6513 | |
| 6514 | // Consider the case where the total number of UserSGPRs with trailing |
| 6515 | // allocated preload SGPRs, is greater than the number of explicitly |
| 6516 | // referenced SGPRs. |
| 6517 | if (PreloadLength) { |
| 6518 | MCContext &Ctx = getContext(); |
| 6519 | NextFreeSGPR = AMDGPUMCExpr::createMax( |
| 6520 | Args: {NextFreeSGPR, MCConstantExpr::create(Value: UserSGPRCount, Ctx)}, Ctx); |
| 6521 | } |
| 6522 | |
| 6523 | const MCExpr *VGPRBlocks; |
| 6524 | const MCExpr *SGPRBlocks; |
| 6525 | if (calculateGPRBlocks(Features: getFeatureBits(), VCCUsed: ReserveVCC, FlatScrUsed: ReserveFlatScr, |
| 6526 | XNACKUsed: getTargetStreamer().getTargetID()->isXnackOnOrAny(), |
| 6527 | EnableWavefrontSize32, NextFreeVGPR, VGPRRange, |
| 6528 | NextFreeSGPR, SGPRRange, VGPRBlocks, SGPRBlocks)) |
| 6529 | return true; |
| 6530 | |
| 6531 | int64_t EvaluatedVGPRBlocks; |
| 6532 | bool VGPRBlocksEvaluatable = |
| 6533 | VGPRBlocks->evaluateAsAbsolute(Res&: EvaluatedVGPRBlocks); |
| 6534 | if (VGPRBlocksEvaluatable && |
| 6535 | !isUInt<COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_WIDTH>( |
| 6536 | x: static_cast<uint64_t>(EvaluatedVGPRBlocks))) { |
| 6537 | return OutOfRangeError(Range: VGPRRange); |
| 6538 | } |
| 6539 | AMDGPU::MCKernelDescriptor::bits_set( |
| 6540 | Dst&: KD.compute_pgm_rsrc1, Value: VGPRBlocks, |
| 6541 | Shift: COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT, |
| 6542 | Mask: COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, Ctx&: getContext()); |
| 6543 | |
| 6544 | int64_t EvaluatedSGPRBlocks; |
| 6545 | if (SGPRBlocks->evaluateAsAbsolute(Res&: EvaluatedSGPRBlocks) && |
| 6546 | !isUInt<COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_WIDTH>( |
| 6547 | x: static_cast<uint64_t>(EvaluatedSGPRBlocks))) |
| 6548 | return OutOfRangeError(Range: SGPRRange); |
| 6549 | AMDGPU::MCKernelDescriptor::bits_set( |
| 6550 | Dst&: KD.compute_pgm_rsrc1, Value: SGPRBlocks, |
| 6551 | Shift: COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT, |
| 6552 | Mask: COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, Ctx&: getContext()); |
| 6553 | |
| 6554 | if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount) |
| 6555 | return TokError(Msg: "amdgpu_user_sgpr_count smaller than implied by " |
| 6556 | "enabled user SGPRs" ); |
| 6557 | |
| 6558 | if (isGFX1250Plus()) { |
| 6559 | AMDGPU::MCKernelDescriptor::bits_set( |
| 6560 | Dst&: KD.compute_pgm_rsrc2, |
| 6561 | Value: MCConstantExpr::create(Value: UserSGPRCount, Ctx&: getContext()), |
| 6562 | Shift: COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT_SHIFT, |
| 6563 | Mask: COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT, Ctx&: getContext()); |
| 6564 | } else { |
| 6565 | AMDGPU::MCKernelDescriptor::bits_set( |
| 6566 | Dst&: KD.compute_pgm_rsrc2, |
| 6567 | Value: MCConstantExpr::create(Value: UserSGPRCount, Ctx&: getContext()), |
| 6568 | Shift: COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT_SHIFT, |
| 6569 | Mask: COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT, Ctx&: getContext()); |
| 6570 | } |
| 6571 | |
| 6572 | int64_t IVal = 0; |
| 6573 | if (!KD.kernarg_size->evaluateAsAbsolute(Res&: IVal)) |
| 6574 | return TokError(Msg: "Kernarg size should be resolvable" ); |
| 6575 | uint64_t kernarg_size = IVal; |
| 6576 | if (PreloadLength && kernarg_size && |
| 6577 | (PreloadLength * 4 + PreloadOffset * 4 > kernarg_size)) |
| 6578 | return TokError(Msg: "Kernarg preload length + offset is larger than the " |
| 6579 | "kernarg segment size" ); |
| 6580 | |
| 6581 | if (isGFX90A()) { |
| 6582 | if (!Seen.contains(key: ".amdhsa_accum_offset" )) |
| 6583 | return TokError(Msg: ".amdhsa_accum_offset directive is required" ); |
| 6584 | int64_t EvaluatedAccum; |
| 6585 | bool AccumEvaluatable = AccumOffset->evaluateAsAbsolute(Res&: EvaluatedAccum); |
| 6586 | uint64_t UEvaluatedAccum = EvaluatedAccum; |
| 6587 | if (AccumEvaluatable && |
| 6588 | (UEvaluatedAccum < 4 || UEvaluatedAccum > 256 || (UEvaluatedAccum & 3))) |
| 6589 | return TokError(Msg: "accum_offset should be in range [4..256] in " |
| 6590 | "increments of 4" ); |
| 6591 | |
| 6592 | int64_t EvaluatedNumVGPR; |
| 6593 | if (NextFreeVGPR->evaluateAsAbsolute(Res&: EvaluatedNumVGPR) && |
| 6594 | AccumEvaluatable && |
| 6595 | UEvaluatedAccum > |
| 6596 | alignTo(Value: std::max(a: (uint64_t)1, b: (uint64_t)EvaluatedNumVGPR), Align: 4)) |
| 6597 | return TokError(Msg: "accum_offset exceeds total VGPR allocation" ); |
| 6598 | const MCExpr *AdjustedAccum = MCBinaryExpr::createSub( |
| 6599 | LHS: MCBinaryExpr::createDiv( |
| 6600 | LHS: AccumOffset, RHS: MCConstantExpr::create(Value: 4, Ctx&: getContext()), Ctx&: getContext()), |
| 6601 | RHS: MCConstantExpr::create(Value: 1, Ctx&: getContext()), Ctx&: getContext()); |
| 6602 | MCKernelDescriptor::bits_set(Dst&: KD.compute_pgm_rsrc3, Value: AdjustedAccum, |
| 6603 | Shift: COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT, |
| 6604 | Mask: COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, |
| 6605 | Ctx&: getContext()); |
| 6606 | } |
| 6607 | |
| 6608 | if (isGFX1250Plus()) |
| 6609 | MCKernelDescriptor::bits_set(Dst&: KD.compute_pgm_rsrc3, Value: NamedBarCnt, |
| 6610 | Shift: COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT, |
| 6611 | Mask: COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT, |
| 6612 | Ctx&: getContext()); |
| 6613 | |
| 6614 | if (ISA.Major >= 10 && ISA.Major < 12) { |
| 6615 | // SharedVGPRCount < 16 checked by PARSE_ENTRY_BITS |
| 6616 | if (SharedVGPRCount && EnableWavefrontSize32 && *EnableWavefrontSize32) { |
| 6617 | return TokError(Msg: "shared_vgpr_count directive not valid on " |
| 6618 | "wavefront size 32" ); |
| 6619 | } |
| 6620 | |
| 6621 | if (VGPRBlocksEvaluatable && |
| 6622 | (SharedVGPRCount * 2 + static_cast<uint64_t>(EvaluatedVGPRBlocks) > |
| 6623 | 63)) { |
| 6624 | return TokError(Msg: "shared_vgpr_count*2 + " |
| 6625 | "compute_pgm_rsrc1.GRANULATED_WORKITEM_VGPR_COUNT cannot " |
| 6626 | "exceed 63\n" ); |
| 6627 | } |
| 6628 | } |
| 6629 | |
| 6630 | emitTargetDirective(); |
| 6631 | getTargetStreamer().EmitAmdhsaKernelDescriptor(STI: getSTI(), KernelName, KernelDescriptor: KD, |
| 6632 | NextVGPR: NextFreeVGPR, NextSGPR: NextFreeSGPR, |
| 6633 | ReserveVCC, ReserveFlatScr); |
| 6634 | return false; |
| 6635 | } |
| 6636 | |
| 6637 | bool AMDGPUAsmParser::ParseDirectiveAMDHSACodeObjectVersion() { |
| 6638 | uint32_t Version; |
| 6639 | if (ParseAsAbsoluteExpression(Ret&: Version)) |
| 6640 | return true; |
| 6641 | |
| 6642 | getTargetStreamer().EmitDirectiveAMDHSACodeObjectVersion(COV: Version); |
| 6643 | emitTargetDirective(); |
| 6644 | return false; |
| 6645 | } |
| 6646 | |
| 6647 | bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID, |
| 6648 | AMDGPUMCKernelCodeT &C) { |
| 6649 | // max_scratch_backing_memory_byte_size is deprecated. Ignore it while parsing |
| 6650 | // assembly for backwards compatibility. |
| 6651 | if (ID == "max_scratch_backing_memory_byte_size" ) { |
| 6652 | Parser.eatToEndOfStatement(); |
| 6653 | return false; |
| 6654 | } |
| 6655 | |
| 6656 | SmallString<40> ErrStr; |
| 6657 | raw_svector_ostream Err(ErrStr); |
| 6658 | if (!C.ParseKernelCodeT(ID, MCParser&: getParser(), Err)) { |
| 6659 | return TokError(Msg: Err.str()); |
| 6660 | } |
| 6661 | Lex(); |
| 6662 | |
| 6663 | if (ID == "enable_wavefront_size32" ) { |
| 6664 | if (C.code_properties & AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32) { |
| 6665 | if (!isGFX10Plus()) |
| 6666 | return TokError(Msg: "enable_wavefront_size32=1 is only allowed on GFX10+" ); |
| 6667 | if (!isWave32()) |
| 6668 | return TokError(Msg: "enable_wavefront_size32=1 requires +WavefrontSize32" ); |
| 6669 | } else { |
| 6670 | if (!isWave64()) |
| 6671 | return TokError(Msg: "enable_wavefront_size32=0 requires +WavefrontSize64" ); |
| 6672 | } |
| 6673 | } |
| 6674 | |
| 6675 | if (ID == "wavefront_size" ) { |
| 6676 | if (C.wavefront_size == 5) { |
| 6677 | if (!isGFX10Plus()) |
| 6678 | return TokError(Msg: "wavefront_size=5 is only allowed on GFX10+" ); |
| 6679 | if (!isWave32()) |
| 6680 | return TokError(Msg: "wavefront_size=5 requires +WavefrontSize32" ); |
| 6681 | } else if (C.wavefront_size == 6) { |
| 6682 | if (!isWave64()) |
| 6683 | return TokError(Msg: "wavefront_size=6 requires +WavefrontSize64" ); |
| 6684 | } |
| 6685 | } |
| 6686 | |
| 6687 | return false; |
| 6688 | } |
| 6689 | |
| 6690 | bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() { |
| 6691 | AMDGPUMCKernelCodeT KernelCode; |
| 6692 | KernelCode.initDefault(STI: getSTI(), Ctx&: getContext()); |
| 6693 | |
| 6694 | while (true) { |
| 6695 | // Lex EndOfStatement. This is in a while loop, because lexing a comment |
| 6696 | // will set the current token to EndOfStatement. |
| 6697 | while (trySkipToken(Kind: AsmToken::EndOfStatement)) |
| 6698 | ; |
| 6699 | |
| 6700 | StringRef ID; |
| 6701 | if (!parseId(Val&: ID, ErrMsg: "expected value identifier or .end_amd_kernel_code_t" )) |
| 6702 | return true; |
| 6703 | |
| 6704 | if (ID == ".end_amd_kernel_code_t" ) |
| 6705 | break; |
| 6706 | |
| 6707 | if (ParseAMDKernelCodeTValue(ID, C&: KernelCode)) |
| 6708 | return true; |
| 6709 | } |
| 6710 | |
| 6711 | KernelCode.validate(STI: &getSTI(), Ctx&: getContext()); |
| 6712 | getTargetStreamer().EmitAMDKernelCodeT(Header&: KernelCode); |
| 6713 | |
| 6714 | return false; |
| 6715 | } |
| 6716 | |
| 6717 | bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() { |
| 6718 | StringRef KernelName; |
| 6719 | if (!parseId(Val&: KernelName, ErrMsg: "expected symbol name" )) |
| 6720 | return true; |
| 6721 | |
| 6722 | getTargetStreamer().EmitAMDGPUSymbolType(SymbolName: KernelName, |
| 6723 | Type: ELF::STT_AMDGPU_HSA_KERNEL); |
| 6724 | |
| 6725 | KernelScope.initialize(Context&: getContext()); |
| 6726 | return false; |
| 6727 | } |
| 6728 | |
| 6729 | bool AMDGPUAsmParser::ParseDirectiveISAVersion() { |
| 6730 | if (!getSTI().getTargetTriple().isAMDGCN()) { |
| 6731 | return Error(L: getLoc(), |
| 6732 | Msg: ".amd_amdgpu_isa directive is not available on non-amdgcn " |
| 6733 | "architectures" ); |
| 6734 | } |
| 6735 | |
| 6736 | StringRef TargetIDDirective = getLexer().getTok().getStringContents(); |
| 6737 | |
| 6738 | std::optional<AMDGPU::TargetID> MaybeParsed = |
| 6739 | AMDGPU::TargetID::parseTargetIDString(TargetIDDirective); |
| 6740 | if (!MaybeParsed) |
| 6741 | return Error(L: getParser().getTok().getLoc(), |
| 6742 | Msg: "malformed target id '" + TargetIDDirective + "'" ); |
| 6743 | |
| 6744 | const AMDGPU::TargetID &ParsedTargetID = *MaybeParsed; |
| 6745 | const Triple &TT = getSTI().getTargetTriple(); |
| 6746 | |
| 6747 | // The processor named in the target id must be covered by the triple's |
| 6748 | // subarch. |
| 6749 | if (!AMDGPU::isCPUValidForSubArch(SubArch: TT.getSubArch(), |
| 6750 | AK: ParsedTargetID.getGPUKind())) { |
| 6751 | return Error(L: getParser().getTok().getLoc(), |
| 6752 | Msg: "target id '" + TargetIDDirective + |
| 6753 | "' specifies a processor that is not valid for subarch '" + |
| 6754 | TT.getArchName() + "'" ); |
| 6755 | } |
| 6756 | |
| 6757 | const std::optional<AMDGPU::TargetID> &CurrentTargetID = |
| 6758 | getTargetStreamer().getTargetID(); |
| 6759 | |
| 6760 | Triple DirectiveTriple(ParsedTargetID.getTargetTripleString()); |
| 6761 | const Triple &STITriple = getSTI().getTargetTriple(); |
| 6762 | if (!DirectiveTriple.isCompatibleWith(Other: STITriple)) { |
| 6763 | return Error(L: getParser().getTok().getLoc(), |
| 6764 | Msg: ".amd_amdgpu_isa " + Twine(ParsedTargetID.toString()) + |
| 6765 | " is incompatible with " + |
| 6766 | Twine(CurrentTargetID->toString())); |
| 6767 | } |
| 6768 | |
| 6769 | // Error if the ISA version doesn't match |
| 6770 | StringRef DirectiveProcessor = |
| 6771 | AMDGPU::getArchNameAMDGCN(AK: ParsedTargetID.getGPUKind()); |
| 6772 | AMDGPU::IsaVersion DirectiveISA = AMDGPU::getIsaVersion(GPU: DirectiveProcessor); |
| 6773 | if (DirectiveISA != ISA) { |
| 6774 | return Error(L: getParser().getTok().getLoc(), |
| 6775 | Msg: ".amd_amdgpu_isa directive processor " + |
| 6776 | Twine(DirectiveProcessor) + |
| 6777 | " does not match the specified processor " + |
| 6778 | Twine(getSTI().getCPU())); |
| 6779 | } |
| 6780 | |
| 6781 | getTargetStreamer().EmitISAVersion(); |
| 6782 | Lex(); |
| 6783 | |
| 6784 | return false; |
| 6785 | } |
| 6786 | |
| 6787 | bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() { |
| 6788 | assert(isHsaAbi(getSTI())); |
| 6789 | |
| 6790 | std::string HSAMetadataString; |
| 6791 | if (ParseToEndDirective(AssemblerDirectiveBegin: HSAMD::V3::AssemblerDirectiveBegin, |
| 6792 | AssemblerDirectiveEnd: HSAMD::V3::AssemblerDirectiveEnd, CollectString&: HSAMetadataString)) |
| 6793 | return true; |
| 6794 | |
| 6795 | if (!getTargetStreamer().EmitHSAMetadataV3(HSAMetadataString)) |
| 6796 | return Error(L: getLoc(), Msg: "invalid HSA metadata" ); |
| 6797 | |
| 6798 | return false; |
| 6799 | } |
| 6800 | |
| 6801 | /// Common code to parse out a block of text (typically YAML) between start and |
| 6802 | /// end directives. |
| 6803 | bool AMDGPUAsmParser::ParseToEndDirective(const char *AssemblerDirectiveBegin, |
| 6804 | const char *AssemblerDirectiveEnd, |
| 6805 | std::string &CollectString) { |
| 6806 | |
| 6807 | raw_string_ostream CollectStream(CollectString); |
| 6808 | |
| 6809 | getLexer().setSkipSpace(false); |
| 6810 | |
| 6811 | bool FoundEnd = false; |
| 6812 | while (!isToken(Kind: AsmToken::Eof)) { |
| 6813 | while (isToken(Kind: AsmToken::Space)) { |
| 6814 | CollectStream << getTokenStr(); |
| 6815 | Lex(); |
| 6816 | } |
| 6817 | |
| 6818 | if (trySkipId(Id: AssemblerDirectiveEnd)) { |
| 6819 | FoundEnd = true; |
| 6820 | break; |
| 6821 | } |
| 6822 | |
| 6823 | CollectStream << Parser.parseStringToEndOfStatement() |
| 6824 | << getContext().getAsmInfo().getSeparatorString(); |
| 6825 | |
| 6826 | Parser.eatToEndOfStatement(); |
| 6827 | } |
| 6828 | |
| 6829 | getLexer().setSkipSpace(true); |
| 6830 | |
| 6831 | if (isToken(Kind: AsmToken::Eof) && !FoundEnd) { |
| 6832 | return TokError(Msg: Twine("expected directive " ) + |
| 6833 | Twine(AssemblerDirectiveEnd) + Twine(" not found" )); |
| 6834 | } |
| 6835 | |
| 6836 | return false; |
| 6837 | } |
| 6838 | |
| 6839 | /// Parse the assembler directive for new MsgPack-format PAL metadata. |
| 6840 | bool AMDGPUAsmParser::ParseDirectivePALMetadataBegin() { |
| 6841 | std::string String; |
| 6842 | if (ParseToEndDirective(AssemblerDirectiveBegin: AMDGPU::PALMD::AssemblerDirectiveBegin, |
| 6843 | AssemblerDirectiveEnd: AMDGPU::PALMD::AssemblerDirectiveEnd, CollectString&: String)) |
| 6844 | return true; |
| 6845 | |
| 6846 | auto *PALMetadata = getTargetStreamer().getPALMetadata(); |
| 6847 | if (!PALMetadata->setFromString(String)) |
| 6848 | return Error(L: getLoc(), Msg: "invalid PAL metadata" ); |
| 6849 | return false; |
| 6850 | } |
| 6851 | |
| 6852 | /// Parse the assembler directive for old linear-format PAL metadata. |
| 6853 | bool AMDGPUAsmParser::ParseDirectivePALMetadata() { |
| 6854 | if (getSTI().getTargetTriple().getOS() != Triple::AMDPAL) { |
| 6855 | return Error(L: getLoc(), Msg: (Twine(PALMD::AssemblerDirective) + |
| 6856 | Twine(" directive is " |
| 6857 | "not available on non-amdpal OSes" )) |
| 6858 | .str()); |
| 6859 | } |
| 6860 | |
| 6861 | auto *PALMetadata = getTargetStreamer().getPALMetadata(); |
| 6862 | PALMetadata->setLegacy(); |
| 6863 | for (;;) { |
| 6864 | uint32_t Key, Value; |
| 6865 | if (ParseAsAbsoluteExpression(Ret&: Key)) { |
| 6866 | return TokError(Msg: Twine("invalid value in " ) + |
| 6867 | Twine(PALMD::AssemblerDirective)); |
| 6868 | } |
| 6869 | if (!trySkipToken(Kind: AsmToken::Comma)) { |
| 6870 | return TokError(Msg: Twine("expected an even number of values in " ) + |
| 6871 | Twine(PALMD::AssemblerDirective)); |
| 6872 | } |
| 6873 | if (ParseAsAbsoluteExpression(Ret&: Value)) { |
| 6874 | return TokError(Msg: Twine("invalid value in " ) + |
| 6875 | Twine(PALMD::AssemblerDirective)); |
| 6876 | } |
| 6877 | PALMetadata->setRegister(Reg: Key, Val: Value); |
| 6878 | if (!trySkipToken(Kind: AsmToken::Comma)) |
| 6879 | break; |
| 6880 | } |
| 6881 | return false; |
| 6882 | } |
| 6883 | |
| 6884 | /// ParseDirectiveAMDGPULDS |
| 6885 | /// ::= .amdgpu_lds identifier ',' size_expression [',' align_expression] |
| 6886 | bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() { |
| 6887 | if (getParser().checkForValidSection()) |
| 6888 | return true; |
| 6889 | |
| 6890 | StringRef Name; |
| 6891 | SMLoc NameLoc = getLoc(); |
| 6892 | if (getParser().parseIdentifier(Res&: Name)) |
| 6893 | return TokError(Msg: "expected identifier in directive" ); |
| 6894 | |
| 6895 | MCSymbol *Symbol = getContext().getOrCreateSymbol(Name); |
| 6896 | if (getParser().parseComma()) |
| 6897 | return true; |
| 6898 | |
| 6899 | unsigned LocalMemorySize = AMDGPU::IsaInfo::getLocalMemorySize(STI: getSTI()); |
| 6900 | |
| 6901 | int64_t Size; |
| 6902 | SMLoc SizeLoc = getLoc(); |
| 6903 | if (getParser().parseAbsoluteExpression(Res&: Size)) |
| 6904 | return true; |
| 6905 | if (Size < 0) |
| 6906 | return Error(L: SizeLoc, Msg: "size must be non-negative" ); |
| 6907 | if (Size > LocalMemorySize) |
| 6908 | return Error(L: SizeLoc, Msg: "size is too large" ); |
| 6909 | |
| 6910 | int64_t Alignment = 4; |
| 6911 | if (trySkipToken(Kind: AsmToken::Comma)) { |
| 6912 | SMLoc AlignLoc = getLoc(); |
| 6913 | if (getParser().parseAbsoluteExpression(Res&: Alignment)) |
| 6914 | return true; |
| 6915 | if (Alignment < 0 || !isPowerOf2_64(Value: Alignment)) |
| 6916 | return Error(L: AlignLoc, Msg: "alignment must be a power of two" ); |
| 6917 | |
| 6918 | // Alignment larger than the size of LDS is possible in theory, as long |
| 6919 | // as the linker manages to place to symbol at address 0, but we do want |
| 6920 | // to make sure the alignment fits nicely into a 32-bit integer. |
| 6921 | if (Alignment >= 1u << 31) |
| 6922 | return Error(L: AlignLoc, Msg: "alignment is too large" ); |
| 6923 | } |
| 6924 | |
| 6925 | if (parseEOL()) |
| 6926 | return true; |
| 6927 | |
| 6928 | Symbol->redefineIfPossible(); |
| 6929 | if (!Symbol->isUndefined()) |
| 6930 | return Error(L: NameLoc, Msg: "invalid symbol redefinition" ); |
| 6931 | |
| 6932 | getTargetStreamer().emitAMDGPULDS(Symbol, Size, Alignment: Align(Alignment)); |
| 6933 | return false; |
| 6934 | } |
| 6935 | |
| 6936 | bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() { |
| 6937 | if (getParser().checkForValidSection()) |
| 6938 | return true; |
| 6939 | |
| 6940 | StringRef FuncName; |
| 6941 | if (getParser().parseIdentifier(Res&: FuncName)) |
| 6942 | return TokError(Msg: "expected symbol name after .amdgpu_info" ); |
| 6943 | |
| 6944 | MCSymbol *FuncSym = getContext().getOrCreateSymbol(Name: FuncName); |
| 6945 | AMDGPU::InfoSectionData ParsedInfoData; |
| 6946 | AMDGPU::FuncInfo FI; |
| 6947 | FI.Sym = FuncSym; |
| 6948 | bool HasScalarAttrs = false; |
| 6949 | |
| 6950 | while (true) { |
| 6951 | while (trySkipToken(Kind: AsmToken::EndOfStatement)) |
| 6952 | ; |
| 6953 | |
| 6954 | StringRef ID; |
| 6955 | SMLoc IDLoc = getLoc(); |
| 6956 | if (!parseId(Val&: ID, ErrMsg: "expected directive or .end_amdgpu_info" )) |
| 6957 | return true; |
| 6958 | |
| 6959 | if (ID == ".end_amdgpu_info" ) |
| 6960 | break; |
| 6961 | |
| 6962 | // Every per-entry directive shares the `.amdgpu_` namespace prefix; strip |
| 6963 | // it once and dispatch on the distinguishing suffix below. The unstripped |
| 6964 | // ID is preserved for diagnostics. |
| 6965 | StringRef Dir = ID; |
| 6966 | if (!Dir.consume_front(Prefix: ".amdgpu_" )) |
| 6967 | return Error(L: IDLoc, Msg: "unknown .amdgpu_info directive '" + ID + "'" ); |
| 6968 | |
| 6969 | if (Dir == "flags" ) { |
| 6970 | int64_t Val; |
| 6971 | if (getParser().parseAbsoluteExpression(Res&: Val)) |
| 6972 | return true; |
| 6973 | auto Flags = static_cast<AMDGPU::FuncInfoFlags>(Val); |
| 6974 | FI.UsesVCC = !!(Flags & AMDGPU::FuncInfoFlags::FUNC_USES_VCC); |
| 6975 | FI.UsesFlatScratch = |
| 6976 | !!(Flags & AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH); |
| 6977 | FI.HasDynStack = !!(Flags & AMDGPU::FuncInfoFlags::FUNC_HAS_DYN_STACK); |
| 6978 | HasScalarAttrs = true; |
| 6979 | } else if (Dir == "num_sgpr" ) { |
| 6980 | int64_t Val; |
| 6981 | if (getParser().parseAbsoluteExpression(Res&: Val)) |
| 6982 | return true; |
| 6983 | FI.NumSGPR = static_cast<uint32_t>(Val); |
| 6984 | HasScalarAttrs = true; |
| 6985 | } else if (Dir == "num_vgpr" ) { |
| 6986 | int64_t Val; |
| 6987 | if (getParser().parseAbsoluteExpression(Res&: Val)) |
| 6988 | return true; |
| 6989 | FI.NumArchVGPR = static_cast<uint32_t>(Val); |
| 6990 | HasScalarAttrs = true; |
| 6991 | } else if (Dir == "num_agpr" ) { |
| 6992 | int64_t Val; |
| 6993 | if (getParser().parseAbsoluteExpression(Res&: Val)) |
| 6994 | return true; |
| 6995 | FI.NumAccVGPR = static_cast<uint32_t>(Val); |
| 6996 | HasScalarAttrs = true; |
| 6997 | } else if (Dir == "private_segment_size" ) { |
| 6998 | int64_t Val; |
| 6999 | if (getParser().parseAbsoluteExpression(Res&: Val)) |
| 7000 | return true; |
| 7001 | FI.PrivateSegmentSize = static_cast<uint32_t>(Val); |
| 7002 | HasScalarAttrs = true; |
| 7003 | } else if (Dir == "use" ) { |
| 7004 | StringRef ResName; |
| 7005 | if (getParser().parseIdentifier(Res&: ResName)) |
| 7006 | return TokError(Msg: "expected resource symbol for .amdgpu_use" ); |
| 7007 | ParsedInfoData.Uses.push_back( |
| 7008 | Elt: {FuncSym, getContext().getOrCreateSymbol(Name: ResName)}); |
| 7009 | } else if (Dir == "call" ) { |
| 7010 | StringRef DstName; |
| 7011 | if (getParser().parseIdentifier(Res&: DstName)) |
| 7012 | return TokError(Msg: "expected callee symbol for .amdgpu_call" ); |
| 7013 | ParsedInfoData.Calls.push_back( |
| 7014 | Elt: {FuncSym, getContext().getOrCreateSymbol(Name: DstName)}); |
| 7015 | } else if (Dir == "indirect_call" ) { |
| 7016 | std::string TypeId; |
| 7017 | if (getParser().parseEscapedString(Data&: TypeId)) |
| 7018 | return TokError(Msg: "expected type ID string for .amdgpu_indirect_call" ); |
| 7019 | ParsedInfoData.IndirectCalls.push_back(Elt: {FuncSym, std::move(TypeId)}); |
| 7020 | } else if (Dir == "typeid" ) { |
| 7021 | std::string TypeId; |
| 7022 | if (getParser().parseEscapedString(Data&: TypeId)) |
| 7023 | return TokError(Msg: "expected type ID string for .amdgpu_typeid" ); |
| 7024 | ParsedInfoData.TypeIds.push_back(Elt: {FuncSym, std::move(TypeId)}); |
| 7025 | } else { |
| 7026 | return Error(L: IDLoc, Msg: "unknown .amdgpu_info directive '" + ID + "'" ); |
| 7027 | } |
| 7028 | } |
| 7029 | |
| 7030 | if (HasScalarAttrs) |
| 7031 | ParsedInfoData.Funcs.push_back(Elt: std::move(FI)); |
| 7032 | |
| 7033 | AMDGPU::InfoSectionData &Data = InfoData ? *InfoData : InfoData.emplace(); |
| 7034 | for (AMDGPU::FuncInfo &Func : ParsedInfoData.Funcs) |
| 7035 | Data.Funcs.push_back(Elt: std::move(Func)); |
| 7036 | for (std::pair<MCSymbol *, MCSymbol *> &Use : ParsedInfoData.Uses) |
| 7037 | Data.Uses.push_back(Elt: Use); |
| 7038 | for (std::pair<MCSymbol *, MCSymbol *> &Call : ParsedInfoData.Calls) |
| 7039 | Data.Calls.push_back(Elt: Call); |
| 7040 | for (std::pair<MCSymbol *, std::string> &IndirectCall : |
| 7041 | ParsedInfoData.IndirectCalls) |
| 7042 | Data.IndirectCalls.push_back(Elt: std::move(IndirectCall)); |
| 7043 | for (std::pair<MCSymbol *, std::string> &TypeId : ParsedInfoData.TypeIds) |
| 7044 | Data.TypeIds.push_back(Elt: std::move(TypeId)); |
| 7045 | |
| 7046 | return false; |
| 7047 | } |
| 7048 | |
| 7049 | void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) { |
| 7050 | // Record every parsed label in the timeline so that, at end of file, the |
| 7051 | // instructions following a kernel's label can be located regardless of |
| 7052 | // whether the .amdhsa_kernel directive came before or after the label. |
| 7053 | OpcodeStreamSymbols.emplace_back(Args&: Symbol, Args&: IDLoc, Args: OpcodeStream.size()); |
| 7054 | } |
| 7055 | |
| 7056 | void AMDGPUAsmParser::checkKernelPrologues() { |
| 7057 | if (getFeatureBits()[AMDGPU::FeatureRequiresInitialUnclausedVmem]) { |
| 7058 | static const unsigned Required[] = {S_MOV_B64_gfx12, V_NOP_e32_gfx12, |
| 7059 | GLOBAL_PREFETCH_B8_SADDR_gfx1250}; |
| 7060 | for (auto [Sym, Loc, Offset] : OpcodeStreamSymbols) { |
| 7061 | if (!AMDHSAKernelSymbols.contains(Ptr: Sym)) |
| 7062 | continue; |
| 7063 | ArrayRef<unsigned> Prologue = ArrayRef(OpcodeStream).drop_front(N: Offset); |
| 7064 | if (!Prologue.empty() && Prologue.front() == S_SETREG_IMM32_B32_gfx12) |
| 7065 | Prologue = Prologue.drop_front(); |
| 7066 | if (Prologue.take_front(N: std::size(Required)) != ArrayRef(Required)) { |
| 7067 | Warning(L: Loc, Msg: "kernel '" + Sym->getName() + |
| 7068 | "' does not begin with the required prologue " |
| 7069 | "sequence: s_mov_b64 followed by v_nop and " |
| 7070 | "global_prefetch_b8" ); |
| 7071 | } |
| 7072 | } |
| 7073 | } |
| 7074 | OpcodeStream.clear(); |
| 7075 | OpcodeStreamSymbols.clear(); |
| 7076 | AMDHSAKernelSymbols.clear(); |
| 7077 | } |
| 7078 | |
| 7079 | void AMDGPUAsmParser::onEndOfFile() { |
| 7080 | emitTargetDirective(); |
| 7081 | checkKernelPrologues(); |
| 7082 | if (InfoData) |
| 7083 | getTargetStreamer().emitAMDGPUInfo(Data: *InfoData); |
| 7084 | } |
| 7085 | |
| 7086 | bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) { |
| 7087 | StringRef IDVal = DirectiveID.getString(); |
| 7088 | |
| 7089 | if (isHsaAbi(STI: getSTI())) { |
| 7090 | if (IDVal == ".amdhsa_kernel" ) |
| 7091 | return ParseDirectiveAMDHSAKernel(); |
| 7092 | |
| 7093 | if (IDVal == ".amdhsa_code_object_version" ) |
| 7094 | return ParseDirectiveAMDHSACodeObjectVersion(); |
| 7095 | |
| 7096 | // TODO: Restructure/combine with PAL metadata directive. |
| 7097 | if (IDVal == AMDGPU::HSAMD::V3::AssemblerDirectiveBegin) |
| 7098 | return ParseDirectiveHSAMetadata(); |
| 7099 | } else { |
| 7100 | if (IDVal == ".amd_kernel_code_t" ) |
| 7101 | return ParseDirectiveAMDKernelCodeT(); |
| 7102 | |
| 7103 | if (IDVal == ".amdgpu_hsa_kernel" ) |
| 7104 | return ParseDirectiveAMDGPUHsaKernel(); |
| 7105 | |
| 7106 | if (IDVal == ".amd_amdgpu_isa" ) |
| 7107 | return ParseDirectiveISAVersion(); |
| 7108 | |
| 7109 | if (IDVal == AMDGPU::HSAMD::AssemblerDirectiveBegin) { |
| 7110 | return Error(L: getLoc(), Msg: (Twine(HSAMD::AssemblerDirectiveBegin) + |
| 7111 | Twine(" directive is " |
| 7112 | "not available on non-amdhsa OSes" )) |
| 7113 | .str()); |
| 7114 | } |
| 7115 | } |
| 7116 | |
| 7117 | if (IDVal == ".amdgcn_target" ) |
| 7118 | return ParseDirectiveAMDGCNTarget(); |
| 7119 | |
| 7120 | if (IDVal == ".amdgpu_lds" ) |
| 7121 | return ParseDirectiveAMDGPULDS(); |
| 7122 | |
| 7123 | if (IDVal == ".amdgpu_info" ) |
| 7124 | return ParseDirectiveAMDGPUInfo(); |
| 7125 | |
| 7126 | if (IDVal == PALMD::AssemblerDirectiveBegin) |
| 7127 | return ParseDirectivePALMetadataBegin(); |
| 7128 | |
| 7129 | if (IDVal == PALMD::AssemblerDirective) |
| 7130 | return ParseDirectivePALMetadata(); |
| 7131 | |
| 7132 | return true; |
| 7133 | } |
| 7134 | |
| 7135 | bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI, |
| 7136 | MCRegister Reg) { |
| 7137 | if (MRI.regsOverlap(RegA: TTMP12_TTMP13_TTMP14_TTMP15, RegB: Reg)) |
| 7138 | return isGFX9Plus(); |
| 7139 | |
| 7140 | // GFX10+ has 2 more SGPRs 104 and 105. |
| 7141 | if (MRI.regsOverlap(RegA: SGPR104_SGPR105, RegB: Reg)) |
| 7142 | return hasSGPR104_SGPR105(); |
| 7143 | |
| 7144 | switch (Reg.id()) { |
| 7145 | case SRC_SHARED_BASE_LO: |
| 7146 | case SRC_SHARED_BASE: |
| 7147 | case SRC_SHARED_LIMIT_LO: |
| 7148 | case SRC_SHARED_LIMIT: |
| 7149 | return isGFX9Plus(); |
| 7150 | case SRC_PRIVATE_BASE_LO: |
| 7151 | case SRC_PRIVATE_BASE: |
| 7152 | case SRC_PRIVATE_LIMIT_LO: |
| 7153 | case SRC_PRIVATE_LIMIT: |
| 7154 | return AMDGPU::hasPrivateApertureRegs(STI: getSTI()); |
| 7155 | case SRC_FLAT_SCRATCH_BASE_LO: |
| 7156 | case SRC_FLAT_SCRATCH_BASE_HI: |
| 7157 | return hasGloballyAddressableScratch(); |
| 7158 | case SRC_POPS_EXITING_WAVE_ID: |
| 7159 | return hasPopsExitingWaveID(STI: getSTI()); |
| 7160 | case TBA: |
| 7161 | case TBA_LO: |
| 7162 | case TBA_HI: |
| 7163 | case TMA: |
| 7164 | case TMA_LO: |
| 7165 | case TMA_HI: |
| 7166 | return !isGFX9Plus(); |
| 7167 | case XNACK_MASK: |
| 7168 | case XNACK_MASK_LO: |
| 7169 | case XNACK_MASK_HI: |
| 7170 | return (isVI() || isGFX9()) && |
| 7171 | getTargetStreamer().getTargetID()->isXnackSupported(); |
| 7172 | case SGPR_NULL: |
| 7173 | return isGFX10Plus(); |
| 7174 | case SRC_EXECZ: |
| 7175 | case SRC_VCCZ: |
| 7176 | return !isGFX11Plus(); |
| 7177 | default: |
| 7178 | break; |
| 7179 | } |
| 7180 | |
| 7181 | if (isCI()) |
| 7182 | return true; |
| 7183 | |
| 7184 | if (isSI() || isGFX10Plus()) { |
| 7185 | // No flat_scr on SI. |
| 7186 | // On GFX10Plus flat scratch is not a valid register operand and can only be |
| 7187 | // accessed with s_setreg/s_getreg. |
| 7188 | switch (Reg.id()) { |
| 7189 | case FLAT_SCR: |
| 7190 | case FLAT_SCR_LO: |
| 7191 | case FLAT_SCR_HI: |
| 7192 | return false; |
| 7193 | default: |
| 7194 | return true; |
| 7195 | } |
| 7196 | } |
| 7197 | |
| 7198 | // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that |
| 7199 | // SI/CI have. |
| 7200 | if (MRI.regsOverlap(RegA: SGPR102_SGPR103, RegB: Reg)) |
| 7201 | return hasSGPR102_SGPR103(); |
| 7202 | |
| 7203 | return true; |
| 7204 | } |
| 7205 | |
| 7206 | ParseStatus AMDGPUAsmParser::parseOperand(OperandVector &Operands, |
| 7207 | StringRef Mnemonic, |
| 7208 | OperandMode Mode) { |
| 7209 | ParseStatus Res = parseVOPD(Operands); |
| 7210 | if (Res.isSuccess() || Res.isFailure() || isToken(Kind: AsmToken::EndOfStatement)) |
| 7211 | return Res; |
| 7212 | |
| 7213 | // Try to parse with a custom parser |
| 7214 | Res = MatchOperandParserImpl(Operands, Mnemonic); |
| 7215 | |
| 7216 | // If we successfully parsed the operand or if there as an error parsing, |
| 7217 | // we are done. |
| 7218 | // |
| 7219 | // If we are parsing after we reach EndOfStatement then this means we |
| 7220 | // are appending default values to the Operands list. This is only done |
| 7221 | // by custom parser, so we shouldn't continue on to the generic parsing. |
| 7222 | if (Res.isSuccess() || Res.isFailure() || isToken(Kind: AsmToken::EndOfStatement)) |
| 7223 | return Res; |
| 7224 | |
| 7225 | SMLoc RBraceLoc; |
| 7226 | SMLoc LBraceLoc = getLoc(); |
| 7227 | if (Mode == OperandMode_NSA && trySkipToken(Kind: AsmToken::LBrac)) { |
| 7228 | unsigned Prefix = Operands.size(); |
| 7229 | |
| 7230 | for (;;) { |
| 7231 | auto Loc = getLoc(); |
| 7232 | Res = parseReg(Operands); |
| 7233 | if (Res.isNoMatch()) |
| 7234 | Error(L: Loc, Msg: "expected a register" ); |
| 7235 | if (!Res.isSuccess()) |
| 7236 | return ParseStatus::Failure; |
| 7237 | |
| 7238 | RBraceLoc = getLoc(); |
| 7239 | if (trySkipToken(Kind: AsmToken::RBrac)) |
| 7240 | break; |
| 7241 | |
| 7242 | if (!skipToken(Kind: AsmToken::Comma, |
| 7243 | ErrMsg: "expected a comma or a closing square bracket" )) |
| 7244 | return ParseStatus::Failure; |
| 7245 | } |
| 7246 | |
| 7247 | if (Operands.size() - Prefix > 1) { |
| 7248 | Operands.insert(I: Operands.begin() + Prefix, |
| 7249 | Elt: AMDGPUOperand::CreateToken(AsmParser: this, Str: "[" , Loc: LBraceLoc)); |
| 7250 | Operands.push_back(Elt: AMDGPUOperand::CreateToken(AsmParser: this, Str: "]" , Loc: RBraceLoc)); |
| 7251 | } |
| 7252 | |
| 7253 | return ParseStatus::Success; |
| 7254 | } |
| 7255 | |
| 7256 | return parseRegOrImm(Operands); |
| 7257 | } |
| 7258 | |
| 7259 | StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) { |
| 7260 | // Clear any forced encodings from the previous instruction. |
| 7261 | setForcedEncodingSize(0); |
| 7262 | setForcedDPP(false); |
| 7263 | setForcedSDWA(false); |
| 7264 | |
| 7265 | if (Name.consume_back(Suffix: "_e64_dpp" )) { |
| 7266 | setForcedDPP(true); |
| 7267 | setForcedEncodingSize(64); |
| 7268 | return Name; |
| 7269 | } |
| 7270 | if (Name.consume_back(Suffix: "_e64" )) { |
| 7271 | setForcedEncodingSize(64); |
| 7272 | return Name; |
| 7273 | } |
| 7274 | if (Name.consume_back(Suffix: "_e32" )) { |
| 7275 | setForcedEncodingSize(32); |
| 7276 | return Name; |
| 7277 | } |
| 7278 | if (Name.consume_back(Suffix: "_dpp" )) { |
| 7279 | setForcedDPP(true); |
| 7280 | return Name; |
| 7281 | } |
| 7282 | if (Name.consume_back(Suffix: "_sdwa" )) { |
| 7283 | setForcedSDWA(true); |
| 7284 | return Name; |
| 7285 | } |
| 7286 | return Name; |
| 7287 | } |
| 7288 | |
| 7289 | static void applyMnemonicAliases(StringRef &Mnemonic, |
| 7290 | const FeatureBitset &Features, |
| 7291 | unsigned VariantID); |
| 7292 | |
| 7293 | bool AMDGPUAsmParser::parseInstruction(ParseInstructionInfo &Info, |
| 7294 | StringRef Name, SMLoc NameLoc, |
| 7295 | OperandVector &Operands) { |
| 7296 | // Add the instruction mnemonic |
| 7297 | Name = parseMnemonicSuffix(Name); |
| 7298 | |
| 7299 | // If the target architecture uses MnemonicAlias, call it here to parse |
| 7300 | // operands correctly. |
| 7301 | applyMnemonicAliases(Mnemonic&: Name, Features: getAvailableFeatures(), VariantID: 0); |
| 7302 | |
| 7303 | Operands.push_back(Elt: AMDGPUOperand::CreateToken(AsmParser: this, Str: Name, Loc: NameLoc)); |
| 7304 | |
| 7305 | bool IsMIMG = Name.starts_with(Prefix: "image_" ); |
| 7306 | |
| 7307 | while (!trySkipToken(Kind: AsmToken::EndOfStatement)) { |
| 7308 | OperandMode Mode = OperandMode_Default; |
| 7309 | if (IsMIMG && isGFX10Plus() && Operands.size() == 2) |
| 7310 | Mode = OperandMode_NSA; |
| 7311 | ParseStatus Res = parseOperand(Operands, Mnemonic: Name, Mode); |
| 7312 | |
| 7313 | if (!Res.isSuccess()) { |
| 7314 | checkUnsupportedInstruction(Mnemo: Name, IDLoc: NameLoc); |
| 7315 | if (!Parser.hasPendingError()) { |
| 7316 | // FIXME: use real operand location rather than the current location. |
| 7317 | StringRef Msg = Res.isFailure() ? "failed parsing operand." |
| 7318 | : "not a valid operand." ; |
| 7319 | Error(L: getLoc(), Msg); |
| 7320 | } |
| 7321 | while (!trySkipToken(Kind: AsmToken::EndOfStatement)) { |
| 7322 | lex(); |
| 7323 | } |
| 7324 | return true; |
| 7325 | } |
| 7326 | |
| 7327 | // Eat the comma or space if there is one. |
| 7328 | trySkipToken(Kind: AsmToken::Comma); |
| 7329 | } |
| 7330 | |
| 7331 | return false; |
| 7332 | } |
| 7333 | |
| 7334 | //===----------------------------------------------------------------------===// |
| 7335 | // Utility functions |
| 7336 | //===----------------------------------------------------------------------===// |
| 7337 | |
| 7338 | ParseStatus AMDGPUAsmParser::parseTokenOp(StringRef Name, |
| 7339 | OperandVector &Operands) { |
| 7340 | SMLoc S = getLoc(); |
| 7341 | if (!trySkipId(Id: Name)) |
| 7342 | return ParseStatus::NoMatch; |
| 7343 | |
| 7344 | Operands.push_back(Elt: AMDGPUOperand::CreateToken(AsmParser: this, Str: Name, Loc: S)); |
| 7345 | return ParseStatus::Success; |
| 7346 | } |
| 7347 | |
| 7348 | ParseStatus AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, |
| 7349 | int64_t &IntVal) { |
| 7350 | |
| 7351 | if (!trySkipId(Id: Prefix, Kind: AsmToken::Colon)) |
| 7352 | return ParseStatus::NoMatch; |
| 7353 | |
| 7354 | return parseExpr(Imm&: IntVal) ? ParseStatus::Success : ParseStatus::Failure; |
| 7355 | } |
| 7356 | |
| 7357 | ParseStatus AMDGPUAsmParser::parseIntWithPrefix( |
| 7358 | const char *Prefix, OperandVector &Operands, AMDGPUOperand::ImmTy ImmTy, |
| 7359 | std::function<bool(int64_t &)> ConvertResult) { |
| 7360 | SMLoc S = getLoc(); |
| 7361 | int64_t Value = 0; |
| 7362 | |
| 7363 | ParseStatus Res = parseIntWithPrefix(Prefix, IntVal&: Value); |
| 7364 | if (!Res.isSuccess()) |
| 7365 | return Res; |
| 7366 | |
| 7367 | if (ConvertResult && !ConvertResult(Value)) { |
| 7368 | Error(L: S, Msg: "invalid " + StringRef(Prefix) + " value." ); |
| 7369 | } |
| 7370 | |
| 7371 | Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Value, Loc: S, Type: ImmTy)); |
| 7372 | return ParseStatus::Success; |
| 7373 | } |
| 7374 | |
| 7375 | ParseStatus AMDGPUAsmParser::parseOperandArrayWithPrefix( |
| 7376 | const char *Prefix, OperandVector &Operands, AMDGPUOperand::ImmTy ImmTy, |
| 7377 | bool (*ConvertResult)(int64_t &)) { |
| 7378 | SMLoc S = getLoc(); |
| 7379 | if (!trySkipId(Id: Prefix, Kind: AsmToken::Colon)) |
| 7380 | return ParseStatus::NoMatch; |
| 7381 | |
| 7382 | if (!skipToken(Kind: AsmToken::LBrac, ErrMsg: "expected a left square bracket" )) |
| 7383 | return ParseStatus::Failure; |
| 7384 | |
| 7385 | unsigned Val = 0; |
| 7386 | const unsigned MaxSize = 4; |
| 7387 | |
| 7388 | // FIXME: How to verify the number of elements matches the number of src |
| 7389 | // operands? |
| 7390 | for (int I = 0;; ++I) { |
| 7391 | int64_t Op; |
| 7392 | SMLoc Loc = getLoc(); |
| 7393 | if (!parseExpr(Imm&: Op)) |
| 7394 | return ParseStatus::Failure; |
| 7395 | |
| 7396 | if (Op != 0 && Op != 1) |
| 7397 | return Error(L: Loc, Msg: "invalid " + StringRef(Prefix) + " value." ); |
| 7398 | |
| 7399 | Val |= (Op << I); |
| 7400 | |
| 7401 | if (trySkipToken(Kind: AsmToken::RBrac)) |
| 7402 | break; |
| 7403 | |
| 7404 | if (I + 1 == MaxSize) |
| 7405 | return Error(L: getLoc(), Msg: "expected a closing square bracket" ); |
| 7406 | |
| 7407 | if (!skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma" )) |
| 7408 | return ParseStatus::Failure; |
| 7409 | } |
| 7410 | |
| 7411 | Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val, Loc: S, Type: ImmTy)); |
| 7412 | return ParseStatus::Success; |
| 7413 | } |
| 7414 | |
| 7415 | ParseStatus AMDGPUAsmParser::parseNamedBit(StringRef Name, |
| 7416 | OperandVector &Operands, |
| 7417 | AMDGPUOperand::ImmTy ImmTy, |
| 7418 | bool IgnoreNegative) { |
| 7419 | int64_t Bit; |
| 7420 | SMLoc S = getLoc(); |
| 7421 | |
| 7422 | if (trySkipId(Id: Name)) { |
| 7423 | Bit = 1; |
| 7424 | } else if (trySkipId(Pref: "no" , Id: Name)) { |
| 7425 | if (IgnoreNegative) |
| 7426 | return ParseStatus::Success; |
| 7427 | Bit = 0; |
| 7428 | } else { |
| 7429 | return ParseStatus::NoMatch; |
| 7430 | } |
| 7431 | |
| 7432 | if (Name == "r128" && !hasMIMG_R128()) |
| 7433 | return Error(L: S, Msg: "r128 modifier is not supported on this GPU" ); |
| 7434 | if (Name == "a16" && !hasA16()) |
| 7435 | return Error(L: S, Msg: "a16 modifier is not supported on this GPU" ); |
| 7436 | |
| 7437 | if (Bit == 0 && Name == "gds" ) { |
| 7438 | StringRef Mnemo = ((AMDGPUOperand &)*Operands[0]).getToken(); |
| 7439 | if (Mnemo.starts_with(Prefix: "ds_gws" )) |
| 7440 | return Error(L: S, Msg: "nogds is not allowed" ); |
| 7441 | } |
| 7442 | |
| 7443 | if (isGFX9() && ImmTy == AMDGPUOperand::ImmTyA16) |
| 7444 | ImmTy = AMDGPUOperand::ImmTyR128A16; |
| 7445 | |
| 7446 | Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Bit, Loc: S, Type: ImmTy)); |
| 7447 | return ParseStatus::Success; |
| 7448 | } |
| 7449 | |
| 7450 | unsigned AMDGPUAsmParser::getCPolKind(StringRef Id, StringRef Mnemo, |
| 7451 | bool &Disabling) const { |
| 7452 | Disabling = Id.consume_front(Prefix: "no" ); |
| 7453 | |
| 7454 | if (isGFX940() && !Mnemo.starts_with(Prefix: "s_" )) { |
| 7455 | return StringSwitch<unsigned>(Id) |
| 7456 | .Case(S: "nt" , Value: AMDGPU::CPol::NT) |
| 7457 | .Case(S: "sc0" , Value: AMDGPU::CPol::SC0) |
| 7458 | .Case(S: "sc1" , Value: AMDGPU::CPol::SC1) |
| 7459 | .Default(Value: 0); |
| 7460 | } |
| 7461 | |
| 7462 | return StringSwitch<unsigned>(Id) |
| 7463 | .Case(S: "dlc" , Value: AMDGPU::CPol::DLC) |
| 7464 | .Case(S: "glc" , Value: AMDGPU::CPol::GLC) |
| 7465 | .Case(S: "scc" , Value: AMDGPU::CPol::SCC) |
| 7466 | .Case(S: "slc" , Value: AMDGPU::CPol::SLC) |
| 7467 | .Default(Value: 0); |
| 7468 | } |
| 7469 | |
| 7470 | ParseStatus AMDGPUAsmParser::parseCPol(OperandVector &Operands) { |
| 7471 | if (isGFX12Plus()) { |
| 7472 | SMLoc StringLoc = getLoc(); |
| 7473 | |
| 7474 | int64_t CPolVal = 0; |
| 7475 | ParseStatus ResTH = ParseStatus::NoMatch; |
| 7476 | ParseStatus ResScope = ParseStatus::NoMatch; |
| 7477 | ParseStatus ResNV = ParseStatus::NoMatch; |
| 7478 | ParseStatus ResScal = ParseStatus::NoMatch; |
| 7479 | ParseStatus ResCFS = ParseStatus::NoMatch; |
| 7480 | |
| 7481 | for (;;) { |
| 7482 | if (ResTH.isNoMatch()) { |
| 7483 | int64_t TH; |
| 7484 | ResTH = parseTH(Operands, TH); |
| 7485 | if (ResTH.isFailure()) |
| 7486 | return ResTH; |
| 7487 | if (ResTH.isSuccess()) { |
| 7488 | CPolVal |= TH; |
| 7489 | continue; |
| 7490 | } |
| 7491 | } |
| 7492 | |
| 7493 | if (ResScope.isNoMatch()) { |
| 7494 | int64_t Scope; |
| 7495 | ResScope = parseScope(Operands, Scope); |
| 7496 | if (ResScope.isFailure()) |
| 7497 | return ResScope; |
| 7498 | if (ResScope.isSuccess()) { |
| 7499 | CPolVal |= Scope; |
| 7500 | continue; |
| 7501 | } |
| 7502 | } |
| 7503 | |
| 7504 | // NV bit exists on GFX12+, but does something starting from GFX1250. |
| 7505 | // Allow parsing on all GFX12 and fail on validation for better |
| 7506 | // diagnostics. |
| 7507 | if (ResNV.isNoMatch()) { |
| 7508 | if (trySkipId(Id: "nv" )) { |
| 7509 | ResNV = ParseStatus::Success; |
| 7510 | CPolVal |= CPol::NV; |
| 7511 | continue; |
| 7512 | } else if (trySkipId(Pref: "no" , Id: "nv" )) { |
| 7513 | ResNV = ParseStatus::Success; |
| 7514 | continue; |
| 7515 | } |
| 7516 | } |
| 7517 | |
| 7518 | if (ResScal.isNoMatch()) { |
| 7519 | if (trySkipId(Id: "scale_offset" )) { |
| 7520 | ResScal = ParseStatus::Success; |
| 7521 | CPolVal |= CPol::SCAL; |
| 7522 | continue; |
| 7523 | } else if (trySkipId(Pref: "no" , Id: "scale_offset" )) { |
| 7524 | ResScal = ParseStatus::Success; |
| 7525 | continue; |
| 7526 | } |
| 7527 | } |
| 7528 | |
| 7529 | if (ResCFS.isNoMatch()) { |
| 7530 | int64_t CFS; |
| 7531 | ResCFS = parseCFS(Operands, CFS); |
| 7532 | if (ResCFS.isFailure()) |
| 7533 | return ResCFS; |
| 7534 | if (ResCFS.isSuccess()) { |
| 7535 | CPolVal |= CFS; |
| 7536 | continue; |
| 7537 | } |
| 7538 | } |
| 7539 | |
| 7540 | break; |
| 7541 | } |
| 7542 | |
| 7543 | if (ResTH.isNoMatch() && ResScope.isNoMatch() && ResNV.isNoMatch() && |
| 7544 | ResScal.isNoMatch() && ResCFS.isNoMatch()) |
| 7545 | return ParseStatus::NoMatch; |
| 7546 | |
| 7547 | Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: CPolVal, Loc: StringLoc, |
| 7548 | Type: AMDGPUOperand::ImmTyCPol)); |
| 7549 | return ParseStatus::Success; |
| 7550 | } |
| 7551 | |
| 7552 | StringRef Mnemo = ((AMDGPUOperand &)*Operands[0]).getToken(); |
| 7553 | SMLoc OpLoc = getLoc(); |
| 7554 | unsigned Enabled = 0, Seen = 0; |
| 7555 | for (;;) { |
| 7556 | SMLoc S = getLoc(); |
| 7557 | bool Disabling; |
| 7558 | unsigned CPol = getCPolKind(Id: getId(), Mnemo, Disabling); |
| 7559 | if (!CPol) |
| 7560 | break; |
| 7561 | |
| 7562 | lex(); |
| 7563 | |
| 7564 | if (!isGFX10Plus() && CPol == AMDGPU::CPol::DLC) |
| 7565 | return Error(L: S, Msg: "dlc modifier is not supported on this GPU" ); |
| 7566 | |
| 7567 | if (!isGFX90A() && CPol == AMDGPU::CPol::SCC) |
| 7568 | return Error(L: S, Msg: "scc modifier is not supported on this GPU" ); |
| 7569 | |
| 7570 | if (Seen & CPol) |
| 7571 | return Error(L: S, Msg: "duplicate cache policy modifier" ); |
| 7572 | |
| 7573 | if (!Disabling) |
| 7574 | Enabled |= CPol; |
| 7575 | |
| 7576 | Seen |= CPol; |
| 7577 | } |
| 7578 | |
| 7579 | if (!Seen) |
| 7580 | return ParseStatus::NoMatch; |
| 7581 | |
| 7582 | Operands.push_back( |
| 7583 | Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Enabled, Loc: OpLoc, Type: AMDGPUOperand::ImmTyCPol)); |
| 7584 | return ParseStatus::Success; |
| 7585 | } |
| 7586 | |
| 7587 | ParseStatus AMDGPUAsmParser::parseScope(OperandVector &Operands, |
| 7588 | int64_t &Scope) { |
| 7589 | static const unsigned Scopes[] = {CPol::SCOPE_CU, CPol::SCOPE_SE, |
| 7590 | CPol::SCOPE_DEV, CPol::SCOPE_SYS}; |
| 7591 | |
| 7592 | ParseStatus Res = parseStringOrIntWithPrefix( |
| 7593 | Operands, Name: "scope" , Ids: {"SCOPE_CU" , "SCOPE_SE" , "SCOPE_DEV" , "SCOPE_SYS" }, |
| 7594 | IntVal&: Scope); |
| 7595 | |
| 7596 | if (Res.isSuccess()) |
| 7597 | Scope = Scopes[Scope]; |
| 7598 | |
| 7599 | return Res; |
| 7600 | } |
| 7601 | |
| 7602 | ParseStatus AMDGPUAsmParser::parseTH(OperandVector &Operands, int64_t &TH) { |
| 7603 | TH = AMDGPU::CPol::TH_RT; // default |
| 7604 | |
| 7605 | StringRef Value; |
| 7606 | SMLoc StringLoc; |
| 7607 | ParseStatus Res = parseStringWithPrefix(Prefix: "th" , Value, StringLoc); |
| 7608 | if (!Res.isSuccess()) |
| 7609 | return Res; |
| 7610 | |
| 7611 | if (Value == "TH_DEFAULT" ) |
| 7612 | TH = AMDGPU::CPol::TH_RT; |
| 7613 | else if (Value == "TH_STORE_LU" || Value == "TH_LOAD_WB" || |
| 7614 | Value == "TH_LOAD_NT_WB" ) { |
| 7615 | return Error(L: StringLoc, Msg: "invalid th value" ); |
| 7616 | } else if (Value.consume_front(Prefix: "TH_ATOMIC_" )) { |
| 7617 | TH = AMDGPU::CPol::TH_TYPE_ATOMIC; |
| 7618 | } else if (Value.consume_front(Prefix: "TH_LOAD_" )) { |
| 7619 | TH = AMDGPU::CPol::TH_TYPE_LOAD; |
| 7620 | } else if (Value.consume_front(Prefix: "TH_STORE_" )) { |
| 7621 | TH = AMDGPU::CPol::TH_TYPE_STORE; |
| 7622 | } else { |
| 7623 | return Error(L: StringLoc, Msg: "invalid th value" ); |
| 7624 | } |
| 7625 | |
| 7626 | if (Value == "BYPASS" ) |
| 7627 | TH |= AMDGPU::CPol::TH_REAL_BYPASS; |
| 7628 | |
| 7629 | if (TH != 0) { |
| 7630 | if (TH & AMDGPU::CPol::TH_TYPE_ATOMIC) |
| 7631 | TH |= StringSwitch<int64_t>(Value) |
| 7632 | .Case(S: "RETURN" , Value: AMDGPU::CPol::TH_ATOMIC_RETURN) |
| 7633 | .Case(S: "RT" , Value: AMDGPU::CPol::TH_RT) |
| 7634 | .Case(S: "RT_RETURN" , Value: AMDGPU::CPol::TH_ATOMIC_RETURN) |
| 7635 | .Case(S: "NT" , Value: AMDGPU::CPol::TH_ATOMIC_NT) |
| 7636 | .Case(S: "NT_RETURN" , Value: AMDGPU::CPol::TH_ATOMIC_NT | |
| 7637 | AMDGPU::CPol::TH_ATOMIC_RETURN) |
| 7638 | .Case(S: "CASCADE_RT" , Value: AMDGPU::CPol::TH_ATOMIC_CASCADE) |
| 7639 | .Case(S: "CASCADE_NT" , Value: AMDGPU::CPol::TH_ATOMIC_CASCADE | |
| 7640 | AMDGPU::CPol::TH_ATOMIC_NT) |
| 7641 | .Default(Value: 0xffffffff); |
| 7642 | else |
| 7643 | TH |= StringSwitch<int64_t>(Value) |
| 7644 | .Case(S: "RT" , Value: AMDGPU::CPol::TH_RT) |
| 7645 | .Case(S: "NT" , Value: AMDGPU::CPol::TH_NT) |
| 7646 | .Case(S: "HT" , Value: AMDGPU::CPol::TH_HT) |
| 7647 | .Case(S: "LU" , Value: AMDGPU::CPol::TH_LU) |
| 7648 | .Case(S: "WB" , Value: AMDGPU::CPol::TH_WB) |
| 7649 | .Case(S: "NT_RT" , Value: AMDGPU::CPol::TH_NT_RT) |
| 7650 | .Case(S: "RT_NT" , Value: AMDGPU::CPol::TH_RT_NT) |
| 7651 | .Case(S: "NT_HT" , Value: AMDGPU::CPol::TH_NT_HT) |
| 7652 | .Case(S: "NT_WB" , Value: AMDGPU::CPol::TH_NT_WB) |
| 7653 | .Case(S: "BYPASS" , Value: AMDGPU::CPol::TH_BYPASS) |
| 7654 | .Default(Value: 0xffffffff); |
| 7655 | } |
| 7656 | |
| 7657 | if (TH == 0xffffffff) |
| 7658 | return Error(L: StringLoc, Msg: "invalid th value" ); |
| 7659 | |
| 7660 | return ParseStatus::Success; |
| 7661 | } |
| 7662 | |
| 7663 | ParseStatus AMDGPUAsmParser::parseCFS(OperandVector &Operands, int64_t &CFS) { |
| 7664 | static const unsigned CFSVals[] = {CPol::CFS_256B, CPol::CFS_128B, |
| 7665 | CPol::CFS_64B, CPol::CFS_32B}; |
| 7666 | |
| 7667 | ParseStatus Res = parseStringOrIntWithPrefix( |
| 7668 | Operands, Name: "cfs" , Ids: {"CFS_256B" , "CFS_128B" , "CFS_64B" , "CFS_32B" }, IntVal&: CFS); |
| 7669 | |
| 7670 | if (Res.isSuccess()) |
| 7671 | CFS = CFSVals[CFS]; |
| 7672 | |
| 7673 | return Res; |
| 7674 | } |
| 7675 | |
| 7676 | static void |
| 7677 | addOptionalImmOperand(MCInst &Inst, const OperandVector &Operands, |
| 7678 | AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx, |
| 7679 | AMDGPUOperand::ImmTy ImmT, int64_t Default = 0, |
| 7680 | std::optional<unsigned> InsertAt = std::nullopt) { |
| 7681 | auto i = OptionalIdx.find(x: ImmT); |
| 7682 | if (i != OptionalIdx.end()) { |
| 7683 | unsigned Idx = i->second; |
| 7684 | const AMDGPUOperand &Op = |
| 7685 | static_cast<const AMDGPUOperand &>(*Operands[Idx]); |
| 7686 | if (InsertAt) |
| 7687 | Inst.insert(I: Inst.begin() + *InsertAt, Op: MCOperand::createImm(Val: Op.getImm())); |
| 7688 | else |
| 7689 | Op.addImmOperands(Inst, N: 1); |
| 7690 | } else { |
| 7691 | if (InsertAt.has_value()) |
| 7692 | Inst.insert(I: Inst.begin() + *InsertAt, Op: MCOperand::createImm(Val: Default)); |
| 7693 | else |
| 7694 | Inst.addOperand(Op: MCOperand::createImm(Val: Default)); |
| 7695 | } |
| 7696 | } |
| 7697 | |
| 7698 | ParseStatus AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, |
| 7699 | StringRef &Value, |
| 7700 | SMLoc &StringLoc) { |
| 7701 | if (!trySkipId(Id: Prefix, Kind: AsmToken::Colon)) |
| 7702 | return ParseStatus::NoMatch; |
| 7703 | |
| 7704 | StringLoc = getLoc(); |
| 7705 | return parseId(Val&: Value, ErrMsg: "expected an identifier" ) ? ParseStatus::Success |
| 7706 | : ParseStatus::Failure; |
| 7707 | } |
| 7708 | |
| 7709 | ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix( |
| 7710 | OperandVector &Operands, StringRef Name, ArrayRef<const char *> Ids, |
| 7711 | int64_t &IntVal) { |
| 7712 | if (!trySkipId(Id: Name, Kind: AsmToken::Colon)) |
| 7713 | return ParseStatus::NoMatch; |
| 7714 | |
| 7715 | SMLoc StringLoc = getLoc(); |
| 7716 | |
| 7717 | StringRef Value; |
| 7718 | if (isToken(Kind: AsmToken::Identifier)) { |
| 7719 | Value = getTokenStr(); |
| 7720 | lex(); |
| 7721 | |
| 7722 | for (IntVal = 0; IntVal < (int64_t)Ids.size(); ++IntVal) |
| 7723 | if (Value == Ids[IntVal]) |
| 7724 | break; |
| 7725 | } else if (!parseExpr(Imm&: IntVal)) |
| 7726 | return ParseStatus::Failure; |
| 7727 | |
| 7728 | if (IntVal < 0 || IntVal >= (int64_t)Ids.size()) |
| 7729 | return Error(L: StringLoc, Msg: "invalid " + Twine(Name) + " value" ); |
| 7730 | |
| 7731 | return ParseStatus::Success; |
| 7732 | } |
| 7733 | |
| 7734 | ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix( |
| 7735 | OperandVector &Operands, StringRef Name, ArrayRef<const char *> Ids, |
| 7736 | AMDGPUOperand::ImmTy Type) { |
| 7737 | SMLoc S = getLoc(); |
| 7738 | int64_t IntVal; |
| 7739 | |
| 7740 | ParseStatus Res = parseStringOrIntWithPrefix(Operands, Name, Ids, IntVal); |
| 7741 | if (Res.isSuccess()) |
| 7742 | Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: IntVal, Loc: S, Type)); |
| 7743 | |
| 7744 | return Res; |
| 7745 | } |
| 7746 | |
| 7747 | //===----------------------------------------------------------------------===// |
| 7748 | // MTBUF format |
| 7749 | //===----------------------------------------------------------------------===// |
| 7750 | |
| 7751 | ParseStatus AMDGPUAsmParser::tryParseFmt(const char *Pref, int64_t MaxVal, |
| 7752 | int64_t &Fmt) { |
| 7753 | int64_t Val; |
| 7754 | SMLoc Loc = getLoc(); |
| 7755 | |
| 7756 | ParseStatus Res = parseIntWithPrefix(Prefix: Pref, IntVal&: Val); |
| 7757 | if (!Res.isSuccess()) |
| 7758 | return Res; |
| 7759 | |
| 7760 | if (Val < 0 || Val > MaxVal) |
| 7761 | return Error(L: Loc, Msg: Twine("out of range " , StringRef(Pref))); |
| 7762 | |
| 7763 | Fmt = Val; |
| 7764 | return ParseStatus::Success; |
| 7765 | } |
| 7766 | |
| 7767 | ParseStatus AMDGPUAsmParser::tryParseIndexKey(OperandVector &Operands, |
| 7768 | AMDGPUOperand::ImmTy ImmTy) { |
| 7769 | const char *Pref = "index_key" ; |
| 7770 | int64_t ImmVal = 0; |
| 7771 | SMLoc Loc = getLoc(); |
| 7772 | auto Res = parseIntWithPrefix(Prefix: Pref, IntVal&: ImmVal); |
| 7773 | if (!Res.isSuccess()) |
| 7774 | return Res; |
| 7775 | |
| 7776 | if ((ImmTy == AMDGPUOperand::ImmTyIndexKey16bit || |
| 7777 | ImmTy == AMDGPUOperand::ImmTyIndexKey32bit) && |
| 7778 | (ImmVal < 0 || ImmVal > 1)) |
| 7779 | return Error(L: Loc, Msg: Twine("out of range " , StringRef(Pref))); |
| 7780 | |
| 7781 | if (ImmTy == AMDGPUOperand::ImmTyIndexKey8bit && (ImmVal < 0 || ImmVal > 3)) |
| 7782 | return Error(L: Loc, Msg: Twine("out of range " , StringRef(Pref))); |
| 7783 | |
| 7784 | Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: ImmVal, Loc, Type: ImmTy)); |
| 7785 | return ParseStatus::Success; |
| 7786 | } |
| 7787 | |
| 7788 | ParseStatus AMDGPUAsmParser::parseIndexKey8bit(OperandVector &Operands) { |
| 7789 | return tryParseIndexKey(Operands, ImmTy: AMDGPUOperand::ImmTyIndexKey8bit); |
| 7790 | } |
| 7791 | |
| 7792 | ParseStatus AMDGPUAsmParser::parseIndexKey16bit(OperandVector &Operands) { |
| 7793 | return tryParseIndexKey(Operands, ImmTy: AMDGPUOperand::ImmTyIndexKey16bit); |
| 7794 | } |
| 7795 | |
| 7796 | ParseStatus AMDGPUAsmParser::parseIndexKey32bit(OperandVector &Operands) { |
| 7797 | return tryParseIndexKey(Operands, ImmTy: AMDGPUOperand::ImmTyIndexKey32bit); |
| 7798 | } |
| 7799 | |
| 7800 | ParseStatus AMDGPUAsmParser::tryParseMatrixFMT(OperandVector &Operands, |
| 7801 | StringRef Name, |
| 7802 | AMDGPUOperand::ImmTy Type) { |
| 7803 | return parseStringOrIntWithPrefix(Operands, Name, Ids: WMMAMods::ModMatrixFmt, |
| 7804 | Type); |
| 7805 | } |
| 7806 | |
| 7807 | ParseStatus AMDGPUAsmParser::parseMatrixAFMT(OperandVector &Operands) { |
| 7808 | return tryParseMatrixFMT(Operands, Name: "matrix_a_fmt" , |
| 7809 | Type: AMDGPUOperand::ImmTyMatrixAFMT); |
| 7810 | } |
| 7811 | |
| 7812 | ParseStatus AMDGPUAsmParser::parseMatrixBFMT(OperandVector &Operands) { |
| 7813 | return tryParseMatrixFMT(Operands, Name: "matrix_b_fmt" , |
| 7814 | Type: AMDGPUOperand::ImmTyMatrixBFMT); |
| 7815 | } |
| 7816 | |
| 7817 | ParseStatus AMDGPUAsmParser::tryParseMatrixScale(OperandVector &Operands, |
| 7818 | StringRef Name, |
| 7819 | AMDGPUOperand::ImmTy Type) { |
| 7820 | return parseStringOrIntWithPrefix(Operands, Name, Ids: WMMAMods::ModMatrixScale, |
| 7821 | Type); |
| 7822 | } |
| 7823 | |
| 7824 | ParseStatus AMDGPUAsmParser::parseMatrixAScale(OperandVector &Operands) { |
| 7825 | return tryParseMatrixScale(Operands, Name: "matrix_a_scale" , |
| 7826 | Type: AMDGPUOperand::ImmTyMatrixAScale); |
| 7827 | } |
| 7828 | |
| 7829 | ParseStatus AMDGPUAsmParser::parseMatrixBScale(OperandVector &Operands) { |
| 7830 | return tryParseMatrixScale(Operands, Name: "matrix_b_scale" , |
| 7831 | Type: AMDGPUOperand::ImmTyMatrixBScale); |
| 7832 | } |
| 7833 | |
| 7834 | ParseStatus AMDGPUAsmParser::tryParseMatrixScaleFmt(OperandVector &Operands, |
| 7835 | StringRef Name, |
| 7836 | AMDGPUOperand::ImmTy Type) { |
| 7837 | return parseStringOrIntWithPrefix(Operands, Name, Ids: WMMAMods::ModMatrixScaleFmt, |
| 7838 | Type); |
| 7839 | } |
| 7840 | |
| 7841 | ParseStatus AMDGPUAsmParser::parseMatrixAScaleFmt(OperandVector &Operands) { |
| 7842 | return tryParseMatrixScaleFmt(Operands, Name: "matrix_a_scale_fmt" , |
| 7843 | Type: AMDGPUOperand::ImmTyMatrixAScaleFmt); |
| 7844 | } |
| 7845 | |
| 7846 | ParseStatus AMDGPUAsmParser::parseMatrixBScaleFmt(OperandVector &Operands) { |
| 7847 | return tryParseMatrixScaleFmt(Operands, Name: "matrix_b_scale_fmt" , |
| 7848 | Type: AMDGPUOperand::ImmTyMatrixBScaleFmt); |
| 7849 | } |
| 7850 | |
| 7851 | // dfmt and nfmt (in a tbuffer instruction) are parsed as one to allow their |
| 7852 | // values to live in a joint format operand in the MCInst encoding. |
| 7853 | ParseStatus AMDGPUAsmParser::parseDfmtNfmt(int64_t &Format) { |
| 7854 | using namespace llvm::AMDGPU::MTBUFFormat; |
| 7855 | |
| 7856 | int64_t Dfmt = DFMT_DEFAULT; |
| 7857 | int64_t Nfmt = NFMT_DEFAULT; |
| 7858 | |
| 7859 | auto TryAfterComma = [&](const char *Pref, int64_t MaxVal, |
| 7860 | int64_t &Fmt) -> ParseStatus { |
| 7861 | if (isId(Id: Pref) && peekToken().is(K: AsmToken::Colon)) |
| 7862 | return Error(L: getLoc(), Msg: "expected a comma" ); |
| 7863 | AsmToken NextTokens[2]; |
| 7864 | peekTokens(Tokens: NextTokens); |
| 7865 | if (!isToken(Kind: AsmToken::Comma) || !isId(Token: NextTokens[0], Id: Pref) || |
| 7866 | !NextTokens[1].is(K: AsmToken::Colon)) |
| 7867 | return ParseStatus::Success; |
| 7868 | lex(); |
| 7869 | return tryParseFmt(Pref, MaxVal, Fmt); |
| 7870 | }; |
| 7871 | |
| 7872 | // dfmt and nfmt can appear in either order, and each is optional. If both |
| 7873 | // are present, they are separated by a comma. |
| 7874 | ParseStatus Res = tryParseFmt(Pref: "dfmt" , MaxVal: DFMT_MAX, Fmt&: Dfmt); |
| 7875 | if (Res.isSuccess()) { |
| 7876 | Res = TryAfterComma("nfmt" , NFMT_MAX, Nfmt); |
| 7877 | } else if (Res.isNoMatch()) { |
| 7878 | Res = tryParseFmt(Pref: "nfmt" , MaxVal: NFMT_MAX, Fmt&: Nfmt); |
| 7879 | if (Res.isSuccess()) |
| 7880 | Res = TryAfterComma("dfmt" , DFMT_MAX, Dfmt); |
| 7881 | } |
| 7882 | if (!Res.isSuccess()) |
| 7883 | return Res; |
| 7884 | |
| 7885 | Format = encodeDfmtNfmt(Dfmt, Nfmt); |
| 7886 | return ParseStatus::Success; |
| 7887 | } |
| 7888 | |
| 7889 | ParseStatus AMDGPUAsmParser::parseUfmt(int64_t &Format) { |
| 7890 | return tryParseFmt(Pref: "format" , MaxVal: MTBUFFormat::UFMT_MAX, Fmt&: Format); |
| 7891 | } |
| 7892 | |
| 7893 | bool AMDGPUAsmParser::matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt, |
| 7894 | StringRef FormatStr, SMLoc Loc) { |
| 7895 | using namespace llvm::AMDGPU::MTBUFFormat; |
| 7896 | int64_t Format; |
| 7897 | |
| 7898 | Format = getDfmt(Name: FormatStr); |
| 7899 | if (Format != DFMT_UNDEF) { |
| 7900 | Dfmt = Format; |
| 7901 | return true; |
| 7902 | } |
| 7903 | |
| 7904 | Format = getNfmt(Name: FormatStr, STI: getSTI()); |
| 7905 | if (Format != NFMT_UNDEF) { |
| 7906 | Nfmt = Format; |
| 7907 | return true; |
| 7908 | } |
| 7909 | |
| 7910 | Error(L: Loc, Msg: "unsupported format" ); |
| 7911 | return false; |
| 7912 | } |
| 7913 | |
| 7914 | ParseStatus AMDGPUAsmParser::parseSymbolicSplitFormat(StringRef FormatStr, |
| 7915 | SMLoc FormatLoc, |
| 7916 | int64_t &Format) { |
| 7917 | using namespace llvm::AMDGPU::MTBUFFormat; |
| 7918 | |
| 7919 | int64_t Dfmt = DFMT_UNDEF; |
| 7920 | int64_t Nfmt = NFMT_UNDEF; |
| 7921 | if (!matchDfmtNfmt(Dfmt, Nfmt, FormatStr, Loc: FormatLoc)) |
| 7922 | return ParseStatus::Failure; |
| 7923 | |
| 7924 | if (trySkipToken(Kind: AsmToken::Comma)) { |
| 7925 | StringRef Str; |
| 7926 | SMLoc Loc = getLoc(); |
| 7927 | if (!parseId(Val&: Str, ErrMsg: "expected a format string" ) || |
| 7928 | !matchDfmtNfmt(Dfmt, Nfmt, FormatStr: Str, Loc)) |
| 7929 | return ParseStatus::Failure; |
| 7930 | if (Dfmt == DFMT_UNDEF) |
| 7931 | return Error(L: Loc, Msg: "duplicate numeric format" ); |
| 7932 | if (Nfmt == NFMT_UNDEF) |
| 7933 | return Error(L: Loc, Msg: "duplicate data format" ); |
| 7934 | } |
| 7935 | |
| 7936 | Dfmt = (Dfmt == DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt; |
| 7937 | Nfmt = (Nfmt == NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt; |
| 7938 | |
| 7939 | if (isGFX10Plus()) { |
| 7940 | auto Ufmt = convertDfmtNfmt2Ufmt(Dfmt, Nfmt, STI: getSTI()); |
| 7941 | if (Ufmt == UFMT_UNDEF) |
| 7942 | return Error(L: FormatLoc, Msg: "unsupported format" ); |
| 7943 | Format = Ufmt; |
| 7944 | } else { |
| 7945 | Format = encodeDfmtNfmt(Dfmt, Nfmt); |
| 7946 | } |
| 7947 | |
| 7948 | return ParseStatus::Success; |
| 7949 | } |
| 7950 | |
| 7951 | ParseStatus AMDGPUAsmParser::parseSymbolicUnifiedFormat(StringRef FormatStr, |
| 7952 | SMLoc Loc, |
| 7953 | int64_t &Format) { |
| 7954 | using namespace llvm::AMDGPU::MTBUFFormat; |
| 7955 | |
| 7956 | auto Id = getUnifiedFormat(Name: FormatStr, STI: getSTI()); |
| 7957 | if (Id == UFMT_UNDEF) |
| 7958 | return ParseStatus::NoMatch; |
| 7959 | |
| 7960 | if (!isGFX10Plus()) |
| 7961 | return Error(L: Loc, Msg: "unified format is not supported on this GPU" ); |
| 7962 | |
| 7963 | Format = Id; |
| 7964 | return ParseStatus::Success; |
| 7965 | } |
| 7966 | |
| 7967 | ParseStatus AMDGPUAsmParser::parseNumericFormat(int64_t &Format) { |
| 7968 | using namespace llvm::AMDGPU::MTBUFFormat; |
| 7969 | SMLoc Loc = getLoc(); |
| 7970 | |
| 7971 | if (!parseExpr(Imm&: Format)) |
| 7972 | return ParseStatus::Failure; |
| 7973 | if (!isValidFormatEncoding(Val: Format, STI: getSTI())) |
| 7974 | return Error(L: Loc, Msg: "out of range format" ); |
| 7975 | |
| 7976 | return ParseStatus::Success; |
| 7977 | } |
| 7978 | |
| 7979 | ParseStatus AMDGPUAsmParser::parseSymbolicOrNumericFormat(int64_t &Format) { |
| 7980 | using namespace llvm::AMDGPU::MTBUFFormat; |
| 7981 | |
| 7982 | if (!trySkipId(Id: "format" , Kind: AsmToken::Colon)) |
| 7983 | return ParseStatus::NoMatch; |
| 7984 | |
| 7985 | if (trySkipToken(Kind: AsmToken::LBrac)) { |
| 7986 | StringRef FormatStr; |
| 7987 | SMLoc Loc = getLoc(); |
| 7988 | if (!parseId(Val&: FormatStr, ErrMsg: "expected a format string" )) |
| 7989 | return ParseStatus::Failure; |
| 7990 | |
| 7991 | auto Res = parseSymbolicUnifiedFormat(FormatStr, Loc, Format); |
| 7992 | if (Res.isNoMatch()) |
| 7993 | Res = parseSymbolicSplitFormat(FormatStr, FormatLoc: Loc, Format); |
| 7994 | if (!Res.isSuccess()) |
| 7995 | return Res; |
| 7996 | |
| 7997 | if (!skipToken(Kind: AsmToken::RBrac, ErrMsg: "expected a closing square bracket" )) |
| 7998 | return ParseStatus::Failure; |
| 7999 | |
| 8000 | return ParseStatus::Success; |
| 8001 | } |
| 8002 | |
| 8003 | return parseNumericFormat(Format); |
| 8004 | } |
| 8005 | |
| 8006 | ParseStatus AMDGPUAsmParser::parseFormat(OperandVector &Operands) { |
| 8007 | SMLoc Loc = getLoc(); |
| 8008 | int64_t Format; |
| 8009 | ParseStatus Res = parseSymbolicOrNumericFormat(Format); |
| 8010 | if (!Res.isSuccess()) |
| 8011 | return Res; |
| 8012 | |
| 8013 | Operands.push_back( |
| 8014 | Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Format, Loc, Type: AMDGPUOperand::ImmTyFormat)); |
| 8015 | return ParseStatus::Success; |
| 8016 | } |
| 8017 | |
| 8018 | ParseStatus AMDGPUAsmParser::parseFormatOp(OperandVector &Operands) { |
| 8019 | SMLoc Loc = getLoc(); |
| 8020 | int64_t Format; |
| 8021 | ParseStatus Res = isGFX10Plus() ? parseUfmt(Format) : parseDfmtNfmt(Format); |
| 8022 | if (!Res.isSuccess()) |
| 8023 | return Res; |
| 8024 | |
| 8025 | Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Format, Loc, |
| 8026 | Type: AMDGPUOperand::ImmTyFormatOp)); |
| 8027 | return ParseStatus::Success; |
| 8028 | } |
| 8029 | |
| 8030 | ParseStatus AMDGPUAsmParser::parseFlatOffset(OperandVector &Operands) { |
| 8031 | ParseStatus Res = |
| 8032 | parseIntWithPrefix(Prefix: "offset" , Operands, ImmTy: AMDGPUOperand::ImmTyOffset); |
| 8033 | if (Res.isNoMatch()) { |
| 8034 | Res = parseIntWithPrefix(Prefix: "inst_offset" , Operands, |
| 8035 | ImmTy: AMDGPUOperand::ImmTyInstOffset); |
| 8036 | } |
| 8037 | return Res; |
| 8038 | } |
| 8039 | |
| 8040 | ParseStatus AMDGPUAsmParser::parseR128A16(OperandVector &Operands) { |
| 8041 | ParseStatus Res = |
| 8042 | parseNamedBit(Name: "r128" , Operands, ImmTy: AMDGPUOperand::ImmTyR128A16); |
| 8043 | if (Res.isNoMatch()) |
| 8044 | Res = parseNamedBit(Name: "a16" , Operands, ImmTy: AMDGPUOperand::ImmTyA16); |
| 8045 | return Res; |
| 8046 | } |
| 8047 | |
| 8048 | ParseStatus AMDGPUAsmParser::parseBLGP(OperandVector &Operands) { |
| 8049 | ParseStatus Res = |
| 8050 | parseIntWithPrefix(Prefix: "blgp" , Operands, ImmTy: AMDGPUOperand::ImmTyBLGP); |
| 8051 | if (Res.isNoMatch()) { |
| 8052 | Res = |
| 8053 | parseOperandArrayWithPrefix(Prefix: "neg" , Operands, ImmTy: AMDGPUOperand::ImmTyBLGP); |
| 8054 | } |
| 8055 | return Res; |
| 8056 | } |
| 8057 | |
| 8058 | //===----------------------------------------------------------------------===// |
| 8059 | // Exp |
| 8060 | //===----------------------------------------------------------------------===// |
| 8061 | |
| 8062 | void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) { |
| 8063 | OptionalImmIndexMap OptionalIdx; |
| 8064 | |
| 8065 | unsigned OperandIdx[4]; |
| 8066 | unsigned EnMask = 0; |
| 8067 | int SrcIdx = 0; |
| 8068 | |
| 8069 | for (unsigned i = 1, e = Operands.size(); i != e; ++i) { |
| 8070 | AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]); |
| 8071 | |
| 8072 | // Add the register arguments |
| 8073 | if (Op.isReg()) { |
| 8074 | assert(SrcIdx < 4); |
| 8075 | OperandIdx[SrcIdx] = Inst.size(); |
| 8076 | Op.addRegOperands(Inst, N: 1); |
| 8077 | ++SrcIdx; |
| 8078 | continue; |
| 8079 | } |
| 8080 | |
| 8081 | if (Op.isOff()) { |
| 8082 | assert(SrcIdx < 4); |
| 8083 | OperandIdx[SrcIdx] = Inst.size(); |
| 8084 | Inst.addOperand(Op: MCOperand::createReg(Reg: MCRegister())); |
| 8085 | ++SrcIdx; |
| 8086 | continue; |
| 8087 | } |
| 8088 | |
| 8089 | if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) { |
| 8090 | Op.addImmOperands(Inst, N: 1); |
| 8091 | continue; |
| 8092 | } |
| 8093 | |
| 8094 | if (Op.isToken() && (Op.getToken() == "done" || Op.getToken() == "row_en" )) |
| 8095 | continue; |
| 8096 | |
| 8097 | // Handle optional arguments |
| 8098 | OptionalIdx[Op.getImmTy()] = i; |
| 8099 | } |
| 8100 | |
| 8101 | assert(SrcIdx == 4); |
| 8102 | |
| 8103 | bool Compr = false; |
| 8104 | if (OptionalIdx.find(x: AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) { |
| 8105 | Compr = true; |
| 8106 | Inst.getOperand(i: OperandIdx[1]) = Inst.getOperand(i: OperandIdx[2]); |
| 8107 | Inst.getOperand(i: OperandIdx[2]).setReg(MCRegister()); |
| 8108 | Inst.getOperand(i: OperandIdx[3]).setReg(MCRegister()); |
| 8109 | } |
| 8110 | |
| 8111 | for (auto i = 0; i < SrcIdx; ++i) { |
| 8112 | if (Inst.getOperand(i: OperandIdx[i]).getReg()) { |
| 8113 | EnMask |= Compr ? (0x3 << i * 2) : (0x1 << i); |
| 8114 | } |
| 8115 | } |
| 8116 | |
| 8117 | addOptionalImmOperand(Inst, Operands, OptionalIdx, ImmT: AMDGPUOperand::ImmTyExpVM); |
| 8118 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 8119 | ImmT: AMDGPUOperand::ImmTyExpCompr); |
| 8120 | |
| 8121 | Inst.addOperand(Op: MCOperand::createImm(Val: EnMask)); |
| 8122 | } |
| 8123 | |
| 8124 | //===----------------------------------------------------------------------===// |
| 8125 | // s_waitcnt |
| 8126 | //===----------------------------------------------------------------------===// |
| 8127 | |
| 8128 | static bool encodeCnt(const AMDGPU::IsaVersion ISA, int64_t &IntVal, |
| 8129 | int64_t CntVal, bool Saturate, |
| 8130 | unsigned (*encode)(const IsaVersion &Version, unsigned, |
| 8131 | unsigned), |
| 8132 | unsigned (*decode)(const IsaVersion &Version, unsigned)) { |
| 8133 | bool Failed = false; |
| 8134 | |
| 8135 | IntVal = encode(ISA, IntVal, CntVal); |
| 8136 | if (CntVal != decode(ISA, IntVal)) { |
| 8137 | if (Saturate) { |
| 8138 | IntVal = encode(ISA, IntVal, -1); |
| 8139 | } else { |
| 8140 | Failed = true; |
| 8141 | } |
| 8142 | } |
| 8143 | return Failed; |
| 8144 | } |
| 8145 | |
| 8146 | bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) { |
| 8147 | |
| 8148 | SMLoc CntLoc = getLoc(); |
| 8149 | StringRef CntName = getTokenStr(); |
| 8150 | |
| 8151 | if (!skipToken(Kind: AsmToken::Identifier, ErrMsg: "expected a counter name" ) || |
| 8152 | !skipToken(Kind: AsmToken::LParen, ErrMsg: "expected a left parenthesis" )) |
| 8153 | return false; |
| 8154 | |
| 8155 | int64_t CntVal; |
| 8156 | SMLoc ValLoc = getLoc(); |
| 8157 | if (!parseExpr(Imm&: CntVal)) |
| 8158 | return false; |
| 8159 | |
| 8160 | bool Failed = true; |
| 8161 | bool Sat = CntName.ends_with(Suffix: "_sat" ); |
| 8162 | |
| 8163 | if (CntName == "vmcnt" || CntName == "vmcnt_sat" ) { |
| 8164 | Failed = encodeCnt(ISA, IntVal, CntVal, Saturate: Sat, encode: encodeVmcnt, decode: decodeVmcnt); |
| 8165 | } else if (CntName == "expcnt" || CntName == "expcnt_sat" ) { |
| 8166 | Failed = encodeCnt(ISA, IntVal, CntVal, Saturate: Sat, encode: encodeExpcnt, decode: decodeExpcnt); |
| 8167 | } else if (CntName == "lgkmcnt" || CntName == "lgkmcnt_sat" ) { |
| 8168 | Failed = encodeCnt(ISA, IntVal, CntVal, Saturate: Sat, encode: encodeLgkmcnt, decode: decodeLgkmcnt); |
| 8169 | } else { |
| 8170 | Error(L: CntLoc, Msg: "invalid counter name " + CntName); |
| 8171 | return false; |
| 8172 | } |
| 8173 | |
| 8174 | if (Failed) { |
| 8175 | Error(L: ValLoc, Msg: "too large value for " + CntName); |
| 8176 | return false; |
| 8177 | } |
| 8178 | |
| 8179 | if (!skipToken(Kind: AsmToken::RParen, ErrMsg: "expected a closing parenthesis" )) |
| 8180 | return false; |
| 8181 | |
| 8182 | if (trySkipToken(Kind: AsmToken::Amp) || trySkipToken(Kind: AsmToken::Comma)) { |
| 8183 | if (isToken(Kind: AsmToken::EndOfStatement)) { |
| 8184 | Error(L: getLoc(), Msg: "expected a counter name" ); |
| 8185 | return false; |
| 8186 | } |
| 8187 | } |
| 8188 | |
| 8189 | return true; |
| 8190 | } |
| 8191 | |
| 8192 | ParseStatus AMDGPUAsmParser::parseSWaitCnt(OperandVector &Operands) { |
| 8193 | int64_t Waitcnt = getWaitcntBitMask(Version: ISA); |
| 8194 | SMLoc S = getLoc(); |
| 8195 | |
| 8196 | if (isToken(Kind: AsmToken::Identifier) && peekToken().is(K: AsmToken::LParen)) { |
| 8197 | while (!isToken(Kind: AsmToken::EndOfStatement)) { |
| 8198 | if (!parseCnt(IntVal&: Waitcnt)) |
| 8199 | return ParseStatus::Failure; |
| 8200 | } |
| 8201 | } else { |
| 8202 | if (!parseExpr(Imm&: Waitcnt)) |
| 8203 | return ParseStatus::Failure; |
| 8204 | } |
| 8205 | |
| 8206 | Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Waitcnt, Loc: S)); |
| 8207 | return ParseStatus::Success; |
| 8208 | } |
| 8209 | |
| 8210 | bool AMDGPUAsmParser::parseDelay(int64_t &Delay) { |
| 8211 | SMLoc FieldLoc = getLoc(); |
| 8212 | StringRef FieldName = getTokenStr(); |
| 8213 | if (!skipToken(Kind: AsmToken::Identifier, ErrMsg: "expected a field name" ) || |
| 8214 | !skipToken(Kind: AsmToken::LParen, ErrMsg: "expected a left parenthesis" )) |
| 8215 | return false; |
| 8216 | |
| 8217 | SMLoc ValueLoc = getLoc(); |
| 8218 | StringRef ValueName = getTokenStr(); |
| 8219 | if (!skipToken(Kind: AsmToken::Identifier, ErrMsg: "expected a value name" ) || |
| 8220 | !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected a right parenthesis" )) |
| 8221 | return false; |
| 8222 | |
| 8223 | unsigned Shift; |
| 8224 | if (FieldName == "instid0" ) { |
| 8225 | Shift = 0; |
| 8226 | } else if (FieldName == "instskip" ) { |
| 8227 | Shift = 4; |
| 8228 | } else if (FieldName == "instid1" ) { |
| 8229 | Shift = 7; |
| 8230 | } else { |
| 8231 | Error(L: FieldLoc, Msg: "invalid field name " + FieldName); |
| 8232 | return false; |
| 8233 | } |
| 8234 | |
| 8235 | int Value; |
| 8236 | if (Shift == 4) { |
| 8237 | // Parse values for instskip. |
| 8238 | Value = StringSwitch<int>(ValueName) |
| 8239 | .Case(S: "SAME" , Value: 0) |
| 8240 | .Case(S: "NEXT" , Value: 1) |
| 8241 | .Case(S: "SKIP_1" , Value: 2) |
| 8242 | .Case(S: "SKIP_2" , Value: 3) |
| 8243 | .Case(S: "SKIP_3" , Value: 4) |
| 8244 | .Case(S: "SKIP_4" , Value: 5) |
| 8245 | .Default(Value: -1); |
| 8246 | } else { |
| 8247 | // Parse values for instid0 and instid1. |
| 8248 | Value = StringSwitch<int>(ValueName) |
| 8249 | .Case(S: "NO_DEP" , Value: 0) |
| 8250 | .Case(S: "VALU_DEP_1" , Value: 1) |
| 8251 | .Case(S: "VALU_DEP_2" , Value: 2) |
| 8252 | .Case(S: "VALU_DEP_3" , Value: 3) |
| 8253 | .Case(S: "VALU_DEP_4" , Value: 4) |
| 8254 | .Case(S: "TRANS32_DEP_1" , Value: 5) |
| 8255 | .Case(S: "TRANS32_DEP_2" , Value: 6) |
| 8256 | .Case(S: "TRANS32_DEP_3" , Value: 7) |
| 8257 | .Case(S: "FMA_ACCUM_CYCLE_1" , Value: 8) |
| 8258 | .Case(S: "SALU_CYCLE_1" , Value: 9) |
| 8259 | .Case(S: "SALU_CYCLE_2" , Value: 10) |
| 8260 | .Case(S: "SALU_CYCLE_3" , Value: 11) |
| 8261 | .Default(Value: -1); |
| 8262 | } |
| 8263 | if (Value < 0) { |
| 8264 | Error(L: ValueLoc, Msg: "invalid value name " + ValueName); |
| 8265 | return false; |
| 8266 | } |
| 8267 | |
| 8268 | Delay |= Value << Shift; |
| 8269 | return true; |
| 8270 | } |
| 8271 | |
| 8272 | ParseStatus AMDGPUAsmParser::parseSDelayALU(OperandVector &Operands) { |
| 8273 | int64_t Delay = 0; |
| 8274 | SMLoc S = getLoc(); |
| 8275 | |
| 8276 | if (isToken(Kind: AsmToken::Identifier) && peekToken().is(K: AsmToken::LParen)) { |
| 8277 | do { |
| 8278 | if (!parseDelay(Delay)) |
| 8279 | return ParseStatus::Failure; |
| 8280 | } while (trySkipToken(Kind: AsmToken::Pipe)); |
| 8281 | } else { |
| 8282 | if (!parseExpr(Imm&: Delay)) |
| 8283 | return ParseStatus::Failure; |
| 8284 | } |
| 8285 | |
| 8286 | Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Delay, Loc: S)); |
| 8287 | return ParseStatus::Success; |
| 8288 | } |
| 8289 | |
| 8290 | bool AMDGPUOperand::isSWaitCnt() const { return isImm(); } |
| 8291 | |
| 8292 | bool AMDGPUOperand::isSDelayALU() const { return isImm(); } |
| 8293 | |
| 8294 | //===----------------------------------------------------------------------===// |
| 8295 | // DepCtr |
| 8296 | //===----------------------------------------------------------------------===// |
| 8297 | |
| 8298 | void AMDGPUAsmParser::depCtrError(SMLoc Loc, int ErrorId, |
| 8299 | StringRef DepCtrName) { |
| 8300 | switch (ErrorId) { |
| 8301 | case OPR_ID_UNKNOWN: |
| 8302 | Error(L: Loc, Msg: Twine("invalid counter name " , DepCtrName)); |
| 8303 | return; |
| 8304 | case OPR_ID_UNSUPPORTED: |
| 8305 | Error(L: Loc, Msg: Twine(DepCtrName, " is not supported on this GPU" )); |
| 8306 | return; |
| 8307 | case OPR_ID_DUPLICATE: |
| 8308 | Error(L: Loc, Msg: Twine("duplicate counter name " , DepCtrName)); |
| 8309 | return; |
| 8310 | case OPR_VAL_INVALID: |
| 8311 | Error(L: Loc, Msg: Twine("invalid value for " , DepCtrName)); |
| 8312 | return; |
| 8313 | default: |
| 8314 | assert(false); |
| 8315 | } |
| 8316 | } |
| 8317 | |
| 8318 | bool AMDGPUAsmParser::parseDepCtr(int64_t &DepCtr, unsigned &UsedOprMask) { |
| 8319 | |
| 8320 | using namespace llvm::AMDGPU::DepCtr; |
| 8321 | |
| 8322 | SMLoc DepCtrLoc = getLoc(); |
| 8323 | StringRef DepCtrName = getTokenStr(); |
| 8324 | |
| 8325 | if (!skipToken(Kind: AsmToken::Identifier, ErrMsg: "expected a counter name" ) || |
| 8326 | !skipToken(Kind: AsmToken::LParen, ErrMsg: "expected a left parenthesis" )) |
| 8327 | return false; |
| 8328 | |
| 8329 | int64_t ExprVal; |
| 8330 | if (!parseExpr(Imm&: ExprVal)) |
| 8331 | return false; |
| 8332 | |
| 8333 | unsigned PrevOprMask = UsedOprMask; |
| 8334 | int CntVal = encodeDepCtr(Name: DepCtrName, Val: ExprVal, UsedOprMask, STI: getSTI()); |
| 8335 | |
| 8336 | if (CntVal < 0) { |
| 8337 | depCtrError(Loc: DepCtrLoc, ErrorId: CntVal, DepCtrName); |
| 8338 | return false; |
| 8339 | } |
| 8340 | |
| 8341 | if (!skipToken(Kind: AsmToken::RParen, ErrMsg: "expected a closing parenthesis" )) |
| 8342 | return false; |
| 8343 | |
| 8344 | if (trySkipToken(Kind: AsmToken::Amp) || trySkipToken(Kind: AsmToken::Comma)) { |
| 8345 | if (isToken(Kind: AsmToken::EndOfStatement)) { |
| 8346 | Error(L: getLoc(), Msg: "expected a counter name" ); |
| 8347 | return false; |
| 8348 | } |
| 8349 | } |
| 8350 | |
| 8351 | int64_t CntValMask = PrevOprMask ^ UsedOprMask; |
| 8352 | DepCtr = (DepCtr & ~CntValMask) | CntVal; |
| 8353 | return true; |
| 8354 | } |
| 8355 | |
| 8356 | ParseStatus AMDGPUAsmParser::parseDepCtr(OperandVector &Operands) { |
| 8357 | using namespace llvm::AMDGPU::DepCtr; |
| 8358 | |
| 8359 | int64_t DepCtr = getDefaultDepCtrEncoding(STI: getSTI()); |
| 8360 | SMLoc Loc = getLoc(); |
| 8361 | |
| 8362 | if (isToken(Kind: AsmToken::Identifier) && peekToken().is(K: AsmToken::LParen)) { |
| 8363 | unsigned UsedOprMask = 0; |
| 8364 | while (!isToken(Kind: AsmToken::EndOfStatement)) { |
| 8365 | if (!parseDepCtr(DepCtr, UsedOprMask)) |
| 8366 | return ParseStatus::Failure; |
| 8367 | } |
| 8368 | } else { |
| 8369 | if (!parseExpr(Imm&: DepCtr)) |
| 8370 | return ParseStatus::Failure; |
| 8371 | } |
| 8372 | |
| 8373 | Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: DepCtr, Loc)); |
| 8374 | return ParseStatus::Success; |
| 8375 | } |
| 8376 | |
| 8377 | bool AMDGPUOperand::isDepCtr() const { return isS16Imm(); } |
| 8378 | |
| 8379 | //===----------------------------------------------------------------------===// |
| 8380 | // hwreg |
| 8381 | //===----------------------------------------------------------------------===// |
| 8382 | |
| 8383 | ParseStatus AMDGPUAsmParser::parseHwregFunc(OperandInfoTy &HwReg, |
| 8384 | OperandInfoTy &Offset, |
| 8385 | OperandInfoTy &Width) { |
| 8386 | using namespace llvm::AMDGPU::Hwreg; |
| 8387 | |
| 8388 | if (!trySkipId(Id: "hwreg" , Kind: AsmToken::LParen)) |
| 8389 | return ParseStatus::NoMatch; |
| 8390 | |
| 8391 | // The register may be specified by name or using a numeric code |
| 8392 | HwReg.Loc = getLoc(); |
| 8393 | if (isToken(Kind: AsmToken::Identifier) && |
| 8394 | (HwReg.Val = getHwregId(Name: getTokenStr(), STI: getSTI())) != OPR_ID_UNKNOWN) { |
| 8395 | HwReg.IsSymbolic = true; |
| 8396 | lex(); // skip register name |
| 8397 | } else if (!parseExpr(Imm&: HwReg.Val, Expected: "a register name" )) { |
| 8398 | return ParseStatus::Failure; |
| 8399 | } |
| 8400 | |
| 8401 | if (trySkipToken(Kind: AsmToken::RParen)) |
| 8402 | return ParseStatus::Success; |
| 8403 | |
| 8404 | // parse optional params |
| 8405 | if (!skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma or a closing parenthesis" )) |
| 8406 | return ParseStatus::Failure; |
| 8407 | |
| 8408 | Offset.Loc = getLoc(); |
| 8409 | if (!parseExpr(Imm&: Offset.Val)) |
| 8410 | return ParseStatus::Failure; |
| 8411 | |
| 8412 | if (!skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma" )) |
| 8413 | return ParseStatus::Failure; |
| 8414 | |
| 8415 | Width.Loc = getLoc(); |
| 8416 | if (!parseExpr(Imm&: Width.Val) || |
| 8417 | !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected a closing parenthesis" )) |
| 8418 | return ParseStatus::Failure; |
| 8419 | |
| 8420 | return ParseStatus::Success; |
| 8421 | } |
| 8422 | |
| 8423 | ParseStatus AMDGPUAsmParser::parseHwreg(OperandVector &Operands) { |
| 8424 | using namespace llvm::AMDGPU::Hwreg; |
| 8425 | |
| 8426 | int64_t ImmVal = 0; |
| 8427 | SMLoc Loc = getLoc(); |
| 8428 | |
| 8429 | StructuredOpField HwReg("id" , "hardware register" , HwregId::Width, |
| 8430 | HwregId::Default); |
| 8431 | StructuredOpField Offset("offset" , "bit offset" , HwregOffset::Width, |
| 8432 | HwregOffset::Default); |
| 8433 | struct : StructuredOpField { |
| 8434 | using StructuredOpField::StructuredOpField; |
| 8435 | bool validate(AMDGPUAsmParser &Parser) const override { |
| 8436 | if (!isUIntN(N: Width, x: Val - 1)) |
| 8437 | return Error(Parser, Err: "only values from 1 to 32 are legal" ); |
| 8438 | return true; |
| 8439 | } |
| 8440 | } Width("size" , "bitfield width" , HwregSize::Width, HwregSize::Default); |
| 8441 | ParseStatus Res = parseStructuredOpFields(Fields: {&HwReg, &Offset, &Width}); |
| 8442 | |
| 8443 | if (Res.isNoMatch()) |
| 8444 | Res = parseHwregFunc(HwReg, Offset, Width); |
| 8445 | |
| 8446 | if (Res.isSuccess()) { |
| 8447 | if (!validateStructuredOpFields(Fields: {&HwReg, &Offset, &Width})) |
| 8448 | return ParseStatus::Failure; |
| 8449 | ImmVal = HwregEncoding::encode(Values: HwReg.Val, Values: Offset.Val, Values: Width.Val); |
| 8450 | } |
| 8451 | |
| 8452 | if (Res.isNoMatch() && |
| 8453 | parseExpr(Imm&: ImmVal, Expected: "a hwreg macro, structured immediate" )) |
| 8454 | Res = ParseStatus::Success; |
| 8455 | |
| 8456 | if (!Res.isSuccess()) |
| 8457 | return ParseStatus::Failure; |
| 8458 | |
| 8459 | if (!isUInt<16>(x: ImmVal)) |
| 8460 | return Error(L: Loc, Msg: "invalid immediate: only 16-bit values are legal" ); |
| 8461 | Operands.push_back( |
| 8462 | Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: ImmVal, Loc, Type: AMDGPUOperand::ImmTyHwreg)); |
| 8463 | return ParseStatus::Success; |
| 8464 | } |
| 8465 | |
| 8466 | bool AMDGPUOperand::isHwreg() const { return isImmTy(ImmT: ImmTyHwreg); } |
| 8467 | |
| 8468 | //===----------------------------------------------------------------------===// |
| 8469 | // sendmsg |
| 8470 | //===----------------------------------------------------------------------===// |
| 8471 | |
| 8472 | bool AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &Msg, OperandInfoTy &Op, |
| 8473 | OperandInfoTy &Stream) { |
| 8474 | using namespace llvm::AMDGPU::SendMsg; |
| 8475 | |
| 8476 | Msg.Loc = getLoc(); |
| 8477 | if (isToken(Kind: AsmToken::Identifier) && |
| 8478 | (Msg.Val = getMsgId(Name: getTokenStr(), STI: getSTI())) != OPR_ID_UNKNOWN) { |
| 8479 | Msg.IsSymbolic = true; |
| 8480 | lex(); // skip message name |
| 8481 | } else if (!parseExpr(Imm&: Msg.Val, Expected: "a message name" )) { |
| 8482 | return false; |
| 8483 | } |
| 8484 | |
| 8485 | if (trySkipToken(Kind: AsmToken::Comma)) { |
| 8486 | Op.IsDefined = true; |
| 8487 | Op.Loc = getLoc(); |
| 8488 | if (isToken(Kind: AsmToken::Identifier) && |
| 8489 | (Op.Val = getMsgOpId(MsgId: Msg.Val, Name: getTokenStr(), STI: getSTI())) != |
| 8490 | OPR_ID_UNKNOWN) { |
| 8491 | lex(); // skip operation name |
| 8492 | } else if (!parseExpr(Imm&: Op.Val, Expected: "an operation name" )) { |
| 8493 | return false; |
| 8494 | } |
| 8495 | |
| 8496 | if (trySkipToken(Kind: AsmToken::Comma)) { |
| 8497 | Stream.IsDefined = true; |
| 8498 | Stream.Loc = getLoc(); |
| 8499 | if (!parseExpr(Imm&: Stream.Val)) |
| 8500 | return false; |
| 8501 | } |
| 8502 | } |
| 8503 | |
| 8504 | return skipToken(Kind: AsmToken::RParen, ErrMsg: "expected a closing parenthesis" ); |
| 8505 | } |
| 8506 | |
| 8507 | bool AMDGPUAsmParser::validateSendMsg(const OperandInfoTy &Msg, |
| 8508 | const OperandInfoTy &Op, |
| 8509 | const OperandInfoTy &Stream) { |
| 8510 | using namespace llvm::AMDGPU::SendMsg; |
| 8511 | |
| 8512 | // Validation strictness depends on whether message is specified |
| 8513 | // in a symbolic or in a numeric form. In the latter case |
| 8514 | // only encoding possibility is checked. |
| 8515 | bool Strict = Msg.IsSymbolic; |
| 8516 | |
| 8517 | if (Strict) { |
| 8518 | if (Msg.Val == OPR_ID_UNSUPPORTED) { |
| 8519 | Error(L: Msg.Loc, Msg: "specified message id is not supported on this GPU" ); |
| 8520 | return false; |
| 8521 | } |
| 8522 | } else { |
| 8523 | if (!isValidMsgId(MsgId: Msg.Val, STI: getSTI())) { |
| 8524 | Error(L: Msg.Loc, Msg: "invalid message id" ); |
| 8525 | return false; |
| 8526 | } |
| 8527 | } |
| 8528 | if (Strict && (msgRequiresOp(MsgId: Msg.Val, STI: getSTI()) != Op.IsDefined)) { |
| 8529 | if (Op.IsDefined) { |
| 8530 | Error(L: Op.Loc, Msg: "message does not support operations" ); |
| 8531 | } else { |
| 8532 | Error(L: Msg.Loc, Msg: "missing message operation" ); |
| 8533 | } |
| 8534 | return false; |
| 8535 | } |
| 8536 | if (!isValidMsgOp(MsgId: Msg.Val, OpId: Op.Val, STI: getSTI(), Strict)) { |
| 8537 | if (Op.Val == OPR_ID_UNSUPPORTED) |
| 8538 | Error(L: Op.Loc, Msg: "specified operation id is not supported on this GPU" ); |
| 8539 | else |
| 8540 | Error(L: Op.Loc, Msg: "invalid operation id" ); |
| 8541 | return false; |
| 8542 | } |
| 8543 | if (Strict && !msgSupportsStream(MsgId: Msg.Val, OpId: Op.Val, STI: getSTI()) && |
| 8544 | Stream.IsDefined) { |
| 8545 | Error(L: Stream.Loc, Msg: "message operation does not support streams" ); |
| 8546 | return false; |
| 8547 | } |
| 8548 | if (!isValidMsgStream(MsgId: Msg.Val, OpId: Op.Val, StreamId: Stream.Val, STI: getSTI(), Strict)) { |
| 8549 | Error(L: Stream.Loc, Msg: "invalid message stream id" ); |
| 8550 | return false; |
| 8551 | } |
| 8552 | return true; |
| 8553 | } |
| 8554 | |
| 8555 | ParseStatus AMDGPUAsmParser::parseSendMsg(OperandVector &Operands) { |
| 8556 | using namespace llvm::AMDGPU::SendMsg; |
| 8557 | |
| 8558 | int64_t ImmVal = 0; |
| 8559 | SMLoc Loc = getLoc(); |
| 8560 | |
| 8561 | if (trySkipId(Id: "sendmsg" , Kind: AsmToken::LParen)) { |
| 8562 | OperandInfoTy Msg(OPR_ID_UNKNOWN); |
| 8563 | OperandInfoTy Op(OP_NONE_); |
| 8564 | OperandInfoTy Stream(STREAM_ID_NONE_); |
| 8565 | if (parseSendMsgBody(Msg, Op, Stream) && validateSendMsg(Msg, Op, Stream)) { |
| 8566 | ImmVal = encodeMsg(MsgId: Msg.Val, OpId: Op.Val, StreamId: Stream.Val); |
| 8567 | } else { |
| 8568 | return ParseStatus::Failure; |
| 8569 | } |
| 8570 | } else if (parseExpr(Imm&: ImmVal, Expected: "a sendmsg macro" )) { |
| 8571 | if (ImmVal < 0 || !isUInt<16>(x: ImmVal)) |
| 8572 | return Error(L: Loc, Msg: "invalid immediate: only 16-bit values are legal" ); |
| 8573 | } else { |
| 8574 | return ParseStatus::Failure; |
| 8575 | } |
| 8576 | |
| 8577 | Operands.push_back( |
| 8578 | Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: ImmVal, Loc, Type: AMDGPUOperand::ImmTySendMsg)); |
| 8579 | return ParseStatus::Success; |
| 8580 | } |
| 8581 | |
| 8582 | bool AMDGPUOperand::isSendMsg() const { return isImmTy(ImmT: ImmTySendMsg); } |
| 8583 | |
| 8584 | ParseStatus AMDGPUAsmParser::parseWaitEvent(OperandVector &Operands) { |
| 8585 | using namespace llvm::AMDGPU::WaitEvent; |
| 8586 | |
| 8587 | SMLoc Loc = getLoc(); |
| 8588 | int64_t ImmVal = 0; |
| 8589 | |
| 8590 | StructuredOpField DontWaitExportReady("dont_wait_export_ready" , "bit value" , |
| 8591 | 1, 0); |
| 8592 | StructuredOpField ExportReady("export_ready" , "bit value" , 1, 0); |
| 8593 | |
| 8594 | StructuredOpField *TargetBitfield = |
| 8595 | isGFX11() ? &DontWaitExportReady : &ExportReady; |
| 8596 | |
| 8597 | ParseStatus Res = parseStructuredOpFields(Fields: {TargetBitfield}); |
| 8598 | if (Res.isNoMatch() && parseExpr(Imm&: ImmVal, Expected: "structured immediate" )) |
| 8599 | Res = ParseStatus::Success; |
| 8600 | else if (Res.isSuccess()) { |
| 8601 | if (!validateStructuredOpFields(Fields: {TargetBitfield})) |
| 8602 | return ParseStatus::Failure; |
| 8603 | ImmVal = TargetBitfield->Val; |
| 8604 | } |
| 8605 | |
| 8606 | if (!Res.isSuccess()) |
| 8607 | return ParseStatus::Failure; |
| 8608 | |
| 8609 | if (!isUInt<16>(x: ImmVal)) |
| 8610 | return Error(L: Loc, Msg: "invalid immediate: only 16-bit values are legal" ); |
| 8611 | |
| 8612 | Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: ImmVal, Loc, |
| 8613 | Type: AMDGPUOperand::ImmTyWaitEvent)); |
| 8614 | return ParseStatus::Success; |
| 8615 | } |
| 8616 | |
| 8617 | bool AMDGPUOperand::isWaitEvent() const { return isImmTy(ImmT: ImmTyWaitEvent); } |
| 8618 | |
| 8619 | //===----------------------------------------------------------------------===// |
| 8620 | // v_interp |
| 8621 | //===----------------------------------------------------------------------===// |
| 8622 | |
| 8623 | ParseStatus AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) { |
| 8624 | StringRef Str; |
| 8625 | SMLoc S = getLoc(); |
| 8626 | |
| 8627 | if (!parseId(Val&: Str)) |
| 8628 | return ParseStatus::NoMatch; |
| 8629 | |
| 8630 | int Slot = StringSwitch<int>(Str) |
| 8631 | .Case(S: "p10" , Value: 0) |
| 8632 | .Case(S: "p20" , Value: 1) |
| 8633 | .Case(S: "p0" , Value: 2) |
| 8634 | .Default(Value: -1); |
| 8635 | |
| 8636 | if (Slot == -1) |
| 8637 | return Error(L: S, Msg: "invalid interpolation slot" ); |
| 8638 | |
| 8639 | Operands.push_back( |
| 8640 | Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Slot, Loc: S, Type: AMDGPUOperand::ImmTyInterpSlot)); |
| 8641 | return ParseStatus::Success; |
| 8642 | } |
| 8643 | |
| 8644 | ParseStatus AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) { |
| 8645 | StringRef Str; |
| 8646 | SMLoc S = getLoc(); |
| 8647 | |
| 8648 | if (!parseId(Val&: Str)) |
| 8649 | return ParseStatus::NoMatch; |
| 8650 | |
| 8651 | if (!Str.starts_with(Prefix: "attr" )) |
| 8652 | return Error(L: S, Msg: "invalid interpolation attribute" ); |
| 8653 | |
| 8654 | StringRef Chan = Str.take_back(N: 2); |
| 8655 | int AttrChan = StringSwitch<int>(Chan) |
| 8656 | .Case(S: ".x" , Value: 0) |
| 8657 | .Case(S: ".y" , Value: 1) |
| 8658 | .Case(S: ".z" , Value: 2) |
| 8659 | .Case(S: ".w" , Value: 3) |
| 8660 | .Default(Value: -1); |
| 8661 | if (AttrChan == -1) |
| 8662 | return Error(L: S, Msg: "invalid or missing interpolation attribute channel" ); |
| 8663 | |
| 8664 | Str = Str.drop_back(N: 2).drop_front(N: 4); |
| 8665 | |
| 8666 | uint8_t Attr; |
| 8667 | if (Str.getAsInteger(Radix: 10, Result&: Attr)) |
| 8668 | return Error(L: S, Msg: "invalid or missing interpolation attribute number" ); |
| 8669 | |
| 8670 | if (Attr > 32) |
| 8671 | return Error(L: S, Msg: "out of bounds interpolation attribute number" ); |
| 8672 | |
| 8673 | SMLoc SChan = SMLoc::getFromPointer(Ptr: Chan.data()); |
| 8674 | |
| 8675 | Operands.push_back( |
| 8676 | Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Attr, Loc: S, Type: AMDGPUOperand::ImmTyInterpAttr)); |
| 8677 | Operands.push_back(Elt: AMDGPUOperand::CreateImm( |
| 8678 | AsmParser: this, Val: AttrChan, Loc: SChan, Type: AMDGPUOperand::ImmTyInterpAttrChan)); |
| 8679 | return ParseStatus::Success; |
| 8680 | } |
| 8681 | |
| 8682 | //===----------------------------------------------------------------------===// |
| 8683 | // exp |
| 8684 | //===----------------------------------------------------------------------===// |
| 8685 | |
| 8686 | ParseStatus AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) { |
| 8687 | using namespace llvm::AMDGPU::Exp; |
| 8688 | |
| 8689 | StringRef Str; |
| 8690 | SMLoc S = getLoc(); |
| 8691 | |
| 8692 | if (!parseId(Val&: Str)) |
| 8693 | return ParseStatus::NoMatch; |
| 8694 | |
| 8695 | unsigned Id = getTgtId(Name: Str); |
| 8696 | if (Id == ET_INVALID || !isSupportedTgtId(Id, STI: getSTI())) |
| 8697 | return Error(L: S, Msg: (Id == ET_INVALID) |
| 8698 | ? "invalid exp target" |
| 8699 | : "exp target is not supported on this GPU" ); |
| 8700 | |
| 8701 | Operands.push_back( |
| 8702 | Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Id, Loc: S, Type: AMDGPUOperand::ImmTyExpTgt)); |
| 8703 | return ParseStatus::Success; |
| 8704 | } |
| 8705 | |
| 8706 | //===----------------------------------------------------------------------===// |
| 8707 | // parser helpers |
| 8708 | //===----------------------------------------------------------------------===// |
| 8709 | |
| 8710 | bool AMDGPUAsmParser::isId(const AsmToken &Token, const StringRef Id) const { |
| 8711 | return Token.is(K: AsmToken::Identifier) && Token.getString() == Id; |
| 8712 | } |
| 8713 | |
| 8714 | bool AMDGPUAsmParser::isId(const StringRef Id) const { |
| 8715 | return isId(Token: getToken(), Id); |
| 8716 | } |
| 8717 | |
| 8718 | bool AMDGPUAsmParser::isToken(const AsmToken::TokenKind Kind) const { |
| 8719 | return getTokenKind() == Kind; |
| 8720 | } |
| 8721 | |
| 8722 | StringRef AMDGPUAsmParser::getId() const { |
| 8723 | return isToken(Kind: AsmToken::Identifier) ? getTokenStr() : StringRef(); |
| 8724 | } |
| 8725 | |
| 8726 | bool AMDGPUAsmParser::trySkipId(const StringRef Id) { |
| 8727 | if (isId(Id)) { |
| 8728 | lex(); |
| 8729 | return true; |
| 8730 | } |
| 8731 | return false; |
| 8732 | } |
| 8733 | |
| 8734 | bool AMDGPUAsmParser::trySkipId(const StringRef Pref, const StringRef Id) { |
| 8735 | if (isToken(Kind: AsmToken::Identifier)) { |
| 8736 | StringRef Tok = getTokenStr(); |
| 8737 | if (Tok.starts_with(Prefix: Pref) && Tok.drop_front(N: Pref.size()) == Id) { |
| 8738 | lex(); |
| 8739 | return true; |
| 8740 | } |
| 8741 | } |
| 8742 | return false; |
| 8743 | } |
| 8744 | |
| 8745 | bool AMDGPUAsmParser::trySkipId(const StringRef Id, |
| 8746 | const AsmToken::TokenKind Kind) { |
| 8747 | if (isId(Id) && peekToken().is(K: Kind)) { |
| 8748 | lex(); |
| 8749 | lex(); |
| 8750 | return true; |
| 8751 | } |
| 8752 | return false; |
| 8753 | } |
| 8754 | |
| 8755 | bool AMDGPUAsmParser::trySkipToken(const AsmToken::TokenKind Kind) { |
| 8756 | if (isToken(Kind)) { |
| 8757 | lex(); |
| 8758 | return true; |
| 8759 | } |
| 8760 | return false; |
| 8761 | } |
| 8762 | |
| 8763 | bool AMDGPUAsmParser::skipToken(const AsmToken::TokenKind Kind, |
| 8764 | const StringRef ErrMsg) { |
| 8765 | if (!trySkipToken(Kind)) { |
| 8766 | Error(L: getLoc(), Msg: ErrMsg); |
| 8767 | return false; |
| 8768 | } |
| 8769 | return true; |
| 8770 | } |
| 8771 | |
| 8772 | bool AMDGPUAsmParser::parseExpr(int64_t &Imm, StringRef Expected) { |
| 8773 | SMLoc S = getLoc(); |
| 8774 | |
| 8775 | const MCExpr *Expr; |
| 8776 | if (Parser.parseExpression(Res&: Expr)) |
| 8777 | return false; |
| 8778 | |
| 8779 | if (Expr->evaluateAsAbsolute(Res&: Imm)) |
| 8780 | return true; |
| 8781 | |
| 8782 | if (Expected.empty()) { |
| 8783 | Error(L: S, Msg: "expected absolute expression" ); |
| 8784 | } else { |
| 8785 | Error(L: S, |
| 8786 | Msg: Twine("expected " , Expected) + Twine(" or an absolute expression" )); |
| 8787 | } |
| 8788 | return false; |
| 8789 | } |
| 8790 | |
| 8791 | bool AMDGPUAsmParser::parseExpr(OperandVector &Operands) { |
| 8792 | SMLoc S = getLoc(); |
| 8793 | |
| 8794 | const MCExpr *Expr; |
| 8795 | if (Parser.parseExpression(Res&: Expr)) |
| 8796 | return false; |
| 8797 | |
| 8798 | int64_t IntVal; |
| 8799 | if (Expr->evaluateAsAbsolute(Res&: IntVal)) { |
| 8800 | Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: IntVal, Loc: S)); |
| 8801 | } else { |
| 8802 | Operands.push_back(Elt: AMDGPUOperand::CreateExpr(AsmParser: this, Expr, S)); |
| 8803 | } |
| 8804 | return true; |
| 8805 | } |
| 8806 | |
| 8807 | bool AMDGPUAsmParser::parseString(StringRef &Val, const StringRef ErrMsg) { |
| 8808 | if (isToken(Kind: AsmToken::String)) { |
| 8809 | Val = getToken().getStringContents(); |
| 8810 | lex(); |
| 8811 | return true; |
| 8812 | } |
| 8813 | Error(L: getLoc(), Msg: ErrMsg); |
| 8814 | return false; |
| 8815 | } |
| 8816 | |
| 8817 | bool AMDGPUAsmParser::parseId(StringRef &Val, const StringRef ErrMsg) { |
| 8818 | if (isToken(Kind: AsmToken::Identifier)) { |
| 8819 | Val = getTokenStr(); |
| 8820 | lex(); |
| 8821 | return true; |
| 8822 | } |
| 8823 | if (!ErrMsg.empty()) |
| 8824 | Error(L: getLoc(), Msg: ErrMsg); |
| 8825 | return false; |
| 8826 | } |
| 8827 | |
| 8828 | AsmToken AMDGPUAsmParser::getToken() const { return Parser.getTok(); } |
| 8829 | |
| 8830 | AsmToken AMDGPUAsmParser::peekToken(bool ShouldSkipSpace) { |
| 8831 | return isToken(Kind: AsmToken::EndOfStatement) |
| 8832 | ? getToken() |
| 8833 | : getLexer().peekTok(ShouldSkipSpace); |
| 8834 | } |
| 8835 | |
| 8836 | void AMDGPUAsmParser::peekTokens(MutableArrayRef<AsmToken> Tokens) { |
| 8837 | auto TokCount = getLexer().peekTokens(Buf: Tokens); |
| 8838 | |
| 8839 | for (auto Idx = TokCount; Idx < Tokens.size(); ++Idx) |
| 8840 | Tokens[Idx] = AsmToken(AsmToken::Error, "" ); |
| 8841 | } |
| 8842 | |
| 8843 | AsmToken::TokenKind AMDGPUAsmParser::getTokenKind() const { |
| 8844 | return getLexer().getKind(); |
| 8845 | } |
| 8846 | |
| 8847 | SMLoc AMDGPUAsmParser::getLoc() const { return getToken().getLoc(); } |
| 8848 | |
| 8849 | StringRef AMDGPUAsmParser::getTokenStr() const { |
| 8850 | return getToken().getString(); |
| 8851 | } |
| 8852 | |
| 8853 | void AMDGPUAsmParser::lex() { Parser.Lex(); } |
| 8854 | |
| 8855 | const AMDGPUOperand & |
| 8856 | AMDGPUAsmParser::findMCOperand(const OperandVector &Operands, |
| 8857 | int MCOpIdx) const { |
| 8858 | for (const auto &Op : Operands) { |
| 8859 | const AMDGPUOperand &TargetOp = static_cast<AMDGPUOperand &>(*Op); |
| 8860 | if (TargetOp.getMCOpIdx() == MCOpIdx) |
| 8861 | return TargetOp; |
| 8862 | } |
| 8863 | llvm_unreachable("no such MC operand!" ); |
| 8864 | } |
| 8865 | |
| 8866 | SMLoc AMDGPUAsmParser::getInstLoc(const OperandVector &Operands) const { |
| 8867 | return ((AMDGPUOperand &)*Operands[0]).getStartLoc(); |
| 8868 | } |
| 8869 | |
| 8870 | // Returns one of the given locations that comes later in the source. |
| 8871 | SMLoc AMDGPUAsmParser::getLaterLoc(SMLoc a, SMLoc b) { |
| 8872 | return a.getPointer() < b.getPointer() ? b : a; |
| 8873 | } |
| 8874 | |
| 8875 | SMLoc AMDGPUAsmParser::getOperandLoc(const OperandVector &Operands, |
| 8876 | int MCOpIdx) const { |
| 8877 | return findMCOperand(Operands, MCOpIdx).getStartLoc(); |
| 8878 | } |
| 8879 | |
| 8880 | SMLoc AMDGPUAsmParser::getOperandLoc( |
| 8881 | std::function<bool(const AMDGPUOperand &)> Test, |
| 8882 | const OperandVector &Operands) const { |
| 8883 | for (unsigned i = Operands.size() - 1; i > 0; --i) { |
| 8884 | AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]); |
| 8885 | if (Test(Op)) |
| 8886 | return Op.getStartLoc(); |
| 8887 | } |
| 8888 | return getInstLoc(Operands); |
| 8889 | } |
| 8890 | |
| 8891 | SMLoc AMDGPUAsmParser::getImmLoc(AMDGPUOperand::ImmTy Type, |
| 8892 | const OperandVector &Operands) const { |
| 8893 | auto Test = [=](const AMDGPUOperand &Op) { return Op.isImmTy(ImmT: Type); }; |
| 8894 | return getOperandLoc(Test, Operands); |
| 8895 | } |
| 8896 | |
| 8897 | ParseStatus |
| 8898 | AMDGPUAsmParser::parseStructuredOpFields(ArrayRef<StructuredOpField *> Fields) { |
| 8899 | if (!trySkipToken(Kind: AsmToken::LCurly)) |
| 8900 | return ParseStatus::NoMatch; |
| 8901 | |
| 8902 | bool First = true; |
| 8903 | while (!trySkipToken(Kind: AsmToken::RCurly)) { |
| 8904 | if (!First && |
| 8905 | !skipToken(Kind: AsmToken::Comma, ErrMsg: "comma or closing brace expected" )) |
| 8906 | return ParseStatus::Failure; |
| 8907 | |
| 8908 | StringRef Id = getTokenStr(); |
| 8909 | SMLoc IdLoc = getLoc(); |
| 8910 | if (!skipToken(Kind: AsmToken::Identifier, ErrMsg: "field name expected" ) || |
| 8911 | !skipToken(Kind: AsmToken::Colon, ErrMsg: "colon expected" )) |
| 8912 | return ParseStatus::Failure; |
| 8913 | |
| 8914 | const auto *I = |
| 8915 | find_if(Range&: Fields, P: [Id](StructuredOpField *F) { return F->Id == Id; }); |
| 8916 | if (I == Fields.end()) |
| 8917 | return Error(L: IdLoc, Msg: "unknown field" ); |
| 8918 | if ((*I)->IsDefined) |
| 8919 | return Error(L: IdLoc, Msg: "duplicate field" ); |
| 8920 | |
| 8921 | // TODO: Support symbolic values. |
| 8922 | (*I)->Loc = getLoc(); |
| 8923 | if (!parseExpr(Imm&: (*I)->Val)) |
| 8924 | return ParseStatus::Failure; |
| 8925 | (*I)->IsDefined = true; |
| 8926 | |
| 8927 | First = false; |
| 8928 | } |
| 8929 | return ParseStatus::Success; |
| 8930 | } |
| 8931 | |
| 8932 | bool AMDGPUAsmParser::validateStructuredOpFields( |
| 8933 | ArrayRef<const StructuredOpField *> Fields) { |
| 8934 | return all_of(Range&: Fields, P: [this](const StructuredOpField *F) { |
| 8935 | return F->validate(Parser&: *this); |
| 8936 | }); |
| 8937 | } |
| 8938 | |
| 8939 | //===----------------------------------------------------------------------===// |
| 8940 | // swizzle |
| 8941 | //===----------------------------------------------------------------------===// |
| 8942 | |
| 8943 | LLVM_READNONE |
| 8944 | static unsigned encodeBitmaskPerm(const unsigned AndMask, const unsigned OrMask, |
| 8945 | const unsigned XorMask) { |
| 8946 | using namespace llvm::AMDGPU::Swizzle; |
| 8947 | |
| 8948 | return BITMASK_PERM_ENC | (AndMask << BITMASK_AND_SHIFT) | |
| 8949 | (OrMask << BITMASK_OR_SHIFT) | (XorMask << BITMASK_XOR_SHIFT); |
| 8950 | } |
| 8951 | |
| 8952 | bool AMDGPUAsmParser::parseSwizzleOperand(int64_t &Op, const unsigned MinVal, |
| 8953 | const unsigned MaxVal, |
| 8954 | const Twine &ErrMsg, SMLoc &Loc) { |
| 8955 | if (!skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma" )) { |
| 8956 | return false; |
| 8957 | } |
| 8958 | Loc = getLoc(); |
| 8959 | if (!parseExpr(Imm&: Op)) { |
| 8960 | return false; |
| 8961 | } |
| 8962 | if (Op < MinVal || Op > MaxVal) { |
| 8963 | Error(L: Loc, Msg: ErrMsg); |
| 8964 | return false; |
| 8965 | } |
| 8966 | |
| 8967 | return true; |
| 8968 | } |
| 8969 | |
| 8970 | bool AMDGPUAsmParser::parseSwizzleOperands(const unsigned OpNum, int64_t *Op, |
| 8971 | const unsigned MinVal, |
| 8972 | const unsigned MaxVal, |
| 8973 | const StringRef ErrMsg) { |
| 8974 | SMLoc Loc; |
| 8975 | for (unsigned i = 0; i < OpNum; ++i) { |
| 8976 | if (!parseSwizzleOperand(Op&: Op[i], MinVal, MaxVal, ErrMsg, Loc)) |
| 8977 | return false; |
| 8978 | } |
| 8979 | |
| 8980 | return true; |
| 8981 | } |
| 8982 | |
| 8983 | bool AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &Imm) { |
| 8984 | using namespace llvm::AMDGPU::Swizzle; |
| 8985 | |
| 8986 | int64_t Lane[LANE_NUM]; |
| 8987 | if (parseSwizzleOperands(OpNum: LANE_NUM, Op: Lane, MinVal: 0, MaxVal: LANE_MAX, |
| 8988 | ErrMsg: "expected a 2-bit lane id" )) { |
| 8989 | Imm = QUAD_PERM_ENC; |
| 8990 | for (unsigned I = 0; I < LANE_NUM; ++I) { |
| 8991 | Imm |= Lane[I] << (LANE_SHIFT * I); |
| 8992 | } |
| 8993 | return true; |
| 8994 | } |
| 8995 | return false; |
| 8996 | } |
| 8997 | |
| 8998 | bool AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &Imm) { |
| 8999 | using namespace llvm::AMDGPU::Swizzle; |
| 9000 | |
| 9001 | SMLoc Loc; |
| 9002 | int64_t GroupSize; |
| 9003 | int64_t LaneIdx; |
| 9004 | |
| 9005 | if (!parseSwizzleOperand(Op&: GroupSize, MinVal: 2, MaxVal: 32, |
| 9006 | ErrMsg: "group size must be in the interval [2,32]" , Loc)) { |
| 9007 | return false; |
| 9008 | } |
| 9009 | if (!isPowerOf2_64(Value: GroupSize)) { |
| 9010 | Error(L: Loc, Msg: "group size must be a power of two" ); |
| 9011 | return false; |
| 9012 | } |
| 9013 | if (parseSwizzleOperand(Op&: LaneIdx, MinVal: 0, MaxVal: GroupSize - 1, |
| 9014 | ErrMsg: "lane id must be in the interval [0,group size - 1]" , |
| 9015 | Loc)) { |
| 9016 | Imm = encodeBitmaskPerm(AndMask: BITMASK_MAX - GroupSize + 1, OrMask: LaneIdx, XorMask: 0); |
| 9017 | return true; |
| 9018 | } |
| 9019 | return false; |
| 9020 | } |
| 9021 | |
| 9022 | bool AMDGPUAsmParser::parseSwizzleReverse(int64_t &Imm) { |
| 9023 | using namespace llvm::AMDGPU::Swizzle; |
| 9024 | |
| 9025 | SMLoc Loc; |
| 9026 | int64_t GroupSize; |
| 9027 | |
| 9028 | if (!parseSwizzleOperand(Op&: GroupSize, MinVal: 2, MaxVal: 32, |
| 9029 | ErrMsg: "group size must be in the interval [2,32]" , Loc)) { |
| 9030 | return false; |
| 9031 | } |
| 9032 | if (!isPowerOf2_64(Value: GroupSize)) { |
| 9033 | Error(L: Loc, Msg: "group size must be a power of two" ); |
| 9034 | return false; |
| 9035 | } |
| 9036 | |
| 9037 | Imm = encodeBitmaskPerm(AndMask: BITMASK_MAX, OrMask: 0, XorMask: GroupSize - 1); |
| 9038 | return true; |
| 9039 | } |
| 9040 | |
| 9041 | bool AMDGPUAsmParser::parseSwizzleSwap(int64_t &Imm) { |
| 9042 | using namespace llvm::AMDGPU::Swizzle; |
| 9043 | |
| 9044 | SMLoc Loc; |
| 9045 | int64_t GroupSize; |
| 9046 | |
| 9047 | if (!parseSwizzleOperand(Op&: GroupSize, MinVal: 1, MaxVal: 16, |
| 9048 | ErrMsg: "group size must be in the interval [1,16]" , Loc)) { |
| 9049 | return false; |
| 9050 | } |
| 9051 | if (!isPowerOf2_64(Value: GroupSize)) { |
| 9052 | Error(L: Loc, Msg: "group size must be a power of two" ); |
| 9053 | return false; |
| 9054 | } |
| 9055 | |
| 9056 | Imm = encodeBitmaskPerm(AndMask: BITMASK_MAX, OrMask: 0, XorMask: GroupSize); |
| 9057 | return true; |
| 9058 | } |
| 9059 | |
| 9060 | bool AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &Imm) { |
| 9061 | using namespace llvm::AMDGPU::Swizzle; |
| 9062 | |
| 9063 | if (!skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma" )) { |
| 9064 | return false; |
| 9065 | } |
| 9066 | |
| 9067 | StringRef Ctl; |
| 9068 | SMLoc StrLoc = getLoc(); |
| 9069 | if (!parseString(Val&: Ctl)) { |
| 9070 | return false; |
| 9071 | } |
| 9072 | if (Ctl.size() != BITMASK_WIDTH) { |
| 9073 | Error(L: StrLoc, Msg: "expected a 5-character mask" ); |
| 9074 | return false; |
| 9075 | } |
| 9076 | |
| 9077 | unsigned AndMask = 0; |
| 9078 | unsigned OrMask = 0; |
| 9079 | unsigned XorMask = 0; |
| 9080 | |
| 9081 | for (size_t i = 0; i < Ctl.size(); ++i) { |
| 9082 | unsigned Mask = 1 << (BITMASK_WIDTH - 1 - i); |
| 9083 | switch (Ctl[i]) { |
| 9084 | default: |
| 9085 | Error(L: StrLoc, Msg: "invalid mask" ); |
| 9086 | return false; |
| 9087 | case '0': |
| 9088 | break; |
| 9089 | case '1': |
| 9090 | OrMask |= Mask; |
| 9091 | break; |
| 9092 | case 'p': |
| 9093 | AndMask |= Mask; |
| 9094 | break; |
| 9095 | case 'i': |
| 9096 | AndMask |= Mask; |
| 9097 | XorMask |= Mask; |
| 9098 | break; |
| 9099 | } |
| 9100 | } |
| 9101 | |
| 9102 | Imm = encodeBitmaskPerm(AndMask, OrMask, XorMask); |
| 9103 | return true; |
| 9104 | } |
| 9105 | |
| 9106 | bool AMDGPUAsmParser::parseSwizzleFFT(int64_t &Imm) { |
| 9107 | using namespace llvm::AMDGPU::Swizzle; |
| 9108 | |
| 9109 | if (!AMDGPU::isGFX9Plus(STI: getSTI())) { |
| 9110 | Error(L: getLoc(), Msg: "FFT mode swizzle not supported on this GPU" ); |
| 9111 | return false; |
| 9112 | } |
| 9113 | |
| 9114 | int64_t Swizzle; |
| 9115 | SMLoc Loc; |
| 9116 | if (!parseSwizzleOperand(Op&: Swizzle, MinVal: 0, MaxVal: FFT_SWIZZLE_MAX, |
| 9117 | ErrMsg: "FFT swizzle must be in the interval [0," + |
| 9118 | Twine(FFT_SWIZZLE_MAX) + Twine(']'), |
| 9119 | Loc)) |
| 9120 | return false; |
| 9121 | |
| 9122 | Imm = FFT_MODE_ENC | Swizzle; |
| 9123 | return true; |
| 9124 | } |
| 9125 | |
| 9126 | bool AMDGPUAsmParser::parseSwizzleRotate(int64_t &Imm) { |
| 9127 | using namespace llvm::AMDGPU::Swizzle; |
| 9128 | |
| 9129 | if (!AMDGPU::isGFX9Plus(STI: getSTI())) { |
| 9130 | Error(L: getLoc(), Msg: "Rotate mode swizzle not supported on this GPU" ); |
| 9131 | return false; |
| 9132 | } |
| 9133 | |
| 9134 | SMLoc Loc; |
| 9135 | int64_t Direction; |
| 9136 | |
| 9137 | if (!parseSwizzleOperand(Op&: Direction, MinVal: 0, MaxVal: 1, |
| 9138 | ErrMsg: "direction must be 0 (left) or 1 (right)" , Loc)) |
| 9139 | return false; |
| 9140 | |
| 9141 | int64_t RotateSize; |
| 9142 | if (!parseSwizzleOperand( |
| 9143 | Op&: RotateSize, MinVal: 0, MaxVal: ROTATE_MAX_SIZE, |
| 9144 | ErrMsg: "number of threads to rotate must be in the interval [0," + |
| 9145 | Twine(ROTATE_MAX_SIZE) + Twine(']'), |
| 9146 | Loc)) |
| 9147 | return false; |
| 9148 | |
| 9149 | Imm = ROTATE_MODE_ENC | (Direction << ROTATE_DIR_SHIFT) | |
| 9150 | (RotateSize << ROTATE_SIZE_SHIFT); |
| 9151 | return true; |
| 9152 | } |
| 9153 | |
| 9154 | bool AMDGPUAsmParser::parseSwizzleOffset(int64_t &Imm) { |
| 9155 | |
| 9156 | SMLoc OffsetLoc = getLoc(); |
| 9157 | |
| 9158 | if (!parseExpr(Imm, Expected: "a swizzle macro" )) { |
| 9159 | return false; |
| 9160 | } |
| 9161 | if (!isUInt<16>(x: Imm)) { |
| 9162 | Error(L: OffsetLoc, Msg: "expected a 16-bit offset" ); |
| 9163 | return false; |
| 9164 | } |
| 9165 | return true; |
| 9166 | } |
| 9167 | |
| 9168 | bool AMDGPUAsmParser::parseSwizzleMacro(int64_t &Imm) { |
| 9169 | using namespace llvm::AMDGPU::Swizzle; |
| 9170 | |
| 9171 | if (skipToken(Kind: AsmToken::LParen, ErrMsg: "expected a left parentheses" )) { |
| 9172 | |
| 9173 | SMLoc ModeLoc = getLoc(); |
| 9174 | bool Ok = false; |
| 9175 | |
| 9176 | if (trySkipId(Id: IdSymbolic[ID_QUAD_PERM])) { |
| 9177 | Ok = parseSwizzleQuadPerm(Imm); |
| 9178 | } else if (trySkipId(Id: IdSymbolic[ID_BITMASK_PERM])) { |
| 9179 | Ok = parseSwizzleBitmaskPerm(Imm); |
| 9180 | } else if (trySkipId(Id: IdSymbolic[ID_BROADCAST])) { |
| 9181 | Ok = parseSwizzleBroadcast(Imm); |
| 9182 | } else if (trySkipId(Id: IdSymbolic[ID_SWAP])) { |
| 9183 | Ok = parseSwizzleSwap(Imm); |
| 9184 | } else if (trySkipId(Id: IdSymbolic[ID_REVERSE])) { |
| 9185 | Ok = parseSwizzleReverse(Imm); |
| 9186 | } else if (trySkipId(Id: IdSymbolic[ID_FFT])) { |
| 9187 | Ok = parseSwizzleFFT(Imm); |
| 9188 | } else if (trySkipId(Id: IdSymbolic[ID_ROTATE])) { |
| 9189 | Ok = parseSwizzleRotate(Imm); |
| 9190 | } else { |
| 9191 | Error(L: ModeLoc, Msg: "expected a swizzle mode" ); |
| 9192 | } |
| 9193 | |
| 9194 | return Ok && skipToken(Kind: AsmToken::RParen, ErrMsg: "expected a closing parentheses" ); |
| 9195 | } |
| 9196 | |
| 9197 | return false; |
| 9198 | } |
| 9199 | |
| 9200 | ParseStatus AMDGPUAsmParser::parseSwizzle(OperandVector &Operands) { |
| 9201 | SMLoc S = getLoc(); |
| 9202 | int64_t Imm = 0; |
| 9203 | |
| 9204 | if (trySkipId(Id: "offset" )) { |
| 9205 | |
| 9206 | bool Ok = false; |
| 9207 | if (skipToken(Kind: AsmToken::Colon, ErrMsg: "expected a colon" )) { |
| 9208 | if (trySkipId(Id: "swizzle" )) { |
| 9209 | Ok = parseSwizzleMacro(Imm); |
| 9210 | } else { |
| 9211 | Ok = parseSwizzleOffset(Imm); |
| 9212 | } |
| 9213 | } |
| 9214 | |
| 9215 | Operands.push_back( |
| 9216 | Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Imm, Loc: S, Type: AMDGPUOperand::ImmTySwizzle)); |
| 9217 | |
| 9218 | return Ok ? ParseStatus::Success : ParseStatus::Failure; |
| 9219 | } |
| 9220 | return ParseStatus::NoMatch; |
| 9221 | } |
| 9222 | |
| 9223 | bool AMDGPUOperand::isSwizzle() const { return isImmTy(ImmT: ImmTySwizzle); } |
| 9224 | |
| 9225 | //===----------------------------------------------------------------------===// |
| 9226 | // VGPR Index Mode |
| 9227 | //===----------------------------------------------------------------------===// |
| 9228 | |
| 9229 | int64_t AMDGPUAsmParser::parseGPRIdxMacro() { |
| 9230 | |
| 9231 | using namespace llvm::AMDGPU::VGPRIndexMode; |
| 9232 | |
| 9233 | if (trySkipToken(Kind: AsmToken::RParen)) { |
| 9234 | return OFF; |
| 9235 | } |
| 9236 | |
| 9237 | int64_t Imm = 0; |
| 9238 | |
| 9239 | while (true) { |
| 9240 | unsigned Mode = 0; |
| 9241 | SMLoc S = getLoc(); |
| 9242 | |
| 9243 | for (unsigned ModeId = ID_MIN; ModeId <= ID_MAX; ++ModeId) { |
| 9244 | if (trySkipId(Id: IdSymbolic[ModeId])) { |
| 9245 | Mode = 1 << ModeId; |
| 9246 | break; |
| 9247 | } |
| 9248 | } |
| 9249 | |
| 9250 | if (Mode == 0) { |
| 9251 | Error(L: S, Msg: (Imm == 0) |
| 9252 | ? "expected a VGPR index mode or a closing parenthesis" |
| 9253 | : "expected a VGPR index mode" ); |
| 9254 | return UNDEF; |
| 9255 | } |
| 9256 | |
| 9257 | if (Imm & Mode) { |
| 9258 | Error(L: S, Msg: "duplicate VGPR index mode" ); |
| 9259 | return UNDEF; |
| 9260 | } |
| 9261 | Imm |= Mode; |
| 9262 | |
| 9263 | if (trySkipToken(Kind: AsmToken::RParen)) |
| 9264 | break; |
| 9265 | if (!skipToken(Kind: AsmToken::Comma, |
| 9266 | ErrMsg: "expected a comma or a closing parenthesis" )) |
| 9267 | return UNDEF; |
| 9268 | } |
| 9269 | |
| 9270 | return Imm; |
| 9271 | } |
| 9272 | |
| 9273 | ParseStatus AMDGPUAsmParser::parseGPRIdxMode(OperandVector &Operands) { |
| 9274 | |
| 9275 | using namespace llvm::AMDGPU::VGPRIndexMode; |
| 9276 | |
| 9277 | int64_t Imm = 0; |
| 9278 | SMLoc S = getLoc(); |
| 9279 | |
| 9280 | if (trySkipId(Id: "gpr_idx" , Kind: AsmToken::LParen)) { |
| 9281 | Imm = parseGPRIdxMacro(); |
| 9282 | if (Imm == UNDEF) |
| 9283 | return ParseStatus::Failure; |
| 9284 | } else { |
| 9285 | if (getParser().parseAbsoluteExpression(Res&: Imm)) |
| 9286 | return ParseStatus::Failure; |
| 9287 | if (Imm < 0 || !isUInt<4>(x: Imm)) |
| 9288 | return Error(L: S, Msg: "invalid immediate: only 4-bit values are legal" ); |
| 9289 | } |
| 9290 | |
| 9291 | Operands.push_back( |
| 9292 | Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Imm, Loc: S, Type: AMDGPUOperand::ImmTyGprIdxMode)); |
| 9293 | return ParseStatus::Success; |
| 9294 | } |
| 9295 | |
| 9296 | bool AMDGPUOperand::isGPRIdxMode() const { return isImmTy(ImmT: ImmTyGprIdxMode); } |
| 9297 | |
| 9298 | //===----------------------------------------------------------------------===// |
| 9299 | // sopp branch targets |
| 9300 | //===----------------------------------------------------------------------===// |
| 9301 | |
| 9302 | ParseStatus AMDGPUAsmParser::parseSOPPBrTarget(OperandVector &Operands) { |
| 9303 | |
| 9304 | // Make sure we are not parsing something |
| 9305 | // that looks like a label or an expression but is not. |
| 9306 | // This will improve error messages. |
| 9307 | if (isRegister() || isModifier()) |
| 9308 | return ParseStatus::NoMatch; |
| 9309 | |
| 9310 | if (!parseExpr(Operands)) |
| 9311 | return ParseStatus::Failure; |
| 9312 | |
| 9313 | AMDGPUOperand &Opr = ((AMDGPUOperand &)*Operands[Operands.size() - 1]); |
| 9314 | assert(Opr.isImm() || Opr.isExpr()); |
| 9315 | SMLoc Loc = Opr.getStartLoc(); |
| 9316 | |
| 9317 | // Currently we do not support arbitrary expressions as branch targets. |
| 9318 | // Only labels and absolute expressions are accepted. |
| 9319 | if (Opr.isExpr() && !Opr.isSymbolRefExpr()) { |
| 9320 | Error(L: Loc, Msg: "expected an absolute expression or a label" ); |
| 9321 | } else if (Opr.isImm() && !Opr.isS16Imm()) { |
| 9322 | Error(L: Loc, Msg: "expected a 16-bit signed jump offset" ); |
| 9323 | } |
| 9324 | |
| 9325 | return ParseStatus::Success; |
| 9326 | } |
| 9327 | |
| 9328 | //===----------------------------------------------------------------------===// |
| 9329 | // Boolean holding registers |
| 9330 | //===----------------------------------------------------------------------===// |
| 9331 | |
| 9332 | ParseStatus AMDGPUAsmParser::parseBoolReg(OperandVector &Operands) { |
| 9333 | return parseReg(Operands); |
| 9334 | } |
| 9335 | |
| 9336 | //===----------------------------------------------------------------------===// |
| 9337 | // mubuf |
| 9338 | //===----------------------------------------------------------------------===// |
| 9339 | |
| 9340 | void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, |
| 9341 | bool IsAtomic) { |
| 9342 | OptionalImmIndexMap OptionalIdx; |
| 9343 | unsigned FirstOperandIdx = 1; |
| 9344 | bool IsAtomicReturn = false; |
| 9345 | |
| 9346 | if (IsAtomic) { |
| 9347 | IsAtomicReturn = SIInstrFlags::isAtomicRet(O: MII, O: Inst); |
| 9348 | } |
| 9349 | |
| 9350 | for (unsigned i = FirstOperandIdx, e = Operands.size(); i != e; ++i) { |
| 9351 | AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]); |
| 9352 | |
| 9353 | // Add the register arguments |
| 9354 | if (Op.isReg()) { |
| 9355 | Op.addRegOperands(Inst, N: 1); |
| 9356 | // Insert a tied src for atomic return dst. |
| 9357 | // This cannot be postponed as subsequent calls to |
| 9358 | // addImmOperands rely on correct number of MC operands. |
| 9359 | if (IsAtomicReturn && i == FirstOperandIdx) |
| 9360 | Op.addRegOperands(Inst, N: 1); |
| 9361 | continue; |
| 9362 | } |
| 9363 | |
| 9364 | // Handle the case where soffset is an immediate |
| 9365 | if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) { |
| 9366 | Op.addImmOperands(Inst, N: 1); |
| 9367 | continue; |
| 9368 | } |
| 9369 | |
| 9370 | // Handle tokens like 'offen' which are sometimes hard-coded into the |
| 9371 | // asm string. There are no MCInst operands for these. |
| 9372 | if (Op.isToken()) { |
| 9373 | continue; |
| 9374 | } |
| 9375 | assert(Op.isImm()); |
| 9376 | |
| 9377 | // Handle optional arguments |
| 9378 | OptionalIdx[Op.getImmTy()] = i; |
| 9379 | } |
| 9380 | |
| 9381 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 9382 | ImmT: AMDGPUOperand::ImmTyOffset); |
| 9383 | addOptionalImmOperand(Inst, Operands, OptionalIdx, ImmT: AMDGPUOperand::ImmTyCPol, |
| 9384 | Default: 0); |
| 9385 | // Parse a dummy operand as a placeholder for the SWZ operand. This enforces |
| 9386 | // agreement between MCInstrDesc.getNumOperands and MCInst.getNumOperands. |
| 9387 | Inst.addOperand(Op: MCOperand::createImm(Val: 0)); |
| 9388 | // The LDS variants carry a trailing IsAsync operand. Parse a dummy the same |
| 9389 | // way as the SWZ operand. |
| 9390 | if (AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::IsAsync)) |
| 9391 | Inst.addOperand(Op: MCOperand::createImm(Val: 0)); |
| 9392 | } |
| 9393 | |
| 9394 | //===----------------------------------------------------------------------===// |
| 9395 | // smrd |
| 9396 | //===----------------------------------------------------------------------===// |
| 9397 | |
| 9398 | bool AMDGPUOperand::isSMRDOffset8() const { |
| 9399 | return isImmLiteral() && isUInt<8>(x: getImm()); |
| 9400 | } |
| 9401 | |
| 9402 | bool AMDGPUOperand::isSMEMOffset() const { |
| 9403 | // Offset range is checked later by validator. |
| 9404 | return isImmLiteral(); |
| 9405 | } |
| 9406 | |
| 9407 | bool AMDGPUOperand::isSMRDLiteralOffset() const { |
| 9408 | // 32-bit literals are only supported on CI and we only want to use them |
| 9409 | // when the offset is > 8-bits. |
| 9410 | return isImmLiteral() && !isUInt<8>(x: getImm()) && isUInt<32>(x: getImm()); |
| 9411 | } |
| 9412 | |
| 9413 | //===----------------------------------------------------------------------===// |
| 9414 | // vop3 |
| 9415 | //===----------------------------------------------------------------------===// |
| 9416 | |
| 9417 | static bool ConvertOmodMul(int64_t &Mul) { |
| 9418 | if (Mul != 1 && Mul != 2 && Mul != 4) |
| 9419 | return false; |
| 9420 | |
| 9421 | Mul >>= 1; |
| 9422 | return true; |
| 9423 | } |
| 9424 | |
| 9425 | static bool ConvertOmodDiv(int64_t &Div) { |
| 9426 | if (Div == 1) { |
| 9427 | Div = 0; |
| 9428 | return true; |
| 9429 | } |
| 9430 | |
| 9431 | if (Div == 2) { |
| 9432 | Div = 3; |
| 9433 | return true; |
| 9434 | } |
| 9435 | |
| 9436 | return false; |
| 9437 | } |
| 9438 | |
| 9439 | // For pre-gfx11 targets, both bound_ctrl:0 and bound_ctrl:1 are encoded as 1. |
| 9440 | // This is intentional and ensures compatibility with sp3. |
| 9441 | // See bug 35397 for details. |
| 9442 | bool AMDGPUAsmParser::convertDppBoundCtrl(int64_t &BoundCtrl) { |
| 9443 | if (BoundCtrl == 0 || BoundCtrl == 1) { |
| 9444 | if (!isGFX11Plus()) |
| 9445 | BoundCtrl = 1; |
| 9446 | return true; |
| 9447 | } |
| 9448 | return false; |
| 9449 | } |
| 9450 | |
| 9451 | void AMDGPUAsmParser::onBeginOfFile() { |
| 9452 | if (!getParser().getStreamer().getTargetStreamer()) |
| 9453 | return; |
| 9454 | |
| 9455 | if (!getTargetStreamer().getTargetID()) |
| 9456 | getTargetStreamer().initializeTargetID(STI: getSTI(), |
| 9457 | /*ApplyFeatureString=*/true); |
| 9458 | } |
| 9459 | |
| 9460 | void AMDGPUAsmParser::emitTargetDirective() { |
| 9461 | if (!getParser().getStreamer().getTargetStreamer() || |
| 9462 | getSTI().getTargetTriple().getArch() == Triple::r600) |
| 9463 | return; |
| 9464 | |
| 9465 | if (isHsaAbi(STI: getSTI())) |
| 9466 | getTargetStreamer().emitDirectiveAMDGCNTargetOnce(); |
| 9467 | } |
| 9468 | |
| 9469 | /// Parse AMDGPU specific expressions. |
| 9470 | /// |
| 9471 | /// expr ::= or(expr, ...) | |
| 9472 | /// max(expr, ...) | |
| 9473 | /// min(expr, ...) |
| 9474 | /// |
| 9475 | bool AMDGPUAsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) { |
| 9476 | using AGVK = AMDGPUMCExpr::VariantKind; |
| 9477 | |
| 9478 | if (isToken(Kind: AsmToken::Identifier)) { |
| 9479 | StringRef TokenId = getTokenStr(); |
| 9480 | AGVK VK = StringSwitch<AGVK>(TokenId) |
| 9481 | .Case(S: "max" , Value: AGVK::AGVK_Max) |
| 9482 | .Case(S: "min" , Value: AGVK::AGVK_Min) |
| 9483 | .Case(S: "or" , Value: AGVK::AGVK_Or) |
| 9484 | .Case(S: "extrasgprs" , Value: AGVK::AGVK_ExtraSGPRs) |
| 9485 | .Case(S: "totalnumvgprs" , Value: AGVK::AGVK_TotalNumVGPRs) |
| 9486 | .Case(S: "alignto" , Value: AGVK::AGVK_AlignTo) |
| 9487 | .Case(S: "occupancy" , Value: AGVK::AGVK_Occupancy) |
| 9488 | .Case(S: "instprefsize" , Value: AGVK::AGVK_InstPrefSize) |
| 9489 | .Default(Value: AGVK::AGVK_None); |
| 9490 | |
| 9491 | if (VK != AGVK::AGVK_None && peekToken().is(K: AsmToken::LParen)) { |
| 9492 | SmallVector<const MCExpr *, 4> Exprs; |
| 9493 | uint64_t CommaCount = 0; |
| 9494 | lex(); // Eat Arg ('or', 'max', 'occupancy', etc.) |
| 9495 | lex(); // Eat '(' |
| 9496 | while (true) { |
| 9497 | if (trySkipToken(Kind: AsmToken::RParen)) { |
| 9498 | if (Exprs.empty()) { |
| 9499 | Error(L: getToken().getLoc(), |
| 9500 | Msg: "empty " + Twine(TokenId) + " expression" ); |
| 9501 | return true; |
| 9502 | } |
| 9503 | if (CommaCount + 1 != Exprs.size()) { |
| 9504 | Error(L: getToken().getLoc(), |
| 9505 | Msg: "mismatch of commas in " + Twine(TokenId) + " expression" ); |
| 9506 | return true; |
| 9507 | } |
| 9508 | if (unsigned Expected = AMDGPUMCExpr::getNumExpectedArgs(Kind: VK); |
| 9509 | Expected && Exprs.size() != Expected) { |
| 9510 | Error(L: getToken().getLoc(), Msg: Twine(TokenId) + " expression expects " + |
| 9511 | Twine(Expected) + " operands" ); |
| 9512 | return true; |
| 9513 | } |
| 9514 | Res = AMDGPUMCExpr::create(Kind: VK, Args: Exprs, Ctx&: getContext()); |
| 9515 | return false; |
| 9516 | } |
| 9517 | const MCExpr *Expr; |
| 9518 | if (getParser().parseExpression(Res&: Expr, EndLoc)) |
| 9519 | return true; |
| 9520 | Exprs.push_back(Elt: Expr); |
| 9521 | bool LastTokenWasComma = trySkipToken(Kind: AsmToken::Comma); |
| 9522 | if (LastTokenWasComma) |
| 9523 | CommaCount++; |
| 9524 | if (!LastTokenWasComma && !isToken(Kind: AsmToken::RParen)) { |
| 9525 | Error(L: getToken().getLoc(), |
| 9526 | Msg: "unexpected token in " + Twine(TokenId) + " expression" ); |
| 9527 | return true; |
| 9528 | } |
| 9529 | } |
| 9530 | } |
| 9531 | } |
| 9532 | return getParser().parsePrimaryExpr(Res, EndLoc, TypeInfo: nullptr); |
| 9533 | } |
| 9534 | |
| 9535 | ParseStatus AMDGPUAsmParser::parseOModSI(OperandVector &Operands) { |
| 9536 | StringRef Name = getTokenStr(); |
| 9537 | if (Name == "mul" ) { |
| 9538 | return parseIntWithPrefix(Prefix: "mul" , Operands, ImmTy: AMDGPUOperand::ImmTyOModSI, |
| 9539 | ConvertResult: ConvertOmodMul); |
| 9540 | } |
| 9541 | |
| 9542 | if (Name == "div" ) { |
| 9543 | return parseIntWithPrefix(Prefix: "div" , Operands, ImmTy: AMDGPUOperand::ImmTyOModSI, |
| 9544 | ConvertResult: ConvertOmodDiv); |
| 9545 | } |
| 9546 | |
| 9547 | return ParseStatus::NoMatch; |
| 9548 | } |
| 9549 | |
| 9550 | // Determines which bit DST_OP_SEL occupies in the op_sel operand according to |
| 9551 | // the number of src operands present, then copies that bit into src0_modifiers. |
| 9552 | static void cvtVOP3DstOpSelOnly(MCInst &Inst, const MCRegisterInfo &MRI) { |
| 9553 | int Opc = Inst.getOpcode(); |
| 9554 | int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel); |
| 9555 | if (OpSelIdx == -1) |
| 9556 | return; |
| 9557 | |
| 9558 | int SrcNum; |
| 9559 | const AMDGPU::OpName Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1, |
| 9560 | AMDGPU::OpName::src2}; |
| 9561 | for (SrcNum = 0; SrcNum < 3 && AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: Ops[SrcNum]); |
| 9562 | ++SrcNum) |
| 9563 | ; |
| 9564 | assert(SrcNum > 0); |
| 9565 | |
| 9566 | unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm(); |
| 9567 | |
| 9568 | int DstIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst); |
| 9569 | if (DstIdx == -1) |
| 9570 | return; |
| 9571 | |
| 9572 | const MCOperand &DstOp = Inst.getOperand(i: DstIdx); |
| 9573 | int ModIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0_modifiers); |
| 9574 | uint32_t ModVal = Inst.getOperand(i: ModIdx).getImm(); |
| 9575 | if (DstOp.isReg() && |
| 9576 | MRI.getRegClass(i: AMDGPU::VGPR_16RegClassID).contains(Reg: DstOp.getReg())) { |
| 9577 | if (AMDGPU::isHi16Reg(Reg: DstOp.getReg(), MRI)) |
| 9578 | ModVal |= SISrcMods::DST_OP_SEL; |
| 9579 | } else { |
| 9580 | if ((OpSel & (1 << SrcNum)) != 0) |
| 9581 | ModVal |= SISrcMods::DST_OP_SEL; |
| 9582 | } |
| 9583 | Inst.getOperand(i: ModIdx).setImm(ModVal); |
| 9584 | } |
| 9585 | |
| 9586 | void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst, |
| 9587 | const OperandVector &Operands) { |
| 9588 | cvtVOP3P(Inst, Operands); |
| 9589 | cvtVOP3DstOpSelOnly(Inst, MRI: *getMRI()); |
| 9590 | } |
| 9591 | |
| 9592 | void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands, |
| 9593 | OptionalImmIndexMap &OptionalIdx) { |
| 9594 | cvtVOP3P(Inst, Operands, OptionalIdx); |
| 9595 | cvtVOP3DstOpSelOnly(Inst, MRI: *getMRI()); |
| 9596 | } |
| 9597 | |
| 9598 | static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) { |
| 9599 | return |
| 9600 | // 1. This operand is input modifiers |
| 9601 | Desc.operands()[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS |
| 9602 | // 2. This is not last operand |
| 9603 | && Desc.NumOperands > (OpNum + 1) |
| 9604 | // 3. Next operand is register class |
| 9605 | && Desc.operands()[OpNum + 1].RegClass != -1 |
| 9606 | // 4. Next register is not tied to any other operand |
| 9607 | && Desc.getOperandConstraint(OpNum: OpNum + 1, |
| 9608 | Constraint: MCOI::OperandConstraint::TIED_TO) == -1; |
| 9609 | } |
| 9610 | |
| 9611 | void AMDGPUAsmParser::cvtOpSelHelper(MCInst &Inst, unsigned OpSel) { |
| 9612 | unsigned Opc = Inst.getOpcode(); |
| 9613 | constexpr AMDGPU::OpName Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1, |
| 9614 | AMDGPU::OpName::src2}; |
| 9615 | constexpr AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers, |
| 9616 | AMDGPU::OpName::src1_modifiers, |
| 9617 | AMDGPU::OpName::src2_modifiers}; |
| 9618 | for (int J = 0; J < 3; ++J) { |
| 9619 | int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: Ops[J]); |
| 9620 | if (OpIdx == -1) |
| 9621 | // Some instructions, e.g. v_interp_p2_f16 in GFX9, have src0, src2, but |
| 9622 | // no src1. So continue instead of break. |
| 9623 | continue; |
| 9624 | |
| 9625 | int ModIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: ModOps[J]); |
| 9626 | uint32_t ModVal = Inst.getOperand(i: ModIdx).getImm(); |
| 9627 | |
| 9628 | if ((OpSel & (1 << J)) != 0) |
| 9629 | ModVal |= SISrcMods::OP_SEL_0; |
| 9630 | // op_sel[3] is encoded in src0_modifiers. |
| 9631 | if (ModOps[J] == AMDGPU::OpName::src0_modifiers && (OpSel & (1 << 3)) != 0) |
| 9632 | ModVal |= SISrcMods::DST_OP_SEL; |
| 9633 | |
| 9634 | Inst.getOperand(i: ModIdx).setImm(ModVal); |
| 9635 | } |
| 9636 | } |
| 9637 | |
| 9638 | void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst, |
| 9639 | const OperandVector &Operands) { |
| 9640 | OptionalImmIndexMap OptionalIdx; |
| 9641 | unsigned Opc = Inst.getOpcode(); |
| 9642 | |
| 9643 | unsigned I = 1; |
| 9644 | const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode()); |
| 9645 | for (unsigned J = 0; J < Desc.getNumDefs(); ++J) { |
| 9646 | ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, N: 1); |
| 9647 | } |
| 9648 | |
| 9649 | for (unsigned E = Operands.size(); I != E; ++I) { |
| 9650 | AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]); |
| 9651 | if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) { |
| 9652 | Op.addRegOrImmWithFPInputModsOperands(Inst, N: 2); |
| 9653 | } else if (Op.isInterpSlot() || Op.isInterpAttr() || |
| 9654 | Op.isInterpAttrChan()) { |
| 9655 | Inst.addOperand(Op: MCOperand::createImm(Val: Op.getImm())); |
| 9656 | } else if (Op.isImmModifier()) { |
| 9657 | OptionalIdx[Op.getImmTy()] = I; |
| 9658 | } else { |
| 9659 | llvm_unreachable("unhandled operand type" ); |
| 9660 | } |
| 9661 | } |
| 9662 | |
| 9663 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::high)) |
| 9664 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 9665 | ImmT: AMDGPUOperand::ImmTyHigh); |
| 9666 | |
| 9667 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::clamp)) |
| 9668 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 9669 | ImmT: AMDGPUOperand::ImmTyClamp); |
| 9670 | |
| 9671 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::omod)) |
| 9672 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 9673 | ImmT: AMDGPUOperand::ImmTyOModSI); |
| 9674 | |
| 9675 | // Some v_interp instructions use op_sel[3] for dst. |
| 9676 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::op_sel)) { |
| 9677 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 9678 | ImmT: AMDGPUOperand::ImmTyOpSel); |
| 9679 | int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel); |
| 9680 | unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm(); |
| 9681 | |
| 9682 | cvtOpSelHelper(Inst, OpSel); |
| 9683 | } |
| 9684 | } |
| 9685 | |
| 9686 | void AMDGPUAsmParser::cvtVINTERP(MCInst &Inst, const OperandVector &Operands) { |
| 9687 | OptionalImmIndexMap OptionalIdx; |
| 9688 | unsigned Opc = Inst.getOpcode(); |
| 9689 | |
| 9690 | unsigned I = 1; |
| 9691 | const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode()); |
| 9692 | for (unsigned J = 0; J < Desc.getNumDefs(); ++J) { |
| 9693 | ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, N: 1); |
| 9694 | } |
| 9695 | |
| 9696 | for (unsigned E = Operands.size(); I != E; ++I) { |
| 9697 | AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]); |
| 9698 | if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) { |
| 9699 | Op.addRegOrImmWithFPInputModsOperands(Inst, N: 2); |
| 9700 | } else if (Op.isImmModifier()) { |
| 9701 | OptionalIdx[Op.getImmTy()] = I; |
| 9702 | } else { |
| 9703 | llvm_unreachable("unhandled operand type" ); |
| 9704 | } |
| 9705 | } |
| 9706 | |
| 9707 | addOptionalImmOperand(Inst, Operands, OptionalIdx, ImmT: AMDGPUOperand::ImmTyClamp); |
| 9708 | |
| 9709 | int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel); |
| 9710 | if (OpSelIdx != -1) |
| 9711 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 9712 | ImmT: AMDGPUOperand::ImmTyOpSel); |
| 9713 | |
| 9714 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 9715 | ImmT: AMDGPUOperand::ImmTyWaitEXP); |
| 9716 | |
| 9717 | if (OpSelIdx == -1) |
| 9718 | return; |
| 9719 | |
| 9720 | unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm(); |
| 9721 | cvtOpSelHelper(Inst, OpSel); |
| 9722 | } |
| 9723 | |
| 9724 | void AMDGPUAsmParser::cvtScaledMFMA(MCInst &Inst, |
| 9725 | const OperandVector &Operands) { |
| 9726 | OptionalImmIndexMap OptionalIdx; |
| 9727 | unsigned Opc = Inst.getOpcode(); |
| 9728 | unsigned I = 1; |
| 9729 | int CbszOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::cbsz); |
| 9730 | |
| 9731 | const MCInstrDesc &Desc = MII.get(Opcode: Opc); |
| 9732 | |
| 9733 | for (unsigned J = 0; J < Desc.getNumDefs(); ++J) |
| 9734 | static_cast<AMDGPUOperand &>(*Operands[I++]).addRegOperands(Inst, N: 1); |
| 9735 | |
| 9736 | for (unsigned E = Operands.size(); I != E; ++I) { |
| 9737 | AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands[I]); |
| 9738 | int NumOperands = Inst.getNumOperands(); |
| 9739 | // The order of operands in MCInst and parsed operands are different. |
| 9740 | // Adding dummy cbsz and blgp operands at corresponding MCInst operand |
| 9741 | // indices for parsing scale values correctly. |
| 9742 | if (NumOperands == CbszOpIdx) { |
| 9743 | Inst.addOperand(Op: MCOperand::createImm(Val: 0)); |
| 9744 | Inst.addOperand(Op: MCOperand::createImm(Val: 0)); |
| 9745 | } |
| 9746 | if (isRegOrImmWithInputMods(Desc, OpNum: NumOperands)) { |
| 9747 | Op.addRegOrImmWithFPInputModsOperands(Inst, N: 2); |
| 9748 | } else if (Op.isImmModifier()) { |
| 9749 | OptionalIdx[Op.getImmTy()] = I; |
| 9750 | } else { |
| 9751 | Op.addRegOrImmOperands(Inst, N: 1); |
| 9752 | } |
| 9753 | } |
| 9754 | |
| 9755 | // Insert CBSZ and BLGP operands for F8F6F4 variants |
| 9756 | auto CbszIdx = OptionalIdx.find(x: AMDGPUOperand::ImmTyCBSZ); |
| 9757 | if (CbszIdx != OptionalIdx.end()) { |
| 9758 | int CbszVal = ((AMDGPUOperand &)*Operands[CbszIdx->second]).getImm(); |
| 9759 | Inst.getOperand(i: CbszOpIdx).setImm(CbszVal); |
| 9760 | } |
| 9761 | |
| 9762 | int BlgpOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::blgp); |
| 9763 | auto BlgpIdx = OptionalIdx.find(x: AMDGPUOperand::ImmTyBLGP); |
| 9764 | if (BlgpIdx != OptionalIdx.end()) { |
| 9765 | int BlgpVal = ((AMDGPUOperand &)*Operands[BlgpIdx->second]).getImm(); |
| 9766 | Inst.getOperand(i: BlgpOpIdx).setImm(BlgpVal); |
| 9767 | } |
| 9768 | |
| 9769 | // Add dummy src_modifiers |
| 9770 | Inst.addOperand(Op: MCOperand::createImm(Val: 0)); |
| 9771 | Inst.addOperand(Op: MCOperand::createImm(Val: 0)); |
| 9772 | |
| 9773 | // Handle op_sel fields |
| 9774 | |
| 9775 | unsigned OpSel = 0; |
| 9776 | auto OpselIdx = OptionalIdx.find(x: AMDGPUOperand::ImmTyOpSel); |
| 9777 | if (OpselIdx != OptionalIdx.end()) { |
| 9778 | OpSel = static_cast<const AMDGPUOperand &>(*Operands[OpselIdx->second]) |
| 9779 | .getImm(); |
| 9780 | } |
| 9781 | |
| 9782 | unsigned OpSelHi = 0; |
| 9783 | auto OpselHiIdx = OptionalIdx.find(x: AMDGPUOperand::ImmTyOpSelHi); |
| 9784 | if (OpselHiIdx != OptionalIdx.end()) { |
| 9785 | OpSelHi = static_cast<const AMDGPUOperand &>(*Operands[OpselHiIdx->second]) |
| 9786 | .getImm(); |
| 9787 | } |
| 9788 | const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers, |
| 9789 | AMDGPU::OpName::src1_modifiers}; |
| 9790 | |
| 9791 | for (unsigned J = 0; J < 2; ++J) { |
| 9792 | unsigned ModVal = 0; |
| 9793 | if (OpSel & (1 << J)) |
| 9794 | ModVal |= SISrcMods::OP_SEL_0; |
| 9795 | if (OpSelHi & (1 << J)) |
| 9796 | ModVal |= SISrcMods::OP_SEL_1; |
| 9797 | |
| 9798 | const int ModIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: ModOps[J]); |
| 9799 | Inst.getOperand(i: ModIdx).setImm(ModVal); |
| 9800 | } |
| 9801 | } |
| 9802 | |
| 9803 | void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands, |
| 9804 | OptionalImmIndexMap &OptionalIdx) { |
| 9805 | unsigned Opc = Inst.getOpcode(); |
| 9806 | |
| 9807 | unsigned I = 1; |
| 9808 | const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode()); |
| 9809 | for (unsigned J = 0; J < Desc.getNumDefs(); ++J) { |
| 9810 | ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, N: 1); |
| 9811 | } |
| 9812 | |
| 9813 | for (unsigned E = Operands.size(); I != E; ++I) { |
| 9814 | AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]); |
| 9815 | if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) { |
| 9816 | Op.addRegOrImmWithFPInputModsOperands(Inst, N: 2); |
| 9817 | } else if (Op.isImmModifier()) { |
| 9818 | OptionalIdx[Op.getImmTy()] = I; |
| 9819 | } else { |
| 9820 | Op.addRegOrImmOperands(Inst, N: 1); |
| 9821 | } |
| 9822 | } |
| 9823 | |
| 9824 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::scale_sel)) |
| 9825 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 9826 | ImmT: AMDGPUOperand::ImmTyScaleSel); |
| 9827 | |
| 9828 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::clamp)) |
| 9829 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 9830 | ImmT: AMDGPUOperand::ImmTyClamp); |
| 9831 | |
| 9832 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::byte_sel)) { |
| 9833 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::vdst_in)) |
| 9834 | Inst.addOperand(Op: Inst.getOperand(i: 0)); |
| 9835 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 9836 | ImmT: AMDGPUOperand::ImmTyByteSel); |
| 9837 | } |
| 9838 | |
| 9839 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::omod)) |
| 9840 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 9841 | ImmT: AMDGPUOperand::ImmTyOModSI); |
| 9842 | |
| 9843 | // Special case v_mac_{f16, f32} and v_fmac_{f16, f32} (gfx906/gfx10+): |
| 9844 | // it has src2 register operand that is tied to dst operand |
| 9845 | // we don't allow modifiers for this operand in assembler so src2_modifiers |
| 9846 | // should be 0. |
| 9847 | if (isMAC(Opc)) { |
| 9848 | auto *it = Inst.begin(); |
| 9849 | std::advance( |
| 9850 | i&: it, n: AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2_modifiers)); |
| 9851 | it = Inst.insert(I: it, Op: MCOperand::createImm(Val: 0)); // no modifiers for src2 |
| 9852 | ++it; |
| 9853 | // Copy the operand to ensure it's not invalidated when Inst grows. |
| 9854 | Inst.insert(I: it, Op: MCOperand(Inst.getOperand(i: 0))); // src2 = dst |
| 9855 | } |
| 9856 | } |
| 9857 | |
| 9858 | void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) { |
| 9859 | OptionalImmIndexMap OptionalIdx; |
| 9860 | cvtVOP3(Inst, Operands, OptionalIdx); |
| 9861 | } |
| 9862 | |
| 9863 | void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands, |
| 9864 | OptionalImmIndexMap &OptIdx) { |
| 9865 | const int Opc = Inst.getOpcode(); |
| 9866 | |
| 9867 | const bool IsPacked = SIInstrFlags::isPacked(O: MII, O: Inst); |
| 9868 | |
| 9869 | if (Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_F16_vi || |
| 9870 | Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_BF16_vi || |
| 9871 | Opc == AMDGPU::V_CVT_SR_BF8_F32_vi || |
| 9872 | Opc == AMDGPU::V_CVT_SR_FP8_F32_vi || |
| 9873 | Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx11 || |
| 9874 | Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx11 || |
| 9875 | Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx12 || |
| 9876 | Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx12 || |
| 9877 | Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx13 || |
| 9878 | Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx13) { |
| 9879 | Inst.addOperand(Op: MCOperand::createImm(Val: 0)); // Placeholder for src2_mods |
| 9880 | Inst.addOperand(Op: Inst.getOperand(i: 0)); |
| 9881 | } |
| 9882 | |
| 9883 | // Append vdst_in only if a previous converter (cvtVOP3DPP for DPP variants, |
| 9884 | // cvtVOP3 for byte_sel variants) hasn't already placed it. Use the position |
| 9885 | // of the named operand to detect that, the same way cvtVOP3DPP does |
| 9886 | // internally. |
| 9887 | int VdstInIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst_in); |
| 9888 | if (VdstInIdx != -1 && VdstInIdx == static_cast<int>(Inst.getNumOperands())) |
| 9889 | Inst.addOperand(Op: Inst.getOperand(i: 0)); |
| 9890 | |
| 9891 | int BitOp3Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::bitop3); |
| 9892 | if (BitOp3Idx != -1) { |
| 9893 | addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyBitOp3); |
| 9894 | } |
| 9895 | |
| 9896 | // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3 |
| 9897 | // instruction, and then figure out where to actually put the modifiers |
| 9898 | |
| 9899 | int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel); |
| 9900 | if (OpSelIdx != -1) { |
| 9901 | addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyOpSel); |
| 9902 | } |
| 9903 | |
| 9904 | int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel_hi); |
| 9905 | if (OpSelHiIdx != -1) { |
| 9906 | int DefaultVal = IsPacked ? -1 : 0; |
| 9907 | addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyOpSelHi, |
| 9908 | Default: DefaultVal); |
| 9909 | } |
| 9910 | |
| 9911 | int MatrixAFMTIdx = |
| 9912 | AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_a_fmt); |
| 9913 | if (MatrixAFMTIdx != -1) { |
| 9914 | addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, |
| 9915 | ImmT: AMDGPUOperand::ImmTyMatrixAFMT, Default: 0); |
| 9916 | } |
| 9917 | |
| 9918 | int MatrixBFMTIdx = |
| 9919 | AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_b_fmt); |
| 9920 | if (MatrixBFMTIdx != -1) { |
| 9921 | addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, |
| 9922 | ImmT: AMDGPUOperand::ImmTyMatrixBFMT, Default: 0); |
| 9923 | } |
| 9924 | |
| 9925 | int MatrixAScaleIdx = |
| 9926 | AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_a_scale); |
| 9927 | if (MatrixAScaleIdx != -1) { |
| 9928 | addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, |
| 9929 | ImmT: AMDGPUOperand::ImmTyMatrixAScale, Default: 0); |
| 9930 | } |
| 9931 | |
| 9932 | int MatrixBScaleIdx = |
| 9933 | AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_b_scale); |
| 9934 | if (MatrixBScaleIdx != -1) { |
| 9935 | addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, |
| 9936 | ImmT: AMDGPUOperand::ImmTyMatrixBScale, Default: 0); |
| 9937 | } |
| 9938 | |
| 9939 | int MatrixAScaleFmtIdx = |
| 9940 | AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_a_scale_fmt); |
| 9941 | if (MatrixAScaleFmtIdx != -1) { |
| 9942 | addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, |
| 9943 | ImmT: AMDGPUOperand::ImmTyMatrixAScaleFmt, Default: 0); |
| 9944 | } |
| 9945 | |
| 9946 | int MatrixBScaleFmtIdx = |
| 9947 | AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_b_scale_fmt); |
| 9948 | if (MatrixBScaleFmtIdx != -1) { |
| 9949 | addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, |
| 9950 | ImmT: AMDGPUOperand::ImmTyMatrixBScaleFmt, Default: 0); |
| 9951 | } |
| 9952 | |
| 9953 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::matrix_a_reuse)) |
| 9954 | addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, |
| 9955 | ImmT: AMDGPUOperand::ImmTyMatrixAReuse, Default: 0); |
| 9956 | |
| 9957 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::matrix_b_reuse)) |
| 9958 | addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, |
| 9959 | ImmT: AMDGPUOperand::ImmTyMatrixBReuse, Default: 0); |
| 9960 | |
| 9961 | int NegLoIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::neg_lo); |
| 9962 | if (NegLoIdx != -1) |
| 9963 | addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyNegLo); |
| 9964 | |
| 9965 | int NegHiIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::neg_hi); |
| 9966 | if (NegHiIdx != -1) |
| 9967 | addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyNegHi); |
| 9968 | |
| 9969 | const AMDGPU::OpName Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1, |
| 9970 | AMDGPU::OpName::src2}; |
| 9971 | const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers, |
| 9972 | AMDGPU::OpName::src1_modifiers, |
| 9973 | AMDGPU::OpName::src2_modifiers}; |
| 9974 | |
| 9975 | unsigned OpSel = 0; |
| 9976 | unsigned OpSelHi = 0; |
| 9977 | unsigned NegLo = 0; |
| 9978 | unsigned NegHi = 0; |
| 9979 | |
| 9980 | if (OpSelIdx != -1) |
| 9981 | OpSel = Inst.getOperand(i: OpSelIdx).getImm(); |
| 9982 | |
| 9983 | if (OpSelHiIdx != -1) |
| 9984 | OpSelHi = Inst.getOperand(i: OpSelHiIdx).getImm(); |
| 9985 | |
| 9986 | if (NegLoIdx != -1) |
| 9987 | NegLo = Inst.getOperand(i: NegLoIdx).getImm(); |
| 9988 | |
| 9989 | if (NegHiIdx != -1) |
| 9990 | NegHi = Inst.getOperand(i: NegHiIdx).getImm(); |
| 9991 | |
| 9992 | for (int J = 0; J < 3; ++J) { |
| 9993 | int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: Ops[J]); |
| 9994 | if (OpIdx == -1) |
| 9995 | break; |
| 9996 | |
| 9997 | int ModIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: ModOps[J]); |
| 9998 | |
| 9999 | if (ModIdx == -1) |
| 10000 | continue; |
| 10001 | |
| 10002 | // For MAC instructions, src2 is tied to vdst and its op_sel bit |
| 10003 | // is not encoded. |
| 10004 | if (AMDGPU::isMAC(Opc) && ModOps[J] == AMDGPU::OpName::src2_modifiers) |
| 10005 | continue; |
| 10006 | |
| 10007 | uint32_t ModVal = 0; |
| 10008 | |
| 10009 | const MCOperand &SrcOp = Inst.getOperand(i: OpIdx); |
| 10010 | if (SrcOp.isReg() && getMRI() |
| 10011 | ->getRegClass(i: AMDGPU::VGPR_16RegClassID) |
| 10012 | .contains(Reg: SrcOp.getReg())) { |
| 10013 | bool VGPRSuffixIsHi = AMDGPU::isHi16Reg(Reg: SrcOp.getReg(), MRI: *getMRI()); |
| 10014 | if (VGPRSuffixIsHi) |
| 10015 | ModVal |= SISrcMods::OP_SEL_0; |
| 10016 | } else { |
| 10017 | if ((OpSel & (1 << J)) != 0) |
| 10018 | ModVal |= SISrcMods::OP_SEL_0; |
| 10019 | } |
| 10020 | |
| 10021 | if ((OpSelHi & (1 << J)) != 0) |
| 10022 | ModVal |= SISrcMods::OP_SEL_1; |
| 10023 | |
| 10024 | if ((NegLo & (1 << J)) != 0) |
| 10025 | ModVal |= SISrcMods::NEG; |
| 10026 | |
| 10027 | if ((NegHi & (1 << J)) != 0) |
| 10028 | ModVal |= SISrcMods::NEG_HI; |
| 10029 | |
| 10030 | Inst.getOperand(i: ModIdx).setImm(Inst.getOperand(i: ModIdx).getImm() | ModVal); |
| 10031 | } |
| 10032 | } |
| 10033 | |
| 10034 | void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands) { |
| 10035 | OptionalImmIndexMap OptIdx; |
| 10036 | cvtVOP3(Inst, Operands, OptionalIdx&: OptIdx); |
| 10037 | cvtVOP3P(Inst, Operands, OptIdx); |
| 10038 | } |
| 10039 | |
| 10040 | static void addSrcModifiersAndSrc(MCInst &Inst, const OperandVector &Operands, |
| 10041 | unsigned i, unsigned Opc, |
| 10042 | AMDGPU::OpName OpName) { |
| 10043 | if (AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: OpName) != -1) |
| 10044 | ((AMDGPUOperand &)*Operands[i]).addRegOrImmWithFPInputModsOperands(Inst, N: 2); |
| 10045 | else |
| 10046 | ((AMDGPUOperand &)*Operands[i]).addRegOperands(Inst, N: 1); |
| 10047 | } |
| 10048 | |
| 10049 | void AMDGPUAsmParser::cvtSWMMAC(MCInst &Inst, const OperandVector &Operands) { |
| 10050 | unsigned Opc = Inst.getOpcode(); |
| 10051 | |
| 10052 | ((AMDGPUOperand &)*Operands[1]).addRegOperands(Inst, N: 1); |
| 10053 | addSrcModifiersAndSrc(Inst, Operands, i: 2, Opc, OpName: AMDGPU::OpName::src0_modifiers); |
| 10054 | addSrcModifiersAndSrc(Inst, Operands, i: 3, Opc, OpName: AMDGPU::OpName::src1_modifiers); |
| 10055 | ((AMDGPUOperand &)*Operands[1]).addRegOperands(Inst, N: 1); // srcTiedDef |
| 10056 | ((AMDGPUOperand &)*Operands[4]).addRegOperands(Inst, N: 1); // src2 |
| 10057 | |
| 10058 | OptionalImmIndexMap OptIdx; |
| 10059 | for (unsigned i = 5; i < Operands.size(); ++i) { |
| 10060 | AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]); |
| 10061 | OptIdx[Op.getImmTy()] = i; |
| 10062 | } |
| 10063 | |
| 10064 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::index_key_8bit)) |
| 10065 | addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, |
| 10066 | ImmT: AMDGPUOperand::ImmTyIndexKey8bit); |
| 10067 | |
| 10068 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::index_key_16bit)) |
| 10069 | addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, |
| 10070 | ImmT: AMDGPUOperand::ImmTyIndexKey16bit); |
| 10071 | |
| 10072 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::index_key_32bit)) |
| 10073 | addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, |
| 10074 | ImmT: AMDGPUOperand::ImmTyIndexKey32bit); |
| 10075 | |
| 10076 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::clamp)) |
| 10077 | addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyClamp); |
| 10078 | |
| 10079 | cvtVOP3P(Inst, Operands, OptIdx); |
| 10080 | } |
| 10081 | |
| 10082 | //===----------------------------------------------------------------------===// |
| 10083 | // VOPD |
| 10084 | //===----------------------------------------------------------------------===// |
| 10085 | |
| 10086 | ParseStatus AMDGPUAsmParser::parseVOPD(OperandVector &Operands) { |
| 10087 | if (!hasVOPD(STI: getSTI())) |
| 10088 | return ParseStatus::NoMatch; |
| 10089 | |
| 10090 | if (isToken(Kind: AsmToken::Colon) && peekToken(ShouldSkipSpace: false).is(K: AsmToken::Colon)) { |
| 10091 | SMLoc S = getLoc(); |
| 10092 | lex(); |
| 10093 | lex(); |
| 10094 | Operands.push_back(Elt: AMDGPUOperand::CreateToken(AsmParser: this, Str: "::" , Loc: S)); |
| 10095 | SMLoc OpYLoc = getLoc(); |
| 10096 | StringRef OpYName; |
| 10097 | if (isToken(Kind: AsmToken::Identifier) && !Parser.parseIdentifier(Res&: OpYName)) { |
| 10098 | Operands.push_back(Elt: AMDGPUOperand::CreateToken(AsmParser: this, Str: OpYName, Loc: OpYLoc)); |
| 10099 | return ParseStatus::Success; |
| 10100 | } |
| 10101 | return Error(L: OpYLoc, Msg: "expected a VOPDY instruction after ::" ); |
| 10102 | } |
| 10103 | return ParseStatus::NoMatch; |
| 10104 | } |
| 10105 | |
| 10106 | // Create VOPD MCInst operands using parsed assembler operands. |
| 10107 | void AMDGPUAsmParser::cvtVOPD(MCInst &Inst, const OperandVector &Operands) { |
| 10108 | const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode()); |
| 10109 | |
| 10110 | auto addOp = [&](uint16_t ParsedOprIdx) { // NOLINT:function pointer |
| 10111 | AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[ParsedOprIdx]); |
| 10112 | if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) { |
| 10113 | Op.addRegOrImmWithFPInputModsOperands(Inst, N: 2); |
| 10114 | return; |
| 10115 | } |
| 10116 | if (Op.isReg()) { |
| 10117 | Op.addRegOperands(Inst, N: 1); |
| 10118 | return; |
| 10119 | } |
| 10120 | if (Op.isImm() || Op.isExpr()) { |
| 10121 | Op.addImmOperands(Inst, N: 1); |
| 10122 | return; |
| 10123 | } |
| 10124 | llvm_unreachable("Unhandled operand type in cvtVOPD" ); |
| 10125 | }; |
| 10126 | |
| 10127 | const auto &InstInfo = getVOPDInstInfo(VOPDOpcode: Inst.getOpcode(), InstrInfo: &MII); |
| 10128 | |
| 10129 | // MCInst operands are ordered as follows: |
| 10130 | // dstX, dstY, src0X [, other OpX operands], src0Y [, other OpY operands] |
| 10131 | |
| 10132 | for (auto CompIdx : VOPD::COMPONENTS) { |
| 10133 | addOp(InstInfo[CompIdx].getIndexOfDstInParsedOperands()); |
| 10134 | } |
| 10135 | |
| 10136 | for (auto CompIdx : VOPD::COMPONENTS) { |
| 10137 | const auto &CInfo = InstInfo[CompIdx]; |
| 10138 | auto CompSrcOperandsNum = InstInfo[CompIdx].getCompParsedSrcOperandsNum(); |
| 10139 | for (unsigned CompSrcIdx = 0; CompSrcIdx < CompSrcOperandsNum; ++CompSrcIdx) |
| 10140 | addOp(CInfo.getIndexOfSrcInParsedOperands(CompSrcIdx)); |
| 10141 | if (CInfo.hasSrc2Acc()) |
| 10142 | addOp(CInfo.getIndexOfDstInParsedOperands()); |
| 10143 | } |
| 10144 | |
| 10145 | int BitOp3Idx = |
| 10146 | AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::bitop3); |
| 10147 | if (BitOp3Idx != -1) { |
| 10148 | OptionalImmIndexMap OptIdx; |
| 10149 | AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands.back()); |
| 10150 | if (Op.isImm()) |
| 10151 | OptIdx[Op.getImmTy()] = Operands.size() - 1; |
| 10152 | |
| 10153 | addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyBitOp3); |
| 10154 | } |
| 10155 | } |
| 10156 | |
| 10157 | //===----------------------------------------------------------------------===// |
| 10158 | // dpp |
| 10159 | //===----------------------------------------------------------------------===// |
| 10160 | |
| 10161 | bool AMDGPUOperand::isDPP8() const { return isImmTy(ImmT: ImmTyDPP8); } |
| 10162 | |
| 10163 | bool AMDGPUOperand::isDPPCtrl() const { |
| 10164 | using namespace AMDGPU::DPP; |
| 10165 | |
| 10166 | bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(x: getImm()); |
| 10167 | if (result) { |
| 10168 | int64_t Imm = getImm(); |
| 10169 | return (Imm >= DppCtrl::QUAD_PERM_FIRST && |
| 10170 | Imm <= DppCtrl::QUAD_PERM_LAST) || |
| 10171 | (Imm >= DppCtrl::ROW_SHL_FIRST && Imm <= DppCtrl::ROW_SHL_LAST) || |
| 10172 | (Imm >= DppCtrl::ROW_SHR_FIRST && Imm <= DppCtrl::ROW_SHR_LAST) || |
| 10173 | (Imm >= DppCtrl::ROW_ROR_FIRST && Imm <= DppCtrl::ROW_ROR_LAST) || |
| 10174 | (Imm == DppCtrl::WAVE_SHL1) || (Imm == DppCtrl::WAVE_ROL1) || |
| 10175 | (Imm == DppCtrl::WAVE_SHR1) || (Imm == DppCtrl::WAVE_ROR1) || |
| 10176 | (Imm == DppCtrl::ROW_MIRROR) || (Imm == DppCtrl::ROW_HALF_MIRROR) || |
| 10177 | (Imm == DppCtrl::BCAST15) || (Imm == DppCtrl::BCAST31) || |
| 10178 | (Imm >= DppCtrl::ROW_SHARE_FIRST && |
| 10179 | Imm <= DppCtrl::ROW_SHARE_LAST) || |
| 10180 | (Imm >= DppCtrl::ROW_XMASK_FIRST && Imm <= DppCtrl::ROW_XMASK_LAST); |
| 10181 | } |
| 10182 | return false; |
| 10183 | } |
| 10184 | |
| 10185 | //===----------------------------------------------------------------------===// |
| 10186 | // mAI |
| 10187 | //===----------------------------------------------------------------------===// |
| 10188 | |
| 10189 | bool AMDGPUOperand::isBLGP() const { |
| 10190 | return isImm() && getImmTy() == ImmTyBLGP && isUInt<3>(x: getImm()); |
| 10191 | } |
| 10192 | |
| 10193 | bool AMDGPUOperand::isS16Imm() const { |
| 10194 | return isImmLiteral() && (isInt<16>(x: getImm()) || isUInt<16>(x: getImm())); |
| 10195 | } |
| 10196 | |
| 10197 | bool AMDGPUOperand::isU16Imm() const { |
| 10198 | return isImmLiteral() && isUInt<16>(x: getImm()); |
| 10199 | } |
| 10200 | |
| 10201 | //===----------------------------------------------------------------------===// |
| 10202 | // dim |
| 10203 | //===----------------------------------------------------------------------===// |
| 10204 | |
| 10205 | bool AMDGPUAsmParser::parseDimId(unsigned &Encoding) { |
| 10206 | // We want to allow "dim:1D" etc., |
| 10207 | // but the initial 1 is tokenized as an integer. |
| 10208 | std::string Token; |
| 10209 | if (isToken(Kind: AsmToken::Integer)) { |
| 10210 | SMLoc Loc = getToken().getEndLoc(); |
| 10211 | Token = std::string(getTokenStr()); |
| 10212 | lex(); |
| 10213 | if (getLoc() != Loc) |
| 10214 | return false; |
| 10215 | } |
| 10216 | |
| 10217 | StringRef Suffix; |
| 10218 | if (!parseId(Val&: Suffix)) |
| 10219 | return false; |
| 10220 | Token += Suffix; |
| 10221 | |
| 10222 | StringRef DimId = Token; |
| 10223 | DimId.consume_front(Prefix: "SQ_RSRC_IMG_" ); |
| 10224 | |
| 10225 | const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfoByAsmSuffix(AsmSuffix: DimId); |
| 10226 | if (!DimInfo) |
| 10227 | return false; |
| 10228 | |
| 10229 | Encoding = DimInfo->Encoding; |
| 10230 | return true; |
| 10231 | } |
| 10232 | |
| 10233 | ParseStatus AMDGPUAsmParser::parseDim(OperandVector &Operands) { |
| 10234 | if (!isGFX10Plus()) |
| 10235 | return ParseStatus::NoMatch; |
| 10236 | |
| 10237 | SMLoc S = getLoc(); |
| 10238 | |
| 10239 | if (!trySkipId(Id: "dim" , Kind: AsmToken::Colon)) |
| 10240 | return ParseStatus::NoMatch; |
| 10241 | |
| 10242 | unsigned Encoding; |
| 10243 | SMLoc Loc = getLoc(); |
| 10244 | if (!parseDimId(Encoding)) |
| 10245 | return Error(L: Loc, Msg: "invalid dim value" ); |
| 10246 | |
| 10247 | Operands.push_back( |
| 10248 | Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Encoding, Loc: S, Type: AMDGPUOperand::ImmTyDim)); |
| 10249 | return ParseStatus::Success; |
| 10250 | } |
| 10251 | |
| 10252 | //===----------------------------------------------------------------------===// |
| 10253 | // dpp |
| 10254 | //===----------------------------------------------------------------------===// |
| 10255 | |
| 10256 | ParseStatus AMDGPUAsmParser::parseDPP8(OperandVector &Operands) { |
| 10257 | SMLoc S = getLoc(); |
| 10258 | |
| 10259 | if (!isGFX10Plus() || !trySkipId(Id: "dpp8" , Kind: AsmToken::Colon)) |
| 10260 | return ParseStatus::NoMatch; |
| 10261 | |
| 10262 | // dpp8:[%d,%d,%d,%d,%d,%d,%d,%d] |
| 10263 | |
| 10264 | int64_t Sels[8]; |
| 10265 | |
| 10266 | if (!skipToken(Kind: AsmToken::LBrac, ErrMsg: "expected an opening square bracket" )) |
| 10267 | return ParseStatus::Failure; |
| 10268 | |
| 10269 | for (size_t i = 0; i < 8; ++i) { |
| 10270 | if (i > 0 && !skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma" )) |
| 10271 | return ParseStatus::Failure; |
| 10272 | |
| 10273 | SMLoc Loc = getLoc(); |
| 10274 | if (getParser().parseAbsoluteExpression(Res&: Sels[i])) |
| 10275 | return ParseStatus::Failure; |
| 10276 | if (0 > Sels[i] || 7 < Sels[i]) |
| 10277 | return Error(L: Loc, Msg: "expected a 3-bit value" ); |
| 10278 | } |
| 10279 | |
| 10280 | if (!skipToken(Kind: AsmToken::RBrac, ErrMsg: "expected a closing square bracket" )) |
| 10281 | return ParseStatus::Failure; |
| 10282 | |
| 10283 | unsigned DPP8 = 0; |
| 10284 | for (size_t i = 0; i < 8; ++i) |
| 10285 | DPP8 |= (Sels[i] << (i * 3)); |
| 10286 | |
| 10287 | Operands.push_back( |
| 10288 | Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: DPP8, Loc: S, Type: AMDGPUOperand::ImmTyDPP8)); |
| 10289 | return ParseStatus::Success; |
| 10290 | } |
| 10291 | |
| 10292 | bool AMDGPUAsmParser::isSupportedDPPCtrl(StringRef Ctrl, |
| 10293 | const OperandVector &Operands) { |
| 10294 | if (Ctrl == "row_newbcast" ) |
| 10295 | return isGFX90A(); |
| 10296 | |
| 10297 | if (Ctrl == "row_share" || Ctrl == "row_xmask" ) |
| 10298 | return isGFX10Plus(); |
| 10299 | |
| 10300 | if (Ctrl == "wave_shl" || Ctrl == "wave_shr" || Ctrl == "wave_rol" || |
| 10301 | Ctrl == "wave_ror" || Ctrl == "row_bcast" ) |
| 10302 | return isVI() || isGFX9(); |
| 10303 | |
| 10304 | return Ctrl == "row_mirror" || Ctrl == "row_half_mirror" || |
| 10305 | Ctrl == "quad_perm" || Ctrl == "row_shl" || Ctrl == "row_shr" || |
| 10306 | Ctrl == "row_ror" ; |
| 10307 | } |
| 10308 | |
| 10309 | int64_t AMDGPUAsmParser::parseDPPCtrlPerm() { |
| 10310 | // quad_perm:[%d,%d,%d,%d] |
| 10311 | |
| 10312 | if (!skipToken(Kind: AsmToken::LBrac, ErrMsg: "expected an opening square bracket" )) |
| 10313 | return -1; |
| 10314 | |
| 10315 | int64_t Val = 0; |
| 10316 | for (int i = 0; i < 4; ++i) { |
| 10317 | if (i > 0 && !skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma" )) |
| 10318 | return -1; |
| 10319 | |
| 10320 | int64_t Temp; |
| 10321 | SMLoc Loc = getLoc(); |
| 10322 | if (getParser().parseAbsoluteExpression(Res&: Temp)) |
| 10323 | return -1; |
| 10324 | if (Temp < 0 || Temp > 3) { |
| 10325 | Error(L: Loc, Msg: "expected a 2-bit value" ); |
| 10326 | return -1; |
| 10327 | } |
| 10328 | |
| 10329 | Val += (Temp << i * 2); |
| 10330 | } |
| 10331 | |
| 10332 | if (!skipToken(Kind: AsmToken::RBrac, ErrMsg: "expected a closing square bracket" )) |
| 10333 | return -1; |
| 10334 | |
| 10335 | return Val; |
| 10336 | } |
| 10337 | |
| 10338 | int64_t AMDGPUAsmParser::parseDPPCtrlSel(StringRef Ctrl) { |
| 10339 | using namespace AMDGPU::DPP; |
| 10340 | |
| 10341 | // sel:%d |
| 10342 | |
| 10343 | int64_t Val; |
| 10344 | SMLoc Loc = getLoc(); |
| 10345 | |
| 10346 | if (getParser().parseAbsoluteExpression(Res&: Val)) |
| 10347 | return -1; |
| 10348 | |
| 10349 | struct DppCtrlCheck { |
| 10350 | int64_t Ctrl; |
| 10351 | int Lo; |
| 10352 | int Hi; |
| 10353 | }; |
| 10354 | |
| 10355 | DppCtrlCheck Check = |
| 10356 | StringSwitch<DppCtrlCheck>(Ctrl) |
| 10357 | .Case(S: "wave_shl" , Value: {.Ctrl: DppCtrl::WAVE_SHL1, .Lo: 1, .Hi: 1}) |
| 10358 | .Case(S: "wave_rol" , Value: {.Ctrl: DppCtrl::WAVE_ROL1, .Lo: 1, .Hi: 1}) |
| 10359 | .Case(S: "wave_shr" , Value: {.Ctrl: DppCtrl::WAVE_SHR1, .Lo: 1, .Hi: 1}) |
| 10360 | .Case(S: "wave_ror" , Value: {.Ctrl: DppCtrl::WAVE_ROR1, .Lo: 1, .Hi: 1}) |
| 10361 | .Case(S: "row_shl" , Value: {.Ctrl: DppCtrl::ROW_SHL0, .Lo: 1, .Hi: 15}) |
| 10362 | .Case(S: "row_shr" , Value: {.Ctrl: DppCtrl::ROW_SHR0, .Lo: 1, .Hi: 15}) |
| 10363 | .Case(S: "row_ror" , Value: {.Ctrl: DppCtrl::ROW_ROR0, .Lo: 1, .Hi: 15}) |
| 10364 | .Case(S: "row_share" , Value: {.Ctrl: DppCtrl::ROW_SHARE_FIRST, .Lo: 0, .Hi: 15}) |
| 10365 | .Case(S: "row_xmask" , Value: {.Ctrl: DppCtrl::ROW_XMASK_FIRST, .Lo: 0, .Hi: 15}) |
| 10366 | .Case(S: "row_newbcast" , Value: {.Ctrl: DppCtrl::ROW_NEWBCAST_FIRST, .Lo: 0, .Hi: 15}) |
| 10367 | .Default(Value: {.Ctrl: -1, .Lo: 0, .Hi: 0}); |
| 10368 | |
| 10369 | bool Valid; |
| 10370 | if (Check.Ctrl == -1) { |
| 10371 | Valid = (Ctrl == "row_bcast" && (Val == 15 || Val == 31)); |
| 10372 | Val = (Val == 15) ? DppCtrl::BCAST15 : DppCtrl::BCAST31; |
| 10373 | } else { |
| 10374 | Valid = Check.Lo <= Val && Val <= Check.Hi; |
| 10375 | Val = (Check.Lo == Check.Hi) ? Check.Ctrl : (Check.Ctrl | Val); |
| 10376 | } |
| 10377 | |
| 10378 | if (!Valid) { |
| 10379 | Error(L: Loc, Msg: Twine("invalid " , Ctrl) + Twine(" value" )); |
| 10380 | return -1; |
| 10381 | } |
| 10382 | |
| 10383 | return Val; |
| 10384 | } |
| 10385 | |
| 10386 | ParseStatus AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) { |
| 10387 | using namespace AMDGPU::DPP; |
| 10388 | |
| 10389 | if (!isToken(Kind: AsmToken::Identifier) || |
| 10390 | !isSupportedDPPCtrl(Ctrl: getTokenStr(), Operands)) |
| 10391 | return ParseStatus::NoMatch; |
| 10392 | |
| 10393 | SMLoc S = getLoc(); |
| 10394 | int64_t Val = -1; |
| 10395 | StringRef Ctrl; |
| 10396 | |
| 10397 | parseId(Val&: Ctrl); |
| 10398 | |
| 10399 | if (Ctrl == "row_mirror" ) { |
| 10400 | Val = DppCtrl::ROW_MIRROR; |
| 10401 | } else if (Ctrl == "row_half_mirror" ) { |
| 10402 | Val = DppCtrl::ROW_HALF_MIRROR; |
| 10403 | } else { |
| 10404 | if (skipToken(Kind: AsmToken::Colon, ErrMsg: "expected a colon" )) { |
| 10405 | if (Ctrl == "quad_perm" ) { |
| 10406 | Val = parseDPPCtrlPerm(); |
| 10407 | } else { |
| 10408 | Val = parseDPPCtrlSel(Ctrl); |
| 10409 | } |
| 10410 | } |
| 10411 | } |
| 10412 | |
| 10413 | if (Val == -1) |
| 10414 | return ParseStatus::Failure; |
| 10415 | |
| 10416 | Operands.push_back( |
| 10417 | Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val, Loc: S, Type: AMDGPUOperand::ImmTyDppCtrl)); |
| 10418 | return ParseStatus::Success; |
| 10419 | } |
| 10420 | |
| 10421 | void AMDGPUAsmParser::cvtVOP3DPP(MCInst &Inst, const OperandVector &Operands, |
| 10422 | bool IsDPP8) { |
| 10423 | OptionalImmIndexMap OptionalIdx; |
| 10424 | unsigned Opc = Inst.getOpcode(); |
| 10425 | const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode()); |
| 10426 | |
| 10427 | // MAC instructions are special because they have 'old' |
| 10428 | // operand which is not tied to dst (but assumed to be). |
| 10429 | // They also have dummy unused src2_modifiers. |
| 10430 | int OldIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::old); |
| 10431 | int Src2ModIdx = |
| 10432 | AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2_modifiers); |
| 10433 | bool IsMAC = OldIdx != -1 && Src2ModIdx != -1 && |
| 10434 | Desc.getOperandConstraint(OpNum: OldIdx, Constraint: MCOI::TIED_TO) == -1; |
| 10435 | |
| 10436 | unsigned I = 1; |
| 10437 | for (unsigned J = 0; J < Desc.getNumDefs(); ++J) { |
| 10438 | ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, N: 1); |
| 10439 | } |
| 10440 | |
| 10441 | int Fi = 0; |
| 10442 | int VdstInIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst_in); |
| 10443 | bool IsVOP3CvtSrDpp = Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx12 || |
| 10444 | Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx13 || |
| 10445 | Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx12 || |
| 10446 | Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx13 || |
| 10447 | Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx12 || |
| 10448 | Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx13 || |
| 10449 | Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx12 || |
| 10450 | Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx13; |
| 10451 | |
| 10452 | for (unsigned E = Operands.size(); I != E; ++I) { |
| 10453 | |
| 10454 | if (IsMAC) { |
| 10455 | int NumOperands = Inst.getNumOperands(); |
| 10456 | if (OldIdx == NumOperands) { |
| 10457 | // Handle old operand |
| 10458 | constexpr int DST_IDX = 0; |
| 10459 | Inst.addOperand(Op: Inst.getOperand(i: DST_IDX)); |
| 10460 | } else if (Src2ModIdx == NumOperands) { |
| 10461 | // Add unused dummy src2_modifiers |
| 10462 | Inst.addOperand(Op: MCOperand::createImm(Val: 0)); |
| 10463 | } |
| 10464 | } |
| 10465 | |
| 10466 | if (VdstInIdx == static_cast<int>(Inst.getNumOperands())) { |
| 10467 | Inst.addOperand(Op: Inst.getOperand(i: 0)); |
| 10468 | } |
| 10469 | |
| 10470 | if (IsVOP3CvtSrDpp) { |
| 10471 | if (Src2ModIdx == static_cast<int>(Inst.getNumOperands())) { |
| 10472 | Inst.addOperand(Op: MCOperand::createImm(Val: 0)); |
| 10473 | Inst.addOperand(Op: MCOperand::createReg(Reg: MCRegister())); |
| 10474 | } |
| 10475 | } |
| 10476 | |
| 10477 | auto TiedTo = |
| 10478 | Desc.getOperandConstraint(OpNum: Inst.getNumOperands(), Constraint: MCOI::TIED_TO); |
| 10479 | if (TiedTo != -1) { |
| 10480 | assert((unsigned)TiedTo < Inst.getNumOperands()); |
| 10481 | // handle tied old or src2 for MAC instructions |
| 10482 | Inst.addOperand(Op: Inst.getOperand(i: TiedTo)); |
| 10483 | } |
| 10484 | AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]); |
| 10485 | // Add the register arguments |
| 10486 | if (IsDPP8 && Op.isDppFI()) { |
| 10487 | Fi = Op.getImm(); |
| 10488 | } else if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) { |
| 10489 | Op.addRegOrImmWithFPInputModsOperands(Inst, N: 2); |
| 10490 | } else if (Op.isReg()) { |
| 10491 | Op.addRegOperands(Inst, N: 1); |
| 10492 | } else if (Op.isImm() && |
| 10493 | Desc.operands()[Inst.getNumOperands()].RegClass != -1) { |
| 10494 | Op.addImmOperands(Inst, N: 1); |
| 10495 | } else if (Op.isImm()) { |
| 10496 | OptionalIdx[Op.getImmTy()] = I; |
| 10497 | } else { |
| 10498 | llvm_unreachable("unhandled operand type" ); |
| 10499 | } |
| 10500 | } |
| 10501 | |
| 10502 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::clamp) && !IsVOP3CvtSrDpp) |
| 10503 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10504 | ImmT: AMDGPUOperand::ImmTyClamp); |
| 10505 | |
| 10506 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::byte_sel)) { |
| 10507 | if (VdstInIdx == static_cast<int>(Inst.getNumOperands())) |
| 10508 | Inst.addOperand(Op: Inst.getOperand(i: 0)); |
| 10509 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10510 | ImmT: AMDGPUOperand::ImmTyByteSel); |
| 10511 | } |
| 10512 | |
| 10513 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::omod)) |
| 10514 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10515 | ImmT: AMDGPUOperand::ImmTyOModSI); |
| 10516 | |
| 10517 | if (SIInstrFlags::isVOP3P(O: Desc)) |
| 10518 | cvtVOP3P(Inst, Operands, OptIdx&: OptionalIdx); |
| 10519 | else if (SIInstrFlags::isVOP3(O: Desc)) |
| 10520 | cvtVOP3OpSel(Inst, Operands, OptionalIdx); |
| 10521 | else if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::op_sel)) { |
| 10522 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10523 | ImmT: AMDGPUOperand::ImmTyOpSel); |
| 10524 | } |
| 10525 | |
| 10526 | if (IsDPP8) { |
| 10527 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10528 | ImmT: AMDGPUOperand::ImmTyDPP8); |
| 10529 | using namespace llvm::AMDGPU::DPP; |
| 10530 | Inst.addOperand(Op: MCOperand::createImm(Val: Fi ? DPP8_FI_1 : DPP8_FI_0)); |
| 10531 | } else { |
| 10532 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10533 | ImmT: AMDGPUOperand::ImmTyDppCtrl, Default: 0xe4); |
| 10534 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10535 | ImmT: AMDGPUOperand::ImmTyDppRowMask, Default: 0xf); |
| 10536 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10537 | ImmT: AMDGPUOperand::ImmTyDppBankMask, Default: 0xf); |
| 10538 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10539 | ImmT: AMDGPUOperand::ImmTyDppBoundCtrl); |
| 10540 | |
| 10541 | if (AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::fi)) |
| 10542 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10543 | ImmT: AMDGPUOperand::ImmTyDppFI); |
| 10544 | } |
| 10545 | } |
| 10546 | |
| 10547 | void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands, |
| 10548 | bool IsDPP8) { |
| 10549 | OptionalImmIndexMap OptionalIdx; |
| 10550 | |
| 10551 | unsigned I = 1; |
| 10552 | const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode()); |
| 10553 | for (unsigned J = 0; J < Desc.getNumDefs(); ++J) { |
| 10554 | ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, N: 1); |
| 10555 | } |
| 10556 | |
| 10557 | int Fi = 0; |
| 10558 | for (unsigned E = Operands.size(); I != E; ++I) { |
| 10559 | auto TiedTo = |
| 10560 | Desc.getOperandConstraint(OpNum: Inst.getNumOperands(), Constraint: MCOI::TIED_TO); |
| 10561 | if (TiedTo != -1) { |
| 10562 | assert((unsigned)TiedTo < Inst.getNumOperands()); |
| 10563 | // handle tied old or src2 for MAC instructions |
| 10564 | Inst.addOperand(Op: Inst.getOperand(i: TiedTo)); |
| 10565 | } |
| 10566 | AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]); |
| 10567 | // Add the register arguments |
| 10568 | if (Op.isReg() && validateVccOperand(Reg: Op.getReg())) { |
| 10569 | // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token. |
| 10570 | // Skip it. |
| 10571 | continue; |
| 10572 | } |
| 10573 | |
| 10574 | if (IsDPP8) { |
| 10575 | if (Op.isDPP8()) { |
| 10576 | Op.addImmOperands(Inst, N: 1); |
| 10577 | } else if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) { |
| 10578 | Op.addRegWithFPInputModsOperands(Inst, N: 2); |
| 10579 | } else if (Op.isDppFI()) { |
| 10580 | Fi = Op.getImm(); |
| 10581 | } else if (Op.isReg()) { |
| 10582 | Op.addRegOperands(Inst, N: 1); |
| 10583 | } else { |
| 10584 | llvm_unreachable("Invalid operand type" ); |
| 10585 | } |
| 10586 | } else { |
| 10587 | if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) { |
| 10588 | Op.addRegWithFPInputModsOperands(Inst, N: 2); |
| 10589 | } else if (Op.isReg()) { |
| 10590 | Op.addRegOperands(Inst, N: 1); |
| 10591 | } else if (Op.isDPPCtrl()) { |
| 10592 | Op.addImmOperands(Inst, N: 1); |
| 10593 | } else if (Op.isImm()) { |
| 10594 | // Handle optional arguments |
| 10595 | OptionalIdx[Op.getImmTy()] = I; |
| 10596 | } else { |
| 10597 | llvm_unreachable("Invalid operand type" ); |
| 10598 | } |
| 10599 | } |
| 10600 | } |
| 10601 | |
| 10602 | if (IsDPP8) { |
| 10603 | using namespace llvm::AMDGPU::DPP; |
| 10604 | Inst.addOperand(Op: MCOperand::createImm(Val: Fi ? DPP8_FI_1 : DPP8_FI_0)); |
| 10605 | } else { |
| 10606 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10607 | ImmT: AMDGPUOperand::ImmTyDppRowMask, Default: 0xf); |
| 10608 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10609 | ImmT: AMDGPUOperand::ImmTyDppBankMask, Default: 0xf); |
| 10610 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10611 | ImmT: AMDGPUOperand::ImmTyDppBoundCtrl); |
| 10612 | if (AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::fi)) { |
| 10613 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10614 | ImmT: AMDGPUOperand::ImmTyDppFI); |
| 10615 | } |
| 10616 | } |
| 10617 | } |
| 10618 | |
| 10619 | //===----------------------------------------------------------------------===// |
| 10620 | // sdwa |
| 10621 | //===----------------------------------------------------------------------===// |
| 10622 | |
| 10623 | ParseStatus AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, |
| 10624 | StringRef Prefix, |
| 10625 | AMDGPUOperand::ImmTy Type) { |
| 10626 | return parseStringOrIntWithPrefix( |
| 10627 | Operands, Name: Prefix, |
| 10628 | Ids: {"BYTE_0" , "BYTE_1" , "BYTE_2" , "BYTE_3" , "WORD_0" , "WORD_1" , "DWORD" }, |
| 10629 | Type); |
| 10630 | } |
| 10631 | |
| 10632 | ParseStatus AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) { |
| 10633 | return parseStringOrIntWithPrefix( |
| 10634 | Operands, Name: "dst_unused" , Ids: {"UNUSED_PAD" , "UNUSED_SEXT" , "UNUSED_PRESERVE" }, |
| 10635 | Type: AMDGPUOperand::ImmTySDWADstUnused); |
| 10636 | } |
| 10637 | |
| 10638 | void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) { |
| 10639 | cvtSDWA(Inst, Operands, BasicInstType: SDWAInstType::VOP1); |
| 10640 | } |
| 10641 | |
| 10642 | void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) { |
| 10643 | cvtSDWA(Inst, Operands, BasicInstType: SDWAInstType::VOP2); |
| 10644 | } |
| 10645 | |
| 10646 | void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst, |
| 10647 | const OperandVector &Operands) { |
| 10648 | cvtSDWA(Inst, Operands, BasicInstType: SDWAInstType::VOP2, SkipDstVcc: true, SkipSrcVcc: true); |
| 10649 | } |
| 10650 | |
| 10651 | void AMDGPUAsmParser::cvtSdwaVOP2e(MCInst &Inst, |
| 10652 | const OperandVector &Operands) { |
| 10653 | cvtSDWA(Inst, Operands, BasicInstType: SDWAInstType::VOP2, SkipDstVcc: false, SkipSrcVcc: true); |
| 10654 | } |
| 10655 | |
| 10656 | void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) { |
| 10657 | cvtSDWA(Inst, Operands, BasicInstType: SDWAInstType::VOPC, SkipDstVcc: isVI()); |
| 10658 | } |
| 10659 | |
| 10660 | void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands, |
| 10661 | SDWAInstType BasicInstType, bool SkipDstVcc, |
| 10662 | bool SkipSrcVcc) { |
| 10663 | using namespace llvm::AMDGPU::SDWA; |
| 10664 | |
| 10665 | OptionalImmIndexMap OptionalIdx; |
| 10666 | bool SkipVcc = SkipDstVcc || SkipSrcVcc; |
| 10667 | bool SkippedVcc = false; |
| 10668 | |
| 10669 | unsigned I = 1; |
| 10670 | const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode()); |
| 10671 | for (unsigned J = 0; J < Desc.getNumDefs(); ++J) { |
| 10672 | ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, N: 1); |
| 10673 | } |
| 10674 | |
| 10675 | for (unsigned E = Operands.size(); I != E; ++I) { |
| 10676 | AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]); |
| 10677 | if (SkipVcc && !SkippedVcc && Op.isReg() && |
| 10678 | (Op.getReg() == AMDGPU::VCC || Op.getReg() == AMDGPU::VCC_LO)) { |
| 10679 | // VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst. |
| 10680 | // Skip it if it's 2nd (e.g. v_add_i32_sdwa v1, vcc, v2, v3) |
| 10681 | // or 4th (v_addc_u32_sdwa v1, vcc, v2, v3, vcc) operand. |
| 10682 | // Skip VCC only if we didn't skip it on previous iteration. |
| 10683 | // Note that src0 and src1 occupy 2 slots each because of modifiers. |
| 10684 | if (BasicInstType == SDWAInstType::VOP2 && |
| 10685 | ((SkipDstVcc && Inst.getNumOperands() == 1) || |
| 10686 | (SkipSrcVcc && Inst.getNumOperands() == 5))) { |
| 10687 | SkippedVcc = true; |
| 10688 | continue; |
| 10689 | } |
| 10690 | if (BasicInstType == SDWAInstType::VOPC && Inst.getNumOperands() == 0) { |
| 10691 | SkippedVcc = true; |
| 10692 | continue; |
| 10693 | } |
| 10694 | } |
| 10695 | if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) { |
| 10696 | Op.addRegOrImmWithInputModsOperands(Inst, N: 2); |
| 10697 | } else if (Op.isImm()) { |
| 10698 | // Handle optional arguments |
| 10699 | OptionalIdx[Op.getImmTy()] = I; |
| 10700 | } else { |
| 10701 | llvm_unreachable("Invalid operand type" ); |
| 10702 | } |
| 10703 | SkippedVcc = false; |
| 10704 | } |
| 10705 | |
| 10706 | const unsigned Opc = Inst.getOpcode(); |
| 10707 | if (Opc != AMDGPU::V_NOP_sdwa_gfx10 && Opc != AMDGPU::V_NOP_sdwa_gfx9 && |
| 10708 | Opc != AMDGPU::V_NOP_sdwa_vi) { |
| 10709 | // v_nop_sdwa_sdwa_vi/gfx9 has no optional sdwa arguments |
| 10710 | switch (BasicInstType) { |
| 10711 | case SDWAInstType::VOP1: |
| 10712 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::clamp)) |
| 10713 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10714 | ImmT: AMDGPUOperand::ImmTyClamp, Default: 0); |
| 10715 | |
| 10716 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::omod)) |
| 10717 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10718 | ImmT: AMDGPUOperand::ImmTyOModSI, Default: 0); |
| 10719 | |
| 10720 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::dst_sel)) |
| 10721 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10722 | ImmT: AMDGPUOperand::ImmTySDWADstSel, Default: SdwaSel::DWORD); |
| 10723 | |
| 10724 | if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::dst_unused)) |
| 10725 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10726 | ImmT: AMDGPUOperand::ImmTySDWADstUnused, |
| 10727 | Default: DstUnused::UNUSED_PRESERVE); |
| 10728 | |
| 10729 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10730 | ImmT: AMDGPUOperand::ImmTySDWASrc0Sel, Default: SdwaSel::DWORD); |
| 10731 | break; |
| 10732 | |
| 10733 | case SDWAInstType::VOP2: |
| 10734 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10735 | ImmT: AMDGPUOperand::ImmTyClamp, Default: 0); |
| 10736 | |
| 10737 | if (AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::omod)) |
| 10738 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10739 | ImmT: AMDGPUOperand::ImmTyOModSI, Default: 0); |
| 10740 | |
| 10741 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10742 | ImmT: AMDGPUOperand::ImmTySDWADstSel, Default: SdwaSel::DWORD); |
| 10743 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10744 | ImmT: AMDGPUOperand::ImmTySDWADstUnused, |
| 10745 | Default: DstUnused::UNUSED_PRESERVE); |
| 10746 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10747 | ImmT: AMDGPUOperand::ImmTySDWASrc0Sel, Default: SdwaSel::DWORD); |
| 10748 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10749 | ImmT: AMDGPUOperand::ImmTySDWASrc1Sel, Default: SdwaSel::DWORD); |
| 10750 | break; |
| 10751 | |
| 10752 | case SDWAInstType::VOPC: |
| 10753 | if (AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::clamp)) |
| 10754 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10755 | ImmT: AMDGPUOperand::ImmTyClamp, Default: 0); |
| 10756 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10757 | ImmT: AMDGPUOperand::ImmTySDWASrc0Sel, Default: SdwaSel::DWORD); |
| 10758 | addOptionalImmOperand(Inst, Operands, OptionalIdx, |
| 10759 | ImmT: AMDGPUOperand::ImmTySDWASrc1Sel, Default: SdwaSel::DWORD); |
| 10760 | break; |
| 10761 | } |
| 10762 | } |
| 10763 | |
| 10764 | // special case v_mac_{f16, f32}: |
| 10765 | // it has src2 register operand that is tied to dst operand |
| 10766 | if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi || |
| 10767 | Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) { |
| 10768 | auto *it = Inst.begin(); |
| 10769 | std::advance( |
| 10770 | i&: it, n: AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::src2)); |
| 10771 | Inst.insert(I: it, Op: Inst.getOperand(i: 0)); // src2 = dst |
| 10772 | } |
| 10773 | } |
| 10774 | |
| 10775 | /// Force static initialization. |
| 10776 | extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void |
| 10777 | LLVMInitializeAMDGPUAsmParser() { |
| 10778 | RegisterMCAsmParser<AMDGPUAsmParser> A(getTheR600Target()); |
| 10779 | RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget()); |
| 10780 | RegisterMCAsmParser<AMDGPUAsmParser> C(getTheGCNLegacyTarget()); |
| 10781 | } |
| 10782 | |
| 10783 | #define GET_MATCHER_IMPLEMENTATION |
| 10784 | #define GET_MNEMONIC_SPELL_CHECKER |
| 10785 | #define GET_MNEMONIC_CHECKER |
| 10786 | #include "AMDGPUGenAsmMatcher.inc" |
| 10787 | |
| 10788 | ParseStatus AMDGPUAsmParser::parseCustomOperand(OperandVector &Operands, |
| 10789 | unsigned MCK) { |
| 10790 | switch (MCK) { |
| 10791 | case MCK_addr64: |
| 10792 | return parseTokenOp(Name: "addr64" , Operands); |
| 10793 | case MCK_done: |
| 10794 | return parseNamedBit(Name: "done" , Operands, ImmTy: AMDGPUOperand::ImmTyDone, IgnoreNegative: true); |
| 10795 | case MCK_idxen: |
| 10796 | return parseTokenOp(Name: "idxen" , Operands); |
| 10797 | case MCK_lds: |
| 10798 | return parseNamedBit(Name: "lds" , Operands, ImmTy: AMDGPUOperand::ImmTyLDS, |
| 10799 | /*IgnoreNegative=*/true); |
| 10800 | case MCK_offen: |
| 10801 | return parseTokenOp(Name: "offen" , Operands); |
| 10802 | case MCK_off: |
| 10803 | return parseTokenOp(Name: "off" , Operands); |
| 10804 | case MCK_row_95_en: |
| 10805 | return parseNamedBit(Name: "row_en" , Operands, ImmTy: AMDGPUOperand::ImmTyRowEn, IgnoreNegative: true); |
| 10806 | case MCK_gds: |
| 10807 | return parseNamedBit(Name: "gds" , Operands, ImmTy: AMDGPUOperand::ImmTyGDS); |
| 10808 | case MCK_tfe: |
| 10809 | return parseNamedBit(Name: "tfe" , Operands, ImmTy: AMDGPUOperand::ImmTyTFE); |
| 10810 | } |
| 10811 | return tryCustomParseOperand(Operands, MCK); |
| 10812 | } |
| 10813 | |
| 10814 | // This function should be defined after auto-generated include so that we have |
| 10815 | // MatchClassKind enum defined |
| 10816 | unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op, |
| 10817 | unsigned Kind) { |
| 10818 | // Tokens like "glc" would be parsed as immediate operands in ParseOperand(). |
| 10819 | // But MatchInstructionImpl() expects to meet token and fails to validate |
| 10820 | // operand. This method checks if we are given immediate operand but expect to |
| 10821 | // get corresponding token. |
| 10822 | AMDGPUOperand &Operand = (AMDGPUOperand &)Op; |
| 10823 | switch (Kind) { |
| 10824 | case MCK_addr64: |
| 10825 | return Operand.isAddr64() ? Match_Success : Match_InvalidOperand; |
| 10826 | case MCK_gds: |
| 10827 | return Operand.isGDS() ? Match_Success : Match_InvalidOperand; |
| 10828 | case MCK_lds: |
| 10829 | return Operand.isLDS() ? Match_Success : Match_InvalidOperand; |
| 10830 | case MCK_idxen: |
| 10831 | return Operand.isIdxen() ? Match_Success : Match_InvalidOperand; |
| 10832 | case MCK_offen: |
| 10833 | return Operand.isOffen() ? Match_Success : Match_InvalidOperand; |
| 10834 | case MCK_tfe: |
| 10835 | return Operand.isTFE() ? Match_Success : Match_InvalidOperand; |
| 10836 | case MCK_done: |
| 10837 | return Operand.isDone() ? Match_Success : Match_InvalidOperand; |
| 10838 | case MCK_row_95_en: |
| 10839 | return Operand.isRowEn() ? Match_Success : Match_InvalidOperand; |
| 10840 | case MCK_SSrc_b32: |
| 10841 | // When operands have expression values, they will return true for isToken, |
| 10842 | // because it is not possible to distinguish between a token and an |
| 10843 | // expression at parse time. MatchInstructionImpl() will always try to |
| 10844 | // match an operand as a token, when isToken returns true, and when the |
| 10845 | // name of the expression is not a valid token, the match will fail, |
| 10846 | // so we need to handle it here. |
| 10847 | return Operand.isSSrc_b32() ? Match_Success : Match_InvalidOperand; |
| 10848 | case MCK_SSrc_f32: |
| 10849 | return Operand.isSSrc_f32() ? Match_Success : Match_InvalidOperand; |
| 10850 | case MCK_SOPPBrTarget: |
| 10851 | return Operand.isSOPPBrTarget() ? Match_Success : Match_InvalidOperand; |
| 10852 | case MCK_VReg32OrOff: |
| 10853 | return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand; |
| 10854 | case MCK_InterpSlot: |
| 10855 | return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand; |
| 10856 | case MCK_InterpAttr: |
| 10857 | return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand; |
| 10858 | case MCK_InterpAttrChan: |
| 10859 | return Operand.isInterpAttrChan() ? Match_Success : Match_InvalidOperand; |
| 10860 | case MCK_SReg_64: |
| 10861 | case MCK_SReg_64_XEXEC: |
| 10862 | // Null is defined as a 32-bit register but |
| 10863 | // it should also be enabled with 64-bit operands or larger. |
| 10864 | // The following code enables it for SReg_64 and larger operands |
| 10865 | // used as source and destination. Remaining source |
| 10866 | // operands are handled in isInlinableImm. |
| 10867 | case MCK_SReg_96: |
| 10868 | case MCK_SReg_128: |
| 10869 | case MCK_SReg_256: |
| 10870 | case MCK_SReg_512: |
| 10871 | return Operand.isNull() ? Match_Success : Match_InvalidOperand; |
| 10872 | default: |
| 10873 | return Match_InvalidOperand; |
| 10874 | } |
| 10875 | } |
| 10876 | |
| 10877 | //===----------------------------------------------------------------------===// |
| 10878 | // endpgm |
| 10879 | //===----------------------------------------------------------------------===// |
| 10880 | |
| 10881 | ParseStatus AMDGPUAsmParser::parseEndpgm(OperandVector &Operands) { |
| 10882 | SMLoc S = getLoc(); |
| 10883 | int64_t Imm = 0; |
| 10884 | |
| 10885 | if (!parseExpr(Imm)) { |
| 10886 | // The operand is optional, if not present default to 0 |
| 10887 | Imm = 0; |
| 10888 | } |
| 10889 | |
| 10890 | if (!isUInt<16>(x: Imm)) |
| 10891 | return Error(L: S, Msg: "expected a 16-bit value" ); |
| 10892 | |
| 10893 | Operands.push_back( |
| 10894 | Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Imm, Loc: S, Type: AMDGPUOperand::ImmTyEndpgm)); |
| 10895 | return ParseStatus::Success; |
| 10896 | } |
| 10897 | |
| 10898 | bool AMDGPUOperand::isEndpgm() const { return isImmTy(ImmT: ImmTyEndpgm); } |
| 10899 | |
| 10900 | //===----------------------------------------------------------------------===// |
| 10901 | // Split Barrier |
| 10902 | //===----------------------------------------------------------------------===// |
| 10903 | |
| 10904 | bool AMDGPUOperand::isSplitBarrier() const { |
| 10905 | if (!isImm()) |
| 10906 | return false; |
| 10907 | |
| 10908 | int64_t Imm = getImm(); |
| 10909 | return isUInt<5>(x: Imm) || (AMDGPU::Barrier::CLUSTER_TRAP <= Imm && |
| 10910 | Imm <= AMDGPU::Barrier::WORKGROUP); |
| 10911 | } |
| 10912 | |