1//===- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AMDKernelCodeT.h"
10#include "MCTargetDesc/AMDGPUInstPrinter.h"
11#include "MCTargetDesc/AMDGPUMCAsmInfo.h"
12#include "MCTargetDesc/AMDGPUMCExpr.h"
13#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h"
14#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
15#include "MCTargetDesc/AMDGPUTargetStreamer.h"
16#include "SIDefines.h"
17#include "SIInstrInfo.h"
18#include "TargetInfo/AMDGPUTargetInfo.h"
19#include "Utils/AMDGPUAsmUtils.h"
20#include "Utils/AMDGPUBaseInfo.h"
21#include "Utils/AMDKernelCodeTUtils.h"
22#include "llvm/ADT/APFloat.h"
23#include "llvm/ADT/SmallBitVector.h"
24#include "llvm/ADT/Twine.h"
25#include "llvm/BinaryFormat/ELF.h"
26#include "llvm/CodeGenTypes/MachineValueType.h"
27#include "llvm/MC/MCAsmInfo.h"
28#include "llvm/MC/MCContext.h"
29#include "llvm/MC/MCExpr.h"
30#include "llvm/MC/MCInst.h"
31#include "llvm/MC/MCInstrDesc.h"
32#include "llvm/MC/MCParser/AsmLexer.h"
33#include "llvm/MC/MCParser/MCAsmParser.h"
34#include "llvm/MC/MCParser/MCParsedAsmOperand.h"
35#include "llvm/MC/MCParser/MCTargetAsmParser.h"
36#include "llvm/MC/MCRegisterInfo.h"
37#include "llvm/MC/MCSymbol.h"
38#include "llvm/MC/TargetRegistry.h"
39#include "llvm/Support/AMDGPUMetadata.h"
40#include "llvm/Support/AMDGPUObjLinkingInfo.h"
41#include "llvm/Support/AMDHSAKernelDescriptor.h"
42#include "llvm/Support/Casting.h"
43#include "llvm/Support/Compiler.h"
44#include "llvm/Support/MathExtras.h"
45#include "llvm/TargetParser/AMDGPUTargetParser.h"
46#include <optional>
47
48using namespace llvm;
49using namespace llvm::AMDGPU;
50using namespace llvm::amdhsa;
51
52namespace {
53
54class AMDGPUAsmParser;
55
56enum RegisterKind {
57 IS_UNKNOWN,
58 IS_VGPR,
59 IS_SGPR,
60 IS_AGPR,
61 IS_TTMP,
62 IS_SPECIAL
63};
64
65//===----------------------------------------------------------------------===//
66// Operand
67//===----------------------------------------------------------------------===//
68
69class AMDGPUOperand : public MCParsedAsmOperand {
70 enum KindTy { Token, Immediate, Register, Expression } Kind;
71
72 SMLoc StartLoc, EndLoc;
73 const AMDGPUAsmParser *AsmParser;
74
75public:
76 AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
77 : Kind(Kind_), AsmParser(AsmParser_) {}
78
79 using Ptr = std::unique_ptr<AMDGPUOperand>;
80
81 struct Modifiers {
82 bool Abs = false;
83 bool Neg = false;
84 bool Sext = false;
85 LitModifier Lit = LitModifier::None;
86
87 bool hasFPModifiers() const { return Abs || Neg; }
88 bool hasIntModifiers() const { return Sext; }
89 bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
90 bool isForcedLit() const { return Lit == LitModifier::Lit; }
91 bool isForcedLit64() const { return Lit == LitModifier::Lit64; }
92
93 int64_t getFPModifiersOperand() const {
94 int64_t Operand = 0;
95 Operand |= Abs ? SISrcMods::ABS : 0u;
96 Operand |= Neg ? SISrcMods::NEG : 0u;
97 return Operand;
98 }
99
100 int64_t getIntModifiersOperand() const {
101 int64_t Operand = 0;
102 Operand |= Sext ? SISrcMods::SEXT : 0u;
103 return Operand;
104 }
105
106 int64_t getModifiersOperand() const {
107 assert(!(hasFPModifiers() && hasIntModifiers()) &&
108 "fp and int modifiers should not be used simultaneously");
109 if (hasFPModifiers())
110 return getFPModifiersOperand();
111 if (hasIntModifiers())
112 return getIntModifiersOperand();
113 return 0;
114 }
115
116 friend raw_ostream &operator<<(raw_ostream &OS,
117 AMDGPUOperand::Modifiers Mods);
118 };
119
120 enum ImmTy {
121 ImmTyNone,
122 ImmTyGDS,
123 ImmTyLDS,
124 ImmTyOffen,
125 ImmTyIdxen,
126 ImmTyAddr64,
127 ImmTyOffset,
128 ImmTyInstOffset,
129 ImmTyOffset0,
130 ImmTyOffset1,
131 ImmTySMEMOffsetMod,
132 ImmTyCPol,
133 ImmTyTFE,
134 ImmTyIsAsync,
135 ImmTyD16,
136 ImmTyClamp,
137 ImmTyOModSI,
138 ImmTySDWADstSel,
139 ImmTySDWASrc0Sel,
140 ImmTySDWASrc1Sel,
141 ImmTySDWADstUnused,
142 ImmTyDMask,
143 ImmTyDim,
144 ImmTyUNorm,
145 ImmTyDA,
146 ImmTyR128A16,
147 ImmTyA16,
148 ImmTyLWE,
149 ImmTyExpTgt,
150 ImmTyExpCompr,
151 ImmTyExpVM,
152 ImmTyDone,
153 ImmTyRowEn,
154 ImmTyFORMAT,
155 ImmTyHwreg,
156 ImmTyOff,
157 ImmTySendMsg,
158 ImmTyWaitEvent,
159 ImmTyInterpSlot,
160 ImmTyInterpAttr,
161 ImmTyInterpAttrChan,
162 ImmTyOpSel,
163 ImmTyOpSelHi,
164 ImmTyNegLo,
165 ImmTyNegHi,
166 ImmTyIndexKey8bit,
167 ImmTyIndexKey16bit,
168 ImmTyIndexKey32bit,
169 ImmTyDPP8,
170 ImmTyDppCtrl,
171 ImmTyDppRowMask,
172 ImmTyDppBankMask,
173 ImmTyDppBoundCtrl,
174 ImmTyDppFI,
175 ImmTySwizzle,
176 ImmTyGprIdxMode,
177 ImmTyHigh,
178 ImmTyBLGP,
179 ImmTyCBSZ,
180 ImmTyABID,
181 ImmTyEndpgm,
182 ImmTyWaitVDST,
183 ImmTyWaitEXP,
184 ImmTyWaitVAVDst,
185 ImmTyWaitVMVSrc,
186 ImmTyBitOp3,
187 ImmTyMatrixAFMT,
188 ImmTyMatrixBFMT,
189 ImmTyMatrixAScale,
190 ImmTyMatrixBScale,
191 ImmTyMatrixAScaleFmt,
192 ImmTyMatrixBScaleFmt,
193 ImmTyMatrixAReuse,
194 ImmTyMatrixBReuse,
195 ImmTyScaleSel,
196 ImmTyByteSel,
197 };
198
199private:
200 struct TokOp {
201 const char *Data;
202 unsigned Length;
203 };
204
205 struct ImmOp {
206 int64_t Val;
207 ImmTy Type;
208 bool IsFPImm;
209 Modifiers Mods;
210 };
211
212 struct RegOp {
213 MCRegister RegNo;
214 Modifiers Mods;
215 };
216
217 union {
218 TokOp Tok;
219 ImmOp Imm;
220 RegOp Reg;
221 const MCExpr *Expr;
222 };
223
224 // The index of the associated MCInst operand.
225 mutable int MCOpIdx = -1;
226
227public:
228 bool isToken() const override { return Kind == Token; }
229
230 bool isSymbolRefExpr() const {
231 return isExpr() && Expr && isa<MCSymbolRefExpr>(Val: Expr);
232 }
233
234 bool isImm() const override { return Kind == Immediate; }
235
236 bool isInlinableImm(MVT type) const;
237 bool isLiteralImm(MVT type) const;
238
239 bool isRegKind() const { return Kind == Register; }
240
241 bool isReg() const override { return isRegKind() && !hasModifiers(); }
242
243 bool isRegOrInline(unsigned RCID, MVT type) const {
244 return isRegClass(RCID) || isInlinableImm(type);
245 }
246
247 bool isRegOrInlineTarget(unsigned TargetRCIdx, MVT type) const {
248 return isRegClassTarget(TargetRCIdx) || isInlinableImm(type);
249 }
250
251 bool isRegOrImmWithInputMods(unsigned RCID, MVT type) const {
252 return isRegOrInline(RCID, type) || isLiteralImm(type);
253 }
254
255 bool isRegOrImmWithInputModsTarget(unsigned TargetRCIdx, MVT type) const {
256 return isRegOrInlineTarget(TargetRCIdx, type) || isLiteralImm(type);
257 }
258
259 bool isRegOrImmWithInt16InputMods() const {
260 return isRegOrImmWithInputMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::i16);
261 }
262
263 template <bool IsFake16> bool isRegOrImmWithIntT16InputMods() const {
264 return isRegOrImmWithInputMods(
265 RCID: IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, type: MVT::i16);
266 }
267
268 bool isRegOrImmWithInt32InputMods() const {
269 return isRegOrImmWithInputMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::i32);
270 }
271
272 bool isRegOrInlineImmWithInt16InputMods() const {
273 return isRegOrInline(RCID: AMDGPU::VS_32RegClassID, type: MVT::i16);
274 }
275
276 template <bool IsFake16> bool isRegOrInlineImmWithIntT16InputMods() const {
277 return isRegOrInline(
278 RCID: IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, type: MVT::i16);
279 }
280
281 bool isRegOrInlineImmWithInt32InputMods() const {
282 return isRegOrInline(RCID: AMDGPU::VS_32RegClassID, type: MVT::i32);
283 }
284
285 bool isRegOrImmWithInt64InputMods() const {
286 return isRegOrImmWithInputModsTarget(TargetRCIdx: AMDGPU::VS_64_AlignTarget, type: MVT::i64);
287 }
288
289 bool isRegOrImmWithFP16InputMods() const {
290 return isRegOrImmWithInputMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::f16);
291 }
292
293 template <bool IsFake16> bool isRegOrImmWithFPT16InputMods() const {
294 return isRegOrImmWithInputMods(
295 RCID: IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, type: MVT::f16);
296 }
297
298 bool isRegOrImmWithFP32InputMods() const {
299 return isRegOrImmWithInputMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::f32);
300 }
301
302 bool isRegOrImmWithFP64InputMods() const {
303 return isRegOrImmWithInputModsTarget(TargetRCIdx: AMDGPU::VS_64_AlignTarget, type: MVT::f64);
304 }
305
306 template <bool IsFake16> bool isRegOrInlineImmWithFP16InputMods() const {
307 return isRegOrInline(
308 RCID: IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, type: MVT::f16);
309 }
310
311 bool isRegOrInlineImmWithFP32InputMods() const {
312 return isRegOrInline(RCID: AMDGPU::VS_32RegClassID, type: MVT::f32);
313 }
314
315 bool isRegOrInlineImmWithFP64InputMods() const {
316 return isRegOrInlineTarget(TargetRCIdx: AMDGPU::VS_64_AlignTarget, type: MVT::f64);
317 }
318
319 bool isVRegWithInputMods(unsigned RCID) const { return isRegClass(RCID); }
320
321 bool isVRegWithFP32InputMods() const {
322 return isVRegWithInputMods(RCID: AMDGPU::VGPR_32RegClassID);
323 }
324
325 bool isVRegWithFP64InputMods() const {
326 return isRegClassTarget(TargetRCIdx: AMDGPU::VReg_64_AlignTarget);
327 }
328
329 bool isPackedFP16InputMods() const {
330 return isRegOrImmWithInputMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::v2f16);
331 }
332
333 bool isPackedVGPRFP32InputMods() const {
334 return isRegOrImmWithInputMods(RCID: AMDGPU::VReg_64RegClassID, type: MVT::v2f32);
335 }
336
337 bool isVReg() const {
338 return isRegClass(RCID: AMDGPU::VGPR_32RegClassID) ||
339 isRegClass(RCID: AMDGPU::VReg_64RegClassID) ||
340 isRegClass(RCID: AMDGPU::VReg_96RegClassID) ||
341 isRegClass(RCID: AMDGPU::VReg_128RegClassID) ||
342 isRegClass(RCID: AMDGPU::VReg_160RegClassID) ||
343 isRegClass(RCID: AMDGPU::VReg_192RegClassID) ||
344 isRegClass(RCID: AMDGPU::VReg_256RegClassID) ||
345 isRegClass(RCID: AMDGPU::VReg_512RegClassID) ||
346 isRegClass(RCID: AMDGPU::VReg_1024RegClassID);
347 }
348
349 bool isVReg32() const { return isRegClass(RCID: AMDGPU::VGPR_32RegClassID); }
350
351 bool isVReg32OrOff() const { return isOff() || isVReg32(); }
352
353 bool isRsrcReg32() const { return isRegClass(RCID: AMDGPU::RsrcReg32RegClassID); }
354
355 bool isNull() const { return isRegKind() && getReg() == AMDGPU::SGPR_NULL; }
356
357 bool isAV_LdSt_32_Align2_RegOp() const {
358 return isRegClass(RCID: AMDGPU::VGPR_32RegClassID) ||
359 isRegClass(RCID: AMDGPU::AGPR_32RegClassID);
360 }
361
362 bool isVRegWithInputMods() const;
363 template <bool IsFake16> bool isT16_Lo128VRegWithInputMods() const;
364 template <bool IsFake16> bool isT16VRegWithInputMods() const;
365
366 bool isSDWAOperand(MVT type) const;
367 bool isSDWAFP16Operand() const;
368 bool isSDWAFP32Operand() const;
369 bool isSDWAInt16Operand() const;
370 bool isSDWAInt32Operand() const;
371
372 bool isImmTy(ImmTy ImmT) const { return isImm() && Imm.Type == ImmT; }
373
374 template <ImmTy Ty> bool isImmTy() const { return isImmTy(ImmT: Ty); }
375
376 bool isImmLiteral() const { return isImmTy(ImmT: ImmTyNone); }
377
378 bool isImmModifier() const { return isImm() && Imm.Type != ImmTyNone; }
379
380 bool isOModSI() const { return isImmTy(ImmT: ImmTyOModSI); }
381 bool isDim() const { return isImmTy(ImmT: ImmTyDim); }
382 bool isR128A16() const { return isImmTy(ImmT: ImmTyR128A16); }
383 bool isOff() const { return isImmTy(ImmT: ImmTyOff); }
384 bool isExpTgt() const { return isImmTy(ImmT: ImmTyExpTgt); }
385 bool isOffen() const { return isImmTy(ImmT: ImmTyOffen); }
386 bool isIdxen() const { return isImmTy(ImmT: ImmTyIdxen); }
387 bool isAddr64() const { return isImmTy(ImmT: ImmTyAddr64); }
388 bool isSMEMOffsetMod() const { return isImmTy(ImmT: ImmTySMEMOffsetMod); }
389 bool isFlatOffset() const {
390 return isImmTy(ImmT: ImmTyOffset) || isImmTy(ImmT: ImmTyInstOffset);
391 }
392 bool isGDS() const { return isImmTy(ImmT: ImmTyGDS); }
393 bool isLDS() const { return isImmTy(ImmT: ImmTyLDS); }
394 bool isCPol() const { return isImmTy(ImmT: ImmTyCPol); }
395 bool isIndexKey8bit() const { return isImmTy(ImmT: ImmTyIndexKey8bit); }
396 bool isIndexKey16bit() const { return isImmTy(ImmT: ImmTyIndexKey16bit); }
397 bool isIndexKey32bit() const { return isImmTy(ImmT: ImmTyIndexKey32bit); }
398 bool isMatrixAFMT() const { return isImmTy(ImmT: ImmTyMatrixAFMT); }
399 bool isMatrixBFMT() const { return isImmTy(ImmT: ImmTyMatrixBFMT); }
400 bool isMatrixAScale() const { return isImmTy(ImmT: ImmTyMatrixAScale); }
401 bool isMatrixBScale() const { return isImmTy(ImmT: ImmTyMatrixBScale); }
402 bool isMatrixAScaleFmt() const { return isImmTy(ImmT: ImmTyMatrixAScaleFmt); }
403 bool isMatrixBScaleFmt() const { return isImmTy(ImmT: ImmTyMatrixBScaleFmt); }
404 bool isMatrixAReuse() const { return isImmTy(ImmT: ImmTyMatrixAReuse); }
405 bool isMatrixBReuse() const { return isImmTy(ImmT: ImmTyMatrixBReuse); }
406 bool isTFE() const { return isImmTy(ImmT: ImmTyTFE); }
407 bool isFORMAT() const { return isImmTy(ImmT: ImmTyFORMAT) && isUInt<7>(x: getImm()); }
408 bool isDppFI() const { return isImmTy(ImmT: ImmTyDppFI); }
409 bool isSDWADstSel() const { return isImmTy(ImmT: ImmTySDWADstSel); }
410 bool isSDWASrc0Sel() const { return isImmTy(ImmT: ImmTySDWASrc0Sel); }
411 bool isSDWASrc1Sel() const { return isImmTy(ImmT: ImmTySDWASrc1Sel); }
412 bool isSDWADstUnused() const { return isImmTy(ImmT: ImmTySDWADstUnused); }
413 bool isInterpSlot() const { return isImmTy(ImmT: ImmTyInterpSlot); }
414 bool isInterpAttr() const { return isImmTy(ImmT: ImmTyInterpAttr); }
415 bool isInterpAttrChan() const { return isImmTy(ImmT: ImmTyInterpAttrChan); }
416 bool isOpSel() const { return isImmTy(ImmT: ImmTyOpSel); }
417 bool isOpSelHi() const { return isImmTy(ImmT: ImmTyOpSelHi); }
418 bool isNegLo() const { return isImmTy(ImmT: ImmTyNegLo); }
419 bool isNegHi() const { return isImmTy(ImmT: ImmTyNegHi); }
420 bool isBitOp3() const { return isImmTy(ImmT: ImmTyBitOp3) && isUInt<8>(x: getImm()); }
421 bool isDone() const { return isImmTy(ImmT: ImmTyDone); }
422 bool isRowEn() const { return isImmTy(ImmT: ImmTyRowEn); }
423
424 bool isRegOrImm() const { return isReg() || isImm(); }
425
426 bool isRegClass(unsigned RCID) const;
427
428 // Check the register against the HwMode-resolved operand class.
429 bool isRegClassTarget(unsigned TargetRCIdx) const;
430
431 bool isInlineValue() const;
432
433 bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
434 return isRegOrInline(RCID, type) && !hasModifiers();
435 }
436
437 bool isSCSrcB16() const {
438 return isRegOrInlineNoMods(RCID: AMDGPU::SReg_32RegClassID, type: MVT::i16);
439 }
440
441 bool isSCSrcV2B16() const { return isSCSrcB16(); }
442
443 bool isSCSrc_b32() const {
444 return isRegOrInlineNoMods(RCID: AMDGPU::SReg_32RegClassID, type: MVT::i32);
445 }
446
447 bool isSCSrc_b64() const {
448 return isRegOrInlineNoMods(RCID: AMDGPU::SReg_64RegClassID, type: MVT::i64);
449 }
450
451 bool isBoolReg() const;
452
453 bool isSCSrcF16() const {
454 return isRegOrInlineNoMods(RCID: AMDGPU::SReg_32RegClassID, type: MVT::f16);
455 }
456
457 bool isSCSrcV2F16() const { return isSCSrcF16(); }
458
459 bool isSCSrcF32() const {
460 return isRegOrInlineNoMods(RCID: AMDGPU::SReg_32RegClassID, type: MVT::f32);
461 }
462
463 bool isSCSrcF64() const {
464 return isRegOrInlineNoMods(RCID: AMDGPU::SReg_64RegClassID, type: MVT::f64);
465 }
466
467 bool isSSrc_b32() const {
468 return isSCSrc_b32() || isLiteralImm(type: MVT::i32) || isExpr();
469 }
470
471 bool isSSrc_b16() const { return isSCSrcB16() || isLiteralImm(type: MVT::i16); }
472
473 bool isSSrcV2B16() const {
474 llvm_unreachable("cannot happen");
475 return isSSrc_b16();
476 }
477
478 bool isSSrc_b64() const {
479 // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
480 // See isVSrc64().
481 return isSCSrc_b64() || isLiteralImm(type: MVT::i64) ||
482 (((const MCTargetAsmParser *)AsmParser)
483 ->getAvailableFeatures()[AMDGPU::Feature64BitLiterals] &&
484 isExpr());
485 }
486
487 bool isSSrc_f32() const {
488 return isSCSrc_b32() || isLiteralImm(type: MVT::f32) || isExpr();
489 }
490
491 bool isSSrcF64() const { return isSCSrc_b64() || isLiteralImm(type: MVT::f64); }
492
493 bool isSSrc_bf16() const { return isSCSrcB16() || isLiteralImm(type: MVT::bf16); }
494
495 bool isSSrc_f16() const { return isSCSrcB16() || isLiteralImm(type: MVT::f16); }
496
497 bool isSSrc_NoInline_f16() const { return isSSrc_f16(); }
498
499 bool isSSrcV2F16() const {
500 llvm_unreachable("cannot happen");
501 return isSSrc_f16();
502 }
503
504 bool isSSrcV2FP32() const {
505 llvm_unreachable("cannot happen");
506 return isSSrc_f32();
507 }
508
509 bool isSCSrcV2FP32() const {
510 llvm_unreachable("cannot happen");
511 return isSCSrcF32();
512 }
513
514 bool isSSrcV2INT32() const {
515 llvm_unreachable("cannot happen");
516 return isSSrc_b32();
517 }
518
519 bool isSCSrcV2INT32() const {
520 llvm_unreachable("cannot happen");
521 return isSCSrc_b32();
522 }
523
524 bool isSSrcOrLds_b32() const {
525 return isRegOrInlineNoMods(RCID: AMDGPU::SRegOrLds_32RegClassID, type: MVT::i32) ||
526 isLiteralImm(type: MVT::i32) || isExpr();
527 }
528
529 bool isVCSrc_b32() const {
530 return isRegOrInlineNoMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::i32);
531 }
532
533 bool isVCSrc_b32_Lo256() const {
534 return isRegOrInlineNoMods(RCID: AMDGPU::VS_32_Lo256RegClassID, type: MVT::i32);
535 }
536
537 bool isVCSrc_b64_Lo256() const {
538 return isRegOrInlineNoMods(RCID: AMDGPU::VS_64_Lo256RegClassID, type: MVT::i64);
539 }
540
541 bool isVCSrc_b64() const {
542 return isRegOrInlineNoMods(RCID: AMDGPU::VS_64RegClassID, type: MVT::i64);
543 }
544
545 bool isVCSrcT_b16() const {
546 return isRegOrInlineNoMods(RCID: AMDGPU::VS_16RegClassID, type: MVT::i16);
547 }
548
549 bool isVCSrcTB16_Lo128() const {
550 return isRegOrInlineNoMods(RCID: AMDGPU::VS_16_Lo128RegClassID, type: MVT::i16);
551 }
552
553 bool isVCSrcFake16B16_Lo128() const {
554 return isRegOrInlineNoMods(RCID: AMDGPU::VS_32_Lo128RegClassID, type: MVT::i16);
555 }
556
557 bool isVCSrc_b16() const {
558 return isRegOrInlineNoMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::i16);
559 }
560
561 bool isVCSrc_v2b16() const { return isVCSrc_b16(); }
562
563 bool isVCSrc_f32() const {
564 return isRegOrInlineNoMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::f32);
565 }
566
567 bool isVCSrc_f64() const {
568 return isRegOrInlineNoMods(RCID: AMDGPU::VS_64RegClassID, type: MVT::f64);
569 }
570
571 bool isVCSrcTBF16() const {
572 return isRegOrInlineNoMods(RCID: AMDGPU::VS_16RegClassID, type: MVT::bf16);
573 }
574
575 bool isVCSrcT_f16() const {
576 return isRegOrInlineNoMods(RCID: AMDGPU::VS_16RegClassID, type: MVT::f16);
577 }
578
579 bool isVCSrcT_bf16() const {
580 return isRegOrInlineNoMods(RCID: AMDGPU::VS_16RegClassID, type: MVT::f16);
581 }
582
583 bool isVCSrcTBF16_Lo128() const {
584 return isRegOrInlineNoMods(RCID: AMDGPU::VS_16_Lo128RegClassID, type: MVT::bf16);
585 }
586
587 bool isVCSrcTF16_Lo128() const {
588 return isRegOrInlineNoMods(RCID: AMDGPU::VS_16_Lo128RegClassID, type: MVT::f16);
589 }
590
591 bool isVCSrcFake16BF16_Lo128() const {
592 return isRegOrInlineNoMods(RCID: AMDGPU::VS_32_Lo128RegClassID, type: MVT::bf16);
593 }
594
595 bool isVCSrcFake16F16_Lo128() const {
596 return isRegOrInlineNoMods(RCID: AMDGPU::VS_32_Lo128RegClassID, type: MVT::f16);
597 }
598
599 bool isVCSrc_bf16() const {
600 return isRegOrInlineNoMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::bf16);
601 }
602
603 bool isVCSrc_f16() const {
604 return isRegOrInlineNoMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::f16);
605 }
606
607 bool isVCSrc_v2bf16() const { return isVCSrc_bf16(); }
608
609 bool isVCSrc_v2f16() const { return isVCSrc_f16(); }
610
611 bool isVSrc_b32() const {
612 return isVCSrc_f32() || isLiteralImm(type: MVT::i32) || isExpr();
613 }
614
615 bool isVSrc_b64() const { return isVCSrc_f64() || isLiteralImm(type: MVT::i64); }
616
617 bool isVSrc_v2b64() const {
618 return isRegOrInlineNoMods(RCID: AMDGPU::VS_128RegClassID, type: MVT::i64) ||
619 isLiteralImm(type: MVT::i64);
620 }
621
622 bool isVSrc_v2f64() const {
623 return isRegOrInlineNoMods(RCID: AMDGPU::VS_128RegClassID, type: MVT::f64) ||
624 isLiteralImm(type: MVT::f64);
625 }
626
627 bool isVSrcT_b16() const { return isVCSrcT_b16() || isLiteralImm(type: MVT::i16); }
628
629 bool isVSrcT_b16_Lo128() const {
630 return isVCSrcTB16_Lo128() || isLiteralImm(type: MVT::i16);
631 }
632
633 bool isVSrcFake16_b16_Lo128() const {
634 return isVCSrcFake16B16_Lo128() || isLiteralImm(type: MVT::i16);
635 }
636
637 bool isVSrc_b16() const { return isVCSrc_b16() || isLiteralImm(type: MVT::i16); }
638
639 bool isVSrc_v2b16() const { return isVSrc_b16() || isLiteralImm(type: MVT::v2i16); }
640
641 bool isVCSrcV2FP32() const { return isVCSrc_f64(); }
642
643 bool isVSrc_v2f32() const { return isVSrc_f64() || isLiteralImm(type: MVT::v2f32); }
644
645 bool isVCSrc_v2b32() const { return isVCSrc_b64(); }
646
647 bool isVSrc_v2b32() const { return isVSrc_b64() || isLiteralImm(type: MVT::v2i32); }
648
649 bool isVSrc_f32() const {
650 return isVCSrc_f32() || isLiteralImm(type: MVT::f32) || isExpr();
651 }
652
653 bool isVSrc_f64() const { return isVCSrc_f64() || isLiteralImm(type: MVT::f64); }
654
655 bool isVSrcT_bf16() const {
656 return isVCSrcTBF16() || isLiteralImm(type: MVT::bf16);
657 }
658
659 bool isVSrcT_f16() const { return isVCSrcT_f16() || isLiteralImm(type: MVT::f16); }
660
661 bool isVSrcT_bf16_Lo128() const {
662 return isVCSrcTBF16_Lo128() || isLiteralImm(type: MVT::bf16);
663 }
664
665 bool isVSrcT_f16_Lo128() const {
666 return isVCSrcTF16_Lo128() || isLiteralImm(type: MVT::f16);
667 }
668
669 bool isVSrcFake16_bf16_Lo128() const {
670 return isVCSrcFake16BF16_Lo128() || isLiteralImm(type: MVT::bf16);
671 }
672
673 bool isVSrcFake16_f16_Lo128() const {
674 return isVCSrcFake16F16_Lo128() || isLiteralImm(type: MVT::f16);
675 }
676
677 bool isVSrc_bf16() const { return isVCSrc_bf16() || isLiteralImm(type: MVT::bf16); }
678
679 bool isVSrc_f16() const { return isVCSrc_f16() || isLiteralImm(type: MVT::f16); }
680
681 bool isVSrc_v2bf16() const {
682 return isVSrc_bf16() || isLiteralImm(type: MVT::v2bf16);
683 }
684
685 bool isVSrc_v2f16() const { return isVSrc_f16() || isLiteralImm(type: MVT::v2f16); }
686
687 bool isVSrc_v2f16_splat() const { return isVSrc_v2f16(); }
688
689 bool isVSrc_NoInline_v2f16() const { return isVSrc_v2f16(); }
690
691 bool isVISrcB32() const {
692 return isRegOrInlineNoMods(RCID: AMDGPU::VGPR_32RegClassID, type: MVT::i32);
693 }
694
695 bool isVISrcB16() const {
696 return isRegOrInlineNoMods(RCID: AMDGPU::VGPR_32RegClassID, type: MVT::i16);
697 }
698
699 bool isVISrcV2B16() const { return isVISrcB16(); }
700
701 bool isVISrcF32() const {
702 return isRegOrInlineNoMods(RCID: AMDGPU::VGPR_32RegClassID, type: MVT::f32);
703 }
704
705 bool isVISrcF16() const {
706 return isRegOrInlineNoMods(RCID: AMDGPU::VGPR_32RegClassID, type: MVT::f16);
707 }
708
709 bool isVISrcV2F16() const { return isVISrcF16() || isVISrcB32(); }
710
711 bool isVISrc_64_bf16() const {
712 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_64RegClassID, type: MVT::bf16);
713 }
714
715 bool isVISrc_64_f16() const {
716 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_64RegClassID, type: MVT::f16);
717 }
718
719 bool isVISrc_64_b32() const {
720 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_64RegClassID, type: MVT::i32);
721 }
722
723 bool isVISrc_64B64() const {
724 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_64RegClassID, type: MVT::i64);
725 }
726
727 bool isVISrc_64_f64() const {
728 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_64RegClassID, type: MVT::f64);
729 }
730
731 bool isVISrc_64V2FP32() const {
732 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_64RegClassID, type: MVT::f32);
733 }
734
735 bool isVISrc_64V2INT32() const {
736 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_64RegClassID, type: MVT::i32);
737 }
738
739 bool isVISrc_256_b32() const {
740 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_256RegClassID, type: MVT::i32);
741 }
742
743 bool isVISrc_256_f32() const {
744 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_256RegClassID, type: MVT::f32);
745 }
746
747 bool isVISrc_256B64() const {
748 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_256RegClassID, type: MVT::i64);
749 }
750
751 bool isVISrc_256_f64() const {
752 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_256RegClassID, type: MVT::f64);
753 }
754
755 bool isVISrc_512_f64() const {
756 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_512RegClassID, type: MVT::f64);
757 }
758
759 bool isVISrc_128B16() const {
760 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_128RegClassID, type: MVT::i16);
761 }
762
763 bool isVISrc_128V2B16() const { return isVISrc_128B16(); }
764
765 bool isVISrc_128_b32() const {
766 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_128RegClassID, type: MVT::i32);
767 }
768
769 bool isVISrc_128_f32() const {
770 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_128RegClassID, type: MVT::f32);
771 }
772
773 bool isVISrc_256V2FP32() const {
774 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_256RegClassID, type: MVT::f32);
775 }
776
777 bool isVISrc_256V2INT32() const {
778 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_256RegClassID, type: MVT::i32);
779 }
780
781 bool isVISrc_512_b32() const {
782 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_512RegClassID, type: MVT::i32);
783 }
784
785 bool isVISrc_512B16() const {
786 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_512RegClassID, type: MVT::i16);
787 }
788
789 bool isVISrc_512V2B16() const { return isVISrc_512B16(); }
790
791 bool isVISrc_512_f32() const {
792 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_512RegClassID, type: MVT::f32);
793 }
794
795 bool isVISrc_512F16() const {
796 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_512RegClassID, type: MVT::f16);
797 }
798
799 bool isVISrc_512V2F16() const {
800 return isVISrc_512F16() || isVISrc_512_b32();
801 }
802
803 bool isVISrc_1024_b32() const {
804 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_1024RegClassID, type: MVT::i32);
805 }
806
807 bool isVISrc_1024B16() const {
808 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_1024RegClassID, type: MVT::i16);
809 }
810
811 bool isVISrc_1024V2B16() const { return isVISrc_1024B16(); }
812
813 bool isVISrc_1024_f32() const {
814 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_1024RegClassID, type: MVT::f32);
815 }
816
817 bool isVISrc_1024F16() const {
818 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_1024RegClassID, type: MVT::f16);
819 }
820
821 bool isVISrc_1024V2F16() const {
822 return isVISrc_1024F16() || isVISrc_1024_b32();
823 }
824
825 bool isAISrcB32() const {
826 return isRegOrInlineNoMods(RCID: AMDGPU::AGPR_32RegClassID, type: MVT::i32);
827 }
828
829 bool isAISrcB16() const {
830 return isRegOrInlineNoMods(RCID: AMDGPU::AGPR_32RegClassID, type: MVT::i16);
831 }
832
833 bool isAISrcV2B16() const { return isAISrcB16(); }
834
835 bool isAISrcF32() const {
836 return isRegOrInlineNoMods(RCID: AMDGPU::AGPR_32RegClassID, type: MVT::f32);
837 }
838
839 bool isAISrcF16() const {
840 return isRegOrInlineNoMods(RCID: AMDGPU::AGPR_32RegClassID, type: MVT::f16);
841 }
842
843 bool isAISrcV2F16() const { return isAISrcF16() || isAISrcB32(); }
844
845 bool isAISrc_64B64() const {
846 return isRegOrInlineNoMods(RCID: AMDGPU::AReg_64RegClassID, type: MVT::i64);
847 }
848
849 bool isAISrc_64_f64() const {
850 return isRegOrInlineNoMods(RCID: AMDGPU::AReg_64RegClassID, type: MVT::f64);
851 }
852
853 bool isAISrc_128_b32() const {
854 return isRegOrInlineNoMods(RCID: AMDGPU::AReg_128RegClassID, type: MVT::i32);
855 }
856
857 bool isAISrc_128B16() const {
858 return isRegOrInlineNoMods(RCID: AMDGPU::AReg_128RegClassID, type: MVT::i16);
859 }
860
861 bool isAISrc_128V2B16() const { return isAISrc_128B16(); }
862
863 bool isAISrc_128_f32() const {
864 return isRegOrInlineNoMods(RCID: AMDGPU::AReg_128RegClassID, type: MVT::f32);
865 }
866
867 bool isAISrc_128F16() const {
868 return isRegOrInlineNoMods(RCID: AMDGPU::AReg_128RegClassID, type: MVT::f16);
869 }
870
871 bool isAISrc_128V2F16() const {
872 return isAISrc_128F16() || isAISrc_128_b32();
873 }
874
875 bool isVISrc_128_bf16() const {
876 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_128RegClassID, type: MVT::bf16);
877 }
878
879 bool isVISrc_128_f16() const {
880 return isRegOrInlineNoMods(RCID: AMDGPU::VReg_128RegClassID, type: MVT::f16);
881 }
882
883 bool isVISrc_128V2F16() const {
884 return isVISrc_128_f16() || isVISrc_128_b32();
885 }
886
887 bool isAISrc_256B64() const {
888 return isRegOrInlineNoMods(RCID: AMDGPU::AReg_256RegClassID, type: MVT::i64);
889 }
890
891 bool isAISrc_256_f64() const {
892 return isRegOrInlineNoMods(RCID: AMDGPU::AReg_256RegClassID, type: MVT::f64);
893 }
894
895 bool isAISrc_512_b32() const {
896 return isRegOrInlineNoMods(RCID: AMDGPU::AReg_512RegClassID, type: MVT::i32);
897 }
898
899 bool isAISrc_512B16() const {
900 return isRegOrInlineNoMods(RCID: AMDGPU::AReg_512RegClassID, type: MVT::i16);
901 }
902
903 bool isAISrc_512V2B16() const { return isAISrc_512B16(); }
904
905 bool isAISrc_512_f32() const {
906 return isRegOrInlineNoMods(RCID: AMDGPU::AReg_512RegClassID, type: MVT::f32);
907 }
908
909 bool isAISrc_512F16() const {
910 return isRegOrInlineNoMods(RCID: AMDGPU::AReg_512RegClassID, type: MVT::f16);
911 }
912
913 bool isAISrc_512V2F16() const {
914 return isAISrc_512F16() || isAISrc_512_b32();
915 }
916
917 bool isAISrc_1024_b32() const {
918 return isRegOrInlineNoMods(RCID: AMDGPU::AReg_1024RegClassID, type: MVT::i32);
919 }
920
921 bool isAISrc_1024B16() const {
922 return isRegOrInlineNoMods(RCID: AMDGPU::AReg_1024RegClassID, type: MVT::i16);
923 }
924
925 bool isAISrc_1024V2B16() const { return isAISrc_1024B16(); }
926
927 bool isAISrc_1024_f32() const {
928 return isRegOrInlineNoMods(RCID: AMDGPU::AReg_1024RegClassID, type: MVT::f32);
929 }
930
931 bool isAISrc_1024F16() const {
932 return isRegOrInlineNoMods(RCID: AMDGPU::AReg_1024RegClassID, type: MVT::f16);
933 }
934
935 bool isAISrc_1024V2F16() const {
936 return isAISrc_1024F16() || isAISrc_1024_b32();
937 }
938
939 bool isKImmFP32() const { return isLiteralImm(type: MVT::f32); }
940
941 bool isKImmFP16() const { return isLiteralImm(type: MVT::f16); }
942
943 bool isKImmFP64() const { return isLiteralImm(type: MVT::f64); }
944
945 bool isMem() const override { return false; }
946
947 bool isExpr() const { return Kind == Expression; }
948
949 bool isSOPPBrTarget() const { return isExpr() || isImm(); }
950
951 bool isSWaitCnt() const;
952 bool isDepCtr() const;
953 bool isSDelayALU() const;
954 bool isHwreg() const;
955 bool isSendMsg() const;
956 bool isWaitEvent() const;
957 bool isSplitBarrier() const;
958 bool isSwizzle() const;
959 bool isSMRDOffset8() const;
960 bool isSMEMOffset() const;
961 bool isSMRDLiteralOffset() const;
962 bool isDPP8() const;
963 bool isDPPCtrl() const;
964 bool isBLGP() const;
965 bool isGPRIdxMode() const;
966 bool isS16Imm() const;
967 bool isU16Imm() const;
968 bool isEndpgm() const;
969
970 auto getPredicate(std::function<bool(const AMDGPUOperand &Op)> P) const {
971 return [this, P]() { return P(*this); };
972 }
973
974 StringRef getToken() const {
975 assert(isToken());
976 return StringRef(Tok.Data, Tok.Length);
977 }
978
979 int64_t getImm() const {
980 assert(isImm());
981 return Imm.Val;
982 }
983
984 void setImm(int64_t Val) {
985 assert(isImm());
986 Imm.Val = Val;
987 }
988
989 ImmTy getImmTy() const {
990 assert(isImm());
991 return Imm.Type;
992 }
993
994 MCRegister getReg() const override {
995 assert(isRegKind());
996 return Reg.RegNo;
997 }
998
999 SMLoc getStartLoc() const override { return StartLoc; }
1000
1001 SMLoc getEndLoc() const override { return EndLoc; }
1002
1003 SMRange getLocRange() const { return SMRange(StartLoc, EndLoc); }
1004
1005 int getMCOpIdx() const { return MCOpIdx; }
1006
1007 Modifiers getModifiers() const {
1008 assert(isRegKind() || isImmTy(ImmTyNone));
1009 return isRegKind() ? Reg.Mods : Imm.Mods;
1010 }
1011
1012 void setModifiers(Modifiers Mods) {
1013 assert(isRegKind() || isImmTy(ImmTyNone));
1014 if (isRegKind())
1015 Reg.Mods = Mods;
1016 else
1017 Imm.Mods = Mods;
1018 }
1019
1020 bool hasModifiers() const { return getModifiers().hasModifiers(); }
1021
1022 bool hasFPModifiers() const { return getModifiers().hasFPModifiers(); }
1023
1024 bool hasIntModifiers() const { return getModifiers().hasIntModifiers(); }
1025
1026 bool isForcedLit() const {
1027 return isImmLiteral() && getModifiers().isForcedLit();
1028 }
1029
1030 bool isForcedLit64() const {
1031 return isImmLiteral() && getModifiers().isForcedLit64();
1032 }
1033
1034 uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const;
1035
1036 void addImmOperands(MCInst &Inst, unsigned N,
1037 bool ApplyModifiers = true) const;
1038
1039 void addLiteralImmOperand(MCInst &Inst, int64_t Val,
1040 bool ApplyModifiers) const;
1041
1042 void addRegOperands(MCInst &Inst, unsigned N) const;
1043
1044 void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
1045 if (isRegKind())
1046 addRegOperands(Inst, N);
1047 else
1048 addImmOperands(Inst, N);
1049 }
1050
1051 void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
1052 Modifiers Mods = getModifiers();
1053 Inst.addOperand(Op: MCOperand::createImm(Val: Mods.getModifiersOperand()));
1054 if (isRegKind()) {
1055 addRegOperands(Inst, N);
1056 } else {
1057 addImmOperands(Inst, N, ApplyModifiers: false);
1058 }
1059 }
1060
1061 void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
1062 assert(!hasIntModifiers());
1063 addRegOrImmWithInputModsOperands(Inst, N);
1064 }
1065
1066 void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
1067 assert(!hasFPModifiers());
1068 addRegOrImmWithInputModsOperands(Inst, N);
1069 }
1070
1071 void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
1072 Modifiers Mods = getModifiers();
1073 Inst.addOperand(Op: MCOperand::createImm(Val: Mods.getModifiersOperand()));
1074 assert(isRegKind());
1075 addRegOperands(Inst, N);
1076 }
1077
1078 void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
1079 assert(!hasIntModifiers());
1080 addRegWithInputModsOperands(Inst, N);
1081 }
1082
1083 void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
1084 assert(!hasFPModifiers());
1085 addRegWithInputModsOperands(Inst, N);
1086 }
1087
1088 static void printImmTy(raw_ostream &OS, ImmTy Type) {
1089 // clang-format off
1090 switch (Type) {
1091 case ImmTyNone: OS << "None"; break;
1092 case ImmTyGDS: OS << "GDS"; break;
1093 case ImmTyLDS: OS << "LDS"; break;
1094 case ImmTyOffen: OS << "Offen"; break;
1095 case ImmTyIdxen: OS << "Idxen"; break;
1096 case ImmTyAddr64: OS << "Addr64"; break;
1097 case ImmTyOffset: OS << "Offset"; break;
1098 case ImmTyInstOffset: OS << "InstOffset"; break;
1099 case ImmTyOffset0: OS << "Offset0"; break;
1100 case ImmTyOffset1: OS << "Offset1"; break;
1101 case ImmTySMEMOffsetMod: OS << "SMEMOffsetMod"; break;
1102 case ImmTyCPol: OS << "CPol"; break;
1103 case ImmTyIndexKey8bit: OS << "index_key"; break;
1104 case ImmTyIndexKey16bit: OS << "index_key"; break;
1105 case ImmTyIndexKey32bit: OS << "index_key"; break;
1106 case ImmTyTFE: OS << "TFE"; break;
1107 case ImmTyIsAsync: OS << "IsAsync"; break;
1108 case ImmTyD16: OS << "D16"; break;
1109 case ImmTyFORMAT: OS << "FORMAT"; break;
1110 case ImmTyClamp: OS << "Clamp"; break;
1111 case ImmTyOModSI: OS << "OModSI"; break;
1112 case ImmTyDPP8: OS << "DPP8"; break;
1113 case ImmTyDppCtrl: OS << "DppCtrl"; break;
1114 case ImmTyDppRowMask: OS << "DppRowMask"; break;
1115 case ImmTyDppBankMask: OS << "DppBankMask"; break;
1116 case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
1117 case ImmTyDppFI: OS << "DppFI"; break;
1118 case ImmTySDWADstSel: OS << "SDWADstSel"; break;
1119 case ImmTySDWASrc0Sel: OS << "SDWASrc0Sel"; break;
1120 case ImmTySDWASrc1Sel: OS << "SDWASrc1Sel"; break;
1121 case ImmTySDWADstUnused: OS << "SDWADstUnused"; break;
1122 case ImmTyDMask: OS << "DMask"; break;
1123 case ImmTyDim: OS << "Dim"; break;
1124 case ImmTyUNorm: OS << "UNorm"; break;
1125 case ImmTyDA: OS << "DA"; break;
1126 case ImmTyR128A16: OS << "R128A16"; break;
1127 case ImmTyA16: OS << "A16"; break;
1128 case ImmTyLWE: OS << "LWE"; break;
1129 case ImmTyOff: OS << "Off"; break;
1130 case ImmTyExpTgt: OS << "ExpTgt"; break;
1131 case ImmTyExpCompr: OS << "ExpCompr"; break;
1132 case ImmTyExpVM: OS << "ExpVM"; break;
1133 case ImmTyDone: OS << "Done"; break;
1134 case ImmTyRowEn: OS << "RowEn"; break;
1135 case ImmTyHwreg: OS << "Hwreg"; break;
1136 case ImmTySendMsg: OS << "SendMsg"; break;
1137 case ImmTyWaitEvent: OS << "WaitEvent"; break;
1138 case ImmTyInterpSlot: OS << "InterpSlot"; break;
1139 case ImmTyInterpAttr: OS << "InterpAttr"; break;
1140 case ImmTyInterpAttrChan: OS << "InterpAttrChan"; break;
1141 case ImmTyOpSel: OS << "OpSel"; break;
1142 case ImmTyOpSelHi: OS << "OpSelHi"; break;
1143 case ImmTyNegLo: OS << "NegLo"; break;
1144 case ImmTyNegHi: OS << "NegHi"; break;
1145 case ImmTySwizzle: OS << "Swizzle"; break;
1146 case ImmTyGprIdxMode: OS << "GprIdxMode"; break;
1147 case ImmTyHigh: OS << "High"; break;
1148 case ImmTyBLGP: OS << "BLGP"; break;
1149 case ImmTyCBSZ: OS << "CBSZ"; break;
1150 case ImmTyABID: OS << "ABID"; break;
1151 case ImmTyEndpgm: OS << "Endpgm"; break;
1152 case ImmTyWaitVDST: OS << "WaitVDST"; break;
1153 case ImmTyWaitEXP: OS << "WaitEXP"; break;
1154 case ImmTyWaitVAVDst: OS << "WaitVAVDst"; break;
1155 case ImmTyWaitVMVSrc: OS << "WaitVMVSrc"; break;
1156 case ImmTyBitOp3: OS << "BitOp3"; break;
1157 case ImmTyMatrixAFMT: OS << "ImmTyMatrixAFMT"; break;
1158 case ImmTyMatrixBFMT: OS << "ImmTyMatrixBFMT"; break;
1159 case ImmTyMatrixAScale: OS << "ImmTyMatrixAScale"; break;
1160 case ImmTyMatrixBScale: OS << "ImmTyMatrixBScale"; break;
1161 case ImmTyMatrixAScaleFmt: OS << "ImmTyMatrixAScaleFmt"; break;
1162 case ImmTyMatrixBScaleFmt: OS << "ImmTyMatrixBScaleFmt"; break;
1163 case ImmTyMatrixAReuse: OS << "ImmTyMatrixAReuse"; break;
1164 case ImmTyMatrixBReuse: OS << "ImmTyMatrixBReuse"; break;
1165 case ImmTyScaleSel: OS << "ScaleSel" ; break;
1166 case ImmTyByteSel: OS << "ByteSel" ; break;
1167 }
1168 // clang-format on
1169 }
1170
1171 void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
1172 switch (Kind) {
1173 case Register:
1174 OS << "<register " << AMDGPUInstPrinter::getRegisterName(Reg: getReg())
1175 << " mods: " << Reg.Mods << '>';
1176 break;
1177 case Immediate:
1178 OS << '<' << getImm();
1179 if (getImmTy() != ImmTyNone) {
1180 OS << " type: ";
1181 printImmTy(OS, Type: getImmTy());
1182 }
1183 OS << " mods: " << Imm.Mods << '>';
1184 break;
1185 case Token:
1186 OS << '\'' << getToken() << '\'';
1187 break;
1188 case Expression:
1189 OS << "<expr ";
1190 MAI.printExpr(OS, *Expr);
1191 OS << '>';
1192 break;
1193 }
1194 }
1195
1196 static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
1197 int64_t Val, SMLoc Loc,
1198 ImmTy Type = ImmTyNone,
1199 bool IsFPImm = false) {
1200 auto Op = std::make_unique<AMDGPUOperand>(args: Immediate, args&: AsmParser);
1201 Op->Imm.Val = Val;
1202 Op->Imm.IsFPImm = IsFPImm;
1203 Op->Imm.Type = Type;
1204 Op->Imm.Mods = Modifiers();
1205 Op->StartLoc = Loc;
1206 Op->EndLoc = Loc;
1207 return Op;
1208 }
1209
1210 static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
1211 StringRef Str, SMLoc Loc,
1212 bool HasExplicitEncodingSize = true) {
1213 auto Res = std::make_unique<AMDGPUOperand>(args: Token, args&: AsmParser);
1214 Res->Tok.Data = Str.data();
1215 Res->Tok.Length = Str.size();
1216 Res->StartLoc = Loc;
1217 Res->EndLoc = Loc;
1218 return Res;
1219 }
1220
1221 static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
1222 MCRegister Reg, SMLoc S, SMLoc E) {
1223 auto Op = std::make_unique<AMDGPUOperand>(args: Register, args&: AsmParser);
1224 Op->Reg.RegNo = Reg;
1225 Op->Reg.Mods = Modifiers();
1226 Op->StartLoc = S;
1227 Op->EndLoc = E;
1228 return Op;
1229 }
1230
1231 static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
1232 const class MCExpr *Expr, SMLoc S) {
1233 auto Op = std::make_unique<AMDGPUOperand>(args: Expression, args&: AsmParser);
1234 Op->Expr = Expr;
1235 Op->StartLoc = S;
1236 Op->EndLoc = S;
1237 return Op;
1238 }
1239};
1240
1241raw_ostream &operator<<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
1242 OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
1243 return OS;
1244}
1245
1246//===----------------------------------------------------------------------===//
1247// AsmParser
1248//===----------------------------------------------------------------------===//
1249
1250// TODO: define GET_SUBTARGET_FEATURE_NAME
1251#define GET_REGISTER_MATCHER
1252#include "AMDGPUGenAsmMatcher.inc"
1253#undef GET_REGISTER_MATCHER
1254#undef GET_SUBTARGET_FEATURE_NAME
1255
1256// Holds info related to the current kernel, e.g. count of SGPRs used.
1257// Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
1258// .amdgpu_hsa_kernel or at EOF.
1259class KernelScopeInfo {
1260 int SgprIndexUnusedMin = -1;
1261 int VgprIndexUnusedMin = -1;
1262 int AgprIndexUnusedMin = -1;
1263 MCContext *Ctx = nullptr;
1264 MCSubtargetInfo const *MSTI = nullptr;
1265
1266 void usesSgprAt(int i) {
1267 if (i >= SgprIndexUnusedMin) {
1268 SgprIndexUnusedMin = ++i;
1269 if (Ctx) {
1270 MCSymbol *const Sym =
1271 Ctx->getOrCreateSymbol(Name: Twine(".kernel.sgpr_count"));
1272 Sym->setVariableValue(MCConstantExpr::create(Value: SgprIndexUnusedMin, Ctx&: *Ctx));
1273 }
1274 }
1275 }
1276
1277 void usesVgprAt(int i) {
1278 if (i >= VgprIndexUnusedMin) {
1279 VgprIndexUnusedMin = ++i;
1280 if (Ctx) {
1281 MCSymbol *const Sym =
1282 Ctx->getOrCreateSymbol(Name: Twine(".kernel.vgpr_count"));
1283 int totalVGPR = getTotalNumVGPRs(has90AInsts: isGFX90A(STI: *MSTI), ArgNumAGPR: AgprIndexUnusedMin,
1284 ArgNumVGPR: VgprIndexUnusedMin);
1285 Sym->setVariableValue(MCConstantExpr::create(Value: totalVGPR, Ctx&: *Ctx));
1286 }
1287 }
1288 }
1289
1290 void usesAgprAt(int i) {
1291 // Instruction will error in AMDGPUAsmParser::matchAndEmitInstruction
1292 if (!hasMAIInsts(STI: *MSTI))
1293 return;
1294
1295 if (i >= AgprIndexUnusedMin) {
1296 AgprIndexUnusedMin = ++i;
1297 if (Ctx) {
1298 MCSymbol *const Sym =
1299 Ctx->getOrCreateSymbol(Name: Twine(".kernel.agpr_count"));
1300 Sym->setVariableValue(MCConstantExpr::create(Value: AgprIndexUnusedMin, Ctx&: *Ctx));
1301
1302 // Also update vgpr_count (dependent on agpr_count for gfx908/gfx90a)
1303 MCSymbol *const vSym =
1304 Ctx->getOrCreateSymbol(Name: Twine(".kernel.vgpr_count"));
1305 int totalVGPR = getTotalNumVGPRs(has90AInsts: isGFX90A(STI: *MSTI), ArgNumAGPR: AgprIndexUnusedMin,
1306 ArgNumVGPR: VgprIndexUnusedMin);
1307 vSym->setVariableValue(MCConstantExpr::create(Value: totalVGPR, Ctx&: *Ctx));
1308 }
1309 }
1310 }
1311
1312public:
1313 KernelScopeInfo() = default;
1314
1315 void initialize(MCContext &Context) {
1316 Ctx = &Context;
1317 MSTI = Ctx->getSubtargetInfo();
1318
1319 usesSgprAt(i: SgprIndexUnusedMin = -1);
1320 usesVgprAt(i: VgprIndexUnusedMin = -1);
1321 if (hasMAIInsts(STI: *MSTI)) {
1322 usesAgprAt(i: AgprIndexUnusedMin = -1);
1323 }
1324 }
1325
1326 void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex,
1327 unsigned RegWidth) {
1328 switch (RegKind) {
1329 case IS_SGPR:
1330 usesSgprAt(i: DwordRegIndex + divideCeil(Numerator: RegWidth, Denominator: 32) - 1);
1331 break;
1332 case IS_AGPR:
1333 usesAgprAt(i: DwordRegIndex + divideCeil(Numerator: RegWidth, Denominator: 32) - 1);
1334 break;
1335 case IS_VGPR:
1336 usesVgprAt(i: DwordRegIndex + divideCeil(Numerator: RegWidth, Denominator: 32) - 1);
1337 break;
1338 default:
1339 break;
1340 }
1341 }
1342};
1343
1344class AMDGPUAsmParser : public MCTargetAsmParser {
1345 MCAsmParser &Parser;
1346
1347 unsigned ForcedEncodingSize = 0;
1348 bool ForcedDPP = false;
1349 bool ForcedSDWA = false;
1350 KernelScopeInfo KernelScope;
1351 const unsigned HwMode;
1352 const AMDGPU::GPUKind Gfx;
1353 const AMDGPU::IsaVersion ISA;
1354
1355 /// @name Auto-generated Match Functions
1356 /// {
1357
1358#define GET_ASSEMBLER_HEADER
1359#include "AMDGPUGenAsmMatcher.inc"
1360
1361 /// }
1362
1363 /// Get size of register operand
1364 unsigned getRegOperandSize(const MCInstrDesc &Desc, unsigned OpNo) const {
1365 assert(OpNo < Desc.NumOperands);
1366 int16_t RCID = MII.getOpRegClassID(OpInfo: Desc.operands()[OpNo], HwModeId: HwMode);
1367 return getRegBitWidth(RCID) / 8;
1368 }
1369
1370 std::optional<AMDGPU::InfoSectionData> InfoData;
1371
1372 /// Whether the leading .amdgcn_target directive has been emitted to the
1373 /// output streamer yet. The emission is deferred until the first piece of
1374 /// content (instruction or kernel descriptor) so that any leading
1375 /// .amdgcn_target/.amd_amdgpu_isa directive in the source has had a chance to
1376 /// update the target ID first.
1377 bool TargetDirectiveEmitted = false;
1378
1379 /// State for checking that every kernel named in a .amdhsa_kernel directive
1380 /// begins with the required prologue instruction sequence. Because the
1381 /// directive may appear either before or after the kernel's label (it is
1382 /// normally emitted after the function body, in .rodata), validation is
1383 /// deferred to onEndOfFile(). We record an order-independent timeline of
1384 /// parsed labels and emitted instruction opcodes, plus the set of symbols
1385 /// named by .amdhsa_kernel directives, and match them up at end of file.
1386 SmallVector<unsigned> OpcodeStream;
1387 SmallVector<std::tuple<const MCSymbol *, SMLoc, unsigned>>
1388 OpcodeStreamSymbols;
1389 SmallPtrSet<const MCSymbol *, 8> AMDHSAKernelSymbols;
1390
1391 /// Verify recorded kernel prologues.
1392 void checkKernelPrologues();
1393
1394private:
1395 void createConstantSymbol(StringRef Id, int64_t Val);
1396
1397 bool ParseAsAbsoluteExpression(uint32_t &Ret);
1398 bool OutOfRangeError(SMRange Range);
1399 /// Calculate VGPR/SGPR blocks required for given target, reserved
1400 /// registers, and user-specified NextFreeXGPR values.
1401 ///
1402 /// \param Features [in] Target features, used for bug corrections.
1403 /// \param VCCUsed [in] Whether VCC special SGPR is reserved.
1404 /// \param FlatScrUsed [in] Whether FLAT_SCRATCH special SGPR is reserved.
1405 /// \param XNACKUsed [in] Whether XNACK_MASK special SGPR is reserved.
1406 /// \param EnableWavefrontSize32 [in] Value of ENABLE_WAVEFRONT_SIZE32 kernel
1407 /// descriptor field, if valid.
1408 /// \param NextFreeVGPR [in] Max VGPR number referenced, plus one.
1409 /// \param VGPRRange [in] Token range, used for VGPR diagnostics.
1410 /// \param NextFreeSGPR [in] Max SGPR number referenced, plus one.
1411 /// \param SGPRRange [in] Token range, used for SGPR diagnostics.
1412 /// \param VGPRBlocks [out] Result VGPR block count.
1413 /// \param SGPRBlocks [out] Result SGPR block count.
1414 bool calculateGPRBlocks(const FeatureBitset &Features, const MCExpr *VCCUsed,
1415 const MCExpr *FlatScrUsed, bool XNACKUsed,
1416 std::optional<bool> EnableWavefrontSize32,
1417 const MCExpr *NextFreeVGPR, SMRange VGPRRange,
1418 const MCExpr *NextFreeSGPR, SMRange SGPRRange,
1419 const MCExpr *&VGPRBlocks, const MCExpr *&SGPRBlocks);
1420 bool ParseDirectiveAMDGCNTarget();
1421 bool ParseDirectiveAMDHSACodeObjectVersion();
1422 bool ParseDirectiveAMDHSAKernel();
1423 bool ParseAMDKernelCodeTValue(StringRef ID, AMDGPUMCKernelCodeT &Header);
1424 bool ParseDirectiveAMDKernelCodeT();
1425 // TODO: Possibly make subtargetHasRegister const.
1426 bool subtargetHasRegister(const MCRegisterInfo &MRI, MCRegister Reg);
1427 bool ParseDirectiveAMDGPUHsaKernel();
1428
1429 bool ParseDirectiveISAVersion();
1430 bool ParseDirectiveHSAMetadata();
1431 bool ParseDirectivePALMetadataBegin();
1432 bool ParseDirectivePALMetadata();
1433 bool ParseDirectiveAMDGPULDS();
1434 bool ParseDirectiveAMDGPUInfo();
1435
1436 /// Common code to parse out a block of text (typically YAML) between start
1437 /// and end directives.
1438 bool ParseToEndDirective(const char *AssemblerDirectiveBegin,
1439 const char *AssemblerDirectiveEnd,
1440 std::string &CollectString);
1441
1442 bool AddNextRegisterToList(MCRegister &Reg, unsigned &RegWidth,
1443 RegisterKind RegKind, MCRegister Reg1,
1444 RegisterKind RegKind1, SMLoc Loc);
1445 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &Reg,
1446 unsigned &RegNum, unsigned &RegWidth,
1447 bool RestoreOnFailure = false);
1448 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &Reg,
1449 unsigned &RegNum, unsigned &RegWidth,
1450 SmallVectorImpl<AsmToken> &Tokens);
1451 MCRegister ParseRegularReg(RegisterKind &RegKind, unsigned &RegNum,
1452 unsigned &RegWidth,
1453 SmallVectorImpl<AsmToken> &Tokens);
1454 MCRegister ParseSpecialReg(RegisterKind &RegKind, unsigned &RegNum,
1455 unsigned &RegWidth,
1456 SmallVectorImpl<AsmToken> &Tokens);
1457 MCRegister ParseRegList(RegisterKind &RegKind, unsigned &RegNum,
1458 unsigned &RegWidth,
1459 SmallVectorImpl<AsmToken> &Tokens);
1460 bool ParseRegRange(unsigned &Num, unsigned &Width, unsigned &SubReg);
1461 MCRegister getRegularReg(RegisterKind RegKind, unsigned RegNum,
1462 unsigned SubReg, unsigned RegWidth, SMLoc Loc);
1463
1464 bool isRegister();
1465 bool isRegister(const AsmToken &Token, const AsmToken &NextToken) const;
1466 std::optional<StringRef> getGprCountSymbolName(RegisterKind RegKind);
1467 void initializeGprCountSymbol(RegisterKind RegKind);
1468 bool updateGprCountSymbols(RegisterKind RegKind, unsigned DwordRegIndex,
1469 unsigned RegWidth);
1470 void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic);
1471
1472public:
1473 enum OperandMode {
1474 OperandMode_Default,
1475 OperandMode_NSA,
1476 };
1477
1478 using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
1479
1480 AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
1481 const MCInstrInfo &MII)
1482 : MCTargetAsmParser(STI, MII), Parser(_Parser),
1483 HwMode(STI.getHwMode(type: MCSubtargetInfo::HwMode_RegInfo)),
1484 Gfx(AMDGPU::parseArchAMDGCN(CPU: STI.getCPU())),
1485 ISA(AMDGPU::getIsaVersion(GPU: STI.getCPU())) {
1486 MCAsmParserExtension::Initialize(Parser);
1487
1488 setAvailableFeatures(ComputeAvailableFeatures(FB: getFeatureBits()));
1489
1490 if (ISA.Major >= 6 && isHsaAbi(STI: getSTI())) {
1491 createConstantSymbol(Id: ".amdgcn.gfx_generation_number", Val: ISA.Major);
1492 createConstantSymbol(Id: ".amdgcn.gfx_generation_minor", Val: ISA.Minor);
1493 createConstantSymbol(Id: ".amdgcn.gfx_generation_stepping", Val: ISA.Stepping);
1494 } else {
1495 createConstantSymbol(Id: ".option.machine_version_major", Val: ISA.Major);
1496 createConstantSymbol(Id: ".option.machine_version_minor", Val: ISA.Minor);
1497 createConstantSymbol(Id: ".option.machine_version_stepping", Val: ISA.Stepping);
1498 }
1499 if (ISA.Major >= 6 && isHsaAbi(STI: getSTI())) {
1500 initializeGprCountSymbol(RegKind: IS_VGPR);
1501 initializeGprCountSymbol(RegKind: IS_SGPR);
1502 } else
1503 KernelScope.initialize(Context&: getContext());
1504
1505 for (auto [Symbol, Code] : AMDGPU::UCVersion::getGFXVersions())
1506 createConstantSymbol(Id: Symbol, Val: Code);
1507
1508 createConstantSymbol(Id: "UC_VERSION_W64_BIT", Val: 0x2000);
1509 createConstantSymbol(Id: "UC_VERSION_W32_BIT", Val: 0x4000);
1510 createConstantSymbol(Id: "UC_VERSION_MDP_BIT", Val: 0x8000);
1511 }
1512
1513 bool hasMIMG_R128() const { return AMDGPU::hasMIMG_R128(STI: getSTI()); }
1514
1515 bool hasPackedD16() const { return AMDGPU::hasPackedD16(STI: getSTI()); }
1516
1517 bool hasA16() const { return AMDGPU::hasA16(STI: getSTI()); }
1518
1519 bool hasG16() const { return AMDGPU::hasG16(STI: getSTI()); }
1520
1521 bool hasGDS() const { return AMDGPU::hasGDS(STI: getSTI()); }
1522
1523 bool isSI() const { return AMDGPU::isSI(STI: getSTI()); }
1524
1525 bool isCI() const { return AMDGPU::isCI(STI: getSTI()); }
1526
1527 bool isVI() const { return AMDGPU::isVI(STI: getSTI()); }
1528
1529 bool isGFX9() const { return AMDGPU::isGFX9(STI: getSTI()); }
1530
1531 // TODO: isGFX90A is also true for GFX940. We need to clean it.
1532 bool isGFX90A() const { return AMDGPU::isGFX90A(STI: getSTI()); }
1533
1534 bool isGFX940() const { return AMDGPU::isGFX940(STI: getSTI()); }
1535
1536 bool isGFX9Plus() const { return AMDGPU::isGFX9Plus(STI: getSTI()); }
1537
1538 bool isGFX10() const { return AMDGPU::isGFX10(STI: getSTI()); }
1539
1540 bool isGFX10Plus() const { return AMDGPU::isGFX10Plus(STI: getSTI()); }
1541
1542 bool isGFX11() const { return AMDGPU::isGFX11(STI: getSTI()); }
1543
1544 bool isGFX11Plus() const { return AMDGPU::isGFX11Plus(STI: getSTI()); }
1545
1546 bool isGFX12() const { return AMDGPU::isGFX12(STI: getSTI()); }
1547
1548 bool isGFX12Plus() const { return AMDGPU::isGFX12Plus(STI: getSTI()); }
1549
1550 bool isGFX1250() const { return AMDGPU::isGFX1250(STI: getSTI()); }
1551
1552 bool isGFX1250Plus() const { return AMDGPU::isGFX1250Plus(STI: getSTI()); }
1553
1554 bool isGFX13() const { return AMDGPU::isGFX13(STI: getSTI()); }
1555
1556 bool isGFX13Plus() const { return AMDGPU::isGFX13Plus(STI: getSTI()); }
1557
1558 bool hasBVHRayTracingInsts() const {
1559 return getFeatureBits()[AMDGPU::FeatureBVHRayTracingInsts];
1560 }
1561
1562 bool isGFX10_BEncoding() const { return AMDGPU::isGFX10_BEncoding(STI: getSTI()); }
1563
1564 bool isWave32() const { return getAvailableFeatures()[Feature_isWave32Bit]; }
1565
1566 bool isWave64() const { return getAvailableFeatures()[Feature_isWave64Bit]; }
1567
1568 bool hasInv2PiInlineImm() const {
1569 return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
1570 }
1571
1572 bool has64BitLiterals() const {
1573 return getFeatureBits()[AMDGPU::Feature64BitLiterals];
1574 }
1575
1576 bool hasFlatOffsets() const {
1577 return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
1578 }
1579
1580 bool hasTrue16Insts() const {
1581 return getFeatureBits()[AMDGPU::FeatureTrue16BitInsts];
1582 }
1583
1584 bool hasArchitectedFlatScratch() const {
1585 return getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch];
1586 }
1587
1588 bool hasSGPR102_SGPR103() const { return !isVI() && !isGFX9(); }
1589
1590 bool hasSGPR104_SGPR105() const { return isGFX10Plus(); }
1591
1592 bool hasIntClamp() const { return getFeatureBits()[AMDGPU::FeatureIntClamp]; }
1593
1594 bool hasPartialNSAEncoding() const {
1595 return getFeatureBits()[AMDGPU::FeaturePartialNSAEncoding];
1596 }
1597
1598 bool hasGloballyAddressableScratch() const {
1599 return getFeatureBits()[AMDGPU::FeatureGloballyAddressableScratch];
1600 }
1601
1602 unsigned getNSAMaxSize(bool HasSampler = false) const {
1603 return AMDGPU::getNSAMaxSize(STI: getSTI(), HasSampler);
1604 }
1605
1606 unsigned getMaxNumUserSGPRs() const {
1607 return AMDGPU::getMaxNumUserSGPRs(STI: getSTI());
1608 }
1609
1610 bool hasKernargPreload() const { return AMDGPU::hasKernargPreload(STI: getSTI()); }
1611
1612 AMDGPUTargetStreamer &getTargetStreamer() {
1613 MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
1614 return static_cast<AMDGPUTargetStreamer &>(TS);
1615 }
1616
1617 MCContext &getContext() const {
1618 // We need this const_cast because for some reason getContext() is not const
1619 // in MCAsmParser.
1620 return const_cast<AMDGPUAsmParser *>(this)->MCTargetAsmParser::getContext();
1621 }
1622
1623 const MCRegisterInfo *getMRI() const {
1624 return getContext().getRegisterInfo();
1625 }
1626
1627 const MCInstrInfo *getMII() const { return &MII; }
1628
1629 // Resolve a RegClassByHwModeUses index to a register class id for the active
1630 // HwMode; -1 if the mode has no entry.
1631 int16_t getTargetRegClass(unsigned TargetRCIdx) const {
1632 return MII.getRegClassByHwModeTable(ModeId: HwMode)[TargetRCIdx];
1633 }
1634
1635 // FIXME: This should not be used. Instead, should use queries derived from
1636 // getAvailableFeatures().
1637 const FeatureBitset &getFeatureBits() const {
1638 return getSTI().getFeatureBits();
1639 }
1640
1641 void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
1642 void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
1643 void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
1644
1645 unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
1646 bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
1647 bool isForcedDPP() const { return ForcedDPP; }
1648 bool isForcedSDWA() const { return ForcedSDWA; }
1649 ArrayRef<unsigned> getMatchedVariants() const;
1650 StringRef getMatchedVariantName() const;
1651
1652 std::unique_ptr<AMDGPUOperand> parseRegister(bool RestoreOnFailure = false);
1653 bool ParseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
1654 bool RestoreOnFailure);
1655 bool parseRegister(MCRegister &Reg, SMLoc &StartLoc, SMLoc &EndLoc) override;
1656 ParseStatus tryParseRegister(MCRegister &Reg, SMLoc &StartLoc,
1657 SMLoc &EndLoc) override;
1658 unsigned checkTargetMatchPredicate(MCInst &Inst) override;
1659 unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
1660 unsigned Kind) override;
1661 bool matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1662 OperandVector &Operands, MCStreamer &Out,
1663 uint64_t &ErrorInfo,
1664 bool MatchingInlineAsm) override;
1665 bool ParseDirective(AsmToken DirectiveID) override;
1666 void doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) override;
1667 void onEndOfFile() override;
1668 ParseStatus parseOperand(OperandVector &Operands, StringRef Mnemonic,
1669 OperandMode Mode = OperandMode_Default);
1670 StringRef parseMnemonicSuffix(StringRef Name);
1671 bool parseInstruction(ParseInstructionInfo &Info, StringRef Name,
1672 SMLoc NameLoc, OperandVector &Operands) override;
1673 // bool ProcessInstruction(MCInst &Inst);
1674
1675 ParseStatus parseTokenOp(StringRef Name, OperandVector &Operands);
1676
1677 ParseStatus parseIntWithPrefix(const char *Prefix, int64_t &Int);
1678
1679 ParseStatus
1680 parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1681 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1682 std::function<bool(int64_t &)> ConvertResult = nullptr);
1683
1684 ParseStatus parseOperandArrayWithPrefix(
1685 const char *Prefix, OperandVector &Operands,
1686 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1687 bool (*ConvertResult)(int64_t &) = nullptr);
1688
1689 ParseStatus
1690 parseNamedBit(StringRef Name, OperandVector &Operands,
1691 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1692 bool IgnoreNegative = false);
1693 unsigned getCPolKind(StringRef Id, StringRef Mnemo, bool &Disabling) const;
1694 ParseStatus parseCPol(OperandVector &Operands);
1695 ParseStatus parseScope(OperandVector &Operands, int64_t &Scope);
1696 ParseStatus parseTH(OperandVector &Operands, int64_t &TH);
1697 ParseStatus parseStringWithPrefix(StringRef Prefix, StringRef &Value,
1698 SMLoc &StringLoc);
1699 ParseStatus parseStringOrIntWithPrefix(OperandVector &Operands,
1700 StringRef Name,
1701 ArrayRef<const char *> Ids,
1702 int64_t &IntVal);
1703 ParseStatus parseStringOrIntWithPrefix(OperandVector &Operands,
1704 StringRef Name,
1705 ArrayRef<const char *> Ids,
1706 AMDGPUOperand::ImmTy Type);
1707
1708 bool isModifier();
1709 bool isOperandModifier(const AsmToken &Token,
1710 const AsmToken &NextToken) const;
1711 bool isRegOrOperandModifier(const AsmToken &Token,
1712 const AsmToken &NextToken) const;
1713 bool isNamedOperandModifier(const AsmToken &Token,
1714 const AsmToken &NextToken) const;
1715 bool isOpcodeModifierWithVal(const AsmToken &Token,
1716 const AsmToken &NextToken) const;
1717 bool parseSP3NegModifier();
1718 ParseStatus parseImm(OperandVector &Operands, bool HasSP3AbsModifier = false,
1719 LitModifier Lit = LitModifier::None);
1720 ParseStatus parseReg(OperandVector &Operands);
1721 ParseStatus parseRegOrImm(OperandVector &Operands, bool HasSP3AbsMod = false,
1722 LitModifier Lit = LitModifier::None);
1723 ParseStatus parseRegOrImmWithFPInputMods(OperandVector &Operands,
1724 bool AllowImm = true);
1725 ParseStatus parseRegOrImmWithIntInputMods(OperandVector &Operands,
1726 bool AllowImm = true);
1727 ParseStatus parseRegWithFPInputMods(OperandVector &Operands);
1728 ParseStatus parseRegWithIntInputMods(OperandVector &Operands);
1729 ParseStatus parseRsrcReg(OperandVector &Operands);
1730 ParseStatus parseVReg32OrOff(OperandVector &Operands);
1731 ParseStatus tryParseIndexKey(OperandVector &Operands,
1732 AMDGPUOperand::ImmTy ImmTy);
1733 ParseStatus parseIndexKey8bit(OperandVector &Operands);
1734 ParseStatus parseIndexKey16bit(OperandVector &Operands);
1735 ParseStatus parseIndexKey32bit(OperandVector &Operands);
1736 ParseStatus tryParseMatrixFMT(OperandVector &Operands, StringRef Name,
1737 AMDGPUOperand::ImmTy Type);
1738 ParseStatus parseMatrixAFMT(OperandVector &Operands);
1739 ParseStatus parseMatrixBFMT(OperandVector &Operands);
1740 ParseStatus tryParseMatrixScale(OperandVector &Operands, StringRef Name,
1741 AMDGPUOperand::ImmTy Type);
1742 ParseStatus parseMatrixAScale(OperandVector &Operands);
1743 ParseStatus parseMatrixBScale(OperandVector &Operands);
1744 ParseStatus tryParseMatrixScaleFmt(OperandVector &Operands, StringRef Name,
1745 AMDGPUOperand::ImmTy Type);
1746 ParseStatus parseMatrixAScaleFmt(OperandVector &Operands);
1747 ParseStatus parseMatrixBScaleFmt(OperandVector &Operands);
1748
1749 ParseStatus parseDfmtNfmt(int64_t &Format);
1750 ParseStatus parseUfmt(int64_t &Format);
1751 ParseStatus parseSymbolicSplitFormat(StringRef FormatStr, SMLoc Loc,
1752 int64_t &Format);
1753 ParseStatus parseSymbolicUnifiedFormat(StringRef FormatStr, SMLoc Loc,
1754 int64_t &Format);
1755 ParseStatus parseFORMAT(OperandVector &Operands);
1756 ParseStatus parseSymbolicOrNumericFormat(int64_t &Format);
1757 ParseStatus parseNumericFormat(int64_t &Format);
1758 ParseStatus parseFlatOffset(OperandVector &Operands);
1759 ParseStatus parseR128A16(OperandVector &Operands);
1760 ParseStatus parseBLGP(OperandVector &Operands);
1761 bool tryParseFmt(const char *Pref, int64_t MaxVal, int64_t &Val);
1762 bool matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt, StringRef FormatStr,
1763 SMLoc Loc);
1764
1765 void cvtExp(MCInst &Inst, const OperandVector &Operands);
1766
1767 bool parseCnt(int64_t &IntVal);
1768 ParseStatus parseSWaitCnt(OperandVector &Operands);
1769
1770 bool parseDepCtr(int64_t &IntVal, unsigned &Mask);
1771 void depCtrError(SMLoc Loc, int ErrorId, StringRef DepCtrName);
1772 ParseStatus parseDepCtr(OperandVector &Operands);
1773
1774 bool parseDelay(int64_t &Delay);
1775 ParseStatus parseSDelayALU(OperandVector &Operands);
1776
1777 ParseStatus parseHwreg(OperandVector &Operands);
1778
1779private:
1780 struct OperandInfoTy {
1781 SMLoc Loc;
1782 int64_t Val;
1783 bool IsSymbolic = false;
1784 bool IsDefined = false;
1785
1786 constexpr OperandInfoTy(int64_t Val) : Val(Val) {}
1787 };
1788
1789 struct StructuredOpField : OperandInfoTy {
1790 StringLiteral Id;
1791 StringLiteral Desc;
1792 unsigned Width;
1793 bool IsDefined = false;
1794
1795 constexpr StructuredOpField(StringLiteral Id, StringLiteral Desc,
1796 unsigned Width, int64_t Default)
1797 : OperandInfoTy(Default), Id(Id), Desc(Desc), Width(Width) {}
1798 virtual ~StructuredOpField() = default;
1799
1800 bool Error(AMDGPUAsmParser &Parser, const Twine &Err) const {
1801 Parser.Error(L: Loc, Msg: "invalid " + Desc + ": " + Err);
1802 return false;
1803 }
1804
1805 virtual bool validate(AMDGPUAsmParser &Parser) const {
1806 if (IsSymbolic && Val == OPR_ID_UNSUPPORTED)
1807 return Error(Parser, Err: "not supported on this GPU");
1808 if (!isUIntN(N: Width, x: Val))
1809 return Error(Parser, Err: "only " + Twine(Width) + "-bit values are legal");
1810 return true;
1811 }
1812 };
1813
1814 ParseStatus parseStructuredOpFields(ArrayRef<StructuredOpField *> Fields);
1815 bool validateStructuredOpFields(ArrayRef<const StructuredOpField *> Fields);
1816
1817 bool parseSendMsgBody(OperandInfoTy &Msg, OperandInfoTy &Op,
1818 OperandInfoTy &Stream);
1819 bool validateSendMsg(const OperandInfoTy &Msg, const OperandInfoTy &Op,
1820 const OperandInfoTy &Stream);
1821
1822 ParseStatus parseHwregFunc(OperandInfoTy &HwReg, OperandInfoTy &Offset,
1823 OperandInfoTy &Width);
1824
1825 const AMDGPUOperand &findMCOperand(const OperandVector &Operands,
1826 int MCOpIdx) const;
1827
1828 static SMLoc getLaterLoc(SMLoc a, SMLoc b);
1829
1830 SMLoc getFlatOffsetLoc(const OperandVector &Operands) const;
1831 SMLoc getSMEMOffsetLoc(const OperandVector &Operands) const;
1832 SMLoc getBLGPLoc(const OperandVector &Operands) const;
1833
1834 SMLoc getOperandLoc(const OperandVector &Operands, int MCOpIdx) const;
1835 SMLoc getOperandLoc(std::function<bool(const AMDGPUOperand &)> Test,
1836 const OperandVector &Operands) const;
1837 SMLoc getImmLoc(AMDGPUOperand::ImmTy Type,
1838 const OperandVector &Operands) const;
1839 SMLoc getInstLoc(const OperandVector &Operands) const;
1840
1841 bool validateInstruction(const MCInst &Inst, SMLoc IDLoc,
1842 const OperandVector &Operands);
1843 bool validateOffset(const MCInst &Inst, const OperandVector &Operands);
1844 bool validateFlatOffset(const MCInst &Inst, const OperandVector &Operands);
1845 bool validateSMEMOffset(const MCInst &Inst, const OperandVector &Operands);
1846 bool validateBF16InlineConst(const MCInst &Inst,
1847 const OperandVector &Operands);
1848 bool validateSOPLiteral(const MCInst &Inst, const OperandVector &Operands);
1849 bool validateConstantBusLimitations(const MCInst &Inst,
1850 const OperandVector &Operands);
1851 std::optional<unsigned> checkVOPDRegBankConstraints(const MCInst &Inst,
1852 bool AsVOPD3);
1853 bool validateVOPD(const MCInst &Inst, const OperandVector &Operands);
1854 bool tryVOPD(const MCInst &Inst);
1855 bool tryVOPD3(const MCInst &Inst);
1856 bool tryAnotherVOPDEncoding(const MCInst &Inst);
1857
1858 bool validateIntClampSupported(const MCInst &Inst);
1859 bool validateMIMGAtomicDMask(const MCInst &Inst);
1860 bool validateMIMGGatherDMask(const MCInst &Inst);
1861 bool validateMovrels(const MCInst &Inst, const OperandVector &Operands);
1862 bool validateMIMGDataSize(const MCInst &Inst, SMLoc IDLoc);
1863 bool validateMIMGAddrSize(const MCInst &Inst, SMLoc IDLoc);
1864 bool validateMIMGD16(const MCInst &Inst);
1865 bool validateMIMGDim(const MCInst &Inst, const OperandVector &Operands);
1866 bool validateTensorR128(const MCInst &Inst);
1867 bool validateMIMGMSAA(const MCInst &Inst);
1868 bool validateOpSel(const MCInst &Inst);
1869 bool validateTrue16OpSel(const MCInst &Inst);
1870 bool validateNeg(const MCInst &Inst, AMDGPU::OpName OpName);
1871 bool validateDPP(const MCInst &Inst, const OperandVector &Operands);
1872 bool validateVccOperand(MCRegister Reg) const;
1873 bool validateVOPLiteral(const MCInst &Inst, const OperandVector &Operands);
1874 bool validateMAIAccWrite(const MCInst &Inst, const OperandVector &Operands);
1875 bool validateMAISrc2(const MCInst &Inst, const OperandVector &Operands);
1876 bool validateMFMA(const MCInst &Inst, const OperandVector &Operands);
1877 bool validateAGPRLdSt(const MCInst &Inst) const;
1878 bool validateVGPRAlign(const MCInst &Inst) const;
1879 bool validateBLGP(const MCInst &Inst, const OperandVector &Operands);
1880 bool validateDS(const MCInst &Inst, const OperandVector &Operands);
1881 bool validateGWS(const MCInst &Inst, const OperandVector &Operands);
1882 bool validateDivScale(const MCInst &Inst);
1883 bool validateWaitCnt(const MCInst &Inst, const OperandVector &Operands);
1884 bool validateCoherencyBits(const MCInst &Inst, const OperandVector &Operands,
1885 SMLoc IDLoc);
1886 bool validateTHAndScopeBits(const MCInst &Inst, const OperandVector &Operands,
1887 const unsigned CPol);
1888 bool validateTFE(const MCInst &Inst, const OperandVector &Operands);
1889 bool validateLdsDirect(const MCInst &Inst, const OperandVector &Operands);
1890 bool validateWMMA(const MCInst &Inst, const OperandVector &Operands);
1891 bool validateMonitorSleep(const MCInst &Inst, const OperandVector &Operands);
1892 bool validateClusterBarrierIsFirst(const MCInst &Inst,
1893 const OperandVector &Operands);
1894 unsigned getConstantBusLimit(unsigned Opcode) const;
1895 bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
1896 bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
1897 MCRegister findImplicitSGPRReadInVOP(const MCInst &Inst) const;
1898
1899 bool isSupportedMnemo(StringRef Mnemo, const FeatureBitset &FBS);
1900 bool isSupportedMnemo(StringRef Mnemo, const FeatureBitset &FBS,
1901 ArrayRef<unsigned> Variants);
1902 bool checkUnsupportedInstruction(StringRef Name, SMLoc IDLoc);
1903
1904 bool isId(const StringRef Id) const;
1905 bool isId(const AsmToken &Token, const StringRef Id) const;
1906 bool isToken(const AsmToken::TokenKind Kind) const;
1907 StringRef getId() const;
1908 bool trySkipId(const StringRef Id);
1909 bool trySkipId(const StringRef Pref, const StringRef Id);
1910 bool trySkipId(const StringRef Id, const AsmToken::TokenKind Kind);
1911 bool trySkipToken(const AsmToken::TokenKind Kind);
1912 bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
1913 bool parseString(StringRef &Val,
1914 const StringRef ErrMsg = "expected a string");
1915 bool parseId(StringRef &Val, const StringRef ErrMsg = "");
1916
1917 void peekTokens(MutableArrayRef<AsmToken> Tokens);
1918 AsmToken::TokenKind getTokenKind() const;
1919 bool parseExpr(int64_t &Imm, StringRef Expected = "");
1920 bool parseExpr(OperandVector &Operands);
1921 StringRef getTokenStr() const;
1922 AsmToken peekToken(bool ShouldSkipSpace = true);
1923 AsmToken getToken() const;
1924 SMLoc getLoc() const;
1925 void lex();
1926
1927public:
1928 void onBeginOfFile() override;
1929 /// Emit the deferred leading .amdgcn_target directive if it has not been
1930 /// emitted yet. Called before emitting the first instruction or kernel
1931 /// descriptor.
1932 void emitTargetDirective();
1933 bool parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) override;
1934
1935 ParseStatus parseCustomOperand(OperandVector &Operands, unsigned MCK);
1936
1937 ParseStatus parseExpTgt(OperandVector &Operands);
1938 ParseStatus parseSendMsg(OperandVector &Operands);
1939 ParseStatus parseWaitEvent(OperandVector &Operands);
1940 ParseStatus parseInterpSlot(OperandVector &Operands);
1941 ParseStatus parseInterpAttr(OperandVector &Operands);
1942 ParseStatus parseSOPPBrTarget(OperandVector &Operands);
1943 ParseStatus parseBoolReg(OperandVector &Operands);
1944
1945 bool parseSwizzleOperand(int64_t &Op, const unsigned MinVal,
1946 const unsigned MaxVal, const Twine &ErrMsg,
1947 SMLoc &Loc);
1948 bool parseSwizzleOperands(const unsigned OpNum, int64_t *Op,
1949 const unsigned MinVal, const unsigned MaxVal,
1950 const StringRef ErrMsg);
1951 ParseStatus parseSwizzle(OperandVector &Operands);
1952 bool parseSwizzleOffset(int64_t &Imm);
1953 bool parseSwizzleMacro(int64_t &Imm);
1954 bool parseSwizzleQuadPerm(int64_t &Imm);
1955 bool parseSwizzleBitmaskPerm(int64_t &Imm);
1956 bool parseSwizzleBroadcast(int64_t &Imm);
1957 bool parseSwizzleSwap(int64_t &Imm);
1958 bool parseSwizzleReverse(int64_t &Imm);
1959 bool parseSwizzleFFT(int64_t &Imm);
1960 bool parseSwizzleRotate(int64_t &Imm);
1961
1962 ParseStatus parseGPRIdxMode(OperandVector &Operands);
1963 int64_t parseGPRIdxMacro();
1964
1965 void cvtMubuf(MCInst &Inst, const OperandVector &Operands) {
1966 cvtMubufImpl(Inst, Operands, IsAtomic: false);
1967 }
1968 void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) {
1969 cvtMubufImpl(Inst, Operands, IsAtomic: true);
1970 }
1971
1972 ParseStatus parseOModSI(OperandVector &Operands);
1973
1974 void cvtVOP3(MCInst &Inst, const OperandVector &Operands,
1975 OptionalImmIndexMap &OptionalIdx);
1976 void cvtScaledMFMA(MCInst &Inst, const OperandVector &Operands);
1977 void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands);
1978 void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1979 void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1980 void cvtSWMMAC(MCInst &Inst, const OperandVector &Operands);
1981
1982 void cvtVOPD(MCInst &Inst, const OperandVector &Operands);
1983 void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands,
1984 OptionalImmIndexMap &OptionalIdx);
1985 void cvtVOP3P(MCInst &Inst, const OperandVector &Operands,
1986 OptionalImmIndexMap &OptionalIdx);
1987
1988 void cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands);
1989 void cvtVINTERP(MCInst &Inst, const OperandVector &Operands);
1990 void cvtOpSelHelper(MCInst &Inst, unsigned OpSel);
1991
1992 bool parseDimId(unsigned &Encoding);
1993 ParseStatus parseDim(OperandVector &Operands);
1994 bool convertDppBoundCtrl(int64_t &BoundCtrl);
1995 ParseStatus parseDPP8(OperandVector &Operands);
1996 ParseStatus parseDPPCtrl(OperandVector &Operands);
1997 bool isSupportedDPPCtrl(StringRef Ctrl, const OperandVector &Operands);
1998 int64_t parseDPPCtrlSel(StringRef Ctrl);
1999 int64_t parseDPPCtrlPerm();
2000 void cvtDPP(MCInst &Inst, const OperandVector &Operands, bool IsDPP8 = false);
2001 void cvtDPP8(MCInst &Inst, const OperandVector &Operands) {
2002 cvtDPP(Inst, Operands, IsDPP8: true);
2003 }
2004 void cvtVOP3DPP(MCInst &Inst, const OperandVector &Operands,
2005 bool IsDPP8 = false);
2006 void cvtVOP3DPP8(MCInst &Inst, const OperandVector &Operands) {
2007 cvtVOP3DPP(Inst, Operands, IsDPP8: true);
2008 }
2009
2010 ParseStatus parseSDWASel(OperandVector &Operands, StringRef Prefix,
2011 AMDGPUOperand::ImmTy Type);
2012 ParseStatus parseSDWADstUnused(OperandVector &Operands);
2013 void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
2014 void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
2015 void cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands);
2016 void cvtSdwaVOP2e(MCInst &Inst, const OperandVector &Operands);
2017 void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
2018
2019 enum class SDWAInstType : unsigned { VOP1 = 0, VOP2 = 1, VOPC = 2 };
2020
2021 void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
2022 SDWAInstType BasicInstType, bool SkipDstVcc = false,
2023 bool SkipSrcVcc = false);
2024
2025 ParseStatus parseEndpgm(OperandVector &Operands);
2026
2027 ParseStatus parseVOPD(OperandVector &Operands);
2028};
2029
2030} // end anonymous namespace
2031
2032// May be called with integer type with equivalent bitwidth.
2033static const fltSemantics *getFltSemantics(unsigned Size) {
2034 switch (Size) {
2035 case 4:
2036 return &APFloat::IEEEsingle();
2037 case 8:
2038 return &APFloat::IEEEdouble();
2039 case 2:
2040 return &APFloat::IEEEhalf();
2041 default:
2042 llvm_unreachable("unsupported fp type");
2043 }
2044}
2045
2046static const fltSemantics *getFltSemantics(MVT VT) {
2047 return getFltSemantics(Size: VT.getScalarSizeInBits() / 8);
2048}
2049
2050static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
2051 switch (OperandType) {
2052 // When floating-point immediate is used as operand of type i16, the 32-bit
2053 // representation of the constant truncated to the 16 LSBs should be used.
2054 case AMDGPU::OPERAND_REG_IMM_INT16:
2055 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
2056 case AMDGPU::OPERAND_REG_IMM_INT32:
2057 case AMDGPU::OPERAND_REG_IMM_FP32:
2058 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
2059 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
2060 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
2061 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
2062 case AMDGPU::OPERAND_REG_IMM_V2FP32:
2063 case AMDGPU::OPERAND_REG_IMM_V2INT32:
2064 case AMDGPU::OPERAND_REG_IMM_V2INT16:
2065 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
2066 case AMDGPU::OPERAND_KIMM32:
2067 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
2068 return &APFloat::IEEEsingle();
2069 case AMDGPU::OPERAND_REG_IMM_INT64:
2070 case AMDGPU::OPERAND_REG_IMM_FP64:
2071 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
2072 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
2073 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
2074 case AMDGPU::OPERAND_REG_IMM_V2FP64:
2075 case AMDGPU::OPERAND_REG_IMM_V2INT64:
2076 case AMDGPU::OPERAND_KIMM64:
2077 return &APFloat::IEEEdouble();
2078 case AMDGPU::OPERAND_REG_IMM_FP16:
2079 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
2080 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
2081 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
2082 case AMDGPU::OPERAND_REG_IMM_V2FP16:
2083 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
2084 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
2085 case AMDGPU::OPERAND_KIMM16:
2086 return &APFloat::IEEEhalf();
2087 case AMDGPU::OPERAND_REG_IMM_BF16:
2088 case AMDGPU::OPERAND_REG_INLINE_C_BF16:
2089 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
2090 case AMDGPU::OPERAND_REG_IMM_V2BF16:
2091 return &APFloat::BFloat();
2092 default:
2093 llvm_unreachable("unsupported fp type");
2094 }
2095}
2096
2097//===----------------------------------------------------------------------===//
2098// Operand
2099//===----------------------------------------------------------------------===//
2100
2101static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
2102 bool Lost;
2103
2104 // Convert literal to single precision
2105 APFloat::opStatus Status = FPLiteral.convert(
2106 ToSemantics: *getFltSemantics(VT), RM: APFloat::rmNearestTiesToEven, losesInfo: &Lost);
2107 // We allow precision lost but not overflow or underflow
2108 if (Status != APFloat::opOK && Lost &&
2109 ((Status & APFloat::opOverflow) != 0 ||
2110 (Status & APFloat::opUnderflow) != 0)) {
2111 return false;
2112 }
2113
2114 return true;
2115}
2116
2117static bool isSafeTruncation(int64_t Val, unsigned Size) {
2118 return isUIntN(N: Size, x: Val) || isIntN(N: Size, x: Val);
2119}
2120
2121static bool isInlineableLiteralOp16(int64_t Val, MVT VT, bool HasInv2Pi) {
2122 if (VT.getScalarType() == MVT::i16)
2123 return isInlinableLiteral32(Literal: Val, HasInv2Pi);
2124
2125 if (VT.getScalarType() == MVT::f16)
2126 return AMDGPU::isInlinableLiteralFP16(Literal: Val, HasInv2Pi);
2127
2128 assert(VT.getScalarType() == MVT::bf16);
2129
2130 return AMDGPU::isInlinableLiteralBF16(Literal: Val, HasInv2Pi);
2131}
2132
2133bool AMDGPUOperand::isInlinableImm(MVT type) const {
2134
2135 // This is a hack to enable named inline values like
2136 // shared_base with both 32-bit and 64-bit operands.
2137 // Note that these values are defined as
2138 // 32-bit operands only.
2139 if (isInlineValue()) {
2140 return true;
2141 }
2142
2143 if (!isImmTy(ImmT: ImmTyNone)) {
2144 // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
2145 return false;
2146 }
2147
2148 if (getModifiers().Lit != LitModifier::None)
2149 return false;
2150
2151 // TODO: We should avoid using host float here. It would be better to
2152 // check the float bit values which is what a few other places do.
2153 // We've had bot failures before due to weird NaN support on mips hosts.
2154
2155 APInt Literal(64, Imm.Val);
2156
2157 if (Imm.IsFPImm) { // We got fp literal token
2158 if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
2159 return AMDGPU::isInlinableLiteral64(Literal: Imm.Val,
2160 HasInv2Pi: AsmParser->hasInv2PiInlineImm());
2161 }
2162
2163 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
2164 if (!canLosslesslyConvertToFPType(FPLiteral, VT: type))
2165 return false;
2166
2167 if (type.getScalarSizeInBits() == 16) {
2168 bool Lost = false;
2169 switch (type.getScalarType().SimpleTy) {
2170 default:
2171 llvm_unreachable("unknown 16-bit type");
2172 case MVT::bf16:
2173 FPLiteral.convert(ToSemantics: APFloatBase::BFloat(), RM: APFloat::rmNearestTiesToEven,
2174 losesInfo: &Lost);
2175 break;
2176 case MVT::f16:
2177 FPLiteral.convert(ToSemantics: APFloatBase::IEEEhalf(), RM: APFloat::rmNearestTiesToEven,
2178 losesInfo: &Lost);
2179 break;
2180 case MVT::i16:
2181 FPLiteral.convert(ToSemantics: APFloatBase::IEEEsingle(),
2182 RM: APFloat::rmNearestTiesToEven, losesInfo: &Lost);
2183 break;
2184 }
2185 // We need to use 32-bit representation here because when a floating-point
2186 // inline constant is used as an i16 operand, its 32-bit representation
2187 // representation will be used. We will need the 32-bit value to check if
2188 // it is FP inline constant.
2189 uint32_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
2190 return isInlineableLiteralOp16(Val: ImmVal, VT: type,
2191 HasInv2Pi: AsmParser->hasInv2PiInlineImm());
2192 }
2193
2194 // Check if single precision literal is inlinable
2195 return AMDGPU::isInlinableLiteral32(
2196 Literal: static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
2197 HasInv2Pi: AsmParser->hasInv2PiInlineImm());
2198 }
2199
2200 // We got int literal token.
2201 if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
2202 return AMDGPU::isInlinableLiteral64(Literal: Imm.Val,
2203 HasInv2Pi: AsmParser->hasInv2PiInlineImm());
2204 }
2205
2206 if (!isSafeTruncation(Val: Imm.Val, Size: type.getScalarSizeInBits())) {
2207 return false;
2208 }
2209
2210 if (type.getScalarSizeInBits() == 16) {
2211 return isInlineableLiteralOp16(
2212 Val: static_cast<int16_t>(Literal.getLoBits(numBits: 16).getSExtValue()), VT: type,
2213 HasInv2Pi: AsmParser->hasInv2PiInlineImm());
2214 }
2215
2216 return AMDGPU::isInlinableLiteral32(
2217 Literal: static_cast<int32_t>(Literal.getLoBits(numBits: 32).getZExtValue()),
2218 HasInv2Pi: AsmParser->hasInv2PiInlineImm());
2219}
2220
2221bool AMDGPUOperand::isLiteralImm(MVT type) const {
2222 // Check that this immediate can be added as literal
2223 if (!isImmTy(ImmT: ImmTyNone)) {
2224 return false;
2225 }
2226
2227 bool Allow64Bit =
2228 (type == MVT::i64 || type == MVT::f64) && AsmParser->has64BitLiterals();
2229
2230 if (!Imm.IsFPImm) {
2231 // We got int literal token.
2232
2233 if (type == MVT::f64 && hasFPModifiers()) {
2234 // Cannot apply fp modifiers to int literals preserving the same semantics
2235 // for VOP1/2/C and VOP3 because of integer truncation. To avoid
2236 // ambiguity, disable these cases.
2237 return false;
2238 }
2239
2240 unsigned Size = type.getSizeInBits();
2241 if (Size == 64) {
2242 if (Allow64Bit && !AMDGPU::isValid32BitLiteral(Val: Imm.Val, IsFP64: false))
2243 return true;
2244 Size = 32;
2245 }
2246
2247 // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
2248 // types.
2249 return isSafeTruncation(Val: Imm.Val, Size);
2250 }
2251
2252 // We got fp literal token
2253 if (type == MVT::f64) { // Expected 64-bit fp operand
2254 // We would set low 64-bits of literal to zeroes but we accept this literals
2255 return true;
2256 }
2257
2258 if (type == MVT::i64) { // Expected 64-bit int operand
2259 // We don't allow fp literals in 64-bit integer instructions. It is
2260 // unclear how we should encode them.
2261 return false;
2262 }
2263
2264 // We allow fp literals with f16x2 operands assuming that the specified
2265 // literal goes into the lower half and the upper half is zero. We also
2266 // require that the literal may be losslessly converted to f16.
2267 //
2268 // For i16x2 operands, we assume that the specified literal is encoded as a
2269 // single-precision float. This is pretty odd, but it matches SP3 and what
2270 // happens in hardware.
2271 MVT ExpectedType = (type == MVT::v2f16) ? MVT::f16
2272 : (type == MVT::v2i16) ? MVT::f32
2273 : (type == MVT::v2f32) ? MVT::f32
2274 : type;
2275
2276 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
2277 return canLosslesslyConvertToFPType(FPLiteral, VT: ExpectedType);
2278}
2279
2280bool AMDGPUOperand::isRegClassTarget(unsigned TargetRCIdx) const {
2281 if (!isRegKind())
2282 return false;
2283 int16_t RCID = AsmParser->getTargetRegClass(TargetRCIdx);
2284 return RCID >= 0 && isRegClass(RCID);
2285}
2286
2287bool AMDGPUOperand::isRegClass(unsigned RCID) const {
2288 return isRegKind() &&
2289 AsmParser->getMRI()->getRegClass(i: RCID).contains(Reg: getReg());
2290}
2291
2292bool AMDGPUOperand::isVRegWithInputMods() const {
2293 return isRegClass(RCID: AMDGPU::VGPR_32RegClassID) ||
2294 // GFX90A allows DPP on 64-bit operands.
2295 (AsmParser->getFeatureBits()[AMDGPU::FeatureDPALU_DPP] &&
2296 isRegClassTarget(TargetRCIdx: AMDGPU::VReg_64_AlignTarget));
2297}
2298
2299template <bool IsFake16>
2300bool AMDGPUOperand::isT16_Lo128VRegWithInputMods() const {
2301 return isRegClass(RCID: IsFake16 ? AMDGPU::VGPR_32_Lo128RegClassID
2302 : AMDGPU::VGPR_16_Lo128RegClassID);
2303}
2304
2305template <bool IsFake16> bool AMDGPUOperand::isT16VRegWithInputMods() const {
2306 return isRegClass(RCID: IsFake16 ? AMDGPU::VGPR_32RegClassID
2307 : AMDGPU::VGPR_16RegClassID);
2308}
2309
2310bool AMDGPUOperand::isSDWAOperand(MVT type) const {
2311 if (AsmParser->isVI())
2312 return isVReg32();
2313 if (AsmParser->isGFX9Plus())
2314 return isRegClass(RCID: AMDGPU::VS_32RegClassID) || isInlinableImm(type);
2315 return false;
2316}
2317
2318bool AMDGPUOperand::isSDWAFP16Operand() const {
2319 return isSDWAOperand(type: MVT::f16);
2320}
2321
2322bool AMDGPUOperand::isSDWAFP32Operand() const {
2323 return isSDWAOperand(type: MVT::f32);
2324}
2325
2326bool AMDGPUOperand::isSDWAInt16Operand() const {
2327 return isSDWAOperand(type: MVT::i16);
2328}
2329
2330bool AMDGPUOperand::isSDWAInt32Operand() const {
2331 return isSDWAOperand(type: MVT::i32);
2332}
2333
2334bool AMDGPUOperand::isBoolReg() const {
2335 return isReg() && ((AsmParser->isWave64() && isSCSrc_b64()) ||
2336 (AsmParser->isWave32() && isSCSrc_b32()));
2337}
2338
2339uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val,
2340 unsigned Size) const {
2341 assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
2342 assert(Size == 2 || Size == 4 || Size == 8);
2343
2344 const uint64_t FpSignMask = (1ULL << (Size * 8 - 1));
2345
2346 if (Imm.Mods.Abs) {
2347 Val &= ~FpSignMask;
2348 }
2349 if (Imm.Mods.Neg) {
2350 Val ^= FpSignMask;
2351 }
2352
2353 return Val;
2354}
2355
2356void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N,
2357 bool ApplyModifiers) const {
2358 MCOpIdx = Inst.getNumOperands();
2359
2360 if (isExpr()) {
2361 Inst.addOperand(Op: MCOperand::createExpr(Val: Expr));
2362 return;
2363 }
2364
2365 if (AMDGPU::isSISrcOperand(Desc: AsmParser->getMII()->get(Opcode: Inst.getOpcode()),
2366 OpNo: Inst.getNumOperands())) {
2367 addLiteralImmOperand(Inst, Val: Imm.Val,
2368 ApplyModifiers: ApplyModifiers & isImmTy(ImmT: ImmTyNone) &&
2369 Imm.Mods.hasFPModifiers());
2370 } else {
2371 assert(!isImmTy(ImmTyNone) || !hasModifiers());
2372 Inst.addOperand(Op: MCOperand::createImm(Val: Imm.Val));
2373 }
2374}
2375
2376void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val,
2377 bool ApplyModifiers) const {
2378 const auto &InstDesc = AsmParser->getMII()->get(Opcode: Inst.getOpcode());
2379 auto OpNum = Inst.getNumOperands();
2380 // Check that this operand accepts literals
2381 assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
2382
2383 if (ApplyModifiers) {
2384 assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum));
2385 const unsigned Size =
2386 Imm.IsFPImm ? sizeof(double) : getOperandSize(Desc: InstDesc, OpNo: OpNum);
2387 Val = applyInputFPModifiers(Val, Size);
2388 }
2389
2390 APInt Literal(64, Val);
2391 uint8_t OpTy = InstDesc.operands()[OpNum].OperandType;
2392
2393 bool CanUse64BitLiterals =
2394 AsmParser->has64BitLiterals() && !SIInstrFlags::isVOP3Like(O: InstDesc);
2395 LitModifier Lit = getModifiers().Lit;
2396 MCContext &Ctx = AsmParser->getContext();
2397
2398 if (Imm.IsFPImm) { // We got fp literal token
2399 switch (OpTy) {
2400 case AMDGPU::OPERAND_REG_IMM_INT64:
2401 case AMDGPU::OPERAND_REG_IMM_FP64:
2402 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
2403 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
2404 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
2405 case AMDGPU::OPERAND_REG_IMM_V2FP64:
2406 case AMDGPU::OPERAND_REG_IMM_V2INT64:
2407 if (Lit == LitModifier::None &&
2408 AMDGPU::isInlinableLiteral64(Literal: Literal.getZExtValue(),
2409 HasInv2Pi: AsmParser->hasInv2PiInlineImm())) {
2410 Inst.addOperand(Op: MCOperand::createImm(Val: Literal.getZExtValue()));
2411 return;
2412 }
2413
2414 // Non-inlineable
2415 if (AMDGPU::isSISrcFPOperand(Desc: InstDesc,
2416 OpNo: OpNum)) { // Expected 64-bit fp operand
2417 bool HasMandatoryLiteral =
2418 AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::imm);
2419 // For fp operands we check if low 32 bits are zeros
2420 if (Literal.getLoBits(numBits: 32) != 0 &&
2421 (InstDesc.getSize() != 4 || !AsmParser->has64BitLiterals()) &&
2422 !HasMandatoryLiteral) {
2423 const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(
2424 L: Inst.getLoc(),
2425 Msg: "Can't encode literal as exact 64-bit floating-point operand. "
2426 "Low 32-bits will be set to zero");
2427 Val &= 0xffffffff00000000u;
2428 }
2429
2430 if ((OpTy == AMDGPU::OPERAND_REG_IMM_FP64 ||
2431 OpTy == AMDGPU::OPERAND_REG_INLINE_C_FP64 ||
2432 OpTy == AMDGPU::OPERAND_REG_INLINE_AC_FP64)) {
2433 if (CanUse64BitLiterals && Lit == LitModifier::None &&
2434 (isInt<32>(x: Val) || isUInt<32>(x: Val))) {
2435 // The floating-point operand will be verbalized as an
2436 // integer one. If that integer happens to fit 32 bits, on
2437 // re-assembling it will be intepreted as the high half of
2438 // the actual value, so we have to wrap it into lit64().
2439 Lit = LitModifier::Lit64;
2440 } else if (Lit == LitModifier::Lit) {
2441 // For FP64 operands lit() specifies the high half of the value.
2442 Val = Hi_32(Value: Val);
2443 }
2444 }
2445 break;
2446 }
2447
2448 // We don't allow fp literals in 64-bit integer instructions. It is
2449 // unclear how we should encode them. This case should be checked earlier
2450 // in predicate methods (isLiteralImm())
2451 llvm_unreachable("fp literal in 64-bit integer instruction.");
2452
2453 case AMDGPU::OPERAND_KIMM64:
2454 if (CanUse64BitLiterals && Lit == LitModifier::None &&
2455 (isInt<32>(x: Val) || isUInt<32>(x: Val)))
2456 Lit = LitModifier::Lit64;
2457 break;
2458
2459 case AMDGPU::OPERAND_REG_IMM_BF16:
2460 case AMDGPU::OPERAND_REG_INLINE_C_BF16:
2461 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
2462 case AMDGPU::OPERAND_REG_IMM_V2BF16:
2463 if (Lit == LitModifier::None && AsmParser->hasInv2PiInlineImm() &&
2464 Literal == 0x3fc45f306725feed) {
2465 // This is the 1/(2*pi) which is going to be truncated to bf16 with the
2466 // loss of precision. The constant represents ideomatic fp32 value of
2467 // 1/(2*pi) = 0.15915494 since bf16 is in fact fp32 with cleared low 16
2468 // bits. Prevent rounding below.
2469 Inst.addOperand(Op: MCOperand::createImm(Val: 0x3e22));
2470 return;
2471 }
2472 [[fallthrough]];
2473
2474 case AMDGPU::OPERAND_REG_IMM_INT32:
2475 case AMDGPU::OPERAND_REG_IMM_FP32:
2476 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
2477 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
2478 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
2479 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
2480 case AMDGPU::OPERAND_REG_IMM_INT16:
2481 case AMDGPU::OPERAND_REG_IMM_FP16:
2482 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
2483 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
2484 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
2485 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
2486 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
2487 case AMDGPU::OPERAND_REG_IMM_V2INT16:
2488 case AMDGPU::OPERAND_REG_IMM_V2FP16:
2489 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
2490 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
2491 case AMDGPU::OPERAND_REG_IMM_V2FP32:
2492 case AMDGPU::OPERAND_REG_IMM_V2INT32:
2493 case AMDGPU::OPERAND_KIMM32:
2494 case AMDGPU::OPERAND_KIMM16:
2495 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: {
2496 bool lost;
2497 APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
2498 // Convert literal to single precision
2499 FPLiteral.convert(ToSemantics: *getOpFltSemantics(OperandType: OpTy), RM: APFloat::rmNearestTiesToEven,
2500 losesInfo: &lost);
2501 // We allow precision lost but not overflow or underflow. This should be
2502 // checked earlier in isLiteralImm()
2503
2504 Val = FPLiteral.bitcastToAPInt().getZExtValue();
2505 break;
2506 }
2507 default:
2508 llvm_unreachable("invalid operand size");
2509 }
2510
2511 if (Lit != LitModifier::None) {
2512 Inst.addOperand(
2513 Op: MCOperand::createExpr(Val: AMDGPUMCExpr::createLit(Lit, Value: Val, Ctx)));
2514 } else {
2515 Inst.addOperand(Op: MCOperand::createImm(Val));
2516 }
2517 return;
2518 }
2519
2520 // We got int literal token.
2521 // Only sign extend inline immediates.
2522 switch (OpTy) {
2523 case AMDGPU::OPERAND_REG_IMM_INT32:
2524 case AMDGPU::OPERAND_REG_IMM_FP32:
2525 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
2526 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
2527 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
2528 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
2529 case AMDGPU::OPERAND_REG_IMM_V2INT16:
2530 case AMDGPU::OPERAND_REG_IMM_V2BF16:
2531 case AMDGPU::OPERAND_REG_IMM_V2FP16:
2532 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
2533 case AMDGPU::OPERAND_REG_IMM_V2FP32:
2534 case AMDGPU::OPERAND_REG_IMM_V2INT32:
2535 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
2536 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
2537 break;
2538
2539 case AMDGPU::OPERAND_REG_IMM_INT64:
2540 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
2541 case AMDGPU::OPERAND_REG_IMM_V2INT64:
2542 if (Lit == LitModifier::None &&
2543 AMDGPU::isInlinableLiteral64(Literal: Val, HasInv2Pi: AsmParser->hasInv2PiInlineImm())) {
2544 Inst.addOperand(Op: MCOperand::createImm(Val));
2545 return;
2546 }
2547
2548 // When the 32 MSBs are not zero (effectively means it can't be safely
2549 // truncated to uint32_t), if the target doesn't support 64-bit literals, or
2550 // the lit modifier is explicitly used, we need to truncate it to the 32
2551 // LSBs.
2552 if (!AsmParser->has64BitLiterals() || Lit == LitModifier::Lit)
2553 Val = Lo_32(Value: Val);
2554 break;
2555
2556 case AMDGPU::OPERAND_REG_IMM_FP64:
2557 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
2558 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
2559 case AMDGPU::OPERAND_REG_IMM_V2FP64:
2560 if (Lit == LitModifier::None &&
2561 AMDGPU::isInlinableLiteral64(Literal: Val, HasInv2Pi: AsmParser->hasInv2PiInlineImm())) {
2562 Inst.addOperand(Op: MCOperand::createImm(Val));
2563 return;
2564 }
2565
2566 // If the target doesn't support 64-bit literals, we need to use the
2567 // constant as the high 32 MSBs of a double-precision floating point value.
2568 if (!AsmParser->has64BitLiterals()) {
2569 Val = static_cast<uint64_t>(Val) << 32;
2570 } else {
2571 // Now the target does support 64-bit literals, there are two cases
2572 // where we still want to use src_literal encoding:
2573 // 1) explicitly forced by using lit modifier;
2574 // 2) the value is a valid 32-bit representation (signed or unsigned),
2575 // meanwhile not forced by lit64 modifier.
2576 if (Lit == LitModifier::Lit ||
2577 (Lit != LitModifier::Lit64 && (isInt<32>(x: Val) || isUInt<32>(x: Val))))
2578 Val = static_cast<uint64_t>(Val) << 32;
2579 }
2580
2581 // For FP64 operands lit() specifies the high half of the value.
2582 if (Lit == LitModifier::Lit)
2583 Val = Hi_32(Value: Val);
2584 break;
2585
2586 case AMDGPU::OPERAND_REG_IMM_INT16:
2587 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
2588 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
2589 case AMDGPU::OPERAND_REG_IMM_FP16:
2590 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
2591 case AMDGPU::OPERAND_REG_IMM_BF16:
2592 case AMDGPU::OPERAND_REG_INLINE_C_BF16:
2593 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
2594 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
2595 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
2596 case AMDGPU::OPERAND_KIMM32:
2597 case AMDGPU::OPERAND_KIMM16:
2598 break;
2599
2600 case AMDGPU::OPERAND_KIMM64:
2601 if ((isInt<32>(x: Val) || isUInt<32>(x: Val)) && Lit != LitModifier::Lit64)
2602 Val <<= 32;
2603 break;
2604
2605 default:
2606 llvm_unreachable("invalid operand type");
2607 }
2608
2609 if (Lit != LitModifier::None) {
2610 Inst.addOperand(
2611 Op: MCOperand::createExpr(Val: AMDGPUMCExpr::createLit(Lit, Value: Val, Ctx)));
2612 } else {
2613 Inst.addOperand(Op: MCOperand::createImm(Val));
2614 }
2615}
2616
2617void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
2618 MCOpIdx = Inst.getNumOperands();
2619 Inst.addOperand(
2620 Op: MCOperand::createReg(Reg: AMDGPU::getMCReg(Reg: getReg(), STI: AsmParser->getSTI())));
2621}
2622
2623bool AMDGPUOperand::isInlineValue() const {
2624 return isRegKind() && ::isInlineValue(Reg: getReg());
2625}
2626
2627//===----------------------------------------------------------------------===//
2628// AsmParser
2629//===----------------------------------------------------------------------===//
2630
2631void AMDGPUAsmParser::createConstantSymbol(StringRef Id, int64_t Val) {
2632 // TODO: make those pre-defined variables read-only.
2633 // Currently there is none suitable machinery in the core llvm-mc for this.
2634 // MCSymbol::isRedefinable is intended for another purpose, and
2635 // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
2636 MCContext &Ctx = getContext();
2637 MCSymbol *Sym = Ctx.getOrCreateSymbol(Name: Id);
2638 Sym->setVariableValue(MCConstantExpr::create(Value: Val, Ctx));
2639}
2640
2641static int getRegClass(RegisterKind Is, unsigned RegWidth) {
2642 if (Is == IS_VGPR) {
2643 switch (RegWidth) {
2644 default:
2645 return -1;
2646 case 32:
2647 return AMDGPU::VGPR_32RegClassID;
2648 case 64:
2649 return AMDGPU::VReg_64RegClassID;
2650 case 96:
2651 return AMDGPU::VReg_96RegClassID;
2652 case 128:
2653 return AMDGPU::VReg_128RegClassID;
2654 case 160:
2655 return AMDGPU::VReg_160RegClassID;
2656 case 192:
2657 return AMDGPU::VReg_192RegClassID;
2658 case 224:
2659 return AMDGPU::VReg_224RegClassID;
2660 case 256:
2661 return AMDGPU::VReg_256RegClassID;
2662 case 288:
2663 return AMDGPU::VReg_288RegClassID;
2664 case 320:
2665 return AMDGPU::VReg_320RegClassID;
2666 case 352:
2667 return AMDGPU::VReg_352RegClassID;
2668 case 384:
2669 return AMDGPU::VReg_384RegClassID;
2670 case 512:
2671 return AMDGPU::VReg_512RegClassID;
2672 case 1024:
2673 return AMDGPU::VReg_1024RegClassID;
2674 }
2675 } else if (Is == IS_TTMP) {
2676 switch (RegWidth) {
2677 default:
2678 return -1;
2679 case 32:
2680 return AMDGPU::TTMP_32RegClassID;
2681 case 64:
2682 return AMDGPU::TTMP_64RegClassID;
2683 case 128:
2684 return AMDGPU::TTMP_128RegClassID;
2685 case 256:
2686 return AMDGPU::TTMP_256RegClassID;
2687 case 512:
2688 return AMDGPU::TTMP_512RegClassID;
2689 }
2690 } else if (Is == IS_SGPR) {
2691 switch (RegWidth) {
2692 default:
2693 return -1;
2694 case 32:
2695 return AMDGPU::SGPR_32RegClassID;
2696 case 64:
2697 return AMDGPU::SGPR_64RegClassID;
2698 case 96:
2699 return AMDGPU::SGPR_96RegClassID;
2700 case 128:
2701 return AMDGPU::SGPR_128RegClassID;
2702 case 160:
2703 return AMDGPU::SGPR_160RegClassID;
2704 case 192:
2705 return AMDGPU::SGPR_192RegClassID;
2706 case 224:
2707 return AMDGPU::SGPR_224RegClassID;
2708 case 256:
2709 return AMDGPU::SGPR_256RegClassID;
2710 case 288:
2711 return AMDGPU::SGPR_288RegClassID;
2712 case 320:
2713 return AMDGPU::SGPR_320RegClassID;
2714 case 352:
2715 return AMDGPU::SGPR_352RegClassID;
2716 case 384:
2717 return AMDGPU::SGPR_384RegClassID;
2718 case 512:
2719 return AMDGPU::SGPR_512RegClassID;
2720 }
2721 } else if (Is == IS_AGPR) {
2722 switch (RegWidth) {
2723 default:
2724 return -1;
2725 case 32:
2726 return AMDGPU::AGPR_32RegClassID;
2727 case 64:
2728 return AMDGPU::AReg_64RegClassID;
2729 case 96:
2730 return AMDGPU::AReg_96RegClassID;
2731 case 128:
2732 return AMDGPU::AReg_128RegClassID;
2733 case 160:
2734 return AMDGPU::AReg_160RegClassID;
2735 case 192:
2736 return AMDGPU::AReg_192RegClassID;
2737 case 224:
2738 return AMDGPU::AReg_224RegClassID;
2739 case 256:
2740 return AMDGPU::AReg_256RegClassID;
2741 case 288:
2742 return AMDGPU::AReg_288RegClassID;
2743 case 320:
2744 return AMDGPU::AReg_320RegClassID;
2745 case 352:
2746 return AMDGPU::AReg_352RegClassID;
2747 case 384:
2748 return AMDGPU::AReg_384RegClassID;
2749 case 512:
2750 return AMDGPU::AReg_512RegClassID;
2751 case 1024:
2752 return AMDGPU::AReg_1024RegClassID;
2753 }
2754 }
2755 return -1;
2756}
2757
2758static MCRegister getSpecialRegForName(StringRef RegName) {
2759 return StringSwitch<unsigned>(RegName)
2760 .Case(S: "exec", Value: AMDGPU::EXEC)
2761 .Case(S: "vcc", Value: AMDGPU::VCC)
2762 .Case(S: "flat_scratch", Value: AMDGPU::FLAT_SCR)
2763 .Case(S: "xnack_mask", Value: AMDGPU::XNACK_MASK)
2764 .Case(S: "shared_base", Value: AMDGPU::SRC_SHARED_BASE)
2765 .Case(S: "src_shared_base", Value: AMDGPU::SRC_SHARED_BASE)
2766 .Case(S: "shared_limit", Value: AMDGPU::SRC_SHARED_LIMIT)
2767 .Case(S: "src_shared_limit", Value: AMDGPU::SRC_SHARED_LIMIT)
2768 .Case(S: "private_base", Value: AMDGPU::SRC_PRIVATE_BASE)
2769 .Case(S: "src_private_base", Value: AMDGPU::SRC_PRIVATE_BASE)
2770 .Case(S: "private_limit", Value: AMDGPU::SRC_PRIVATE_LIMIT)
2771 .Case(S: "src_private_limit", Value: AMDGPU::SRC_PRIVATE_LIMIT)
2772 .Case(S: "src_flat_scratch_base_lo", Value: AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)
2773 .Case(S: "src_flat_scratch_base_hi", Value: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)
2774 .Case(S: "pops_exiting_wave_id", Value: AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2775 .Case(S: "src_pops_exiting_wave_id", Value: AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2776 .Case(S: "lds_direct", Value: AMDGPU::LDS_DIRECT)
2777 .Case(S: "src_lds_direct", Value: AMDGPU::LDS_DIRECT)
2778 .Case(S: "m0", Value: AMDGPU::M0)
2779 .Case(S: "vccz", Value: AMDGPU::SRC_VCCZ)
2780 .Case(S: "src_vccz", Value: AMDGPU::SRC_VCCZ)
2781 .Case(S: "execz", Value: AMDGPU::SRC_EXECZ)
2782 .Case(S: "src_execz", Value: AMDGPU::SRC_EXECZ)
2783 .Case(S: "scc", Value: AMDGPU::SRC_SCC)
2784 .Case(S: "src_scc", Value: AMDGPU::SRC_SCC)
2785 .Case(S: "tba", Value: AMDGPU::TBA)
2786 .Case(S: "tma", Value: AMDGPU::TMA)
2787 .Case(S: "flat_scratch_lo", Value: AMDGPU::FLAT_SCR_LO)
2788 .Case(S: "flat_scratch_hi", Value: AMDGPU::FLAT_SCR_HI)
2789 .Case(S: "xnack_mask_lo", Value: AMDGPU::XNACK_MASK_LO)
2790 .Case(S: "xnack_mask_hi", Value: AMDGPU::XNACK_MASK_HI)
2791 .Case(S: "vcc_lo", Value: AMDGPU::VCC_LO)
2792 .Case(S: "vcc_hi", Value: AMDGPU::VCC_HI)
2793 .Case(S: "exec_lo", Value: AMDGPU::EXEC_LO)
2794 .Case(S: "exec_hi", Value: AMDGPU::EXEC_HI)
2795 .Case(S: "tma_lo", Value: AMDGPU::TMA_LO)
2796 .Case(S: "tma_hi", Value: AMDGPU::TMA_HI)
2797 .Case(S: "tba_lo", Value: AMDGPU::TBA_LO)
2798 .Case(S: "tba_hi", Value: AMDGPU::TBA_HI)
2799 .Case(S: "pc", Value: AMDGPU::PC_REG)
2800 .Case(S: "null", Value: AMDGPU::SGPR_NULL)
2801 .Default(Value: AMDGPU::NoRegister);
2802}
2803
2804bool AMDGPUAsmParser::ParseRegister(MCRegister &RegNo, SMLoc &StartLoc,
2805 SMLoc &EndLoc, bool RestoreOnFailure) {
2806 auto R = parseRegister();
2807 if (!R)
2808 return true;
2809 assert(R->isReg());
2810 RegNo = R->getReg();
2811 StartLoc = R->getStartLoc();
2812 EndLoc = R->getEndLoc();
2813 return false;
2814}
2815
2816bool AMDGPUAsmParser::parseRegister(MCRegister &Reg, SMLoc &StartLoc,
2817 SMLoc &EndLoc) {
2818 return ParseRegister(RegNo&: Reg, StartLoc, EndLoc, /*RestoreOnFailure=*/false);
2819}
2820
2821ParseStatus AMDGPUAsmParser::tryParseRegister(MCRegister &Reg, SMLoc &StartLoc,
2822 SMLoc &EndLoc) {
2823 bool Result = ParseRegister(RegNo&: Reg, StartLoc, EndLoc, /*RestoreOnFailure=*/true);
2824 bool PendingErrors = getParser().hasPendingError();
2825 getParser().clearPendingErrors();
2826 if (PendingErrors)
2827 return ParseStatus::Failure;
2828 if (Result)
2829 return ParseStatus::NoMatch;
2830 return ParseStatus::Success;
2831}
2832
2833bool AMDGPUAsmParser::AddNextRegisterToList(MCRegister &Reg, unsigned &RegWidth,
2834 RegisterKind RegKind,
2835 MCRegister Reg1,
2836 RegisterKind RegKind1, SMLoc Loc) {
2837 // Allow VCC_LO/HI at the end of SGPR lists.
2838 if (RegKind == IS_SGPR) {
2839 unsigned RegIdx = (Reg - AMDGPU::SGPR0) + RegWidth / 32;
2840 if ((RegIdx == 106 && Reg1 == AMDGPU::VCC_LO) ||
2841 (RegIdx == 107 && Reg1 == AMDGPU::VCC_HI)) {
2842 RegWidth += 32;
2843 return true;
2844 }
2845 }
2846
2847 if (RegKind != RegKind1) {
2848 Error(L: Loc, Msg: "registers in a list must be of the same kind");
2849 return false;
2850 }
2851
2852 switch (RegKind) {
2853 case IS_SPECIAL:
2854 if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
2855 Reg = AMDGPU::EXEC;
2856 RegWidth = 64;
2857 return true;
2858 }
2859 if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
2860 Reg = AMDGPU::FLAT_SCR;
2861 RegWidth = 64;
2862 return true;
2863 }
2864 if (Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
2865 Reg = AMDGPU::XNACK_MASK;
2866 RegWidth = 64;
2867 return true;
2868 }
2869 if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
2870 Reg = AMDGPU::VCC;
2871 RegWidth = 64;
2872 return true;
2873 }
2874 if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
2875 Reg = AMDGPU::TBA;
2876 RegWidth = 64;
2877 return true;
2878 }
2879 if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
2880 Reg = AMDGPU::TMA;
2881 RegWidth = 64;
2882 return true;
2883 }
2884 Error(L: Loc, Msg: "register does not fit in the list");
2885 return false;
2886 case IS_VGPR:
2887 case IS_SGPR:
2888 case IS_AGPR:
2889 case IS_TTMP:
2890 if (Reg1 != Reg + RegWidth / 32) {
2891 Error(L: Loc, Msg: "registers in a list must have consecutive indices");
2892 return false;
2893 }
2894 RegWidth += 32;
2895 return true;
2896 default:
2897 llvm_unreachable("unexpected register kind");
2898 }
2899}
2900
2901struct RegInfo {
2902 StringLiteral Name;
2903 RegisterKind Kind;
2904};
2905
2906static constexpr RegInfo RegularRegisters[] = {
2907 {.Name: {"v"}, .Kind: IS_VGPR}, {.Name: {"s"}, .Kind: IS_SGPR}, {.Name: {"ttmp"}, .Kind: IS_TTMP},
2908 {.Name: {"acc"}, .Kind: IS_AGPR}, {.Name: {"a"}, .Kind: IS_AGPR},
2909};
2910
2911static bool isRegularReg(RegisterKind Kind) {
2912 return Kind == IS_VGPR || Kind == IS_SGPR || Kind == IS_TTMP ||
2913 Kind == IS_AGPR;
2914}
2915
2916static const RegInfo *getRegularRegInfo(StringRef Str) {
2917 for (const RegInfo &Reg : RegularRegisters)
2918 if (Str.starts_with(Prefix: Reg.Name))
2919 return &Reg;
2920 return nullptr;
2921}
2922
2923static bool getRegNum(StringRef Str, unsigned &Num) {
2924 return !Str.getAsInteger(Radix: 10, Result&: Num);
2925}
2926
2927bool AMDGPUAsmParser::isRegister(const AsmToken &Token,
2928 const AsmToken &NextToken) const {
2929
2930 // A list of consecutive registers: [s0,s1,s2,s3]
2931 if (Token.is(K: AsmToken::LBrac))
2932 return true;
2933
2934 if (!Token.is(K: AsmToken::Identifier))
2935 return false;
2936
2937 // A single register like s0 or a range of registers like s[0:1]
2938
2939 StringRef Str = Token.getString();
2940 const RegInfo *Reg = getRegularRegInfo(Str);
2941 if (Reg) {
2942 StringRef RegName = Reg->Name;
2943 StringRef RegSuffix = Str.substr(Start: RegName.size());
2944 if (!RegSuffix.empty()) {
2945 RegSuffix.consume_back(Suffix: ".l");
2946 RegSuffix.consume_back(Suffix: ".h");
2947 unsigned Num;
2948 // A single register with an index: rXX
2949 if (getRegNum(Str: RegSuffix, Num))
2950 return true;
2951 } else {
2952 // A range of registers: r[XX:YY].
2953 if (NextToken.is(K: AsmToken::LBrac))
2954 return true;
2955 }
2956 }
2957
2958 return getSpecialRegForName(RegName: Str).isValid();
2959}
2960
2961bool AMDGPUAsmParser::isRegister() {
2962 return isRegister(Token: getToken(), NextToken: peekToken());
2963}
2964
2965MCRegister AMDGPUAsmParser::getRegularReg(RegisterKind RegKind, unsigned RegNum,
2966 unsigned SubReg, unsigned RegWidth,
2967 SMLoc Loc) {
2968 assert(isRegularReg(RegKind));
2969
2970 unsigned AlignSize = 1;
2971 if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
2972 // SGPR and TTMP registers must be aligned.
2973 // Max required alignment is 4 dwords.
2974 AlignSize = std::min(a: llvm::bit_ceil(Value: RegWidth / 32), b: 4u);
2975 }
2976
2977 if (RegNum % AlignSize != 0) {
2978 Error(L: Loc, Msg: "invalid register alignment");
2979 return MCRegister();
2980 }
2981
2982 unsigned RegIdx = RegNum / AlignSize;
2983 int RCID = getRegClass(Is: RegKind, RegWidth);
2984 if (RCID == -1) {
2985 Error(L: Loc, Msg: "invalid or unsupported register size");
2986 return MCRegister();
2987 }
2988
2989 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2990 const MCRegisterClass &RC = TRI->getRegClass(i: RCID);
2991 if (RegIdx >= RC.getNumRegs() || (RegKind == IS_VGPR && RegIdx > 255)) {
2992 Error(L: Loc, Msg: "register index is out of range");
2993 return AMDGPU::NoRegister;
2994 }
2995
2996 if (RegKind == IS_VGPR && !isGFX1250Plus() && RegIdx + RegWidth / 32 > 256) {
2997 Error(L: Loc, Msg: "register index is out of range");
2998 return MCRegister();
2999 }
3000
3001 MCRegister Reg = RC.getRegister(i: RegIdx);
3002
3003 if (SubReg) {
3004 Reg = TRI->getSubReg(Reg, Idx: SubReg);
3005
3006 // Currently all regular registers have their .l and .h subregisters, so
3007 // we should never need to generate an error here.
3008 assert(Reg && "Invalid subregister!");
3009 }
3010
3011 return Reg;
3012}
3013
3014bool AMDGPUAsmParser::ParseRegRange(unsigned &Num, unsigned &RegWidth,
3015 unsigned &SubReg) {
3016 int64_t RegLo, RegHi;
3017 if (!skipToken(Kind: AsmToken::LBrac, ErrMsg: "missing register index"))
3018 return false;
3019
3020 SMLoc FirstIdxLoc = getLoc();
3021 SMLoc SecondIdxLoc;
3022
3023 if (!parseExpr(Imm&: RegLo))
3024 return false;
3025
3026 if (trySkipToken(Kind: AsmToken::Colon)) {
3027 SecondIdxLoc = getLoc();
3028 if (!parseExpr(Imm&: RegHi))
3029 return false;
3030 } else {
3031 RegHi = RegLo;
3032 }
3033
3034 if (!skipToken(Kind: AsmToken::RBrac, ErrMsg: "expected a closing square bracket"))
3035 return false;
3036
3037 if (!isUInt<32>(x: RegLo)) {
3038 Error(L: FirstIdxLoc, Msg: "invalid register index");
3039 return false;
3040 }
3041
3042 if (!isUInt<32>(x: RegHi)) {
3043 Error(L: SecondIdxLoc, Msg: "invalid register index");
3044 return false;
3045 }
3046
3047 if (RegLo > RegHi) {
3048 Error(L: FirstIdxLoc, Msg: "first register index should not exceed second index");
3049 return false;
3050 }
3051
3052 if (RegHi == RegLo) {
3053 StringRef RegSuffix = getTokenStr();
3054 if (RegSuffix == ".l") {
3055 SubReg = AMDGPU::lo16;
3056 lex();
3057 } else if (RegSuffix == ".h") {
3058 SubReg = AMDGPU::hi16;
3059 lex();
3060 }
3061 }
3062
3063 Num = static_cast<unsigned>(RegLo);
3064 RegWidth = 32 * ((RegHi - RegLo) + 1);
3065
3066 return true;
3067}
3068
3069MCRegister AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind,
3070 unsigned &RegNum,
3071 unsigned &RegWidth,
3072 SmallVectorImpl<AsmToken> &Tokens) {
3073 assert(isToken(AsmToken::Identifier));
3074 MCRegister Reg = getSpecialRegForName(RegName: getTokenStr());
3075 if (Reg) {
3076 RegNum = 0;
3077 RegWidth = 32;
3078 RegKind = IS_SPECIAL;
3079 Tokens.push_back(Elt: getToken());
3080 lex(); // skip register name
3081 }
3082 return Reg;
3083}
3084
3085MCRegister AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
3086 unsigned &RegNum,
3087 unsigned &RegWidth,
3088 SmallVectorImpl<AsmToken> &Tokens) {
3089 assert(isToken(AsmToken::Identifier));
3090 StringRef RegName = getTokenStr();
3091 auto Loc = getLoc();
3092
3093 const RegInfo *RI = getRegularRegInfo(Str: RegName);
3094 if (!RI) {
3095 Error(L: Loc, Msg: "invalid register name");
3096 return MCRegister();
3097 }
3098
3099 Tokens.push_back(Elt: getToken());
3100 lex(); // skip register name
3101
3102 RegKind = RI->Kind;
3103 StringRef RegSuffix = RegName.substr(Start: RI->Name.size());
3104 unsigned SubReg = NoSubRegister;
3105 bool IsRange = false;
3106 if (!RegSuffix.empty()) {
3107 if (RegSuffix.consume_back(Suffix: ".l"))
3108 SubReg = AMDGPU::lo16;
3109 else if (RegSuffix.consume_back(Suffix: ".h"))
3110 SubReg = AMDGPU::hi16;
3111
3112 // Single 32-bit register: vXX.
3113 if (!getRegNum(Str: RegSuffix, Num&: RegNum)) {
3114 Error(L: Loc, Msg: "invalid register index");
3115 return MCRegister();
3116 }
3117 RegWidth = 32;
3118 } else {
3119 // Range of registers: v[XX:YY]. ":YY" is optional.
3120 IsRange = true;
3121 if (!ParseRegRange(Num&: RegNum, RegWidth, SubReg))
3122 return MCRegister();
3123 }
3124
3125 // Do not allow vcc_lo/hi be referred as s106/107.
3126 MCRegister Reg = getRegularReg(RegKind, RegNum, SubReg, RegWidth, Loc);
3127 const MCRegisterInfo &TRI = *getContext().getRegisterInfo();
3128 if (RegKind == IS_SGPR && IsRange
3129 ? (TRI.isSubRegister(RegA: Reg, RegB: VCC_LO) || TRI.isSubRegister(RegA: Reg, RegB: VCC_HI))
3130 : (Reg == VCC_LO || Reg == VCC_HI)) {
3131 Error(L: Loc, Msg: "register index is out of range");
3132 return MCRegister();
3133 }
3134
3135 return Reg;
3136}
3137
3138MCRegister AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind,
3139 unsigned &RegNum, unsigned &RegWidth,
3140 SmallVectorImpl<AsmToken> &Tokens) {
3141 MCRegister Reg;
3142 auto ListLoc = getLoc();
3143
3144 if (!skipToken(Kind: AsmToken::LBrac,
3145 ErrMsg: "expected a register or a list of registers")) {
3146 return MCRegister();
3147 }
3148
3149 // List of consecutive registers, e.g.: [s0,s1,s2,s3]
3150
3151 auto Loc = getLoc();
3152 if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth))
3153 return MCRegister();
3154 if (RegWidth != 32) {
3155 Error(L: Loc, Msg: "expected a single 32-bit register");
3156 return MCRegister();
3157 }
3158
3159 for (; trySkipToken(Kind: AsmToken::Comma);) {
3160 RegisterKind NextRegKind;
3161 MCRegister NextReg;
3162 unsigned NextRegNum, NextRegWidth;
3163 Loc = getLoc();
3164
3165 if (!ParseAMDGPURegister(RegKind&: NextRegKind, Reg&: NextReg, RegNum&: NextRegNum, RegWidth&: NextRegWidth,
3166 Tokens)) {
3167 return MCRegister();
3168 }
3169 if (NextRegWidth != 32) {
3170 Error(L: Loc, Msg: "expected a single 32-bit register");
3171 return MCRegister();
3172 }
3173 if (!AddNextRegisterToList(Reg, RegWidth, RegKind, Reg1: NextReg, RegKind1: NextRegKind,
3174 Loc))
3175 return MCRegister();
3176 }
3177
3178 if (!skipToken(Kind: AsmToken::RBrac,
3179 ErrMsg: "expected a comma or a closing square bracket")) {
3180 return MCRegister();
3181 }
3182
3183 if (isRegularReg(Kind: RegKind))
3184 Reg = getRegularReg(RegKind, RegNum, SubReg: NoSubRegister, RegWidth, Loc: ListLoc);
3185
3186 return Reg;
3187}
3188
3189bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3190 MCRegister &Reg, unsigned &RegNum,
3191 unsigned &RegWidth,
3192 SmallVectorImpl<AsmToken> &Tokens) {
3193 auto Loc = getLoc();
3194 Reg = MCRegister();
3195
3196 if (isToken(Kind: AsmToken::Identifier)) {
3197 Reg = ParseSpecialReg(RegKind, RegNum, RegWidth, Tokens);
3198 if (!Reg)
3199 Reg = ParseRegularReg(RegKind, RegNum, RegWidth, Tokens);
3200 } else {
3201 Reg = ParseRegList(RegKind, RegNum, RegWidth, Tokens);
3202 }
3203
3204 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
3205 if (!Reg) {
3206 assert(Parser.hasPendingError());
3207 return false;
3208 }
3209
3210 if (!subtargetHasRegister(MRI: *TRI, Reg)) {
3211 if (Reg == AMDGPU::SGPR_NULL) {
3212 Error(L: Loc, Msg: "'null' operand is not supported on this GPU");
3213 } else {
3214 Error(L: Loc, Msg: Twine(AMDGPUInstPrinter::getRegisterName(Reg)) +
3215 " register not available on this GPU");
3216 }
3217 return false;
3218 }
3219
3220 return true;
3221}
3222
3223bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3224 MCRegister &Reg, unsigned &RegNum,
3225 unsigned &RegWidth,
3226 bool RestoreOnFailure /*=false*/) {
3227 Reg = MCRegister();
3228
3229 SmallVector<AsmToken, 1> Tokens;
3230 if (ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, Tokens)) {
3231 if (RestoreOnFailure) {
3232 while (!Tokens.empty()) {
3233 getLexer().UnLex(Token: Tokens.pop_back_val());
3234 }
3235 }
3236 return true;
3237 }
3238 return false;
3239}
3240
3241std::optional<StringRef>
3242AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) {
3243 switch (RegKind) {
3244 case IS_VGPR:
3245 return StringRef(".amdgcn.next_free_vgpr");
3246 case IS_SGPR:
3247 return StringRef(".amdgcn.next_free_sgpr");
3248 default:
3249 return std::nullopt;
3250 }
3251}
3252
3253void AMDGPUAsmParser::initializeGprCountSymbol(RegisterKind RegKind) {
3254 auto SymbolName = getGprCountSymbolName(RegKind);
3255 assert(SymbolName && "initializing invalid register kind");
3256 MCSymbol *Sym = getContext().getOrCreateSymbol(Name: *SymbolName);
3257 Sym->setVariableValue(MCConstantExpr::create(Value: 0, Ctx&: getContext()));
3258 Sym->setRedefinable(true);
3259}
3260
3261bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind,
3262 unsigned DwordRegIndex,
3263 unsigned RegWidth) {
3264 // Symbols are only defined for GCN targets
3265 if (ISA.Major < 6)
3266 return true;
3267
3268 auto SymbolName = getGprCountSymbolName(RegKind);
3269 if (!SymbolName)
3270 return true;
3271 MCSymbol *Sym = getContext().getOrCreateSymbol(Name: *SymbolName);
3272
3273 int64_t NewMax = DwordRegIndex + divideCeil(Numerator: RegWidth, Denominator: 32) - 1;
3274 int64_t OldCount;
3275
3276 if (!Sym->isVariable())
3277 return !Error(L: getLoc(),
3278 Msg: ".amdgcn.next_free_{v,s}gpr symbols must be variable");
3279 if (!Sym->getVariableValue()->evaluateAsAbsolute(Res&: OldCount))
3280 return !Error(
3281 L: getLoc(),
3282 Msg: ".amdgcn.next_free_{v,s}gpr symbols must be absolute expressions");
3283
3284 if (OldCount <= NewMax)
3285 Sym->setVariableValue(MCConstantExpr::create(Value: NewMax + 1, Ctx&: getContext()));
3286
3287 return true;
3288}
3289
3290std::unique_ptr<AMDGPUOperand>
3291AMDGPUAsmParser::parseRegister(bool RestoreOnFailure) {
3292 const auto &Tok = getToken();
3293 SMLoc StartLoc = Tok.getLoc();
3294 SMLoc EndLoc = Tok.getEndLoc();
3295 RegisterKind RegKind;
3296 MCRegister Reg;
3297 unsigned RegNum, RegWidth;
3298
3299 if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) {
3300 return nullptr;
3301 }
3302 if (isHsaAbi(STI: getSTI())) {
3303 if (!updateGprCountSymbols(RegKind, DwordRegIndex: RegNum, RegWidth))
3304 return nullptr;
3305 } else
3306 KernelScope.usesRegister(RegKind, DwordRegIndex: RegNum, RegWidth);
3307 return AMDGPUOperand::CreateReg(AsmParser: this, Reg, S: StartLoc, E: EndLoc);
3308}
3309
3310ParseStatus AMDGPUAsmParser::parseImm(OperandVector &Operands,
3311 bool HasSP3AbsModifier, LitModifier Lit) {
3312 // TODO: add syntactic sugar for 1/(2*PI)
3313
3314 if (isRegister() || isModifier())
3315 return ParseStatus::NoMatch;
3316
3317 if (Lit == LitModifier::None) {
3318 if (trySkipId(Id: "lit"))
3319 Lit = LitModifier::Lit;
3320 else if (trySkipId(Id: "lit64"))
3321 Lit = LitModifier::Lit64;
3322
3323 if (Lit != LitModifier::None) {
3324 if (!skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after lit"))
3325 return ParseStatus::Failure;
3326 ParseStatus S = parseImm(Operands, HasSP3AbsModifier, Lit);
3327 if (S.isSuccess() &&
3328 !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected closing parentheses"))
3329 return ParseStatus::Failure;
3330 return S;
3331 }
3332 }
3333
3334 const auto &Tok = getToken();
3335 const auto &NextTok = peekToken();
3336 bool IsReal = Tok.is(K: AsmToken::Real);
3337 SMLoc S = getLoc();
3338 bool Negate = false;
3339
3340 if (!IsReal && Tok.is(K: AsmToken::Minus) && NextTok.is(K: AsmToken::Real)) {
3341 lex();
3342 IsReal = true;
3343 Negate = true;
3344 }
3345
3346 AMDGPUOperand::Modifiers Mods;
3347 Mods.Lit = Lit;
3348
3349 if (IsReal) {
3350 // Floating-point expressions are not supported.
3351 // Can only allow floating-point literals with an
3352 // optional sign.
3353
3354 StringRef Num = getTokenStr();
3355 lex();
3356
3357 APFloat RealVal(APFloat::IEEEdouble());
3358 auto roundMode = APFloat::rmNearestTiesToEven;
3359 if (errorToBool(Err: RealVal.convertFromString(Num, roundMode).takeError()))
3360 return ParseStatus::Failure;
3361 if (Negate)
3362 RealVal.changeSign();
3363
3364 Operands.push_back(
3365 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: RealVal.bitcastToAPInt().getZExtValue(),
3366 Loc: S, Type: AMDGPUOperand::ImmTyNone, IsFPImm: true));
3367 AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
3368 Op.setModifiers(Mods);
3369
3370 return ParseStatus::Success;
3371
3372 } else {
3373 int64_t IntVal;
3374 const MCExpr *Expr;
3375 SMLoc S = getLoc();
3376
3377 if (HasSP3AbsModifier) {
3378 // This is a workaround for handling expressions
3379 // as arguments of SP3 'abs' modifier, for example:
3380 // |1.0|
3381 // |-1|
3382 // |1+x|
3383 // This syntax is not compatible with syntax of standard
3384 // MC expressions (due to the trailing '|').
3385 SMLoc EndLoc;
3386 if (getParser().parsePrimaryExpr(Res&: Expr, EndLoc, TypeInfo: nullptr))
3387 return ParseStatus::Failure;
3388 } else {
3389 if (Parser.parseExpression(Res&: Expr))
3390 return ParseStatus::Failure;
3391 }
3392
3393 if (Expr->evaluateAsAbsolute(Res&: IntVal)) {
3394 if (Lit == LitModifier::Lit && !isInt<32>(x: IntVal) && !isUInt<32>(x: IntVal))
3395 return Error(L: S, Msg: "literal value out of range");
3396 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: IntVal, Loc: S));
3397 AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
3398 Op.setModifiers(Mods);
3399 } else {
3400 if (Lit != LitModifier::None)
3401 return ParseStatus::NoMatch;
3402 Operands.push_back(Elt: AMDGPUOperand::CreateExpr(AsmParser: this, Expr, S));
3403 }
3404
3405 return ParseStatus::Success;
3406 }
3407
3408 return ParseStatus::NoMatch;
3409}
3410
3411ParseStatus AMDGPUAsmParser::parseReg(OperandVector &Operands) {
3412 if (!isRegister())
3413 return ParseStatus::NoMatch;
3414
3415 if (auto R = parseRegister()) {
3416 assert(R->isReg());
3417 Operands.push_back(Elt: std::move(R));
3418 return ParseStatus::Success;
3419 }
3420 return ParseStatus::Failure;
3421}
3422
3423ParseStatus AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands,
3424 bool HasSP3AbsMod, LitModifier Lit) {
3425 ParseStatus Res = parseReg(Operands);
3426 if (!Res.isNoMatch())
3427 return Res;
3428 if (isModifier())
3429 return ParseStatus::NoMatch;
3430 return parseImm(Operands, HasSP3AbsModifier: HasSP3AbsMod, Lit);
3431}
3432
3433bool AMDGPUAsmParser::isNamedOperandModifier(const AsmToken &Token,
3434 const AsmToken &NextToken) const {
3435 if (Token.is(K: AsmToken::Identifier) && NextToken.is(K: AsmToken::LParen)) {
3436 const auto &str = Token.getString();
3437 return str == "abs" || str == "neg" || str == "sext";
3438 }
3439 return false;
3440}
3441
3442bool AMDGPUAsmParser::isOpcodeModifierWithVal(const AsmToken &Token,
3443 const AsmToken &NextToken) const {
3444 return Token.is(K: AsmToken::Identifier) && NextToken.is(K: AsmToken::Colon);
3445}
3446
3447bool AMDGPUAsmParser::isOperandModifier(const AsmToken &Token,
3448 const AsmToken &NextToken) const {
3449 return isNamedOperandModifier(Token, NextToken) || Token.is(K: AsmToken::Pipe);
3450}
3451
3452bool AMDGPUAsmParser::isRegOrOperandModifier(const AsmToken &Token,
3453 const AsmToken &NextToken) const {
3454 return isRegister(Token, NextToken) || isOperandModifier(Token, NextToken);
3455}
3456
3457// Check if this is an operand modifier or an opcode modifier
3458// which may look like an expression but it is not. We should
3459// avoid parsing these modifiers as expressions. Currently
3460// recognized sequences are:
3461// |...|
3462// abs(...)
3463// neg(...)
3464// sext(...)
3465// -reg
3466// -|...|
3467// -abs(...)
3468// name:...
3469//
3470bool AMDGPUAsmParser::isModifier() {
3471
3472 AsmToken Tok = getToken();
3473 AsmToken NextToken[2];
3474 peekTokens(Tokens: NextToken);
3475
3476 return isOperandModifier(Token: Tok, NextToken: NextToken[0]) ||
3477 (Tok.is(K: AsmToken::Minus) &&
3478 isRegOrOperandModifier(Token: NextToken[0], NextToken: NextToken[1])) ||
3479 isOpcodeModifierWithVal(Token: Tok, NextToken: NextToken[0]);
3480}
3481
3482// Check if the current token is an SP3 'neg' modifier.
3483// Currently this modifier is allowed in the following context:
3484//
3485// 1. Before a register, e.g. "-v0", "-v[...]" or "-[v0,v1]".
3486// 2. Before an 'abs' modifier: -abs(...)
3487// 3. Before an SP3 'abs' modifier: -|...|
3488//
3489// In all other cases "-" is handled as a part
3490// of an expression that follows the sign.
3491//
3492// Note: When "-" is followed by an integer literal,
3493// this is interpreted as integer negation rather
3494// than a floating-point NEG modifier applied to N.
3495// Beside being contr-intuitive, such use of floating-point
3496// NEG modifier would have resulted in different meaning
3497// of integer literals used with VOP1/2/C and VOP3,
3498// for example:
3499// v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF
3500// v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001
3501// Negative fp literals with preceding "-" are
3502// handled likewise for uniformity
3503//
3504bool AMDGPUAsmParser::parseSP3NegModifier() {
3505
3506 AsmToken NextToken[2];
3507 peekTokens(Tokens: NextToken);
3508
3509 if (isToken(Kind: AsmToken::Minus) &&
3510 (isRegister(Token: NextToken[0], NextToken: NextToken[1]) ||
3511 NextToken[0].is(K: AsmToken::Pipe) || isId(Token: NextToken[0], Id: "abs"))) {
3512 lex();
3513 return true;
3514 }
3515
3516 return false;
3517}
3518
3519ParseStatus
3520AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
3521 bool AllowImm) {
3522 bool Neg, SP3Neg;
3523 bool Abs, SP3Abs;
3524 SMLoc Loc;
3525
3526 // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
3527 if (isToken(Kind: AsmToken::Minus) && peekToken().is(K: AsmToken::Minus))
3528 return Error(L: getLoc(), Msg: "invalid syntax, expected 'neg' modifier");
3529
3530 SP3Neg = parseSP3NegModifier();
3531
3532 Loc = getLoc();
3533 Neg = trySkipId(Id: "neg");
3534 if (Neg && SP3Neg)
3535 return Error(L: Loc, Msg: "expected register or immediate");
3536 if (Neg && !skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after neg"))
3537 return ParseStatus::Failure;
3538
3539 Abs = trySkipId(Id: "abs");
3540 if (Abs && !skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after abs"))
3541 return ParseStatus::Failure;
3542
3543 LitModifier Lit = LitModifier::None;
3544 if (trySkipId(Id: "lit")) {
3545 Lit = LitModifier::Lit;
3546 if (!skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after lit"))
3547 return ParseStatus::Failure;
3548 } else if (trySkipId(Id: "lit64")) {
3549 Lit = LitModifier::Lit64;
3550 if (!skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after lit64"))
3551 return ParseStatus::Failure;
3552 if (!has64BitLiterals())
3553 return Error(L: Loc, Msg: "lit64 is not supported on this GPU");
3554 }
3555
3556 Loc = getLoc();
3557 SP3Abs = trySkipToken(Kind: AsmToken::Pipe);
3558 if (Abs && SP3Abs)
3559 return Error(L: Loc, Msg: "expected register or immediate");
3560
3561 ParseStatus Res;
3562 if (AllowImm) {
3563 Res = parseRegOrImm(Operands, HasSP3AbsMod: SP3Abs, Lit);
3564 } else {
3565 Res = parseReg(Operands);
3566 }
3567 if (!Res.isSuccess())
3568 return (SP3Neg || Neg || SP3Abs || Abs || Lit != LitModifier::None)
3569 ? ParseStatus::Failure
3570 : Res;
3571
3572 if (Lit != LitModifier::None && !Operands.back()->isImm())
3573 Error(L: Loc, Msg: "expected immediate with lit modifier");
3574
3575 if (SP3Abs && !skipToken(Kind: AsmToken::Pipe, ErrMsg: "expected vertical bar"))
3576 return ParseStatus::Failure;
3577 if (Abs && !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected closing parentheses"))
3578 return ParseStatus::Failure;
3579 if (Neg && !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected closing parentheses"))
3580 return ParseStatus::Failure;
3581 if (Lit != LitModifier::None &&
3582 !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected closing parentheses"))
3583 return ParseStatus::Failure;
3584
3585 AMDGPUOperand::Modifiers Mods;
3586 Mods.Abs = Abs || SP3Abs;
3587 Mods.Neg = Neg || SP3Neg;
3588 Mods.Lit = Lit;
3589
3590 if (Mods.hasFPModifiers() || Lit != LitModifier::None) {
3591 AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
3592 if (Op.isExpr())
3593 return Error(L: Op.getStartLoc(), Msg: "expected an absolute expression");
3594 Op.setModifiers(Mods);
3595 }
3596 return ParseStatus::Success;
3597}
3598
3599ParseStatus
3600AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
3601 bool AllowImm) {
3602 bool Sext = trySkipId(Id: "sext");
3603 if (Sext && !skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after sext"))
3604 return ParseStatus::Failure;
3605
3606 ParseStatus Res;
3607 if (AllowImm) {
3608 Res = parseRegOrImm(Operands);
3609 } else {
3610 Res = parseReg(Operands);
3611 }
3612 if (!Res.isSuccess())
3613 return Sext ? ParseStatus::Failure : Res;
3614
3615 if (Sext && !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected closing parentheses"))
3616 return ParseStatus::Failure;
3617
3618 AMDGPUOperand::Modifiers Mods;
3619 Mods.Sext = Sext;
3620
3621 if (Mods.hasIntModifiers()) {
3622 AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
3623 if (Op.isExpr())
3624 return Error(L: Op.getStartLoc(), Msg: "expected an absolute expression");
3625 Op.setModifiers(Mods);
3626 }
3627
3628 return ParseStatus::Success;
3629}
3630
3631ParseStatus AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
3632 return parseRegOrImmWithFPInputMods(Operands, AllowImm: false);
3633}
3634
3635ParseStatus AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
3636 return parseRegOrImmWithIntInputMods(Operands, AllowImm: false);
3637}
3638
3639ParseStatus AMDGPUAsmParser::parseRsrcReg(OperandVector &Operands) {
3640 // Without the marker, fall back to plain register parsing so the legacy
3641 // bare-register form (e.g. `s8`, `v8`) still assembles for indexed
3642 // buffer/image instructions.
3643 if (!trySkipId(Id: "rsrcidx"))
3644 return parseReg(Operands);
3645
3646 if (!skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after rsrcidx"))
3647 return ParseStatus::Failure;
3648
3649 SMLoc RegLoc = getLoc();
3650 std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
3651 if (!Reg)
3652 return ParseStatus::Failure;
3653
3654 // Enforce that the inner register is a valid index register. The matcher
3655 // predicate alone is not sufficient: if it fails, the matcher will fall back
3656 // to a non-indexed instruction variant whose resource operand happens to
3657 // accept the same register, silently dropping the `rsrcidx` intent.
3658 if (!Reg->isRsrcReg32())
3659 return Error(L: RegLoc, Msg: "rsrcidx operand must be a 32-bit SGPR or VGPR");
3660
3661 if (!skipToken(Kind: AsmToken::RParen, ErrMsg: "expected closing parenthesis"))
3662 return ParseStatus::Failure;
3663
3664 Operands.push_back(Elt: std::move(Reg));
3665 return ParseStatus::Success;
3666}
3667
3668ParseStatus AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
3669 auto Loc = getLoc();
3670 if (trySkipId(Id: "off")) {
3671 Operands.push_back(
3672 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: 0, Loc, Type: AMDGPUOperand::ImmTyOff, IsFPImm: false));
3673 return ParseStatus::Success;
3674 }
3675
3676 if (!isRegister())
3677 return ParseStatus::NoMatch;
3678
3679 std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
3680 if (Reg) {
3681 Operands.push_back(Elt: std::move(Reg));
3682 return ParseStatus::Success;
3683 }
3684
3685 return ParseStatus::Failure;
3686}
3687
3688unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
3689 if ((getForcedEncodingSize() == 32 && SIInstrFlags::isVOP3(O: MII, O: Inst)) ||
3690 (getForcedEncodingSize() == 64 && !SIInstrFlags::isVOP3(O: MII, O: Inst)) ||
3691 (isForcedDPP() && !SIInstrFlags::isDPP(O: MII, O: Inst)) ||
3692 (isForcedSDWA() && !SIInstrFlags::isSDWA(O: MII, O: Inst)))
3693 return Match_InvalidOperand;
3694
3695 if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3696 Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
3697 // v_mac_f32/16 allow only dst_sel == DWORD;
3698 auto OpNum =
3699 AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::dst_sel);
3700 const auto &Op = Inst.getOperand(i: OpNum);
3701 if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
3702 return Match_InvalidOperand;
3703 }
3704 }
3705
3706 // Asm can first try to match VOPD or VOPD3. By failing early here with
3707 // Match_InvalidOperand, the parser will retry parsing as VOPD3 or VOPD.
3708 // Checking later during validateInstruction does not give a chance to retry
3709 // parsing as a different encoding.
3710 if (tryAnotherVOPDEncoding(Inst))
3711 return Match_InvalidOperand;
3712
3713 return Match_Success;
3714}
3715
3716static ArrayRef<unsigned> getAllVariants() {
3717 static const unsigned Variants[] = {
3718 AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
3719 AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::SDWA9,
3720 AMDGPUAsmVariants::DPP, AMDGPUAsmVariants::VOP3_DPP};
3721
3722 return ArrayRef(Variants);
3723}
3724
3725// What asm variants we should check
3726ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
3727 if (isForcedDPP() && isForcedVOP3()) {
3728 static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3_DPP};
3729 return ArrayRef(Variants);
3730 }
3731 if (getForcedEncodingSize() == 32) {
3732 static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
3733 return ArrayRef(Variants);
3734 }
3735
3736 if (isForcedVOP3()) {
3737 static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
3738 return ArrayRef(Variants);
3739 }
3740
3741 if (isForcedSDWA()) {
3742 static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA,
3743 AMDGPUAsmVariants::SDWA9};
3744 return ArrayRef(Variants);
3745 }
3746
3747 if (isForcedDPP()) {
3748 static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
3749 return ArrayRef(Variants);
3750 }
3751
3752 return getAllVariants();
3753}
3754
3755StringRef AMDGPUAsmParser::getMatchedVariantName() const {
3756 if (isForcedDPP() && isForcedVOP3())
3757 return "e64_dpp";
3758
3759 if (getForcedEncodingSize() == 32)
3760 return "e32";
3761
3762 if (isForcedVOP3())
3763 return "e64";
3764
3765 if (isForcedSDWA())
3766 return "sdwa";
3767
3768 if (isForcedDPP())
3769 return "dpp";
3770
3771 return "";
3772}
3773
3774MCRegister
3775AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
3776 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
3777 for (MCPhysReg Reg : Desc.implicit_uses()) {
3778 switch (Reg) {
3779 case AMDGPU::FLAT_SCR:
3780 case AMDGPU::VCC:
3781 case AMDGPU::VCC_LO:
3782 case AMDGPU::VCC_HI:
3783 case AMDGPU::M0:
3784 return Reg;
3785 default:
3786 break;
3787 }
3788 }
3789 return MCRegister();
3790}
3791
3792// NB: This code is correct only when used to check constant
3793// bus limitations because GFX7 support no f16 inline constants.
3794// Note that there are no cases when a GFX7 opcode violates
3795// constant bus limitations due to the use of an f16 constant.
3796bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
3797 unsigned OpIdx) const {
3798 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
3799
3800 if (!AMDGPU::isSISrcOperand(Desc, OpNo: OpIdx) ||
3801 AMDGPU::isKImmOperand(Desc, OpNo: OpIdx)) {
3802 return false;
3803 }
3804
3805 const MCOperand &MO = Inst.getOperand(i: OpIdx);
3806
3807 int64_t Val = MO.isImm() ? MO.getImm() : getLitValue(Expr: MO.getExpr());
3808 auto OpSize = AMDGPU::getOperandSize(Desc, OpNo: OpIdx);
3809
3810 switch (OpSize) { // expected operand size
3811 case 8:
3812 return AMDGPU::isInlinableLiteral64(Literal: Val, HasInv2Pi: hasInv2PiInlineImm());
3813 case 4:
3814 return AMDGPU::isInlinableLiteral32(Literal: Val, HasInv2Pi: hasInv2PiInlineImm());
3815 case 2: {
3816 const unsigned OperandType = Desc.operands()[OpIdx].OperandType;
3817 if (OperandType == AMDGPU::OPERAND_REG_IMM_INT16 ||
3818 OperandType == AMDGPU::OPERAND_REG_INLINE_C_INT16)
3819 return AMDGPU::isInlinableLiteralI16(Literal: Val, HasInv2Pi: hasInv2PiInlineImm());
3820
3821 if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
3822 OperandType == AMDGPU::OPERAND_REG_IMM_V2INT16)
3823 return AMDGPU::isInlinableLiteralV2I16(Literal: Val);
3824
3825 if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16 ||
3826 OperandType == AMDGPU::OPERAND_REG_IMM_V2FP16)
3827 return AMDGPU::isInlinableLiteralV2F16(Literal: Val);
3828
3829 if (OperandType == AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT)
3830 return AMDGPU::isPKFMACF16InlineConstant(Literal: Val, IsGFX11Plus: isGFX11Plus());
3831
3832 if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2BF16 ||
3833 OperandType == AMDGPU::OPERAND_REG_IMM_V2BF16)
3834 return AMDGPU::isInlinableLiteralV2BF16(Literal: Val);
3835
3836 if (OperandType == AMDGPU::OPERAND_REG_IMM_FP16 ||
3837 OperandType == AMDGPU::OPERAND_REG_INLINE_C_FP16)
3838 return AMDGPU::isInlinableLiteralFP16(Literal: Val, HasInv2Pi: hasInv2PiInlineImm());
3839
3840 if (OperandType == AMDGPU::OPERAND_REG_IMM_BF16 ||
3841 OperandType == AMDGPU::OPERAND_REG_INLINE_C_BF16)
3842 return AMDGPU::isInlinableLiteralBF16(Literal: Val, HasInv2Pi: hasInv2PiInlineImm());
3843
3844 if (OperandType == AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16 ||
3845 OperandType == AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16)
3846 return false;
3847
3848 llvm_unreachable("invalid operand type");
3849 }
3850 default:
3851 llvm_unreachable("invalid operand size");
3852 }
3853}
3854
3855unsigned AMDGPUAsmParser::getConstantBusLimit(unsigned Opcode) const {
3856 if (!isGFX10Plus())
3857 return 1;
3858
3859 switch (Opcode) {
3860 // 64-bit shift instructions can use only one scalar value input
3861 case AMDGPU::V_LSHLREV_B64_e64:
3862 case AMDGPU::V_LSHLREV_B64_gfx10:
3863 case AMDGPU::V_LSHLREV_B64_e64_gfx11:
3864 case AMDGPU::V_LSHLREV_B64_e32_gfx12:
3865 case AMDGPU::V_LSHLREV_B64_e64_gfx12:
3866 case AMDGPU::V_LSHRREV_B64_e64:
3867 case AMDGPU::V_LSHRREV_B64_gfx10:
3868 case AMDGPU::V_LSHRREV_B64_e64_gfx11:
3869 case AMDGPU::V_LSHRREV_B64_e64_gfx12:
3870 case AMDGPU::V_ASHRREV_I64_e64:
3871 case AMDGPU::V_ASHRREV_I64_gfx10:
3872 case AMDGPU::V_ASHRREV_I64_e64_gfx11:
3873 case AMDGPU::V_ASHRREV_I64_e64_gfx12:
3874 case AMDGPU::V_LSHL_B64_e64:
3875 case AMDGPU::V_LSHR_B64_e64:
3876 case AMDGPU::V_ASHR_I64_e64:
3877 return 1;
3878 default:
3879 return 2;
3880 }
3881}
3882
3883constexpr unsigned MAX_SRC_OPERANDS_NUM = 6;
3884using OperandIndices = SmallVector<int16_t, MAX_SRC_OPERANDS_NUM>;
3885
3886// Get regular operand indices in the same order as specified
3887// in the instruction (but append mandatory literals to the end).
3888static OperandIndices getSrcOperandIndices(unsigned Opcode,
3889 bool AddMandatoryLiterals = false) {
3890
3891 int16_t ImmIdx =
3892 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, Name: OpName::imm) : -1;
3893
3894 if (isVOPD(Opc: Opcode)) {
3895 int16_t ImmXIdx =
3896 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, Name: OpName::immX) : -1;
3897
3898 return {getNamedOperandIdx(Opcode, Name: OpName::src0X),
3899 getNamedOperandIdx(Opcode, Name: OpName::vsrc1X),
3900 getNamedOperandIdx(Opcode, Name: OpName::vsrc2X),
3901 getNamedOperandIdx(Opcode, Name: OpName::src0Y),
3902 getNamedOperandIdx(Opcode, Name: OpName::vsrc1Y),
3903 getNamedOperandIdx(Opcode, Name: OpName::vsrc2Y),
3904 ImmXIdx,
3905 ImmIdx};
3906 }
3907
3908 return {getNamedOperandIdx(Opcode, Name: OpName::src0),
3909 getNamedOperandIdx(Opcode, Name: OpName::src1),
3910 getNamedOperandIdx(Opcode, Name: OpName::src2), ImmIdx};
3911}
3912
3913bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
3914 const MCOperand &MO = Inst.getOperand(i: OpIdx);
3915 if (MO.isImm())
3916 return !isInlineConstant(Inst, OpIdx);
3917 if (MO.isReg()) {
3918 auto Reg = MO.getReg();
3919 if (!Reg)
3920 return false;
3921 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
3922 auto PReg = mc2PseudoReg(Reg);
3923 return isSGPR(Reg: PReg, TRI) && PReg != SGPR_NULL;
3924 }
3925 return true;
3926}
3927
3928// Based on the comment for `AMDGPUInstructionSelector::selectWritelane`:
3929// Writelane is special in that it can use SGPR and M0 (which would normally
3930// count as using the constant bus twice - but in this case it is allowed since
3931// the lane selector doesn't count as a use of the constant bus). However, it is
3932// still required to abide by the 1 SGPR rule.
3933static bool checkWriteLane(const MCInst &Inst) {
3934 const unsigned Opcode = Inst.getOpcode();
3935 if (Opcode != V_WRITELANE_B32_gfx6_gfx7 && Opcode != V_WRITELANE_B32_vi)
3936 return false;
3937 const MCOperand &LaneSelOp = Inst.getOperand(i: 2);
3938 if (!LaneSelOp.isReg())
3939 return false;
3940 auto LaneSelReg = mc2PseudoReg(Reg: LaneSelOp.getReg());
3941 return LaneSelReg == M0 || LaneSelReg == M0_gfxpre11;
3942}
3943
3944bool AMDGPUAsmParser::validateConstantBusLimitations(
3945 const MCInst &Inst, const OperandVector &Operands) {
3946 const unsigned Opcode = Inst.getOpcode();
3947 const MCInstrDesc &Desc = MII.get(Opcode);
3948 MCRegister LastSGPR;
3949 unsigned ConstantBusUseCount = 0;
3950 unsigned NumLiterals = 0;
3951 unsigned LiteralSize;
3952
3953 if (!SIInstrFlags::isVOPC(O: Desc) && !SIInstrFlags::isVOP1(O: Desc) &&
3954 !SIInstrFlags::isVOP2(O: Desc) && !SIInstrFlags::isVOP3Like(O: Desc) &&
3955 !SIInstrFlags::isSDWA(O: Desc) && !isVOPD(Opc: Opcode))
3956 return true;
3957
3958 if (checkWriteLane(Inst))
3959 return true;
3960
3961 // Check special imm operands (used by madmk, etc)
3962 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::imm)) {
3963 ++NumLiterals;
3964 LiteralSize = 4;
3965 }
3966
3967 SmallDenseSet<MCRegister> SGPRsUsed;
3968 MCRegister SGPRUsed = findImplicitSGPRReadInVOP(Inst);
3969 if (SGPRUsed) {
3970 SGPRsUsed.insert(V: SGPRUsed);
3971 ++ConstantBusUseCount;
3972 }
3973
3974 OperandIndices OpIndices = getSrcOperandIndices(Opcode);
3975
3976 unsigned ConstantBusLimit = getConstantBusLimit(Opcode);
3977
3978 for (int OpIdx : OpIndices) {
3979 if (OpIdx == -1)
3980 continue;
3981
3982 const MCOperand &MO = Inst.getOperand(i: OpIdx);
3983 if (usesConstantBus(Inst, OpIdx)) {
3984 if (MO.isReg()) {
3985 LastSGPR = mc2PseudoReg(Reg: MO.getReg());
3986 // Pairs of registers with a partial intersections like these
3987 // s0, s[0:1]
3988 // flat_scratch_lo, flat_scratch
3989 // flat_scratch_lo, flat_scratch_hi
3990 // are theoretically valid but they are disabled anyway.
3991 // Note that this code mimics SIInstrInfo::verifyInstruction
3992 if (SGPRsUsed.insert(V: LastSGPR).second) {
3993 ++ConstantBusUseCount;
3994 }
3995 } else { // Expression or a literal
3996
3997 if (Desc.operands()[OpIdx].OperandType == MCOI::OPERAND_IMMEDIATE)
3998 continue; // special operand like VINTERP attr_chan
3999
4000 // An instruction may use only one literal.
4001 // This has been validated on the previous step.
4002 // See validateVOPLiteral.
4003 // This literal may be used as more than one operand.
4004 // If all these operands are of the same size,
4005 // this literal counts as one scalar value.
4006 // Otherwise it counts as 2 scalar values.
4007 // See "GFX10 Shader Programming", section 3.6.2.3.
4008
4009 unsigned Size = AMDGPU::getOperandSize(Desc, OpNo: OpIdx);
4010 if (Size < 4)
4011 Size = 4;
4012
4013 if (NumLiterals == 0) {
4014 NumLiterals = 1;
4015 LiteralSize = Size;
4016 } else if (LiteralSize != Size) {
4017 NumLiterals = 2;
4018 }
4019 }
4020 }
4021
4022 if (ConstantBusUseCount + NumLiterals > ConstantBusLimit) {
4023 Error(L: getOperandLoc(Operands, MCOpIdx: OpIdx),
4024 Msg: "invalid operand (violates constant bus restrictions)");
4025 return false;
4026 }
4027 }
4028 return true;
4029}
4030
4031std::optional<unsigned>
4032AMDGPUAsmParser::checkVOPDRegBankConstraints(const MCInst &Inst, bool AsVOPD3) {
4033
4034 const unsigned Opcode = Inst.getOpcode();
4035 if (!isVOPD(Opc: Opcode))
4036 return {};
4037
4038 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
4039
4040 auto getVRegIdx = [&](unsigned, unsigned OperandIdx) {
4041 const MCOperand &Opr = Inst.getOperand(i: OperandIdx);
4042 return (Opr.isReg() && !isSGPR(Reg: mc2PseudoReg(Reg: Opr.getReg()), TRI))
4043 ? Opr.getReg()
4044 : MCRegister();
4045 };
4046
4047 // On GFX1170+ if both OpX and OpY are V_MOV_B32 then OPY uses SRC2
4048 // source-cache.
4049 bool SkipSrc =
4050 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1170 ||
4051 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 ||
4052 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 ||
4053 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx13 ||
4054 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 ||
4055 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx13;
4056 bool AllowSameVGPR = isGFX12Plus();
4057
4058 if (AsVOPD3) { // Literal constants are not allowed with VOPD3.
4059 for (auto OpName : {OpName::src0X, OpName::src0Y}) {
4060 int I = getNamedOperandIdx(Opcode, Name: OpName);
4061 const MCOperand &Op = Inst.getOperand(i: I);
4062 if (!Op.isImm())
4063 continue;
4064 int64_t Imm = Op.getImm();
4065 if (!AMDGPU::isInlinableLiteral32(Literal: Imm, HasInv2Pi: hasInv2PiInlineImm()) &&
4066 !AMDGPU::isInlinableLiteral64(Literal: Imm, HasInv2Pi: hasInv2PiInlineImm()))
4067 return (unsigned)I;
4068 }
4069
4070 for (auto OpName : {OpName::vsrc1X, OpName::vsrc1Y, OpName::vsrc2X,
4071 OpName::vsrc2Y, OpName::imm}) {
4072 int I = getNamedOperandIdx(Opcode, Name: OpName);
4073 if (I == -1)
4074 continue;
4075 const MCOperand &Op = Inst.getOperand(i: I);
4076 if (Op.isImm())
4077 return (unsigned)I;
4078 }
4079 }
4080
4081 const auto &InstInfo = getVOPDInstInfo(VOPDOpcode: Opcode, InstrInfo: &MII);
4082 auto InvalidCompOprIdx = InstInfo.getInvalidCompOperandIndex(
4083 GetRegIdx: getVRegIdx, MRI: *TRI, SkipSrc, AllowSameVGPR, VOPD3: AsVOPD3);
4084
4085 return InvalidCompOprIdx;
4086}
4087
4088bool AMDGPUAsmParser::validateVOPD(const MCInst &Inst,
4089 const OperandVector &Operands) {
4090
4091 unsigned Opcode = Inst.getOpcode();
4092 bool AsVOPD3 = SIInstrFlags::isVOPD3(O: MII, O: Inst);
4093
4094 if (AsVOPD3) {
4095 for (const std::unique_ptr<MCParsedAsmOperand> &Operand : Operands) {
4096 AMDGPUOperand &Op = (AMDGPUOperand &)*Operand;
4097 if ((Op.isRegKind() || Op.isImmTy(ImmT: AMDGPUOperand::ImmTyNone)) &&
4098 (Op.getModifiers().getFPModifiersOperand() & SISrcMods::ABS))
4099 Error(L: Op.getStartLoc(), Msg: "ABS not allowed in VOPD3 instructions");
4100 }
4101 }
4102
4103 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3);
4104 if (!InvalidCompOprIdx.has_value())
4105 return true;
4106
4107 auto CompOprIdx = *InvalidCompOprIdx;
4108 const auto &InstInfo = getVOPDInstInfo(VOPDOpcode: Opcode, InstrInfo: &MII);
4109 auto ParsedIdx =
4110 std::max(a: InstInfo[VOPD::X].getIndexInParsedOperands(CompOprIdx),
4111 b: InstInfo[VOPD::Y].getIndexInParsedOperands(CompOprIdx));
4112 assert(ParsedIdx > 0 && ParsedIdx < Operands.size());
4113
4114 auto Loc = ((AMDGPUOperand &)*Operands[ParsedIdx]).getStartLoc();
4115 if (CompOprIdx == VOPD::Component::DST) {
4116 if (AsVOPD3)
4117 Error(L: Loc, Msg: "dst registers must be distinct");
4118 else
4119 Error(L: Loc, Msg: "one dst register must be even and the other odd");
4120 } else {
4121 auto CompSrcIdx = CompOprIdx - VOPD::Component::DST_NUM;
4122 Error(L: Loc, Msg: Twine("src") + Twine(CompSrcIdx) +
4123 " operands must use different VGPR banks");
4124 }
4125
4126 return false;
4127}
4128
4129// \returns true if \p Inst does not satisfy VOPD constraints, but can be
4130// potentially used as VOPD3 with the same operands.
4131bool AMDGPUAsmParser::tryVOPD3(const MCInst &Inst) {
4132 // First check if it fits VOPD
4133 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3: false);
4134 if (!InvalidCompOprIdx.has_value())
4135 return false;
4136
4137 // Then if it fits VOPD3
4138 InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3: true);
4139 if (InvalidCompOprIdx.has_value()) {
4140 // If failed operand is dst it is better to show error about VOPD3
4141 // instruction as it has more capabilities and error message will be
4142 // more informative. If the dst is not legal for VOPD3, then it is not
4143 // legal for VOPD either.
4144 if (*InvalidCompOprIdx == VOPD::Component::DST)
4145 return true;
4146
4147 // Otherwise prefer VOPD as we may find ourselves in an awkward situation
4148 // with a conflict in tied implicit src2 of fmac and no asm operand to
4149 // to point to.
4150 return false;
4151 }
4152 return true;
4153}
4154
4155// \returns true is a VOPD3 instruction can be also represented as a shorter
4156// VOPD encoding.
4157bool AMDGPUAsmParser::tryVOPD(const MCInst &Inst) {
4158 const unsigned Opcode = Inst.getOpcode();
4159 const auto &II = getVOPDInstInfo(VOPDOpcode: Opcode, InstrInfo: &MII);
4160 unsigned EncodingFamily = AMDGPU::getVOPDEncodingFamily(ST: getSTI());
4161 if (!getCanBeVOPD(Opc: II[VOPD::X].getOpcode(), EncodingFamily, VOPD3: false).X ||
4162 !getCanBeVOPD(Opc: II[VOPD::Y].getOpcode(), EncodingFamily, VOPD3: false).Y)
4163 return false;
4164
4165 // This is an awkward exception, VOPD3 variant of V_DUAL_CNDMASK_B32 has
4166 // explicit src2 even if it is vcc_lo. If it was parsed as VOPD3 it cannot
4167 // be parsed as VOPD which does not accept src2.
4168 if (II[VOPD::X].getOpcode() == AMDGPU::V_CNDMASK_B32_e32 ||
4169 II[VOPD::Y].getOpcode() == AMDGPU::V_CNDMASK_B32_e32)
4170 return false;
4171
4172 // If any modifiers are set this cannot be VOPD.
4173 for (auto OpName : {OpName::src0X_modifiers, OpName::src0Y_modifiers,
4174 OpName::vsrc1X_modifiers, OpName::vsrc1Y_modifiers,
4175 OpName::vsrc2X_modifiers, OpName::vsrc2Y_modifiers}) {
4176 int I = getNamedOperandIdx(Opcode, Name: OpName);
4177 if (I == -1)
4178 continue;
4179 if (Inst.getOperand(i: I).getImm())
4180 return false;
4181 }
4182
4183 return !tryVOPD3(Inst);
4184}
4185
4186// VOPD3 has more relaxed register constraints than VOPD. We prefer shorter VOPD
4187// form but switch to VOPD3 otherwise.
4188bool AMDGPUAsmParser::tryAnotherVOPDEncoding(const MCInst &Inst) {
4189 if (!isGFX1250Plus() || !isVOPD(Opc: Inst.getOpcode()))
4190 return false;
4191
4192 if (SIInstrFlags::isVOPD3(O: MII, O: Inst))
4193 return tryVOPD(Inst);
4194 return tryVOPD3(Inst);
4195}
4196
4197bool AMDGPUAsmParser::validateIntClampSupported(const MCInst &Inst) {
4198
4199 const unsigned Opc = Inst.getOpcode();
4200
4201 if (SIInstrFlags::hasIntClamp(O: MII, O: Inst) && !hasIntClamp()) {
4202 int ClampIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::clamp);
4203 assert(ClampIdx != -1);
4204 return Inst.getOperand(i: ClampIdx).getImm() == 0;
4205 }
4206
4207 return true;
4208}
4209
4210bool AMDGPUAsmParser::validateMIMGDataSize(const MCInst &Inst, SMLoc IDLoc) {
4211
4212 const unsigned Opc = Inst.getOpcode();
4213 const MCInstrDesc &Desc = MII.get(Opcode: Opc);
4214
4215 if ((SIInstrFlags::isImage(O: Desc)) == 0)
4216 return true;
4217
4218 int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdata);
4219 int DMaskIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dmask);
4220 int TFEIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::tfe);
4221
4222 if (VDataIdx == -1 && isGFX10Plus()) // no return image_sample
4223 return true;
4224
4225 if ((DMaskIdx == -1 || TFEIdx == -1) &&
4226 hasBVHRayTracingInsts()) // intersect_ray
4227 return true;
4228
4229 unsigned VDataSize = getRegOperandSize(Desc, OpNo: VDataIdx);
4230 unsigned TFESize = (TFEIdx != -1 && Inst.getOperand(i: TFEIdx).getImm()) ? 1 : 0;
4231 unsigned DMask = Inst.getOperand(i: DMaskIdx).getImm() & 0xf;
4232 if (DMask == 0)
4233 DMask = 1;
4234
4235 bool IsPackedD16 = false;
4236 unsigned DataSize = SIInstrFlags::isGather4(O: Desc) ? 4 : llvm::popcount(Value: DMask);
4237 if (hasPackedD16()) {
4238 int D16Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::d16);
4239 IsPackedD16 = D16Idx >= 0;
4240 if (IsPackedD16 && Inst.getOperand(i: D16Idx).getImm())
4241 DataSize = (DataSize + 1) / 2;
4242 }
4243
4244 if ((VDataSize / 4) == DataSize + TFESize)
4245 return true;
4246
4247 StringRef Modifiers;
4248 if (isGFX90A())
4249 Modifiers = IsPackedD16 ? "dmask and d16" : "dmask";
4250 else
4251 Modifiers = IsPackedD16 ? "dmask, d16 and tfe" : "dmask and tfe";
4252
4253 Error(L: IDLoc, Msg: Twine("image data size does not match ") + Modifiers);
4254 return false;
4255}
4256
4257bool AMDGPUAsmParser::validateMIMGAddrSize(const MCInst &Inst, SMLoc IDLoc) {
4258 const unsigned Opc = Inst.getOpcode();
4259 const MCInstrDesc &Desc = MII.get(Opcode: Opc);
4260
4261 if (!SIInstrFlags::isImage(O: Desc) || !isGFX10Plus())
4262 return true;
4263
4264 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc);
4265
4266 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4267 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
4268 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr0);
4269 AMDGPU::OpName RSrcOpName =
4270 SIInstrFlags::isMIMG(O: Desc) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
4271 int SrsrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: RSrcOpName);
4272 int DimIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dim);
4273 int A16Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::a16);
4274
4275 assert(VAddr0Idx != -1);
4276 assert(SrsrcIdx != -1);
4277 assert(SrsrcIdx > VAddr0Idx);
4278
4279 bool IsA16 = (A16Idx != -1 && Inst.getOperand(i: A16Idx).getImm());
4280 if (BaseOpcode->BVH) {
4281 if (IsA16 == BaseOpcode->A16)
4282 return true;
4283 Error(L: IDLoc, Msg: "image address size does not match a16");
4284 return false;
4285 }
4286
4287 unsigned Dim = Inst.getOperand(i: DimIdx).getImm();
4288 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfoByEncoding(DimEnc: Dim);
4289 bool IsNSA = SrsrcIdx - VAddr0Idx > 1;
4290 unsigned ActualAddrSize =
4291 IsNSA ? SrsrcIdx - VAddr0Idx : getRegOperandSize(Desc, OpNo: VAddr0Idx) / 4;
4292
4293 unsigned ExpectedAddrSize =
4294 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim: DimInfo, IsA16, IsG16Supported: hasG16());
4295
4296 if (IsNSA) {
4297 if (hasPartialNSAEncoding() &&
4298 ExpectedAddrSize > getNSAMaxSize(HasSampler: SIInstrFlags::isVSAMPLE(O: Desc))) {
4299 int VAddrLastIdx = SrsrcIdx - 1;
4300 unsigned VAddrLastSize = getRegOperandSize(Desc, OpNo: VAddrLastIdx) / 4;
4301
4302 ActualAddrSize = VAddrLastIdx - VAddr0Idx + VAddrLastSize;
4303 }
4304 } else {
4305 if (ExpectedAddrSize > 12)
4306 ExpectedAddrSize = 16;
4307
4308 // Allow oversized 8 VGPR vaddr when only 5/6/7 VGPRs are required.
4309 // This provides backward compatibility for assembly created
4310 // before 160b/192b/224b types were directly supported.
4311 if (ActualAddrSize == 8 && (ExpectedAddrSize >= 5 && ExpectedAddrSize <= 7))
4312 return true;
4313 }
4314
4315 if (ActualAddrSize == ExpectedAddrSize)
4316 return true;
4317
4318 Error(L: IDLoc, Msg: "image address size does not match dim and a16");
4319 return false;
4320}
4321
4322bool AMDGPUAsmParser::validateMIMGAtomicDMask(const MCInst &Inst) {
4323
4324 const unsigned Opc = Inst.getOpcode();
4325 const MCInstrDesc &Desc = MII.get(Opcode: Opc);
4326
4327 if ((SIInstrFlags::isImage(O: Desc)) == 0)
4328 return true;
4329 if (!Desc.mayLoad() || !Desc.mayStore())
4330 return true; // Not atomic
4331
4332 int DMaskIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dmask);
4333 unsigned DMask = Inst.getOperand(i: DMaskIdx).getImm() & 0xf;
4334
4335 // This is an incomplete check because image_atomic_cmpswap
4336 // may only use 0x3 and 0xf while other atomic operations
4337 // may use 0x1 and 0x3. However these limitations are
4338 // verified when we check that dmask matches dst size.
4339 return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
4340}
4341
4342bool AMDGPUAsmParser::validateMIMGGatherDMask(const MCInst &Inst) {
4343
4344 const unsigned Opc = Inst.getOpcode();
4345
4346 if (!SIInstrFlags::isGather4(O: MII, O: Inst))
4347 return true;
4348
4349 int DMaskIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dmask);
4350 unsigned DMask = Inst.getOperand(i: DMaskIdx).getImm() & 0xf;
4351
4352 // GATHER4 instructions use dmask in a different fashion compared to
4353 // other MIMG instructions. The only useful DMASK values are
4354 // 1=red, 2=green, 4=blue, 8=alpha. (e.g. 1 returns
4355 // (red,red,red,red) etc.) The ISA document doesn't mention
4356 // this.
4357 return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8;
4358}
4359
4360bool AMDGPUAsmParser::validateMIMGDim(const MCInst &Inst,
4361 const OperandVector &Operands) {
4362 if (!isGFX10Plus())
4363 return true;
4364
4365 const unsigned Opc = Inst.getOpcode();
4366
4367 if ((SIInstrFlags::isImage(O: MII, O: Inst)) == 0)
4368 return true;
4369
4370 // image_bvh_intersect_ray instructions do not have dim
4371 if (AMDGPU::getMIMGBaseOpcode(Opc)->BVH)
4372 return true;
4373
4374 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4375 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4376 if (Op.isDim())
4377 return true;
4378 }
4379 return false;
4380}
4381
4382bool AMDGPUAsmParser::validateMIMGMSAA(const MCInst &Inst) {
4383 const unsigned Opc = Inst.getOpcode();
4384
4385 if ((SIInstrFlags::isImage(O: MII, O: Inst)) == 0)
4386 return true;
4387
4388 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc);
4389 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4390 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
4391
4392 if (!BaseOpcode->MSAA)
4393 return true;
4394
4395 int DimIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dim);
4396 assert(DimIdx != -1);
4397
4398 unsigned Dim = Inst.getOperand(i: DimIdx).getImm();
4399 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfoByEncoding(DimEnc: Dim);
4400
4401 return DimInfo->MSAA;
4402}
4403
4404static bool IsMovrelsSDWAOpcode(const unsigned Opcode) {
4405 switch (Opcode) {
4406 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
4407 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
4408 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
4409 return true;
4410 default:
4411 return false;
4412 }
4413}
4414
4415// movrels* opcodes should only allow VGPRS as src0.
4416// This is specified in .td description for vop1/vop3,
4417// but sdwa is handled differently. See isSDWAOperand.
4418bool AMDGPUAsmParser::validateMovrels(const MCInst &Inst,
4419 const OperandVector &Operands) {
4420
4421 const unsigned Opc = Inst.getOpcode();
4422
4423 if (!SIInstrFlags::isSDWA(O: MII, O: Inst) || !IsMovrelsSDWAOpcode(Opcode: Opc))
4424 return true;
4425
4426 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
4427 assert(Src0Idx != -1);
4428
4429 const MCOperand &Src0 = Inst.getOperand(i: Src0Idx);
4430 if (Src0.isReg()) {
4431 auto Reg = mc2PseudoReg(Reg: Src0.getReg());
4432 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
4433 if (!isSGPR(Reg, TRI))
4434 return true;
4435 }
4436
4437 Error(L: getOperandLoc(Operands, MCOpIdx: Src0Idx), Msg: "source operand must be a VGPR");
4438 return false;
4439}
4440
4441bool AMDGPUAsmParser::validateMAIAccWrite(const MCInst &Inst,
4442 const OperandVector &Operands) {
4443
4444 const unsigned Opc = Inst.getOpcode();
4445
4446 if (Opc != AMDGPU::V_ACCVGPR_WRITE_B32_vi)
4447 return true;
4448
4449 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
4450 assert(Src0Idx != -1);
4451
4452 const MCOperand &Src0 = Inst.getOperand(i: Src0Idx);
4453 if (!Src0.isReg())
4454 return true;
4455
4456 auto Reg = mc2PseudoReg(Reg: Src0.getReg());
4457 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
4458 if (!isGFX90A() && isSGPR(Reg, TRI)) {
4459 Error(L: getOperandLoc(Operands, MCOpIdx: Src0Idx),
4460 Msg: "source operand must be either a VGPR or an inline constant");
4461 return false;
4462 }
4463
4464 return true;
4465}
4466
4467bool AMDGPUAsmParser::validateMAISrc2(const MCInst &Inst,
4468 const OperandVector &Operands) {
4469 unsigned Opcode = Inst.getOpcode();
4470
4471 if (!SIInstrFlags::isMAI(O: MII, O: Inst) ||
4472 !getFeatureBits()[FeatureMFMAInlineLiteralBug])
4473 return true;
4474
4475 const int Src2Idx = getNamedOperandIdx(Opcode, Name: OpName::src2);
4476 if (Src2Idx == -1)
4477 return true;
4478
4479 if (Inst.getOperand(i: Src2Idx).isImm() && isInlineConstant(Inst, OpIdx: Src2Idx)) {
4480 Error(L: getOperandLoc(Operands, MCOpIdx: Src2Idx),
4481 Msg: "inline constants are not allowed for this operand");
4482 return false;
4483 }
4484
4485 return true;
4486}
4487
4488bool AMDGPUAsmParser::validateMFMA(const MCInst &Inst,
4489 const OperandVector &Operands) {
4490 const unsigned Opc = Inst.getOpcode();
4491 const MCInstrDesc &Desc = MII.get(Opcode: Opc);
4492
4493 if (!SIInstrFlags::isMAI(O: Desc))
4494 return true;
4495
4496 int BlgpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::blgp);
4497 if (BlgpIdx != -1) {
4498 if (const MFMA_F8F6F4_Info *Info = AMDGPU::isMFMA_F8F6F4(Opcode: Opc)) {
4499 int CbszIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::cbsz);
4500
4501 unsigned CBSZ = Inst.getOperand(i: CbszIdx).getImm();
4502 unsigned BLGP = Inst.getOperand(i: BlgpIdx).getImm();
4503
4504 // Validate the correct register size was used for the floating point
4505 // format operands
4506
4507 bool Success = true;
4508 if (Info->NumRegsSrcA != mfmaScaleF8F6F4FormatToNumRegs(EncodingVal: CBSZ)) {
4509 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
4510 Error(L: getOperandLoc(Operands, MCOpIdx: Src0Idx),
4511 Msg: "wrong register tuple size for cbsz value " + Twine(CBSZ));
4512 Success = false;
4513 }
4514
4515 if (Info->NumRegsSrcB != mfmaScaleF8F6F4FormatToNumRegs(EncodingVal: BLGP)) {
4516 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
4517 Error(L: getOperandLoc(Operands, MCOpIdx: Src1Idx),
4518 Msg: "wrong register tuple size for blgp value " + Twine(BLGP));
4519 Success = false;
4520 }
4521
4522 return Success;
4523 }
4524 }
4525
4526 const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
4527 if (Src2Idx == -1)
4528 return true;
4529
4530 const MCOperand &Src2 = Inst.getOperand(i: Src2Idx);
4531 if (!Src2.isReg())
4532 return true;
4533
4534 MCRegister Src2Reg = Src2.getReg();
4535 MCRegister DstReg = Inst.getOperand(i: 0).getReg();
4536 if (Src2Reg == DstReg)
4537 return true;
4538
4539 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
4540 if (TRI->getRegClass(i: MII.getOpRegClassID(OpInfo: Desc.operands()[0], HwModeId: HwMode))
4541 .getSizeInBits() <= 128)
4542 return true;
4543
4544 if (TRI->regsOverlap(RegA: Src2Reg, RegB: DstReg)) {
4545 Error(L: getOperandLoc(Operands, MCOpIdx: Src2Idx),
4546 Msg: "source 2 operand must not partially overlap with dst");
4547 return false;
4548 }
4549
4550 return true;
4551}
4552
4553bool AMDGPUAsmParser::validateDivScale(const MCInst &Inst) {
4554 switch (Inst.getOpcode()) {
4555 default:
4556 return true;
4557 case V_DIV_SCALE_F32_gfx6_gfx7:
4558 case V_DIV_SCALE_F32_vi:
4559 case V_DIV_SCALE_F32_gfx10:
4560 case V_DIV_SCALE_F64_gfx6_gfx7:
4561 case V_DIV_SCALE_F64_vi:
4562 case V_DIV_SCALE_F64_gfx10:
4563 break;
4564 }
4565
4566 // TODO: Check that src0 = src1 or src2.
4567
4568 for (auto Name :
4569 {AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src2_modifiers,
4570 AMDGPU::OpName::src2_modifiers}) {
4571 if (Inst.getOperand(i: AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name))
4572 .getImm() &
4573 SISrcMods::ABS) {
4574 return false;
4575 }
4576 }
4577
4578 return true;
4579}
4580
4581bool AMDGPUAsmParser::validateMIMGD16(const MCInst &Inst) {
4582
4583 const unsigned Opc = Inst.getOpcode();
4584
4585 if ((SIInstrFlags::isImage(O: MII, O: Inst)) == 0)
4586 return true;
4587
4588 int D16Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::d16);
4589 if (D16Idx >= 0 && Inst.getOperand(i: D16Idx).getImm()) {
4590 if (isCI() || isSI())
4591 return false;
4592 }
4593
4594 return true;
4595}
4596
4597bool AMDGPUAsmParser::validateTensorR128(const MCInst &Inst) {
4598 const unsigned Opc = Inst.getOpcode();
4599
4600 if (!SIInstrFlags::usesTENSOR_CNT(O: MII, O: Inst))
4601 return true;
4602
4603 int R128Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::r128);
4604
4605 return R128Idx < 0 || !Inst.getOperand(i: R128Idx).getImm();
4606}
4607
4608static bool IsRevOpcode(const unsigned Opcode) {
4609 switch (Opcode) {
4610 case AMDGPU::V_SUBREV_F32_e32:
4611 case AMDGPU::V_SUBREV_F32_e64:
4612 case AMDGPU::V_SUBREV_F32_e32_gfx10:
4613 case AMDGPU::V_SUBREV_F32_e32_gfx6_gfx7:
4614 case AMDGPU::V_SUBREV_F32_e32_vi:
4615 case AMDGPU::V_SUBREV_F32_e64_gfx10:
4616 case AMDGPU::V_SUBREV_F32_e64_gfx6_gfx7:
4617 case AMDGPU::V_SUBREV_F32_e64_vi:
4618
4619 case AMDGPU::V_SUBREV_CO_U32_e32:
4620 case AMDGPU::V_SUBREV_CO_U32_e64:
4621 case AMDGPU::V_SUBREV_I32_e32_gfx6_gfx7:
4622 case AMDGPU::V_SUBREV_I32_e64_gfx6_gfx7:
4623
4624 case AMDGPU::V_SUBBREV_U32_e32:
4625 case AMDGPU::V_SUBBREV_U32_e64:
4626 case AMDGPU::V_SUBBREV_U32_e32_gfx6_gfx7:
4627 case AMDGPU::V_SUBBREV_U32_e32_vi:
4628 case AMDGPU::V_SUBBREV_U32_e64_gfx6_gfx7:
4629 case AMDGPU::V_SUBBREV_U32_e64_vi:
4630
4631 case AMDGPU::V_SUBREV_U32_e32:
4632 case AMDGPU::V_SUBREV_U32_e64:
4633 case AMDGPU::V_SUBREV_U32_e32_gfx9:
4634 case AMDGPU::V_SUBREV_U32_e32_vi:
4635 case AMDGPU::V_SUBREV_U32_e64_gfx9:
4636 case AMDGPU::V_SUBREV_U32_e64_vi:
4637
4638 case AMDGPU::V_SUBREV_F16_e32:
4639 case AMDGPU::V_SUBREV_F16_e64:
4640 case AMDGPU::V_SUBREV_F16_e32_gfx10:
4641 case AMDGPU::V_SUBREV_F16_e32_vi:
4642 case AMDGPU::V_SUBREV_F16_e64_gfx10:
4643 case AMDGPU::V_SUBREV_F16_e64_vi:
4644
4645 case AMDGPU::V_SUBREV_U16_e32:
4646 case AMDGPU::V_SUBREV_U16_e64:
4647 case AMDGPU::V_SUBREV_U16_e32_vi:
4648 case AMDGPU::V_SUBREV_U16_e64_vi:
4649
4650 case AMDGPU::V_SUBREV_CO_U32_e32_gfx9:
4651 case AMDGPU::V_SUBREV_CO_U32_e64_gfx10:
4652 case AMDGPU::V_SUBREV_CO_U32_e64_gfx9:
4653
4654 case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9:
4655 case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9:
4656
4657 case AMDGPU::V_SUBREV_NC_U32_e32_gfx10:
4658 case AMDGPU::V_SUBREV_NC_U32_e64_gfx10:
4659
4660 case AMDGPU::V_SUBREV_CO_CI_U32_e32_gfx10:
4661 case AMDGPU::V_SUBREV_CO_CI_U32_e64_gfx10:
4662
4663 case AMDGPU::V_LSHRREV_B32_e32:
4664 case AMDGPU::V_LSHRREV_B32_e64:
4665 case AMDGPU::V_LSHRREV_B32_e32_gfx6_gfx7:
4666 case AMDGPU::V_LSHRREV_B32_e64_gfx6_gfx7:
4667 case AMDGPU::V_LSHRREV_B32_e32_vi:
4668 case AMDGPU::V_LSHRREV_B32_e64_vi:
4669 case AMDGPU::V_LSHRREV_B32_e32_gfx10:
4670 case AMDGPU::V_LSHRREV_B32_e64_gfx10:
4671
4672 case AMDGPU::V_ASHRREV_I32_e32:
4673 case AMDGPU::V_ASHRREV_I32_e64:
4674 case AMDGPU::V_ASHRREV_I32_e32_gfx10:
4675 case AMDGPU::V_ASHRREV_I32_e32_gfx6_gfx7:
4676 case AMDGPU::V_ASHRREV_I32_e32_vi:
4677 case AMDGPU::V_ASHRREV_I32_e64_gfx10:
4678 case AMDGPU::V_ASHRREV_I32_e64_gfx6_gfx7:
4679 case AMDGPU::V_ASHRREV_I32_e64_vi:
4680
4681 case AMDGPU::V_LSHLREV_B32_e32:
4682 case AMDGPU::V_LSHLREV_B32_e64:
4683 case AMDGPU::V_LSHLREV_B32_e32_gfx10:
4684 case AMDGPU::V_LSHLREV_B32_e32_gfx6_gfx7:
4685 case AMDGPU::V_LSHLREV_B32_e32_vi:
4686 case AMDGPU::V_LSHLREV_B32_e64_gfx10:
4687 case AMDGPU::V_LSHLREV_B32_e64_gfx6_gfx7:
4688 case AMDGPU::V_LSHLREV_B32_e64_vi:
4689
4690 case AMDGPU::V_LSHLREV_B16_e32:
4691 case AMDGPU::V_LSHLREV_B16_e64:
4692 case AMDGPU::V_LSHLREV_B16_e32_vi:
4693 case AMDGPU::V_LSHLREV_B16_e64_vi:
4694 case AMDGPU::V_LSHLREV_B16_gfx10:
4695
4696 case AMDGPU::V_LSHRREV_B16_e32:
4697 case AMDGPU::V_LSHRREV_B16_e64:
4698 case AMDGPU::V_LSHRREV_B16_e32_vi:
4699 case AMDGPU::V_LSHRREV_B16_e64_vi:
4700 case AMDGPU::V_LSHRREV_B16_gfx10:
4701
4702 case AMDGPU::V_ASHRREV_I16_e32:
4703 case AMDGPU::V_ASHRREV_I16_e64:
4704 case AMDGPU::V_ASHRREV_I16_e32_vi:
4705 case AMDGPU::V_ASHRREV_I16_e64_vi:
4706 case AMDGPU::V_ASHRREV_I16_gfx10:
4707
4708 case AMDGPU::V_LSHLREV_B64_e64:
4709 case AMDGPU::V_LSHLREV_B64_gfx10:
4710 case AMDGPU::V_LSHLREV_B64_vi:
4711
4712 case AMDGPU::V_LSHRREV_B64_e64:
4713 case AMDGPU::V_LSHRREV_B64_gfx10:
4714 case AMDGPU::V_LSHRREV_B64_vi:
4715
4716 case AMDGPU::V_ASHRREV_I64_e64:
4717 case AMDGPU::V_ASHRREV_I64_gfx10:
4718 case AMDGPU::V_ASHRREV_I64_vi:
4719
4720 case AMDGPU::V_PK_LSHLREV_B16:
4721 case AMDGPU::V_PK_LSHLREV_B16_gfx10:
4722 case AMDGPU::V_PK_LSHLREV_B16_vi:
4723
4724 case AMDGPU::V_PK_LSHRREV_B16:
4725 case AMDGPU::V_PK_LSHRREV_B16_gfx10:
4726 case AMDGPU::V_PK_LSHRREV_B16_vi:
4727 case AMDGPU::V_PK_ASHRREV_I16:
4728 case AMDGPU::V_PK_ASHRREV_I16_gfx10:
4729 case AMDGPU::V_PK_ASHRREV_I16_vi:
4730 return true;
4731 default:
4732 return false;
4733 }
4734}
4735
4736bool AMDGPUAsmParser::validateLdsDirect(const MCInst &Inst,
4737 const OperandVector &Operands) {
4738 const unsigned Opcode = Inst.getOpcode();
4739
4740 // lds_direct register is defined so that it can be used
4741 // with 9-bit operands only. Ignore encodings which do not accept these.
4742 if (!SIInstrFlags::isVOP1(O: MII, O: Inst) && !SIInstrFlags::isVOP2(O: MII, O: Inst) &&
4743 !SIInstrFlags::isVOP3Like(O: MII, O: Inst) &&
4744 !SIInstrFlags::isVOPC(O: MII, O: Inst) && !SIInstrFlags::isSDWA(O: MII, O: Inst))
4745 return true;
4746
4747 for (auto SrcName : {OpName::src0, OpName::src1, OpName::src2}) {
4748 auto SrcIdx = getNamedOperandIdx(Opcode, Name: SrcName);
4749 if (SrcIdx == -1)
4750 break;
4751 const auto &Src = Inst.getOperand(i: SrcIdx);
4752 if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
4753
4754 if (isGFX90A() || isGFX11Plus()) {
4755 Error(L: getOperandLoc(Operands, MCOpIdx: SrcIdx),
4756 Msg: "lds_direct is not supported on this GPU");
4757 return false;
4758 }
4759
4760 if (IsRevOpcode(Opcode) || SIInstrFlags::isSDWA(O: MII, O: Inst)) {
4761 Error(L: getOperandLoc(Operands, MCOpIdx: SrcIdx),
4762 Msg: "lds_direct cannot be used with this instruction");
4763 return false;
4764 }
4765
4766 if (SrcName != OpName::src0) {
4767 Error(L: getOperandLoc(Operands, MCOpIdx: SrcIdx),
4768 Msg: "lds_direct may be used as src0 only");
4769 return false;
4770 }
4771 }
4772 }
4773
4774 return true;
4775}
4776
4777SMLoc AMDGPUAsmParser::getFlatOffsetLoc(const OperandVector &Operands) const {
4778 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4779 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4780 if (Op.isFlatOffset())
4781 return Op.getStartLoc();
4782 }
4783 return getLoc();
4784}
4785
4786bool AMDGPUAsmParser::validateOffset(const MCInst &Inst,
4787 const OperandVector &Operands) {
4788 auto Opcode = Inst.getOpcode();
4789 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::offset);
4790 if (OpNum == -1)
4791 return true;
4792
4793 if (SIInstrFlags::isFLAT(O: MII, O: Inst))
4794 return validateFlatOffset(Inst, Operands);
4795
4796 if (SIInstrFlags::isSMRD(O: MII, O: Inst))
4797 return validateSMEMOffset(Inst, Operands);
4798
4799 const auto &Op = Inst.getOperand(i: OpNum);
4800 // GFX12+ buffer ops: InstOffset is signed 24, but must not be a negative.
4801 if (isGFX12Plus() && SIInstrFlags::isBuffer(O: MII, O: Inst)) {
4802 const unsigned OffsetSize = 24;
4803 if (!isUIntN(N: OffsetSize - 1, x: Op.getImm())) {
4804 Error(L: getFlatOffsetLoc(Operands),
4805 Msg: Twine("expected a ") + Twine(OffsetSize - 1) +
4806 "-bit unsigned offset for buffer ops");
4807 return false;
4808 }
4809 } else {
4810 const unsigned OffsetSize = 16;
4811 if (!isUIntN(N: OffsetSize, x: Op.getImm())) {
4812 Error(L: getFlatOffsetLoc(Operands),
4813 Msg: Twine("expected a ") + Twine(OffsetSize) + "-bit unsigned offset");
4814 return false;
4815 }
4816 }
4817 return true;
4818}
4819
4820bool AMDGPUAsmParser::validateFlatOffset(const MCInst &Inst,
4821 const OperandVector &Operands) {
4822 if (!SIInstrFlags::isFLAT(O: MII, O: Inst))
4823 return true;
4824
4825 auto Opcode = Inst.getOpcode();
4826 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::offset);
4827 assert(OpNum != -1);
4828
4829 const auto &Op = Inst.getOperand(i: OpNum);
4830 if (!hasFlatOffsets() && Op.getImm() != 0) {
4831 Error(L: getFlatOffsetLoc(Operands),
4832 Msg: "flat offset modifier is not supported on this GPU");
4833 return false;
4834 }
4835
4836 // For pre-GFX12 FLAT instructions the offset must be positive;
4837 // MSB is ignored and forced to zero.
4838 unsigned OffsetSize = AMDGPU::getNumFlatOffsetBits(ST: getSTI());
4839 bool AllowNegative =
4840 SIInstrFlags::isSegmentSpecificFLAT(O: MII, O: Inst) || isGFX12Plus();
4841 if (!isIntN(N: OffsetSize, x: Op.getImm()) || (!AllowNegative && Op.getImm() < 0)) {
4842 Error(L: getFlatOffsetLoc(Operands),
4843 Msg: Twine("expected a ") +
4844 (AllowNegative ? Twine(OffsetSize) + "-bit signed offset"
4845 : Twine(OffsetSize - 1) + "-bit unsigned offset"));
4846 return false;
4847 }
4848
4849 return true;
4850}
4851
4852SMLoc AMDGPUAsmParser::getSMEMOffsetLoc(const OperandVector &Operands) const {
4853 // Start with second operand because SMEM Offset cannot be dst or src0.
4854 for (unsigned i = 2, e = Operands.size(); i != e; ++i) {
4855 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4856 if (Op.isSMEMOffset() || Op.isSMEMOffsetMod())
4857 return Op.getStartLoc();
4858 }
4859 return getLoc();
4860}
4861
4862bool AMDGPUAsmParser::validateSMEMOffset(const MCInst &Inst,
4863 const OperandVector &Operands) {
4864 if (isCI() || isSI())
4865 return true;
4866
4867 if (!SIInstrFlags::isSMRD(O: MII, O: Inst))
4868 return true;
4869
4870 auto Opcode = Inst.getOpcode();
4871 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::offset);
4872 if (OpNum == -1)
4873 return true;
4874
4875 const auto &Op = Inst.getOperand(i: OpNum);
4876 if (!Op.isImm())
4877 return true;
4878
4879 uint64_t Offset = Op.getImm();
4880 bool IsBuffer = AMDGPU::getSMEMIsBuffer(Opc: Opcode);
4881 if (AMDGPU::isLegalSMRDEncodedUnsignedOffset(ST: getSTI(), EncodedOffset: Offset) ||
4882 AMDGPU::isLegalSMRDEncodedSignedOffset(ST: getSTI(), EncodedOffset: Offset, IsBuffer))
4883 return true;
4884
4885 Error(L: getSMEMOffsetLoc(Operands),
4886 Msg: isGFX12Plus() && IsBuffer
4887 ? "expected a 23-bit unsigned offset for buffer ops"
4888 : isGFX12Plus() ? "expected a 24-bit signed offset"
4889 : (isVI() || IsBuffer) ? "expected a 20-bit unsigned offset"
4890 : "expected a 21-bit signed offset");
4891
4892 return false;
4893}
4894
4895// On subtargets with FeatureBF16InlineConstFromUpperFP32 the hardware generates
4896// a bf16 inline constant in the high half of the corresponding fp32 inline
4897// constant. A VOP1 bf16 opcode (v_cvt_f32_bf16 and the bf16 transcendentals)
4898// reads the low half of its source, so it must use the VOP3 encoding with
4899// op_sel[0] set in order to see the constant at all.
4900bool AMDGPUAsmParser::validateBF16InlineConst(const MCInst &Inst,
4901 const OperandVector &Operands) {
4902 if (!getFeatureBits()[AMDGPU::FeatureBF16InlineConstFromUpperFP32])
4903 return true;
4904
4905 const unsigned Opc = Inst.getOpcode();
4906 const MCInstrDesc &Desc = MII.get(Opcode: Opc);
4907 const bool IsVOP3 =
4908 SIInstrFlags::isVOP3(O: Desc) && !SIInstrFlags::isVOP3P(O: Desc);
4909 if (!SIInstrFlags::isVOP1(O: Desc) && !IsVOP3)
4910 return true;
4911
4912 // Only the single-source VOP1 bf16 opcodes are affected. Multi-source and
4913 // packed bf16 instructions such as v_fma_mix*_bf16 are not.
4914 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::src1))
4915 return true;
4916
4917 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
4918 if (Src0Idx == -1)
4919 return true;
4920
4921 const MCOperandInfo &Src0Info = Desc.operands()[Src0Idx];
4922 if (!AMDGPU::isBF16SrcOperand(OpInfo: Src0Info))
4923 return true;
4924
4925 const MCOperand &Src0 = Inst.getOperand(i: Src0Idx);
4926 if (!Src0.isImm() ||
4927 !AMDGPU::isInlinableLiteralBF16(Literal: static_cast<int16_t>(Src0.getImm()),
4928 HasInv2Pi: hasInv2PiInlineImm()))
4929 return true;
4930
4931 if (IsVOP3) {
4932 const int ModsIdx =
4933 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0_modifiers);
4934 if (ModsIdx != -1 &&
4935 (Inst.getOperand(i: ModsIdx).getImm() & SISrcMods::OP_SEL_0))
4936 return true;
4937 }
4938
4939 Error(L: getOperandLoc(Operands, MCOpIdx: Src0Idx),
4940 Msg: "bf16 inline constant is read from the high half of the fp32 inline "
4941 "constant on this GPU; use the e64 encoding with op_sel:[1,0]");
4942 return false;
4943}
4944
4945bool AMDGPUAsmParser::validateSOPLiteral(const MCInst &Inst,
4946 const OperandVector &Operands) {
4947 unsigned Opcode = Inst.getOpcode();
4948 const MCInstrDesc &Desc = MII.get(Opcode);
4949 if (!SIInstrFlags::isSOP2(O: Desc) && !SIInstrFlags::isSOPC(O: Desc))
4950 return true;
4951
4952 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0);
4953 const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src1);
4954
4955 const int OpIndices[] = {Src0Idx, Src1Idx};
4956
4957 unsigned NumExprs = 0;
4958 unsigned NumLiterals = 0;
4959 int64_t LiteralValue;
4960
4961 for (int OpIdx : OpIndices) {
4962 if (OpIdx == -1)
4963 break;
4964
4965 const MCOperand &MO = Inst.getOperand(i: OpIdx);
4966 // Exclude special imm operands (like that used by s_set_gpr_idx_on)
4967 if (AMDGPU::isSISrcOperand(Desc, OpNo: OpIdx)) {
4968 bool IsLit = false;
4969 std::optional<int64_t> Imm;
4970 if (MO.isImm()) {
4971 Imm = MO.getImm();
4972 } else if (MO.isExpr()) {
4973 if (isLitExpr(Expr: MO.getExpr())) {
4974 IsLit = true;
4975 Imm = getLitValue(Expr: MO.getExpr());
4976 }
4977 } else {
4978 continue;
4979 }
4980
4981 if (!Imm.has_value()) {
4982 ++NumExprs;
4983 } else if (!isInlineConstant(Inst, OpIdx)) {
4984 auto OpType = static_cast<AMDGPU::OperandType>(
4985 Desc.operands()[OpIdx].OperandType);
4986 int64_t Value = encode32BitLiteral(Imm: *Imm, Type: OpType, IsLit);
4987 if (NumLiterals == 0 || LiteralValue != Value) {
4988 LiteralValue = Value;
4989 ++NumLiterals;
4990 }
4991 }
4992 }
4993 }
4994
4995 if (NumLiterals + NumExprs <= 1)
4996 return true;
4997
4998 Error(L: getOperandLoc(Operands, MCOpIdx: Src1Idx),
4999 Msg: "only one unique literal operand is allowed");
5000 return false;
5001}
5002
5003bool AMDGPUAsmParser::validateOpSel(const MCInst &Inst) {
5004 const unsigned Opc = Inst.getOpcode();
5005 if (isPermlane16(Opc)) {
5006 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel);
5007 unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm();
5008
5009 if (OpSel & ~3)
5010 return false;
5011 }
5012
5013 if (isGFX940() && SIInstrFlags::isDOT(O: MII, O: Inst)) {
5014 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel);
5015 if (OpSelIdx != -1) {
5016 if (Inst.getOperand(i: OpSelIdx).getImm() != 0)
5017 return false;
5018 }
5019 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel_hi);
5020 if (OpSelHiIdx != -1) {
5021 if (Inst.getOperand(i: OpSelHiIdx).getImm() != -1)
5022 return false;
5023 }
5024 }
5025
5026 // op_sel[0:1] must be 0 for v_dot2_bf16_bf16 and v_dot2_f16_f16 (VOP3 Dot).
5027 if (isGFX11Plus() && SIInstrFlags::isDOT(O: MII, O: Inst) &&
5028 SIInstrFlags::isVOP3(O: MII, O: Inst) && !SIInstrFlags::isVOP3P(O: MII, O: Inst)) {
5029 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel);
5030 unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm();
5031 if (OpSel & 3)
5032 return false;
5033 }
5034
5035 // Packed math FP32 instructions typically accept SGPRs or VGPRs as source
5036 // operands. On gfx12+, if a source operand uses SGPRs, the HW can only read
5037 // the first SGPR and use it for both the low and high operations.
5038 if (isPackedSingleSGPRFP32Inst(Opc)) {
5039 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
5040 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
5041 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel);
5042 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel_hi);
5043
5044 const MCOperand &Src0 = Inst.getOperand(i: Src0Idx);
5045 const MCOperand &Src1 = Inst.getOperand(i: Src1Idx);
5046 unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm();
5047 unsigned OpSelHi = Inst.getOperand(i: OpSelHiIdx).getImm();
5048
5049 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
5050
5051 auto VerifyOneSGPR = [OpSel, OpSelHi](unsigned Index) -> bool {
5052 unsigned Mask = 1U << Index;
5053 return ((OpSel & Mask) == 0) && ((OpSelHi & Mask) == 0);
5054 };
5055
5056 if (Src0.isReg() && isSGPR(Reg: Src0.getReg(), TRI) &&
5057 !VerifyOneSGPR(/*Index=*/0))
5058 return false;
5059 if (Src1.isReg() && isSGPR(Reg: Src1.getReg(), TRI) &&
5060 !VerifyOneSGPR(/*Index=*/1))
5061 return false;
5062
5063 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
5064 if (Src2Idx != -1) {
5065 const MCOperand &Src2 = Inst.getOperand(i: Src2Idx);
5066 if (Src2.isReg() && isSGPR(Reg: Src2.getReg(), TRI) &&
5067 !VerifyOneSGPR(/*Index=*/2))
5068 return false;
5069 }
5070 }
5071
5072 return true;
5073}
5074
5075bool AMDGPUAsmParser::validateTrue16OpSel(const MCInst &Inst) {
5076 if (!hasTrue16Insts())
5077 return true;
5078 const MCRegisterInfo *MRI = getMRI();
5079 const unsigned Opc = Inst.getOpcode();
5080 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel);
5081 if (OpSelIdx == -1)
5082 return true;
5083 unsigned OpSelOpValue = Inst.getOperand(i: OpSelIdx).getImm();
5084 // If the value is 0 we could have a default OpSel Operand, so conservatively
5085 // allow it.
5086 if (OpSelOpValue == 0)
5087 return true;
5088 unsigned OpCount = 0;
5089 for (AMDGPU::OpName OpName : {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
5090 AMDGPU::OpName::src2, AMDGPU::OpName::vdst}) {
5091 int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: OpName);
5092 if (OpIdx == -1)
5093 continue;
5094 const MCOperand &Op = Inst.getOperand(i: OpIdx);
5095 if (Op.isReg() &&
5096 MRI->getRegClass(i: AMDGPU::VGPR_16RegClassID).contains(Reg: Op.getReg())) {
5097 bool VGPRSuffixIsHi = AMDGPU::isHi16Reg(Reg: Op.getReg(), MRI: *MRI);
5098 bool OpSelOpIsHi = ((OpSelOpValue & (1 << OpCount)) != 0);
5099 if (OpSelOpIsHi != VGPRSuffixIsHi)
5100 return false;
5101 }
5102 ++OpCount;
5103 }
5104
5105 return true;
5106}
5107
5108bool AMDGPUAsmParser::validateNeg(const MCInst &Inst, AMDGPU::OpName OpName) {
5109 assert(OpName == AMDGPU::OpName::neg_lo || OpName == AMDGPU::OpName::neg_hi);
5110
5111 const unsigned Opc = Inst.getOpcode();
5112
5113 // v_dot4 fp8/bf8 neg_lo/neg_hi not allowed on src0 and src1 (allowed on src2)
5114 // v_wmma iu4/iu8 neg_lo not allowed on src2 (allowed on src0, src1)
5115 // v_swmmac f16/bf16 neg_lo/neg_hi not allowed on src2 (allowed on src0, src1)
5116 // other wmma/swmmac instructions don't have neg_lo/neg_hi operand.
5117 if (!SIInstrFlags::isDOT(O: MII, O: Inst) && !SIInstrFlags::isWMMA(O: MII, O: Inst) &&
5118 !SIInstrFlags::isSWMMAC(O: MII, O: Inst))
5119 return true;
5120
5121 int NegIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: OpName);
5122 if (NegIdx == -1)
5123 return true;
5124
5125 unsigned Neg = Inst.getOperand(i: NegIdx).getImm();
5126
5127 // Instructions that have neg_lo or neg_hi operand but neg modifier is allowed
5128 // on some src operands but not allowed on other.
5129 // It is convenient that such instructions don't have src_modifiers operand
5130 // for src operands that don't allow neg because they also don't allow opsel.
5131
5132 const AMDGPU::OpName SrcMods[3] = {AMDGPU::OpName::src0_modifiers,
5133 AMDGPU::OpName::src1_modifiers,
5134 AMDGPU::OpName::src2_modifiers};
5135
5136 for (unsigned i = 0; i < 3; ++i) {
5137 if (!AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: SrcMods[i])) {
5138 if (Neg & (1 << i))
5139 return false;
5140 }
5141 }
5142
5143 return true;
5144}
5145
5146bool AMDGPUAsmParser::validateDPP(const MCInst &Inst,
5147 const OperandVector &Operands) {
5148 const unsigned Opc = Inst.getOpcode();
5149 int DppCtrlIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dpp_ctrl);
5150 if (DppCtrlIdx >= 0) {
5151 unsigned DppCtrl = Inst.getOperand(i: DppCtrlIdx).getImm();
5152
5153 if (!AMDGPU::isLegalDPALU_DPPControl(ST: getSTI(), DC: DppCtrl) &&
5154 AMDGPU::isDPALU_DPP(OpDesc: MII.get(Opcode: Opc), MII, ST: getSTI())) {
5155 // DP ALU DPP is supported for row_newbcast only on GFX9* and row_share
5156 // only on GFX12.
5157 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyDppCtrl, Operands);
5158 Error(L: S, Msg: isGFX12() ? "DP ALU dpp only supports row_share"
5159 : "DP ALU dpp only supports row_newbcast");
5160 return false;
5161 }
5162 }
5163
5164 int Dpp8Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dpp8);
5165 bool IsDPP = DppCtrlIdx >= 0 || Dpp8Idx >= 0;
5166
5167 if (IsDPP && !hasDPPSrc1SGPR(STI: getSTI())) {
5168 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
5169 if (Src1Idx >= 0) {
5170 const MCOperand &Src1 = Inst.getOperand(i: Src1Idx);
5171 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
5172 if (Src1.isReg() && isSGPR(Reg: mc2PseudoReg(Reg: Src1.getReg()), TRI)) {
5173 Error(L: getOperandLoc(Operands, MCOpIdx: Src1Idx),
5174 Msg: "invalid operand for instruction");
5175 return false;
5176 }
5177 if (Src1.isImm()) {
5178 Error(L: getInstLoc(Operands),
5179 Msg: "src1 immediate operand invalid for instruction");
5180 return false;
5181 }
5182 }
5183 }
5184
5185 return true;
5186}
5187
5188// Check if VCC register matches wavefront size
5189bool AMDGPUAsmParser::validateVccOperand(MCRegister Reg) const {
5190 return (Reg == AMDGPU::VCC && isWave64()) ||
5191 (Reg == AMDGPU::VCC_LO && isWave32());
5192}
5193
5194// One unique literal can be used. VOP3 literal is only allowed in GFX10+
5195bool AMDGPUAsmParser::validateVOPLiteral(const MCInst &Inst,
5196 const OperandVector &Operands) {
5197 unsigned Opcode = Inst.getOpcode();
5198 const MCInstrDesc &Desc = MII.get(Opcode);
5199 bool HasMandatoryLiteral = getNamedOperandIdx(Opcode, Name: OpName::imm) != -1;
5200 if (!SIInstrFlags::isVOP3Like(O: Desc) && !HasMandatoryLiteral &&
5201 !isVOPD(Opc: Opcode))
5202 return true;
5203
5204 OperandIndices OpIndices = getSrcOperandIndices(Opcode, AddMandatoryLiterals: HasMandatoryLiteral);
5205
5206 std::optional<unsigned> LiteralOpIdx;
5207 std::optional<uint64_t> LiteralValue;
5208
5209 for (int OpIdx : OpIndices) {
5210 if (OpIdx == -1)
5211 continue;
5212
5213 const MCOperand &MO = Inst.getOperand(i: OpIdx);
5214 if (!MO.isImm() && !MO.isExpr())
5215 continue;
5216 if (!isSISrcOperand(Desc, OpNo: OpIdx))
5217 continue;
5218
5219 std::optional<int64_t> Imm;
5220 if (MO.isImm())
5221 Imm = MO.getImm();
5222 else if (MO.isExpr() && isLitExpr(Expr: MO.getExpr()))
5223 Imm = getLitValue(Expr: MO.getExpr());
5224
5225 bool IsAnotherLiteral = false;
5226 bool IsForcedLit = findMCOperand(Operands, MCOpIdx: OpIdx).isForcedLit();
5227 bool IsForcedLit64 = findMCOperand(Operands, MCOpIdx: OpIdx).isForcedLit64();
5228 if (!Imm.has_value()) {
5229 // Literal value not known, so we conservately assume it's different.
5230 IsAnotherLiteral = true;
5231 } else if (IsForcedLit || IsForcedLit64 || !isInlineConstant(Inst, OpIdx)) {
5232 uint64_t Value = *Imm;
5233 bool IsForcedFP64 =
5234 Desc.operands()[OpIdx].OperandType == AMDGPU::OPERAND_KIMM64 ||
5235 (Desc.operands()[OpIdx].OperandType == AMDGPU::OPERAND_REG_IMM_FP64 &&
5236 HasMandatoryLiteral);
5237 AMDGPU::OperandType OpTy =
5238 static_cast<AMDGPU::OperandType>(Desc.operands()[OpIdx].OperandType);
5239 bool IsFP64 =
5240 (IsForcedFP64 || (AMDGPU::isSISrcFPOperand(Desc, OpNo: OpIdx) &&
5241 OpTy != AMDGPU::OPERAND_REG_IMM_V2INT64)) &&
5242 AMDGPU::getOperandSize(OpInfo: Desc.operands()[OpIdx]) == 8;
5243 bool IsValid32Op =
5244 IsForcedLit || AMDGPU::isValid32BitLiteral(Val: Value, IsFP64);
5245
5246 if (((!IsValid32Op && !isInt<32>(x: Value) && !isUInt<32>(x: Value) &&
5247 !IsForcedFP64) ||
5248 (IsForcedLit64 && !HasMandatoryLiteral)) &&
5249 (!has64BitLiterals() || Desc.getSize() != 4)) {
5250 Error(L: getOperandLoc(Operands, MCOpIdx: OpIdx),
5251 Msg: "invalid operand for instruction");
5252 return false;
5253 }
5254
5255 // Only src0 can use lit64 in VOP* encoding.
5256 if (!IsForcedFP64 && (IsForcedLit64 || !IsValid32Op) &&
5257 OpIdx != getNamedOperandIdx(Opcode, Name: OpName::src0)) {
5258 Error(L: getOperandLoc(Operands, MCOpIdx: OpIdx),
5259 Msg: "invalid operand for instruction");
5260 return false;
5261 }
5262
5263 // Compare values using the word encoded by a 32-bit literal.
5264 if (IsValid32Op && !IsForcedFP64 && !IsForcedLit64) {
5265 Value = static_cast<uint32_t>(
5266 AMDGPU::encode32BitLiteral(Imm: Value, Type: OpTy, IsLit: IsForcedLit));
5267 }
5268
5269 IsAnotherLiteral = !LiteralValue || *LiteralValue != Value;
5270 LiteralValue = Value;
5271 }
5272
5273 if (IsAnotherLiteral && !HasMandatoryLiteral &&
5274 !getFeatureBits()[FeatureVOP3Literal]) {
5275 Error(L: getOperandLoc(Operands, MCOpIdx: OpIdx),
5276 Msg: "literal operands are not supported");
5277 return false;
5278 }
5279
5280 if (LiteralOpIdx && IsAnotherLiteral) {
5281 Error(L: getLaterLoc(a: getOperandLoc(Operands, MCOpIdx: OpIdx),
5282 b: getOperandLoc(Operands, MCOpIdx: *LiteralOpIdx)),
5283 Msg: "only one unique literal operand is allowed");
5284 return false;
5285 }
5286
5287 if (IsAnotherLiteral)
5288 LiteralOpIdx = OpIdx;
5289 }
5290
5291 return true;
5292}
5293
5294// Returns -1 if not a register, 0 if VGPR and 1 if AGPR.
5295static int IsAGPROperand(const MCInst &Inst, AMDGPU::OpName Name,
5296 const MCRegisterInfo *MRI) {
5297 int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name);
5298 if (OpIdx < 0)
5299 return -1;
5300
5301 const MCOperand &Op = Inst.getOperand(i: OpIdx);
5302 if (!Op.isReg())
5303 return -1;
5304
5305 MCRegister Sub = MRI->getSubReg(Reg: Op.getReg(), Idx: AMDGPU::sub0);
5306 auto Reg = Sub ? Sub : Op.getReg();
5307 const MCRegisterClass &AGPR32 = MRI->getRegClass(i: AMDGPU::AGPR_32RegClassID);
5308 return AGPR32.contains(Reg) ? 1 : 0;
5309}
5310
5311bool AMDGPUAsmParser::validateAGPRLdSt(const MCInst &Inst) const {
5312 if (!SIInstrFlags::isFLAT(O: MII, O: Inst) && !SIInstrFlags::isBuffer(O: MII, O: Inst) &&
5313 !SIInstrFlags::isMIMG(O: MII, O: Inst) && !SIInstrFlags::isDS(O: MII, O: Inst))
5314 return true;
5315
5316 AMDGPU::OpName DataName = SIInstrFlags::isDS(O: MII, O: Inst)
5317 ? AMDGPU::OpName::data0
5318 : AMDGPU::OpName::vdata;
5319
5320 const MCRegisterInfo *MRI = getMRI();
5321 int DstAreg = IsAGPROperand(Inst, Name: AMDGPU::OpName::vdst, MRI);
5322 int DataAreg = IsAGPROperand(Inst, Name: DataName, MRI);
5323
5324 if (SIInstrFlags::isDS(O: MII, O: Inst) && DataAreg >= 0) {
5325 int Data2Areg = IsAGPROperand(Inst, Name: AMDGPU::OpName::data1, MRI);
5326 if (Data2Areg >= 0 && Data2Areg != DataAreg)
5327 return false;
5328 }
5329
5330 auto FB = getFeatureBits();
5331 if (FB[AMDGPU::FeatureGFX90AInsts]) {
5332 if (DataAreg < 0 || DstAreg < 0)
5333 return true;
5334 return DstAreg == DataAreg;
5335 }
5336
5337 return DstAreg < 1 && DataAreg < 1;
5338}
5339
5340bool AMDGPUAsmParser::validateVGPRAlign(const MCInst &Inst) const {
5341 auto FB = getFeatureBits();
5342 if (!FB[AMDGPU::FeatureRequiresAlignedVGPRs])
5343 return true;
5344
5345 unsigned Opc = Inst.getOpcode();
5346 const MCRegisterInfo *MRI = getMRI();
5347 // DS_READ_B96_TR_B6 is the only DS instruction in GFX950, that allows
5348 // unaligned VGPR. All others only allow even aligned VGPRs.
5349 if (FB[AMDGPU::FeatureGFX90AInsts] && Opc == AMDGPU::DS_READ_B96_TR_B6_vi)
5350 return true;
5351
5352 if (FB[AMDGPU::FeatureGFX1250Insts]) {
5353 switch (Opc) {
5354 default:
5355 break;
5356 case AMDGPU::DS_LOAD_TR6_B96:
5357 case AMDGPU::DS_LOAD_TR6_B96_gfx12:
5358 // DS_LOAD_TR6_B96 is the only DS instruction in GFX1250, that
5359 // allows unaligned VGPR. All others only allow even aligned VGPRs.
5360 return true;
5361 case AMDGPU::GLOBAL_LOAD_TR6_B96:
5362 case AMDGPU::GLOBAL_LOAD_TR6_B96_gfx1250: {
5363 // GLOBAL_LOAD_TR6_B96 is the only GLOBAL instruction in GFX1250, that
5364 // allows unaligned VGPR for vdst, but other operands still only allow
5365 // even aligned VGPRs.
5366 int VAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr);
5367 if (VAddrIdx != -1) {
5368 const MCOperand &Op = Inst.getOperand(i: VAddrIdx);
5369 MCRegister Sub = MRI->getSubReg(Reg: Op.getReg(), Idx: AMDGPU::sub0);
5370 if ((Sub - AMDGPU::VGPR0) & 1)
5371 return false;
5372 }
5373 return true;
5374 }
5375 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR:
5376 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR_gfx1250:
5377 return true;
5378 }
5379 }
5380
5381 const MCRegisterClass &VGPR32 = MRI->getRegClass(i: AMDGPU::VGPR_32RegClassID);
5382 const MCRegisterClass &AGPR32 = MRI->getRegClass(i: AMDGPU::AGPR_32RegClassID);
5383 for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) {
5384 const MCOperand &Op = Inst.getOperand(i: I);
5385 if (!Op.isReg())
5386 continue;
5387
5388 MCRegister Sub = MRI->getSubReg(Reg: Op.getReg(), Idx: AMDGPU::sub0);
5389 if (!Sub)
5390 continue;
5391
5392 if (VGPR32.contains(Reg: Sub) && ((Sub - AMDGPU::VGPR0) & 1))
5393 return false;
5394 if (AGPR32.contains(Reg: Sub) && ((Sub - AMDGPU::AGPR0) & 1))
5395 return false;
5396 }
5397
5398 return true;
5399}
5400
5401SMLoc AMDGPUAsmParser::getBLGPLoc(const OperandVector &Operands) const {
5402 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
5403 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
5404 if (Op.isBLGP())
5405 return Op.getStartLoc();
5406 }
5407 return SMLoc();
5408}
5409
5410bool AMDGPUAsmParser::validateBLGP(const MCInst &Inst,
5411 const OperandVector &Operands) {
5412 unsigned Opc = Inst.getOpcode();
5413 int BlgpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::blgp);
5414 if (BlgpIdx == -1)
5415 return true;
5416 SMLoc BLGPLoc = getBLGPLoc(Operands);
5417 if (!BLGPLoc.isValid())
5418 return true;
5419 bool IsNeg = StringRef(BLGPLoc.getPointer()).starts_with(Prefix: "neg:");
5420 auto FB = getFeatureBits();
5421 bool UsesNeg = false;
5422 if (FB[AMDGPU::FeatureGFX940Insts]) {
5423 switch (Opc) {
5424 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_acd:
5425 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_vcd:
5426 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_acd:
5427 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_vcd:
5428 UsesNeg = true;
5429 }
5430 }
5431
5432 if (IsNeg == UsesNeg)
5433 return true;
5434
5435 Error(L: BLGPLoc, Msg: UsesNeg ? "invalid modifier: blgp is not supported"
5436 : "invalid modifier: neg is not supported");
5437
5438 return false;
5439}
5440
5441bool AMDGPUAsmParser::validateWaitCnt(const MCInst &Inst,
5442 const OperandVector &Operands) {
5443 if (!isGFX11Plus())
5444 return true;
5445
5446 unsigned Opc = Inst.getOpcode();
5447 if (Opc != AMDGPU::S_WAITCNT_EXPCNT_gfx11 &&
5448 Opc != AMDGPU::S_WAITCNT_LGKMCNT_gfx11 &&
5449 Opc != AMDGPU::S_WAITCNT_VMCNT_gfx11 &&
5450 Opc != AMDGPU::S_WAITCNT_VSCNT_gfx11)
5451 return true;
5452
5453 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::sdst);
5454 assert(Src0Idx >= 0 && Inst.getOperand(Src0Idx).isReg());
5455 auto Reg = mc2PseudoReg(Reg: Inst.getOperand(i: Src0Idx).getReg());
5456 if (Reg == AMDGPU::SGPR_NULL)
5457 return true;
5458
5459 Error(L: getOperandLoc(Operands, MCOpIdx: Src0Idx), Msg: "src0 must be null");
5460 return false;
5461}
5462
5463bool AMDGPUAsmParser::validateDS(const MCInst &Inst,
5464 const OperandVector &Operands) {
5465 if (!SIInstrFlags::isDS(O: MII, O: Inst))
5466 return true;
5467 if (SIInstrFlags::isGWS(O: MII, O: Inst))
5468 return validateGWS(Inst, Operands);
5469 // Only validate GDS for non-GWS instructions.
5470 if (hasGDS())
5471 return true;
5472 int GDSIdx =
5473 AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::gds);
5474 if (GDSIdx < 0)
5475 return true;
5476 unsigned GDS = Inst.getOperand(i: GDSIdx).getImm();
5477 if (GDS) {
5478 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyGDS, Operands);
5479 Error(L: S, Msg: "gds modifier is not supported on this GPU");
5480 return false;
5481 }
5482 return true;
5483}
5484
5485// gfx90a has an undocumented limitation:
5486// DS_GWS opcodes must use even aligned registers.
5487bool AMDGPUAsmParser::validateGWS(const MCInst &Inst,
5488 const OperandVector &Operands) {
5489 if (!getFeatureBits()[AMDGPU::FeatureGFX90AInsts])
5490 return true;
5491
5492 int Opc = Inst.getOpcode();
5493 if (Opc != AMDGPU::DS_GWS_INIT_vi && Opc != AMDGPU::DS_GWS_BARRIER_vi &&
5494 Opc != AMDGPU::DS_GWS_SEMA_BR_vi)
5495 return true;
5496
5497 const MCRegisterInfo *MRI = getMRI();
5498 const MCRegisterClass &VGPR32 = MRI->getRegClass(i: AMDGPU::VGPR_32RegClassID);
5499 int Data0Pos =
5500 AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::data0);
5501 assert(Data0Pos != -1);
5502 auto Reg = Inst.getOperand(i: Data0Pos).getReg();
5503 auto RegIdx = Reg - (VGPR32.contains(Reg) ? AMDGPU::VGPR0 : AMDGPU::AGPR0);
5504 if (RegIdx & 1) {
5505 Error(L: getOperandLoc(Operands, MCOpIdx: Data0Pos), Msg: "vgpr must be even aligned");
5506 return false;
5507 }
5508
5509 return true;
5510}
5511
5512bool AMDGPUAsmParser::validateCoherencyBits(const MCInst &Inst,
5513 const OperandVector &Operands,
5514 SMLoc IDLoc) {
5515 int CPolPos =
5516 AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::cpol);
5517 if (CPolPos == -1)
5518 return true;
5519
5520 unsigned CPol = Inst.getOperand(i: CPolPos).getImm();
5521
5522 if (!isGFX1250Plus()) {
5523 if (CPol & CPol::SCAL) {
5524 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands);
5525 StringRef CStr(S.getPointer());
5526 S = SMLoc::getFromPointer(Ptr: &CStr.data()[CStr.find(Str: "scale_offset")]);
5527 Error(L: S, Msg: "scale_offset is not supported on this GPU");
5528 }
5529 if (CPol & CPol::NV) {
5530 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands);
5531 StringRef CStr(S.getPointer());
5532 S = SMLoc::getFromPointer(Ptr: &CStr.data()[CStr.find(Str: "nv")]);
5533 Error(L: S, Msg: "nv is not supported on this GPU");
5534 }
5535 }
5536
5537 if ((CPol & CPol::SCAL) && !supportsScaleOffset(MII, Opcode: Inst.getOpcode())) {
5538 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands);
5539 StringRef CStr(S.getPointer());
5540 S = SMLoc::getFromPointer(Ptr: &CStr.data()[CStr.find(Str: "scale_offset")]);
5541 Error(L: S, Msg: "scale_offset is not supported for this instruction");
5542 }
5543
5544 if (isGFX12Plus())
5545 return validateTHAndScopeBits(Inst, Operands, CPol);
5546
5547 if (SIInstrFlags::isSMRD(O: MII, O: Inst)) {
5548 if (CPol && (isSI() || isCI())) {
5549 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands);
5550 Error(L: S, Msg: "cache policy is not supported for SMRD instructions");
5551 return false;
5552 }
5553 if (CPol & ~(AMDGPU::CPol::GLC | AMDGPU::CPol::DLC)) {
5554 Error(L: IDLoc, Msg: "invalid cache policy for SMEM instruction");
5555 return false;
5556 }
5557 }
5558
5559 if (isGFX90A() && !isGFX940() && (CPol & CPol::SCC)) {
5560 if (!SIInstrFlags::isVMEM(O: MII, O: Inst)) {
5561 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands);
5562 StringRef CStr(S.getPointer());
5563 S = SMLoc::getFromPointer(Ptr: &CStr.data()[CStr.find(Str: "scc")]);
5564 Error(L: S,
5565 Msg: "scc modifier is not supported for this instruction on this GPU");
5566 return false;
5567 }
5568 }
5569
5570 if (!SIInstrFlags::isAtomic(O: MII, O: Inst))
5571 return true;
5572
5573 if (SIInstrFlags::isAtomicRet(O: MII, O: Inst)) {
5574 if (!SIInstrFlags::isMIMG(O: MII, O: Inst) && !(CPol & CPol::GLC)) {
5575 Error(L: IDLoc, Msg: isGFX940() ? "instruction must use sc0"
5576 : "instruction must use glc");
5577 return false;
5578 }
5579 } else {
5580 if (CPol & CPol::GLC) {
5581 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands);
5582 StringRef CStr(S.getPointer());
5583 S = SMLoc::getFromPointer(
5584 Ptr: &CStr.data()[CStr.find(Str: isGFX940() ? "sc0" : "glc")]);
5585 Error(L: S, Msg: isGFX940() ? "instruction must not use sc0"
5586 : "instruction must not use glc");
5587 return false;
5588 }
5589 }
5590
5591 return true;
5592}
5593
5594bool AMDGPUAsmParser::validateTHAndScopeBits(const MCInst &Inst,
5595 const OperandVector &Operands,
5596 const unsigned CPol) {
5597 const unsigned TH = CPol & AMDGPU::CPol::TH;
5598 const unsigned Scope = CPol & AMDGPU::CPol::SCOPE;
5599
5600 auto PrintError = [&](StringRef Msg) {
5601 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands);
5602 Error(L: S, Msg);
5603 return false;
5604 };
5605
5606 if ((TH & AMDGPU::CPol::TH_ATOMIC_RETURN) &&
5607 SIInstrFlags::isAtomicNoRet(O: MII, O: Inst))
5608 return PrintError("th:TH_ATOMIC_RETURN requires a destination operand");
5609
5610 if (SIInstrFlags::isAtomicRet(O: MII, O: Inst) &&
5611 (SIInstrFlags::isFLAT(O: MII, O: Inst) || SIInstrFlags::isMUBUF(O: MII, O: Inst)) &&
5612 (!(TH & AMDGPU::CPol::TH_ATOMIC_RETURN)))
5613 return PrintError("instruction must use th:TH_ATOMIC_RETURN");
5614
5615 if (TH == 0)
5616 return true;
5617
5618 if (SIInstrFlags::isSMRD(O: MII, O: Inst) &&
5619 ((TH == AMDGPU::CPol::TH_NT_RT) || (TH == AMDGPU::CPol::TH_RT_NT) ||
5620 (TH == AMDGPU::CPol::TH_NT_HT)))
5621 return PrintError("invalid th value for SMEM instruction");
5622
5623 if (TH == AMDGPU::CPol::TH_BYPASS) {
5624 if ((Scope != AMDGPU::CPol::SCOPE_SYS &&
5625 CPol & AMDGPU::CPol::TH_REAL_BYPASS) ||
5626 (Scope == AMDGPU::CPol::SCOPE_SYS &&
5627 !(CPol & AMDGPU::CPol::TH_REAL_BYPASS)))
5628 return PrintError("scope and th combination is not valid");
5629 }
5630
5631 unsigned THType = AMDGPU::getTemporalHintType(TID: MII.get(Opcode: Inst.getOpcode()));
5632 if (THType == AMDGPU::CPol::TH_TYPE_ATOMIC) {
5633 if (!(CPol & AMDGPU::CPol::TH_TYPE_ATOMIC))
5634 return PrintError("invalid th value for atomic instructions");
5635 } else if (THType == AMDGPU::CPol::TH_TYPE_STORE) {
5636 if (!(CPol & AMDGPU::CPol::TH_TYPE_STORE))
5637 return PrintError("invalid th value for store instructions");
5638 } else {
5639 if (!(CPol & AMDGPU::CPol::TH_TYPE_LOAD))
5640 return PrintError("invalid th value for load instructions");
5641 }
5642
5643 return true;
5644}
5645
5646bool AMDGPUAsmParser::validateTFE(const MCInst &Inst,
5647 const OperandVector &Operands) {
5648 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
5649 if (Desc.mayStore() && SIInstrFlags::isBuffer(O: Desc)) {
5650 SMLoc Loc = getImmLoc(Type: AMDGPUOperand::ImmTyTFE, Operands);
5651 if (Loc != getInstLoc(Operands)) {
5652 Error(L: Loc, Msg: "TFE modifier has no meaning for store instructions");
5653 return false;
5654 }
5655 }
5656
5657 return true;
5658}
5659
5660bool AMDGPUAsmParser::validateWMMA(const MCInst &Inst,
5661 const OperandVector &Operands) {
5662 unsigned Opc = Inst.getOpcode();
5663 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
5664 const MCInstrDesc &Desc = MII.get(Opcode: Opc);
5665
5666 int AFmtIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_a_fmt);
5667 if (AFmtIdx == -1)
5668 return true;
5669 unsigned AFmt = Inst.getOperand(i: AFmtIdx).getImm();
5670 int BFmtIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_b_fmt);
5671 unsigned BFmt = Inst.getOperand(i: BFmtIdx).getImm();
5672
5673 auto validateFmt = [&](unsigned Fmt, AMDGPU::OpName SrcOp) -> bool {
5674 int SrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: SrcOp);
5675 unsigned RegSize =
5676 TRI->getRegClass(i: MII.getOpRegClassID(OpInfo: Desc.operands()[SrcIdx], HwModeId: HwMode))
5677 .getSizeInBits();
5678
5679 if (RegSize == AMDGPU::wmmaScaleF8F6F4FormatToNumRegs(Fmt) * 32)
5680 return true;
5681
5682 Error(L: getOperandLoc(Operands, MCOpIdx: SrcIdx),
5683 Msg: "wrong register tuple size for " +
5684 Twine(WMMAMods::ModMatrixFmt[Fmt]));
5685 return false;
5686 };
5687
5688 if (!validateFmt(AFmt, AMDGPU::OpName::src0) ||
5689 !validateFmt(BFmt, AMDGPU::OpName::src1))
5690 return false;
5691
5692 int AScaleIdx =
5693 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_a_scale_fmt);
5694 if (AScaleIdx == -1)
5695 return true;
5696 unsigned AScale = Inst.getOperand(i: AScaleIdx).getImm();
5697 int BScaleIdx =
5698 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_b_scale_fmt);
5699 unsigned BScale = Inst.getOperand(i: BScaleIdx).getImm();
5700 if (!isValidWMMAScaleFmtCombination(AFmt, AScale, BFmt, BScale)) {
5701 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyMatrixAFMT, Operands),
5702 Msg: "invalid matrix and scale format combination");
5703 return false;
5704 }
5705
5706 return true;
5707}
5708
5709bool AMDGPUAsmParser::validateMonitorSleep(const MCInst &Inst,
5710 const OperandVector &Operands) {
5711 unsigned Opc = Inst.getOpcode();
5712 if (Opc != AMDGPU::S_MONITOR_SLEEP_gfx12 ||
5713 !getSTI().hasFeature(Feature: AMDGPU::FeatureNoSleepForever))
5714 return true;
5715
5716 int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::simm16);
5717 if (Inst.getOperand(i: ImmIdx).getImm() & 0x8000) {
5718 Error(L: getOperandLoc(Operands, MCOpIdx: ImmIdx),
5719 Msg: "sleep forever is unsuported on the target");
5720 return false;
5721 }
5722
5723 return true;
5724}
5725
5726bool AMDGPUAsmParser::validateClusterBarrierIsFirst(
5727 const MCInst &Inst, const OperandVector &Operands) {
5728 unsigned Opc = Inst.getOpcode();
5729 if (Opc != AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM_gfx12 &&
5730 Opc != AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM_gfx13)
5731 return true;
5732
5733 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
5734 int BarrierID = Inst.getOperand(i: Src0Idx).getImm();
5735 if (BarrierID != AMDGPU::Barrier::CLUSTER)
5736 return true;
5737
5738 Error(
5739 L: getOperandLoc(Operands, MCOpIdx: Src0Idx),
5740 Msg: "s_barrier_signal_isfirst does not support user_cluster_barrier_id (-3)");
5741 return false;
5742}
5743
5744bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst, SMLoc IDLoc,
5745 const OperandVector &Operands) {
5746 if (!validateLdsDirect(Inst, Operands))
5747 return false;
5748 if (!validateTrue16OpSel(Inst)) {
5749 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyOpSel, Operands),
5750 Msg: "op_sel operand conflicts with 16-bit operand suffix");
5751 return false;
5752 }
5753 if (!validateSOPLiteral(Inst, Operands))
5754 return false;
5755 if (!validateVOPLiteral(Inst, Operands)) {
5756 return false;
5757 }
5758 if (!validateConstantBusLimitations(Inst, Operands)) {
5759 return false;
5760 }
5761 if (!validateVOPD(Inst, Operands)) {
5762 return false;
5763 }
5764 if (!validateIntClampSupported(Inst)) {
5765 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyClamp, Operands),
5766 Msg: "integer clamping is not supported on this GPU");
5767 return false;
5768 }
5769 if (!validateOpSel(Inst)) {
5770 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyOpSel, Operands),
5771 Msg: "invalid op_sel operand");
5772 return false;
5773 }
5774 if (!validateNeg(Inst, OpName: AMDGPU::OpName::neg_lo)) {
5775 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyNegLo, Operands),
5776 Msg: "invalid neg_lo operand");
5777 return false;
5778 }
5779 if (!validateNeg(Inst, OpName: AMDGPU::OpName::neg_hi)) {
5780 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyNegHi, Operands),
5781 Msg: "invalid neg_hi operand");
5782 return false;
5783 }
5784 if (!validateDPP(Inst, Operands)) {
5785 return false;
5786 }
5787 // For MUBUF/MTBUF d16 is a part of opcode, so there is nothing to validate.
5788 if (!validateMIMGD16(Inst)) {
5789 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyD16, Operands),
5790 Msg: "d16 modifier is not supported on this GPU");
5791 return false;
5792 }
5793 if (!validateMIMGDim(Inst, Operands)) {
5794 Error(L: IDLoc, Msg: "missing dim operand");
5795 return false;
5796 }
5797 if (!validateTensorR128(Inst)) {
5798 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyD16, Operands),
5799 Msg: "instruction must set modifier r128=0");
5800 return false;
5801 }
5802 if (!validateMIMGMSAA(Inst)) {
5803 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyDim, Operands),
5804 Msg: "invalid dim; must be MSAA type");
5805 return false;
5806 }
5807 if (!validateMIMGDataSize(Inst, IDLoc)) {
5808 return false;
5809 }
5810 if (!validateMIMGAddrSize(Inst, IDLoc))
5811 return false;
5812 if (!validateMIMGAtomicDMask(Inst)) {
5813 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyDMask, Operands),
5814 Msg: "invalid atomic image dmask");
5815 return false;
5816 }
5817 if (!validateMIMGGatherDMask(Inst)) {
5818 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyDMask, Operands),
5819 Msg: "invalid image_gather dmask: only one bit must be set");
5820 return false;
5821 }
5822 if (!validateMovrels(Inst, Operands)) {
5823 return false;
5824 }
5825 if (!validateOffset(Inst, Operands)) {
5826 return false;
5827 }
5828 if (!validateBF16InlineConst(Inst, Operands)) {
5829 return false;
5830 }
5831 if (!validateMAIAccWrite(Inst, Operands)) {
5832 return false;
5833 }
5834 if (!validateMAISrc2(Inst, Operands)) {
5835 return false;
5836 }
5837 if (!validateMFMA(Inst, Operands)) {
5838 return false;
5839 }
5840 if (!validateCoherencyBits(Inst, Operands, IDLoc)) {
5841 return false;
5842 }
5843
5844 if (!validateAGPRLdSt(Inst)) {
5845 Error(
5846 L: IDLoc,
5847 Msg: getFeatureBits()[AMDGPU::FeatureGFX90AInsts]
5848 ? "invalid register class: data and dst should be all VGPR or AGPR"
5849 : "invalid register class: agpr loads and stores not supported on "
5850 "this GPU");
5851 return false;
5852 }
5853 if (!validateVGPRAlign(Inst)) {
5854 Error(L: IDLoc, Msg: "invalid register class: vgpr tuples must be 64 bit aligned");
5855 return false;
5856 }
5857 if (!validateDS(Inst, Operands)) {
5858 return false;
5859 }
5860
5861 if (!validateBLGP(Inst, Operands)) {
5862 return false;
5863 }
5864
5865 if (!validateDivScale(Inst)) {
5866 Error(L: IDLoc, Msg: "ABS not allowed in VOP3B instructions");
5867 return false;
5868 }
5869 if (!validateWaitCnt(Inst, Operands)) {
5870 return false;
5871 }
5872 if (!validateTFE(Inst, Operands)) {
5873 return false;
5874 }
5875 if (!validateWMMA(Inst, Operands)) {
5876 return false;
5877 }
5878 if (!validateMonitorSleep(Inst, Operands)) {
5879 return false;
5880 }
5881 if (!validateClusterBarrierIsFirst(Inst, Operands)) {
5882 return false;
5883 }
5884
5885 return true;
5886}
5887
5888static std::string AMDGPUMnemonicSpellCheck(StringRef S,
5889 const FeatureBitset &FBS,
5890 unsigned VariantID = 0);
5891
5892static bool AMDGPUCheckMnemonic(StringRef Mnemonic,
5893 const FeatureBitset &AvailableFeatures,
5894 unsigned VariantID);
5895
5896bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
5897 const FeatureBitset &FBS) {
5898 return isSupportedMnemo(Mnemo, FBS, Variants: getAllVariants());
5899}
5900
5901bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
5902 const FeatureBitset &FBS,
5903 ArrayRef<unsigned> Variants) {
5904 for (auto Variant : Variants) {
5905 if (AMDGPUCheckMnemonic(Mnemonic: Mnemo, AvailableFeatures: FBS, VariantID: Variant))
5906 return true;
5907 }
5908
5909 return false;
5910}
5911
5912bool AMDGPUAsmParser::checkUnsupportedInstruction(StringRef Mnemo,
5913 SMLoc IDLoc) {
5914 FeatureBitset FBS = ComputeAvailableFeatures(FB: getFeatureBits());
5915
5916 // Check if requested instruction variant is supported.
5917 if (isSupportedMnemo(Mnemo, FBS, Variants: getMatchedVariants()))
5918 return false;
5919
5920 // This instruction is not supported.
5921 // Clear any other pending errors because they are no longer relevant.
5922 getParser().clearPendingErrors();
5923
5924 // Requested instruction variant is not supported.
5925 // Check if any other variants are supported.
5926 StringRef VariantName = getMatchedVariantName();
5927 if (!VariantName.empty() && isSupportedMnemo(Mnemo, FBS)) {
5928 return Error(L: IDLoc, Msg: Twine(VariantName,
5929 " variant of this instruction is not supported"));
5930 }
5931
5932 // Check if this instruction may be used with a different wavesize.
5933 if (isGFX10Plus() && getFeatureBits()[AMDGPU::FeatureWavefrontSize64] &&
5934 !getFeatureBits()[AMDGPU::FeatureWavefrontSize32]) {
5935 // FIXME: Use getAvailableFeatures, and do not manually recompute
5936 FeatureBitset FeaturesWS32 = getFeatureBits();
5937 FeaturesWS32.flip(I: AMDGPU::FeatureWavefrontSize64)
5938 .flip(I: AMDGPU::FeatureWavefrontSize32);
5939 FeatureBitset AvailableFeaturesWS32 =
5940 ComputeAvailableFeatures(FB: FeaturesWS32);
5941
5942 if (isSupportedMnemo(Mnemo, FBS: AvailableFeaturesWS32, Variants: getMatchedVariants()))
5943 return Error(L: IDLoc, Msg: "instruction requires wavesize=32");
5944 }
5945
5946 // Finally check if this instruction is supported on any other GPU.
5947 if (isSupportedMnemo(Mnemo, FBS: FeatureBitset().set())) {
5948 return Error(L: IDLoc, Msg: "instruction not supported on this GPU (" +
5949 getSTI().getCPU() + ")" + ": " + Mnemo);
5950 }
5951
5952 // Instruction not supported on any GPU. Probably a typo.
5953 std::string Suggestion = AMDGPUMnemonicSpellCheck(S: Mnemo, FBS);
5954 return Error(L: IDLoc, Msg: "invalid instruction" + Suggestion);
5955}
5956
5957static bool isInvalidVOPDY(const OperandVector &Operands,
5958 uint64_t InvalidOprIdx) {
5959 assert(InvalidOprIdx < Operands.size());
5960 const auto &Op = ((AMDGPUOperand &)*Operands[InvalidOprIdx]);
5961 if (Op.isToken() && InvalidOprIdx > 1) {
5962 const auto &PrevOp = ((AMDGPUOperand &)*Operands[InvalidOprIdx - 1]);
5963 return PrevOp.isToken() && PrevOp.getToken() == "::";
5964 }
5965 return false;
5966}
5967
5968bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
5969 OperandVector &Operands,
5970 MCStreamer &Out,
5971 uint64_t &ErrorInfo,
5972 bool MatchingInlineAsm) {
5973 MCInst Inst;
5974 Inst.setLoc(IDLoc);
5975 unsigned Result = Match_Success;
5976
5977 // Order match statuses from least to most specific and keep the most
5978 // specific one:
5979 // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature
5980 auto atLeastAsSpecific = [](unsigned New, unsigned Cur) {
5981 auto rank = [](unsigned M) {
5982 return M == Match_MnemonicFail ? 1
5983 : M == Match_InvalidOperand ? 2
5984 : M == Match_MissingFeature ? 3
5985 : 0; // Match_Success sentinel
5986 };
5987 return rank(New) >= rank(Cur);
5988 };
5989
5990 for (auto Variant : getMatchedVariants()) {
5991 uint64_t EI;
5992 auto R =
5993 MatchInstructionImpl(Operands, Inst, ErrorInfo&: EI, matchingInlineAsm: MatchingInlineAsm, VariantID: Variant);
5994 if (R == Match_Success || atLeastAsSpecific(R, Result)) {
5995 Result = R;
5996 ErrorInfo = EI;
5997 }
5998 if (R == Match_Success)
5999 break;
6000 }
6001
6002 if (Result == Match_Success) {
6003 if (!validateInstruction(Inst, IDLoc, Operands)) {
6004 return true;
6005 }
6006 emitTargetDirective();
6007 Out.emitInstruction(Inst, STI: getSTI());
6008 // Record for kernel prologue checking.
6009 OpcodeStream.push_back(Elt: Inst.getOpcode());
6010 return false;
6011 }
6012
6013 StringRef Mnemo = ((AMDGPUOperand &)*Operands[0]).getToken();
6014 if (checkUnsupportedInstruction(Mnemo, IDLoc)) {
6015 return true;
6016 }
6017
6018 switch (Result) {
6019 default:
6020 break;
6021 case Match_MissingFeature:
6022 // It has been verified that the specified instruction
6023 // mnemonic is valid. A match was found but it requires
6024 // features which are not supported on this GPU.
6025 return Error(L: IDLoc, Msg: "operands are not valid for this GPU or mode");
6026
6027 case Match_InvalidOperand: {
6028 SMLoc ErrorLoc = IDLoc;
6029 if (ErrorInfo != ~0ULL) {
6030 if (ErrorInfo >= Operands.size()) {
6031 return Error(L: IDLoc, Msg: "too few operands for instruction");
6032 }
6033 AMDGPUOperand &ErrorOp = (AMDGPUOperand &)*Operands[ErrorInfo];
6034 ErrorLoc = ErrorOp.getStartLoc();
6035 if (ErrorLoc == SMLoc())
6036 ErrorLoc = IDLoc;
6037
6038 if (isInvalidVOPDY(Operands, InvalidOprIdx: ErrorInfo))
6039 return Error(L: ErrorLoc, Msg: "invalid VOPDY instruction");
6040 }
6041 return Error(L: ErrorLoc, Msg: "invalid operand for instruction");
6042 }
6043
6044 case Match_MnemonicFail:
6045 llvm_unreachable("Invalid instructions should have been handled already");
6046 }
6047 llvm_unreachable("Implement any new match types added!");
6048}
6049
6050bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
6051 int64_t Tmp = -1;
6052 if (!isToken(Kind: AsmToken::Integer) && !isToken(Kind: AsmToken::Identifier)) {
6053 return true;
6054 }
6055 if (getParser().parseAbsoluteExpression(Res&: Tmp)) {
6056 return true;
6057 }
6058 Ret = static_cast<uint32_t>(Tmp);
6059 return false;
6060}
6061
6062bool AMDGPUAsmParser::ParseDirectiveAMDGCNTarget() {
6063 if (!getSTI().getTargetTriple().isAMDGCN())
6064 return TokError(Msg: "directive only supported for amdgcn architecture");
6065
6066 std::string TargetIDDirective;
6067 SMLoc TargetStart = getTok().getLoc();
6068 if (getParser().parseEscapedString(Data&: TargetIDDirective))
6069 return true;
6070
6071 std::optional<AMDGPU::TargetID> MaybeParsed =
6072 AMDGPU::TargetID::parseTargetIDString(TargetIDDirective);
6073 if (!MaybeParsed)
6074 return getParser().Error(L: TargetStart,
6075 Msg: "malformed target id '" + TargetIDDirective + "'");
6076
6077 const AMDGPU::TargetID &ParsedTargetID = *MaybeParsed;
6078 const Triple &TT = getSTI().getTargetTriple();
6079
6080 // The processor named in the target id must be covered by the triple's
6081 // subarch.
6082 if (!AMDGPU::isCPUValidForSubArch(SubArch: TT.getSubArch(),
6083 AK: ParsedTargetID.getGPUKind())) {
6084 return getParser().Error(
6085 L: TargetStart, Msg: "target id '" + TargetIDDirective +
6086 "' specifies a processor that is not valid for "
6087 "subarch '" +
6088 TT.getArchName() + "'");
6089 }
6090
6091 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
6092 getTargetStreamer().getTargetID();
6093
6094 Triple DirectiveTriple(ParsedTargetID.getTargetTripleString());
6095 const Triple &STITriple = getSTI().getTargetTriple();
6096 if (!DirectiveTriple.isCompatibleWith(Other: STITriple)) {
6097 return getParser().Error(
6098 L: TargetStart, Msg: ".amdgcn_target " + Twine(ParsedTargetID.toString()) +
6099 " is incompatible with " +
6100 Twine(CurrentTargetID->toString()));
6101 }
6102
6103 // Error if the ISA version doesn't match
6104 StringRef DirectiveProcessor =
6105 AMDGPU::getArchNameAMDGCN(AK: ParsedTargetID.getGPUKind());
6106 AMDGPU::IsaVersion DirectiveISA = AMDGPU::getIsaVersion(GPU: DirectiveProcessor);
6107 if (DirectiveISA != ISA) {
6108 return getParser().Error(L: TargetStart,
6109 Msg: ".amdgcn_target directive processor " +
6110 Twine(DirectiveProcessor) +
6111 " does not match the specified processor " +
6112 Twine(getSTI().getCPU()));
6113 }
6114
6115 // Warn if sramecc or xnack mismatch. These do not change the encoding.
6116 if (AMDGPU::IsaInfo::targetIDSettingsConflict(
6117 Lhs: ParsedTargetID.getXnackSetting(),
6118 Rhs: CurrentTargetID->getXnackSetting())) {
6119 Warning(L: TargetStart,
6120 Msg: ".amdgcn_target directive has conflicting xnack settings");
6121 }
6122 if (AMDGPU::IsaInfo::targetIDSettingsConflict(
6123 Lhs: ParsedTargetID.getSramEccSetting(),
6124 Rhs: CurrentTargetID->getSramEccSetting())) {
6125 Warning(L: TargetStart,
6126 Msg: ".amdgcn_target directive has conflicting sramecc settings");
6127 }
6128
6129 // Update the target streamer's TargetID with settings from the directive.
6130 // We don't update the MCSubtargetInfo because we've already validated
6131 // that the directive matches the command-line CPU.
6132 getTargetStreamer().getTargetID()->setXnackSetting(
6133 ParsedTargetID.getXnackSetting());
6134 getTargetStreamer().getTargetID()->setSramEccSetting(
6135 ParsedTargetID.getSramEccSetting());
6136
6137 return false;
6138}
6139
6140bool AMDGPUAsmParser::OutOfRangeError(SMRange Range) {
6141 return Error(L: Range.Start, Msg: "value out of range", Range);
6142}
6143
6144bool AMDGPUAsmParser::calculateGPRBlocks(
6145 const FeatureBitset &Features, const MCExpr *VCCUsed,
6146 const MCExpr *FlatScrUsed, bool XNACKUsed,
6147 std::optional<bool> EnableWavefrontSize32, const MCExpr *NextFreeVGPR,
6148 SMRange VGPRRange, const MCExpr *NextFreeSGPR, SMRange SGPRRange,
6149 const MCExpr *&VGPRBlocks, const MCExpr *&SGPRBlocks) {
6150 // TODO(scott.linder): These calculations are duplicated from
6151 // AMDGPUAsmPrinter::getSIProgramInfo and could be unified.
6152 MCContext &Ctx = getContext();
6153
6154 const MCExpr *NumSGPRs = NextFreeSGPR;
6155 int64_t EvaluatedSGPRs;
6156
6157 if (ISA.Major >= 10)
6158 NumSGPRs = MCConstantExpr::create(Value: 0, Ctx);
6159 else {
6160 unsigned MaxAddressableNumSGPRs = AMDGPU::getAddressableNumSGPRs(AK: Gfx);
6161
6162 if (NumSGPRs->evaluateAsAbsolute(Res&: EvaluatedSGPRs) && ISA.Major >= 8 &&
6163 !Features.test(I: FeatureSGPRInitBug) &&
6164 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6165 return OutOfRangeError(Range: SGPRRange);
6166
6167 const MCExpr *ExtraSGPRs =
6168 AMDGPUMCExpr::createExtraSGPRs(VCCUsed, FlatScrUsed, XNACKUsed, Ctx);
6169 NumSGPRs = MCBinaryExpr::createAdd(LHS: NumSGPRs, RHS: ExtraSGPRs, Ctx);
6170
6171 if (NumSGPRs->evaluateAsAbsolute(Res&: EvaluatedSGPRs) &&
6172 (ISA.Major <= 7 || Features.test(I: FeatureSGPRInitBug)) &&
6173 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6174 return OutOfRangeError(Range: SGPRRange);
6175
6176 if (Features.test(I: FeatureSGPRInitBug))
6177 NumSGPRs =
6178 MCConstantExpr::create(Value: IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG, Ctx);
6179 }
6180
6181 // The MCExpr equivalent of getNumSGPRBlocks/getNumVGPRBlocks:
6182 // (alignTo(max(1u, NumGPR), GPREncodingGranule) / GPREncodingGranule) - 1
6183 auto GetNumGPRBlocks = [&Ctx](const MCExpr *NumGPR,
6184 unsigned Granule) -> const MCExpr * {
6185 const MCExpr *OneConst = MCConstantExpr::create(Value: 1ul, Ctx);
6186 const MCExpr *GranuleConst = MCConstantExpr::create(Value: Granule, Ctx);
6187 const MCExpr *MaxNumGPR = AMDGPUMCExpr::createMax(Args: {NumGPR, OneConst}, Ctx);
6188 const MCExpr *AlignToGPR =
6189 AMDGPUMCExpr::createAlignTo(Value: MaxNumGPR, Align: GranuleConst, Ctx);
6190 const MCExpr *DivGPR =
6191 MCBinaryExpr::createDiv(LHS: AlignToGPR, RHS: GranuleConst, Ctx);
6192 const MCExpr *SubGPR = MCBinaryExpr::createSub(LHS: DivGPR, RHS: OneConst, Ctx);
6193 return SubGPR;
6194 };
6195
6196 VGPRBlocks = GetNumGPRBlocks(
6197 NextFreeVGPR,
6198 IsaInfo::getVGPREncodingGranule(STI: getSTI(), EnableWavefrontSize32));
6199 SGPRBlocks =
6200 GetNumGPRBlocks(NumSGPRs, IsaInfo::getSGPREncodingGranule(STI: getSTI()));
6201
6202 return false;
6203}
6204
6205bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
6206 if (!getSTI().getTargetTriple().isAMDGCN())
6207 return TokError(Msg: "directive only supported for amdgcn architecture");
6208
6209 if (!isHsaAbi(STI: getSTI()))
6210 return TokError(Msg: "directive only supported for amdhsa OS");
6211
6212 StringRef KernelName;
6213 if (getParser().parseIdentifier(Res&: KernelName))
6214 return true;
6215
6216 // Remember the kernel name so its prologue can be checked at end of file.
6217 // The matching label may have been parsed already or may follow later.
6218 AMDHSAKernelSymbols.insert(Ptr: getContext().getOrCreateSymbol(Name: KernelName));
6219
6220 AMDGPU::MCKernelDescriptor KD =
6221 AMDGPU::MCKernelDescriptor::getDefaultAmdhsaKernelDescriptor(
6222 STI: &getSTI(), Ctx&: getContext());
6223
6224 StringSet<> Seen;
6225
6226 const MCExpr *ZeroExpr = MCConstantExpr::create(Value: 0, Ctx&: getContext());
6227 const MCExpr *OneExpr = MCConstantExpr::create(Value: 1, Ctx&: getContext());
6228
6229 SMRange VGPRRange;
6230 const MCExpr *NextFreeVGPR = ZeroExpr;
6231 const MCExpr *AccumOffset = MCConstantExpr::create(Value: 0, Ctx&: getContext());
6232 const MCExpr *NamedBarCnt = ZeroExpr;
6233 uint64_t SharedVGPRCount = 0;
6234 uint64_t PreloadLength = 0;
6235 uint64_t PreloadOffset = 0;
6236 SMRange SGPRRange;
6237 const MCExpr *NextFreeSGPR = ZeroExpr;
6238
6239 // Count the number of user SGPRs implied from the enabled feature bits.
6240 unsigned ImpliedUserSGPRCount = 0;
6241
6242 // Track if the asm explicitly contains the directive for the user SGPR
6243 // count.
6244 std::optional<unsigned> ExplicitUserSGPRCount;
6245 const MCExpr *ReserveVCC = OneExpr;
6246 const MCExpr *ReserveFlatScr = OneExpr;
6247 std::optional<bool> EnableWavefrontSize32;
6248
6249 while (true) {
6250 while (trySkipToken(Kind: AsmToken::EndOfStatement))
6251 ;
6252
6253 StringRef ID;
6254 SMRange IDRange = getTok().getLocRange();
6255 if (!parseId(Val&: ID, ErrMsg: "expected .amdhsa_ directive or .end_amdhsa_kernel"))
6256 return true;
6257
6258 if (ID == ".end_amdhsa_kernel")
6259 break;
6260
6261 if (!Seen.insert(key: ID).second)
6262 return TokError(Msg: ".amdhsa_ directives cannot be repeated");
6263
6264 SMLoc ValStart = getLoc();
6265 const MCExpr *ExprVal;
6266 if (getParser().parseExpression(Res&: ExprVal))
6267 return true;
6268 SMLoc ValEnd = getLoc();
6269 SMRange ValRange = SMRange(ValStart, ValEnd);
6270
6271 int64_t IVal = 0;
6272 uint64_t Val = IVal;
6273 bool EvaluatableExpr;
6274 if ((EvaluatableExpr = ExprVal->evaluateAsAbsolute(Res&: IVal))) {
6275 if (IVal < 0)
6276 return OutOfRangeError(Range: ValRange);
6277 Val = IVal;
6278 }
6279
6280#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE) \
6281 if (!isUInt<ENTRY##_WIDTH>(Val)) \
6282 return OutOfRangeError(RANGE); \
6283 AMDGPU::MCKernelDescriptor::bits_set(FIELD, VALUE, ENTRY##_SHIFT, ENTRY, \
6284 getContext());
6285
6286// Some fields use the parsed value immediately which requires the expression to
6287// be solvable.
6288#define EXPR_RESOLVE_OR_ERROR(RESOLVED) \
6289 if (!(RESOLVED)) \
6290 return Error(IDRange.Start, "directive should have resolvable expression", \
6291 IDRange);
6292
6293 if (ID == ".amdhsa_group_segment_fixed_size") {
6294 if (!isUInt<sizeof(kernel_descriptor_t::group_segment_fixed_size) *
6295 CHAR_BIT>(x: Val))
6296 return OutOfRangeError(Range: ValRange);
6297 KD.group_segment_fixed_size = ExprVal;
6298 } else if (ID == ".amdhsa_private_segment_fixed_size") {
6299 if (!isUInt<sizeof(kernel_descriptor_t::private_segment_fixed_size) *
6300 CHAR_BIT>(x: Val))
6301 return OutOfRangeError(Range: ValRange);
6302 KD.private_segment_fixed_size = ExprVal;
6303 } else if (ID == ".amdhsa_kernarg_size") {
6304 if (!isUInt<sizeof(kernel_descriptor_t::kernarg_size) * CHAR_BIT>(x: Val))
6305 return OutOfRangeError(Range: ValRange);
6306 KD.kernarg_size = ExprVal;
6307 } else if (ID == ".amdhsa_user_sgpr_count") {
6308 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6309 ExplicitUserSGPRCount = Val;
6310 } else if (ID == ".amdhsa_user_sgpr_private_segment_buffer") {
6311 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6312 if (hasArchitectedFlatScratch())
6313 return Error(L: IDRange.Start,
6314 Msg: "directive is not supported with architected flat scratch",
6315 Range: IDRange);
6316 PARSE_BITS_ENTRY(KD.kernel_code_properties,
6317 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
6318 ExprVal, ValRange);
6319 if (Val)
6320 ImpliedUserSGPRCount += 4;
6321 } else if (ID == ".amdhsa_user_sgpr_kernarg_preload_length") {
6322 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6323 if (!hasKernargPreload())
6324 return Error(L: IDRange.Start, Msg: "directive requires gfx90a+", Range: IDRange);
6325
6326 if (Val > getMaxNumUserSGPRs())
6327 return OutOfRangeError(Range: ValRange);
6328 PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_LENGTH, ExprVal,
6329 ValRange);
6330 if (Val) {
6331 ImpliedUserSGPRCount += Val;
6332 PreloadLength = Val;
6333 }
6334 } else if (ID == ".amdhsa_user_sgpr_kernarg_preload_offset") {
6335 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6336 if (!hasKernargPreload())
6337 return Error(L: IDRange.Start, Msg: "directive requires gfx90a+", Range: IDRange);
6338
6339 if (Val >= 1024)
6340 return OutOfRangeError(Range: ValRange);
6341 PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_OFFSET, ExprVal,
6342 ValRange);
6343 if (Val)
6344 PreloadOffset = Val;
6345 } else if (ID == ".amdhsa_user_sgpr_dispatch_ptr") {
6346 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6347 PARSE_BITS_ENTRY(KD.kernel_code_properties,
6348 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, ExprVal,
6349 ValRange);
6350 if (Val)
6351 ImpliedUserSGPRCount += 2;
6352 } else if (ID == ".amdhsa_user_sgpr_queue_ptr") {
6353 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6354 PARSE_BITS_ENTRY(KD.kernel_code_properties,
6355 KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, ExprVal,
6356 ValRange);
6357 if (Val)
6358 ImpliedUserSGPRCount += 2;
6359 } else if (ID == ".amdhsa_user_sgpr_kernarg_segment_ptr") {
6360 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6361 PARSE_BITS_ENTRY(KD.kernel_code_properties,
6362 KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
6363 ExprVal, ValRange);
6364 if (Val)
6365 ImpliedUserSGPRCount += 2;
6366 } else if (ID == ".amdhsa_user_sgpr_dispatch_id") {
6367 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6368 PARSE_BITS_ENTRY(KD.kernel_code_properties,
6369 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, ExprVal,
6370 ValRange);
6371 if (Val)
6372 ImpliedUserSGPRCount += 2;
6373 } else if (ID == ".amdhsa_user_sgpr_flat_scratch_init") {
6374 if (hasArchitectedFlatScratch())
6375 return Error(L: IDRange.Start,
6376 Msg: "directive is not supported with architected flat scratch",
6377 Range: IDRange);
6378 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6379 PARSE_BITS_ENTRY(KD.kernel_code_properties,
6380 KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT,
6381 ExprVal, ValRange);
6382 if (Val)
6383 ImpliedUserSGPRCount += 2;
6384 } else if (ID == ".amdhsa_user_sgpr_private_segment_size") {
6385 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6386 PARSE_BITS_ENTRY(KD.kernel_code_properties,
6387 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
6388 ExprVal, ValRange);
6389 if (Val)
6390 ImpliedUserSGPRCount += 1;
6391 } else if (ID == ".amdhsa_wavefront_size32") {
6392 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6393 if (ISA.Major < 10)
6394 return Error(L: IDRange.Start, Msg: "directive requires gfx10+", Range: IDRange);
6395 EnableWavefrontSize32 = Val;
6396 PARSE_BITS_ENTRY(KD.kernel_code_properties,
6397 KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, ExprVal,
6398 ValRange);
6399 } else if (ID == ".amdhsa_uses_dynamic_stack") {
6400 PARSE_BITS_ENTRY(KD.kernel_code_properties,
6401 KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, ExprVal,
6402 ValRange);
6403 } else if (ID == ".amdhsa_system_sgpr_private_segment_wavefront_offset") {
6404 if (hasArchitectedFlatScratch())
6405 return Error(L: IDRange.Start,
6406 Msg: "directive is not supported with architected flat scratch",
6407 Range: IDRange);
6408 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6409 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6410 ValRange);
6411 } else if (ID == ".amdhsa_enable_private_segment") {
6412 if (!hasArchitectedFlatScratch())
6413 return Error(
6414 L: IDRange.Start,
6415 Msg: "directive is not supported without architected flat scratch",
6416 Range: IDRange);
6417 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6418 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6419 ValRange);
6420 } else if (ID == ".amdhsa_system_sgpr_workgroup_id_x") {
6421 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6422 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, ExprVal,
6423 ValRange);
6424 } else if (ID == ".amdhsa_system_sgpr_workgroup_id_y") {
6425 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6426 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, ExprVal,
6427 ValRange);
6428 } else if (ID == ".amdhsa_system_sgpr_workgroup_id_z") {
6429 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6430 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, ExprVal,
6431 ValRange);
6432 } else if (ID == ".amdhsa_system_sgpr_workgroup_info") {
6433 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6434 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, ExprVal,
6435 ValRange);
6436 } else if (ID == ".amdhsa_system_vgpr_workitem_id") {
6437 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6438 COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, ExprVal,
6439 ValRange);
6440 } else if (ID == ".amdhsa_next_free_vgpr") {
6441 VGPRRange = ValRange;
6442 NextFreeVGPR = ExprVal;
6443 } else if (ID == ".amdhsa_next_free_sgpr") {
6444 SGPRRange = ValRange;
6445 NextFreeSGPR = ExprVal;
6446 } else if (ID == ".amdhsa_accum_offset") {
6447 if (!isGFX90A())
6448 return Error(L: IDRange.Start, Msg: "directive requires gfx90a+", Range: IDRange);
6449 AccumOffset = ExprVal;
6450 } else if (ID == ".amdhsa_named_barrier_count") {
6451 if (!isGFX1250Plus())
6452 return Error(L: IDRange.Start, Msg: "directive requires gfx1250+", Range: IDRange);
6453 NamedBarCnt = ExprVal;
6454 } else if (ID == ".amdhsa_reserve_vcc") {
6455 if (EvaluatableExpr && !isUInt<1>(x: Val))
6456 return OutOfRangeError(Range: ValRange);
6457 ReserveVCC = ExprVal;
6458 } else if (ID == ".amdhsa_reserve_flat_scratch") {
6459 if (ISA.Major < 7)
6460 return Error(L: IDRange.Start, Msg: "directive requires gfx7+", Range: IDRange);
6461 if (hasArchitectedFlatScratch())
6462 return Error(L: IDRange.Start,
6463 Msg: "directive is not supported with architected flat scratch",
6464 Range: IDRange);
6465 if (EvaluatableExpr && !isUInt<1>(x: Val))
6466 return OutOfRangeError(Range: ValRange);
6467 ReserveFlatScr = ExprVal;
6468 } else if (ID == ".amdhsa_reserve_xnack_mask") {
6469 if (ISA.Major < 8)
6470 return Error(L: IDRange.Start, Msg: "directive requires gfx8+", Range: IDRange);
6471 if (!isUInt<1>(x: Val))
6472 return OutOfRangeError(Range: ValRange);
6473 bool XnackOn = getTargetStreamer().getTargetID()->isXnackOnOrAny();
6474 if (Val != XnackOn) {
6475 return getParser().Error(
6476 L: IDRange.Start,
6477 Msg: ".amdhsa_reserve_xnack_mask does not match target id", Range: IDRange);
6478 }
6479 } else if (ID == ".amdhsa_float_round_mode_32") {
6480 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6481 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, ExprVal,
6482 ValRange);
6483 } else if (ID == ".amdhsa_float_round_mode_16_64") {
6484 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6485 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, ExprVal,
6486 ValRange);
6487 } else if (ID == ".amdhsa_float_denorm_mode_32") {
6488 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6489 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, ExprVal,
6490 ValRange);
6491 } else if (ID == ".amdhsa_float_denorm_mode_16_64") {
6492 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6493 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, ExprVal,
6494 ValRange);
6495 } else if (ID == ".amdhsa_dx10_clamp") {
6496 if (!getSTI().hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode))
6497 return Error(L: IDRange.Start, Msg: "directive unsupported on gfx1170+",
6498 Range: IDRange);
6499 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6500 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, ExprVal,
6501 ValRange);
6502 } else if (ID == ".amdhsa_ieee_mode") {
6503 if (!getSTI().hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode))
6504 return Error(L: IDRange.Start, Msg: "directive unsupported on gfx1170+",
6505 Range: IDRange);
6506 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6507 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, ExprVal,
6508 ValRange);
6509 } else if (ID == ".amdhsa_fp16_overflow") {
6510 if (ISA.Major < 9)
6511 return Error(L: IDRange.Start, Msg: "directive requires gfx9+", Range: IDRange);
6512 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6513 COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, ExprVal,
6514 ValRange);
6515 } else if (ID == ".amdhsa_tg_split") {
6516 if (!isGFX90A())
6517 return Error(L: IDRange.Start, Msg: "directive requires gfx90a+", Range: IDRange);
6518 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT,
6519 ExprVal, ValRange);
6520 } else if (ID == ".amdhsa_workgroup_processor_mode") {
6521 if (!supportsWGP(STI: getSTI()))
6522 return Error(L: IDRange.Start,
6523 Msg: "directive unsupported on " + getSTI().getCPU(), Range: IDRange);
6524 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6525 COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal,
6526 ValRange);
6527 } else if (ID == ".amdhsa_memory_ordered") {
6528 if (ISA.Major < 10)
6529 return Error(L: IDRange.Start, Msg: "directive requires gfx10+", Range: IDRange);
6530 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6531 COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, ExprVal,
6532 ValRange);
6533 } else if (ID == ".amdhsa_forward_progress") {
6534 if (ISA.Major < 10)
6535 return Error(L: IDRange.Start, Msg: "directive requires gfx10+", Range: IDRange);
6536 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6537 COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, ExprVal,
6538 ValRange);
6539 } else if (ID == ".amdhsa_shared_vgpr_count") {
6540 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6541 if (ISA.Major < 10 || ISA.Major >= 12)
6542 return Error(L: IDRange.Start, Msg: "directive requires gfx10 or gfx11",
6543 Range: IDRange);
6544 SharedVGPRCount = Val;
6545 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3,
6546 COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal,
6547 ValRange);
6548 } else if (ID == ".amdhsa_inst_pref_size") {
6549 if (ISA.Major < 11)
6550 return Error(L: IDRange.Start, Msg: "directive requires gfx11+", Range: IDRange);
6551 if (ISA.Major == 11) {
6552 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3,
6553 COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE, ExprVal,
6554 ValRange);
6555 } else {
6556 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3,
6557 COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE, ExprVal,
6558 ValRange);
6559 }
6560 } else if (ID == ".amdhsa_exception_fp_ieee_invalid_op") {
6561 PARSE_BITS_ENTRY(
6562 KD.compute_pgm_rsrc2,
6563 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION,
6564 ExprVal, ValRange);
6565 } else if (ID == ".amdhsa_exception_fp_denorm_src") {
6566 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6567 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
6568 ExprVal, ValRange);
6569 } else if (ID == ".amdhsa_exception_fp_ieee_div_zero") {
6570 PARSE_BITS_ENTRY(
6571 KD.compute_pgm_rsrc2,
6572 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO,
6573 ExprVal, ValRange);
6574 } else if (ID == ".amdhsa_exception_fp_ieee_overflow") {
6575 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6576 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
6577 ExprVal, ValRange);
6578 } else if (ID == ".amdhsa_exception_fp_ieee_underflow") {
6579 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6580 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
6581 ExprVal, ValRange);
6582 } else if (ID == ".amdhsa_exception_fp_ieee_inexact") {
6583 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6584 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
6585 ExprVal, ValRange);
6586 } else if (ID == ".amdhsa_exception_int_div_zero") {
6587 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6588 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
6589 ExprVal, ValRange);
6590 } else if (ID == ".amdhsa_round_robin_scheduling") {
6591 if (ISA.Major < 12)
6592 return Error(L: IDRange.Start, Msg: "directive requires gfx12+", Range: IDRange);
6593 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6594 COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, ExprVal,
6595 ValRange);
6596 } else {
6597 return Error(L: IDRange.Start, Msg: "unknown .amdhsa_kernel directive", Range: IDRange);
6598 }
6599
6600#undef PARSE_BITS_ENTRY
6601 }
6602
6603 if (!Seen.contains(key: ".amdhsa_next_free_vgpr"))
6604 return TokError(Msg: ".amdhsa_next_free_vgpr directive is required");
6605
6606 if (!Seen.contains(key: ".amdhsa_next_free_sgpr"))
6607 return TokError(Msg: ".amdhsa_next_free_sgpr directive is required");
6608
6609 unsigned UserSGPRCount = ExplicitUserSGPRCount.value_or(u&: ImpliedUserSGPRCount);
6610 if (UserSGPRCount > getMaxNumUserSGPRs())
6611 return TokError(Msg: "too many user SGPRs enabled, found " +
6612 Twine(UserSGPRCount) + ", but only " +
6613 Twine(getMaxNumUserSGPRs()) + " are supported.");
6614
6615 // Consider the case where the total number of UserSGPRs with trailing
6616 // allocated preload SGPRs, is greater than the number of explicitly
6617 // referenced SGPRs.
6618 if (PreloadLength) {
6619 MCContext &Ctx = getContext();
6620 NextFreeSGPR = AMDGPUMCExpr::createMax(
6621 Args: {NextFreeSGPR, MCConstantExpr::create(Value: UserSGPRCount, Ctx)}, Ctx);
6622 }
6623
6624 const MCExpr *VGPRBlocks;
6625 const MCExpr *SGPRBlocks;
6626 if (calculateGPRBlocks(Features: getFeatureBits(), VCCUsed: ReserveVCC, FlatScrUsed: ReserveFlatScr,
6627 XNACKUsed: getTargetStreamer().getTargetID()->isXnackOnOrAny(),
6628 EnableWavefrontSize32, NextFreeVGPR, VGPRRange,
6629 NextFreeSGPR, SGPRRange, VGPRBlocks, SGPRBlocks))
6630 return true;
6631
6632 int64_t EvaluatedVGPRBlocks;
6633 bool VGPRBlocksEvaluatable =
6634 VGPRBlocks->evaluateAsAbsolute(Res&: EvaluatedVGPRBlocks);
6635 if (VGPRBlocksEvaluatable &&
6636 !isUInt<COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_WIDTH>(
6637 x: static_cast<uint64_t>(EvaluatedVGPRBlocks))) {
6638 return OutOfRangeError(Range: VGPRRange);
6639 }
6640 AMDGPU::MCKernelDescriptor::bits_set(
6641 Dst&: KD.compute_pgm_rsrc1, Value: VGPRBlocks,
6642 Shift: COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT,
6643 Mask: COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, Ctx&: getContext());
6644
6645 int64_t EvaluatedSGPRBlocks;
6646 if (SGPRBlocks->evaluateAsAbsolute(Res&: EvaluatedSGPRBlocks) &&
6647 !isUInt<COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_WIDTH>(
6648 x: static_cast<uint64_t>(EvaluatedSGPRBlocks)))
6649 return OutOfRangeError(Range: SGPRRange);
6650 AMDGPU::MCKernelDescriptor::bits_set(
6651 Dst&: KD.compute_pgm_rsrc1, Value: SGPRBlocks,
6652 Shift: COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT,
6653 Mask: COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, Ctx&: getContext());
6654
6655 if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount)
6656 return TokError(Msg: "amdgpu_user_sgpr_count smaller than implied by "
6657 "enabled user SGPRs");
6658
6659 if (isGFX1250Plus()) {
6660 AMDGPU::MCKernelDescriptor::bits_set(
6661 Dst&: KD.compute_pgm_rsrc2,
6662 Value: MCConstantExpr::create(Value: UserSGPRCount, Ctx&: getContext()),
6663 Shift: COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT_SHIFT,
6664 Mask: COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT, Ctx&: getContext());
6665 } else {
6666 AMDGPU::MCKernelDescriptor::bits_set(
6667 Dst&: KD.compute_pgm_rsrc2,
6668 Value: MCConstantExpr::create(Value: UserSGPRCount, Ctx&: getContext()),
6669 Shift: COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT_SHIFT,
6670 Mask: COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT, Ctx&: getContext());
6671 }
6672
6673 int64_t IVal = 0;
6674 if (!KD.kernarg_size->evaluateAsAbsolute(Res&: IVal))
6675 return TokError(Msg: "Kernarg size should be resolvable");
6676 uint64_t kernarg_size = IVal;
6677 if (PreloadLength && kernarg_size &&
6678 (PreloadLength * 4 + PreloadOffset * 4 > kernarg_size))
6679 return TokError(Msg: "Kernarg preload length + offset is larger than the "
6680 "kernarg segment size");
6681
6682 if (isGFX90A()) {
6683 if (!Seen.contains(key: ".amdhsa_accum_offset"))
6684 return TokError(Msg: ".amdhsa_accum_offset directive is required");
6685 int64_t EvaluatedAccum;
6686 bool AccumEvaluatable = AccumOffset->evaluateAsAbsolute(Res&: EvaluatedAccum);
6687 uint64_t UEvaluatedAccum = EvaluatedAccum;
6688 if (AccumEvaluatable &&
6689 (UEvaluatedAccum < 4 || UEvaluatedAccum > 256 || (UEvaluatedAccum & 3)))
6690 return TokError(Msg: "accum_offset should be in range [4..256] in "
6691 "increments of 4");
6692
6693 int64_t EvaluatedNumVGPR;
6694 if (NextFreeVGPR->evaluateAsAbsolute(Res&: EvaluatedNumVGPR) &&
6695 AccumEvaluatable &&
6696 UEvaluatedAccum >
6697 alignTo(Value: std::max(a: (uint64_t)1, b: (uint64_t)EvaluatedNumVGPR), Align: 4))
6698 return TokError(Msg: "accum_offset exceeds total VGPR allocation");
6699 const MCExpr *AdjustedAccum = MCBinaryExpr::createSub(
6700 LHS: MCBinaryExpr::createDiv(
6701 LHS: AccumOffset, RHS: MCConstantExpr::create(Value: 4, Ctx&: getContext()), Ctx&: getContext()),
6702 RHS: MCConstantExpr::create(Value: 1, Ctx&: getContext()), Ctx&: getContext());
6703 MCKernelDescriptor::bits_set(Dst&: KD.compute_pgm_rsrc3, Value: AdjustedAccum,
6704 Shift: COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
6705 Mask: COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
6706 Ctx&: getContext());
6707 }
6708
6709 if (isGFX1250Plus())
6710 MCKernelDescriptor::bits_set(Dst&: KD.compute_pgm_rsrc3, Value: NamedBarCnt,
6711 Shift: COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT,
6712 Mask: COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
6713 Ctx&: getContext());
6714
6715 if (ISA.Major >= 10 && ISA.Major < 12) {
6716 // SharedVGPRCount < 16 checked by PARSE_ENTRY_BITS
6717 if (SharedVGPRCount && EnableWavefrontSize32 && *EnableWavefrontSize32) {
6718 return TokError(Msg: "shared_vgpr_count directive not valid on "
6719 "wavefront size 32");
6720 }
6721
6722 if (VGPRBlocksEvaluatable &&
6723 (SharedVGPRCount * 2 + static_cast<uint64_t>(EvaluatedVGPRBlocks) >
6724 63)) {
6725 return TokError(Msg: "shared_vgpr_count*2 + "
6726 "compute_pgm_rsrc1.GRANULATED_WORKITEM_VGPR_COUNT cannot "
6727 "exceed 63\n");
6728 }
6729 }
6730
6731 emitTargetDirective();
6732 getTargetStreamer().EmitAmdhsaKernelDescriptor(STI: getSTI(), KernelName, KernelDescriptor: KD,
6733 NextVGPR: NextFreeVGPR, NextSGPR: NextFreeSGPR,
6734 ReserveVCC, ReserveFlatScr);
6735 return false;
6736}
6737
6738bool AMDGPUAsmParser::ParseDirectiveAMDHSACodeObjectVersion() {
6739 uint32_t Version;
6740 if (ParseAsAbsoluteExpression(Ret&: Version))
6741 return true;
6742
6743 getTargetStreamer().EmitDirectiveAMDHSACodeObjectVersion(COV: Version);
6744 emitTargetDirective();
6745 return false;
6746}
6747
6748bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
6749 AMDGPUMCKernelCodeT &C) {
6750 // max_scratch_backing_memory_byte_size is deprecated. Ignore it while parsing
6751 // assembly for backwards compatibility.
6752 if (ID == "max_scratch_backing_memory_byte_size") {
6753 Parser.eatToEndOfStatement();
6754 return false;
6755 }
6756
6757 SmallString<40> ErrStr;
6758 raw_svector_ostream Err(ErrStr);
6759 if (!C.ParseKernelCodeT(ID, MCParser&: getParser(), Err)) {
6760 return TokError(Msg: Err.str());
6761 }
6762 Lex();
6763
6764 if (ID == "enable_wavefront_size32") {
6765 if (C.code_properties & AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32) {
6766 if (!isGFX10Plus())
6767 return TokError(Msg: "enable_wavefront_size32=1 is only allowed on GFX10+");
6768 if (!isWave32())
6769 return TokError(Msg: "enable_wavefront_size32=1 requires +WavefrontSize32");
6770 } else {
6771 if (!isWave64())
6772 return TokError(Msg: "enable_wavefront_size32=0 requires +WavefrontSize64");
6773 }
6774 }
6775
6776 if (ID == "wavefront_size") {
6777 if (C.wavefront_size == 5) {
6778 if (!isGFX10Plus())
6779 return TokError(Msg: "wavefront_size=5 is only allowed on GFX10+");
6780 if (!isWave32())
6781 return TokError(Msg: "wavefront_size=5 requires +WavefrontSize32");
6782 } else if (C.wavefront_size == 6) {
6783 if (!isWave64())
6784 return TokError(Msg: "wavefront_size=6 requires +WavefrontSize64");
6785 }
6786 }
6787
6788 return false;
6789}
6790
6791bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
6792 AMDGPUMCKernelCodeT KernelCode;
6793 KernelCode.initDefault(STI: getSTI(), Ctx&: getContext());
6794
6795 while (true) {
6796 // Lex EndOfStatement. This is in a while loop, because lexing a comment
6797 // will set the current token to EndOfStatement.
6798 while (trySkipToken(Kind: AsmToken::EndOfStatement))
6799 ;
6800
6801 StringRef ID;
6802 if (!parseId(Val&: ID, ErrMsg: "expected value identifier or .end_amd_kernel_code_t"))
6803 return true;
6804
6805 if (ID == ".end_amd_kernel_code_t")
6806 break;
6807
6808 if (ParseAMDKernelCodeTValue(ID, C&: KernelCode))
6809 return true;
6810 }
6811
6812 KernelCode.validate(STI: &getSTI(), Ctx&: getContext());
6813 getTargetStreamer().EmitAMDKernelCodeT(Header&: KernelCode);
6814
6815 return false;
6816}
6817
6818bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
6819 StringRef KernelName;
6820 if (!parseId(Val&: KernelName, ErrMsg: "expected symbol name"))
6821 return true;
6822
6823 getTargetStreamer().EmitAMDGPUSymbolType(SymbolName: KernelName,
6824 Type: ELF::STT_AMDGPU_HSA_KERNEL);
6825
6826 KernelScope.initialize(Context&: getContext());
6827 return false;
6828}
6829
6830bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
6831 if (!getSTI().getTargetTriple().isAMDGCN()) {
6832 return Error(L: getLoc(),
6833 Msg: ".amd_amdgpu_isa directive is not available on non-amdgcn "
6834 "architectures");
6835 }
6836
6837 StringRef TargetIDDirective = getLexer().getTok().getStringContents();
6838
6839 std::optional<AMDGPU::TargetID> MaybeParsed =
6840 AMDGPU::TargetID::parseTargetIDString(TargetIDDirective);
6841 if (!MaybeParsed)
6842 return Error(L: getParser().getTok().getLoc(),
6843 Msg: "malformed target id '" + TargetIDDirective + "'");
6844
6845 const AMDGPU::TargetID &ParsedTargetID = *MaybeParsed;
6846 const Triple &TT = getSTI().getTargetTriple();
6847
6848 // The processor named in the target id must be covered by the triple's
6849 // subarch.
6850 if (!AMDGPU::isCPUValidForSubArch(SubArch: TT.getSubArch(),
6851 AK: ParsedTargetID.getGPUKind())) {
6852 return Error(L: getParser().getTok().getLoc(),
6853 Msg: "target id '" + TargetIDDirective +
6854 "' specifies a processor that is not valid for subarch '" +
6855 TT.getArchName() + "'");
6856 }
6857
6858 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
6859 getTargetStreamer().getTargetID();
6860
6861 Triple DirectiveTriple(ParsedTargetID.getTargetTripleString());
6862 const Triple &STITriple = getSTI().getTargetTriple();
6863 if (!DirectiveTriple.isCompatibleWith(Other: STITriple)) {
6864 return Error(L: getParser().getTok().getLoc(),
6865 Msg: ".amd_amdgpu_isa " + Twine(ParsedTargetID.toString()) +
6866 " is incompatible with " +
6867 Twine(CurrentTargetID->toString()));
6868 }
6869
6870 // Error if the ISA version doesn't match
6871 StringRef DirectiveProcessor =
6872 AMDGPU::getArchNameAMDGCN(AK: ParsedTargetID.getGPUKind());
6873 AMDGPU::IsaVersion DirectiveISA = AMDGPU::getIsaVersion(GPU: DirectiveProcessor);
6874 if (DirectiveISA != ISA) {
6875 return Error(L: getParser().getTok().getLoc(),
6876 Msg: ".amd_amdgpu_isa directive processor " +
6877 Twine(DirectiveProcessor) +
6878 " does not match the specified processor " +
6879 Twine(getSTI().getCPU()));
6880 }
6881
6882 getTargetStreamer().EmitISAVersion();
6883 Lex();
6884
6885 return false;
6886}
6887
6888bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
6889 assert(isHsaAbi(getSTI()));
6890
6891 std::string HSAMetadataString;
6892 if (ParseToEndDirective(AssemblerDirectiveBegin: HSAMD::V3::AssemblerDirectiveBegin,
6893 AssemblerDirectiveEnd: HSAMD::V3::AssemblerDirectiveEnd, CollectString&: HSAMetadataString))
6894 return true;
6895
6896 if (!getTargetStreamer().EmitHSAMetadataV3(HSAMetadataString))
6897 return Error(L: getLoc(), Msg: "invalid HSA metadata");
6898
6899 return false;
6900}
6901
6902/// Common code to parse out a block of text (typically YAML) between start and
6903/// end directives.
6904bool AMDGPUAsmParser::ParseToEndDirective(const char *AssemblerDirectiveBegin,
6905 const char *AssemblerDirectiveEnd,
6906 std::string &CollectString) {
6907
6908 raw_string_ostream CollectStream(CollectString);
6909
6910 getLexer().setSkipSpace(false);
6911
6912 bool FoundEnd = false;
6913 while (!isToken(Kind: AsmToken::Eof)) {
6914 while (isToken(Kind: AsmToken::Space)) {
6915 CollectStream << getTokenStr();
6916 Lex();
6917 }
6918
6919 if (trySkipId(Id: AssemblerDirectiveEnd)) {
6920 FoundEnd = true;
6921 break;
6922 }
6923
6924 CollectStream << Parser.parseStringToEndOfStatement()
6925 << getContext().getAsmInfo().getSeparatorString();
6926
6927 Parser.eatToEndOfStatement();
6928 }
6929
6930 getLexer().setSkipSpace(true);
6931
6932 if (isToken(Kind: AsmToken::Eof) && !FoundEnd) {
6933 return TokError(Msg: Twine("expected directive ") +
6934 Twine(AssemblerDirectiveEnd) + Twine(" not found"));
6935 }
6936
6937 return false;
6938}
6939
6940/// Parse the assembler directive for new MsgPack-format PAL metadata.
6941bool AMDGPUAsmParser::ParseDirectivePALMetadataBegin() {
6942 std::string String;
6943 if (ParseToEndDirective(AssemblerDirectiveBegin: AMDGPU::PALMD::AssemblerDirectiveBegin,
6944 AssemblerDirectiveEnd: AMDGPU::PALMD::AssemblerDirectiveEnd, CollectString&: String))
6945 return true;
6946
6947 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6948 if (!PALMetadata->setFromString(String))
6949 return Error(L: getLoc(), Msg: "invalid PAL metadata");
6950 return false;
6951}
6952
6953/// Parse the assembler directive for old linear-format PAL metadata.
6954bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
6955 if (getSTI().getTargetTriple().getOS() != Triple::AMDPAL) {
6956 return Error(L: getLoc(), Msg: (Twine(PALMD::AssemblerDirective) +
6957 Twine(" directive is "
6958 "not available on non-amdpal OSes"))
6959 .str());
6960 }
6961
6962 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6963 PALMetadata->setLegacy();
6964 for (;;) {
6965 uint32_t Key, Value;
6966 if (ParseAsAbsoluteExpression(Ret&: Key)) {
6967 return TokError(Msg: Twine("invalid value in ") +
6968 Twine(PALMD::AssemblerDirective));
6969 }
6970 if (!trySkipToken(Kind: AsmToken::Comma)) {
6971 return TokError(Msg: Twine("expected an even number of values in ") +
6972 Twine(PALMD::AssemblerDirective));
6973 }
6974 if (ParseAsAbsoluteExpression(Ret&: Value)) {
6975 return TokError(Msg: Twine("invalid value in ") +
6976 Twine(PALMD::AssemblerDirective));
6977 }
6978 PALMetadata->setRegister(Reg: Key, Val: Value);
6979 if (!trySkipToken(Kind: AsmToken::Comma))
6980 break;
6981 }
6982 return false;
6983}
6984
6985/// ParseDirectiveAMDGPULDS
6986/// ::= .amdgpu_lds identifier ',' size_expression [',' align_expression]
6987bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() {
6988 if (getParser().checkForValidSection())
6989 return true;
6990
6991 StringRef Name;
6992 SMLoc NameLoc = getLoc();
6993 if (getParser().parseIdentifier(Res&: Name))
6994 return TokError(Msg: "expected identifier in directive");
6995
6996 MCSymbol *Symbol = getContext().getOrCreateSymbol(Name);
6997 if (getParser().parseComma())
6998 return true;
6999
7000 unsigned LocalMemorySize = AMDGPU::IsaInfo::getLocalMemorySize(STI: getSTI());
7001
7002 int64_t Size;
7003 SMLoc SizeLoc = getLoc();
7004 if (getParser().parseAbsoluteExpression(Res&: Size))
7005 return true;
7006 if (Size < 0)
7007 return Error(L: SizeLoc, Msg: "size must be non-negative");
7008 if (Size > LocalMemorySize)
7009 return Error(L: SizeLoc, Msg: "size is too large");
7010
7011 int64_t Alignment = 4;
7012 if (trySkipToken(Kind: AsmToken::Comma)) {
7013 SMLoc AlignLoc = getLoc();
7014 if (getParser().parseAbsoluteExpression(Res&: Alignment))
7015 return true;
7016 if (Alignment < 0 || !isPowerOf2_64(Value: Alignment))
7017 return Error(L: AlignLoc, Msg: "alignment must be a power of two");
7018
7019 // Alignment larger than the size of LDS is possible in theory, as long
7020 // as the linker manages to place to symbol at address 0, but we do want
7021 // to make sure the alignment fits nicely into a 32-bit integer.
7022 if (Alignment >= 1u << 31)
7023 return Error(L: AlignLoc, Msg: "alignment is too large");
7024 }
7025
7026 if (parseEOL())
7027 return true;
7028
7029 Symbol->redefineIfPossible();
7030 if (!Symbol->isUndefined())
7031 return Error(L: NameLoc, Msg: "invalid symbol redefinition");
7032
7033 getTargetStreamer().emitAMDGPULDS(Symbol, Size, Alignment: Align(Alignment));
7034 return false;
7035}
7036
7037bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
7038 if (getParser().checkForValidSection())
7039 return true;
7040
7041 StringRef FuncName;
7042 if (getParser().parseIdentifier(Res&: FuncName))
7043 return TokError(Msg: "expected symbol name after .amdgpu_info");
7044
7045 MCSymbol *FuncSym = getContext().getOrCreateSymbol(Name: FuncName);
7046 AMDGPU::InfoSectionData ParsedInfoData;
7047 AMDGPU::FuncInfo FI;
7048 FI.Sym = FuncSym;
7049 bool HasScalarAttrs = false;
7050
7051 while (true) {
7052 while (trySkipToken(Kind: AsmToken::EndOfStatement))
7053 ;
7054
7055 StringRef ID;
7056 SMLoc IDLoc = getLoc();
7057 if (!parseId(Val&: ID, ErrMsg: "expected directive or .end_amdgpu_info"))
7058 return true;
7059
7060 if (ID == ".end_amdgpu_info")
7061 break;
7062
7063 // Every per-entry directive shares the `.amdgpu_` namespace prefix; strip
7064 // it once and dispatch on the distinguishing suffix below. The unstripped
7065 // ID is preserved for diagnostics.
7066 StringRef Dir = ID;
7067 if (!Dir.consume_front(Prefix: ".amdgpu_"))
7068 return Error(L: IDLoc, Msg: "unknown .amdgpu_info directive '" + ID + "'");
7069
7070 if (Dir == "flags") {
7071 int64_t Val;
7072 if (getParser().parseAbsoluteExpression(Res&: Val))
7073 return true;
7074 auto Flags = static_cast<AMDGPU::FuncInfoFlags>(Val);
7075 FI.UsesVCC = !!(Flags & AMDGPU::FuncInfoFlags::FUNC_USES_VCC);
7076 FI.UsesFlatScratch =
7077 !!(Flags & AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH);
7078 FI.HasDynStack = !!(Flags & AMDGPU::FuncInfoFlags::FUNC_HAS_DYN_STACK);
7079 HasScalarAttrs = true;
7080 } else if (Dir == "num_sgpr") {
7081 int64_t Val;
7082 if (getParser().parseAbsoluteExpression(Res&: Val))
7083 return true;
7084 FI.NumSGPR = static_cast<uint32_t>(Val);
7085 HasScalarAttrs = true;
7086 } else if (Dir == "num_vgpr") {
7087 int64_t Val;
7088 if (getParser().parseAbsoluteExpression(Res&: Val))
7089 return true;
7090 FI.NumArchVGPR = static_cast<uint32_t>(Val);
7091 HasScalarAttrs = true;
7092 } else if (Dir == "num_agpr") {
7093 int64_t Val;
7094 if (getParser().parseAbsoluteExpression(Res&: Val))
7095 return true;
7096 FI.NumAccVGPR = static_cast<uint32_t>(Val);
7097 HasScalarAttrs = true;
7098 } else if (Dir == "private_segment_size") {
7099 int64_t Val;
7100 if (getParser().parseAbsoluteExpression(Res&: Val))
7101 return true;
7102 FI.PrivateSegmentSize = static_cast<uint32_t>(Val);
7103 HasScalarAttrs = true;
7104 } else if (Dir == "use") {
7105 StringRef ResName;
7106 if (getParser().parseIdentifier(Res&: ResName))
7107 return TokError(Msg: "expected resource symbol for .amdgpu_use");
7108 ParsedInfoData.Uses.push_back(
7109 Elt: {FuncSym, getContext().getOrCreateSymbol(Name: ResName)});
7110 } else if (Dir == "call") {
7111 StringRef DstName;
7112 if (getParser().parseIdentifier(Res&: DstName))
7113 return TokError(Msg: "expected callee symbol for .amdgpu_call");
7114 ParsedInfoData.Calls.push_back(
7115 Elt: {FuncSym, getContext().getOrCreateSymbol(Name: DstName)});
7116 } else if (Dir == "indirect_call") {
7117 std::string TypeId;
7118 if (getParser().parseEscapedString(Data&: TypeId))
7119 return TokError(Msg: "expected type ID string for .amdgpu_indirect_call");
7120 ParsedInfoData.IndirectCalls.push_back(Elt: {FuncSym, std::move(TypeId)});
7121 } else if (Dir == "typeid") {
7122 std::string TypeId;
7123 if (getParser().parseEscapedString(Data&: TypeId))
7124 return TokError(Msg: "expected type ID string for .amdgpu_typeid");
7125 ParsedInfoData.TypeIds.push_back(Elt: {FuncSym, std::move(TypeId)});
7126 } else {
7127 return Error(L: IDLoc, Msg: "unknown .amdgpu_info directive '" + ID + "'");
7128 }
7129 }
7130
7131 if (HasScalarAttrs)
7132 ParsedInfoData.Funcs.push_back(Elt: std::move(FI));
7133
7134 AMDGPU::InfoSectionData &Data = InfoData ? *InfoData : InfoData.emplace();
7135 for (AMDGPU::FuncInfo &Func : ParsedInfoData.Funcs)
7136 Data.Funcs.push_back(Elt: std::move(Func));
7137 for (std::pair<MCSymbol *, MCSymbol *> &Use : ParsedInfoData.Uses)
7138 Data.Uses.push_back(Elt: Use);
7139 for (std::pair<MCSymbol *, MCSymbol *> &Call : ParsedInfoData.Calls)
7140 Data.Calls.push_back(Elt: Call);
7141 for (std::pair<MCSymbol *, std::string> &IndirectCall :
7142 ParsedInfoData.IndirectCalls)
7143 Data.IndirectCalls.push_back(Elt: std::move(IndirectCall));
7144 for (std::pair<MCSymbol *, std::string> &TypeId : ParsedInfoData.TypeIds)
7145 Data.TypeIds.push_back(Elt: std::move(TypeId));
7146
7147 return false;
7148}
7149
7150void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
7151 // Record every parsed label in the timeline so that, at end of file, the
7152 // instructions following a kernel's label can be located regardless of
7153 // whether the .amdhsa_kernel directive came before or after the label.
7154 OpcodeStreamSymbols.emplace_back(Args&: Symbol, Args&: IDLoc, Args: OpcodeStream.size());
7155}
7156
7157void AMDGPUAsmParser::checkKernelPrologues() {
7158 if (getFeatureBits()[AMDGPU::FeatureRequiresInitialUnclausedVmem]) {
7159 static const unsigned Required[] = {S_MOV_B64_gfx12, V_NOP_e32_gfx12,
7160 GLOBAL_PREFETCH_B8_SADDR_gfx1250};
7161 for (auto [Sym, Loc, Offset] : OpcodeStreamSymbols) {
7162 if (!AMDHSAKernelSymbols.contains(Ptr: Sym))
7163 continue;
7164 ArrayRef<unsigned> Prologue = ArrayRef(OpcodeStream).drop_front(N: Offset);
7165 if (!Prologue.empty() && Prologue.front() == S_SETREG_IMM32_B32_gfx12)
7166 Prologue = Prologue.drop_front();
7167 if (Prologue.take_front(N: std::size(Required)) != ArrayRef(Required)) {
7168 Warning(L: Loc, Msg: "kernel '" + Sym->getName() +
7169 "' does not begin with the required prologue "
7170 "sequence: s_mov_b64 followed by v_nop and "
7171 "global_prefetch_b8");
7172 }
7173 }
7174 }
7175 OpcodeStream.clear();
7176 OpcodeStreamSymbols.clear();
7177 AMDHSAKernelSymbols.clear();
7178}
7179
7180void AMDGPUAsmParser::onEndOfFile() {
7181 emitTargetDirective();
7182 checkKernelPrologues();
7183 if (InfoData)
7184 getTargetStreamer().emitAMDGPUInfo(Data: *InfoData);
7185}
7186
7187bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
7188 StringRef IDVal = DirectiveID.getString();
7189
7190 if (isHsaAbi(STI: getSTI())) {
7191 if (IDVal == ".amdhsa_kernel")
7192 return ParseDirectiveAMDHSAKernel();
7193
7194 if (IDVal == ".amdhsa_code_object_version")
7195 return ParseDirectiveAMDHSACodeObjectVersion();
7196
7197 // TODO: Restructure/combine with PAL metadata directive.
7198 if (IDVal == AMDGPU::HSAMD::V3::AssemblerDirectiveBegin)
7199 return ParseDirectiveHSAMetadata();
7200 } else {
7201 if (IDVal == ".amd_kernel_code_t")
7202 return ParseDirectiveAMDKernelCodeT();
7203
7204 if (IDVal == ".amdgpu_hsa_kernel")
7205 return ParseDirectiveAMDGPUHsaKernel();
7206
7207 if (IDVal == ".amd_amdgpu_isa")
7208 return ParseDirectiveISAVersion();
7209
7210 if (IDVal == AMDGPU::HSAMD::AssemblerDirectiveBegin) {
7211 return Error(L: getLoc(), Msg: (Twine(HSAMD::AssemblerDirectiveBegin) +
7212 Twine(" directive is "
7213 "not available on non-amdhsa OSes"))
7214 .str());
7215 }
7216 }
7217
7218 if (IDVal == ".amdgcn_target")
7219 return ParseDirectiveAMDGCNTarget();
7220
7221 if (IDVal == ".amdgpu_lds")
7222 return ParseDirectiveAMDGPULDS();
7223
7224 if (IDVal == ".amdgpu_info")
7225 return ParseDirectiveAMDGPUInfo();
7226
7227 if (IDVal == PALMD::AssemblerDirectiveBegin)
7228 return ParseDirectivePALMetadataBegin();
7229
7230 if (IDVal == PALMD::AssemblerDirective)
7231 return ParseDirectivePALMetadata();
7232
7233 return true;
7234}
7235
7236bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
7237 MCRegister Reg) {
7238 if (MRI.regsOverlap(RegA: TTMP12_TTMP13_TTMP14_TTMP15, RegB: Reg))
7239 return isGFX9Plus();
7240
7241 // GFX10+ has 2 more SGPRs 104 and 105.
7242 if (MRI.regsOverlap(RegA: SGPR104_SGPR105, RegB: Reg))
7243 return hasSGPR104_SGPR105();
7244
7245 switch (Reg.id()) {
7246 case SRC_SHARED_BASE_LO:
7247 case SRC_SHARED_BASE:
7248 case SRC_SHARED_LIMIT_LO:
7249 case SRC_SHARED_LIMIT:
7250 return isGFX9Plus();
7251 case SRC_PRIVATE_BASE_LO:
7252 case SRC_PRIVATE_BASE:
7253 case SRC_PRIVATE_LIMIT_LO:
7254 case SRC_PRIVATE_LIMIT:
7255 return AMDGPU::hasPrivateApertureRegs(STI: getSTI());
7256 case SRC_FLAT_SCRATCH_BASE_LO:
7257 case SRC_FLAT_SCRATCH_BASE_HI:
7258 return hasGloballyAddressableScratch();
7259 case SRC_POPS_EXITING_WAVE_ID:
7260 return hasPopsExitingWaveID(STI: getSTI());
7261 case TBA:
7262 case TBA_LO:
7263 case TBA_HI:
7264 case TMA:
7265 case TMA_LO:
7266 case TMA_HI:
7267 return !isGFX9Plus();
7268 case XNACK_MASK:
7269 case XNACK_MASK_LO:
7270 case XNACK_MASK_HI:
7271 return (isVI() || isGFX9()) &&
7272 getTargetStreamer().getTargetID()->isXnackSupported();
7273 case SGPR_NULL:
7274 return isGFX10Plus();
7275 case SRC_EXECZ:
7276 case SRC_VCCZ:
7277 return !isGFX11Plus();
7278 default:
7279 break;
7280 }
7281
7282 if (isCI())
7283 return true;
7284
7285 if (isSI() || isGFX10Plus()) {
7286 // No flat_scr on SI.
7287 // On GFX10Plus flat scratch is not a valid register operand and can only be
7288 // accessed with s_setreg/s_getreg.
7289 switch (Reg.id()) {
7290 case FLAT_SCR:
7291 case FLAT_SCR_LO:
7292 case FLAT_SCR_HI:
7293 return false;
7294 default:
7295 return true;
7296 }
7297 }
7298
7299 // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
7300 // SI/CI have.
7301 if (MRI.regsOverlap(RegA: SGPR102_SGPR103, RegB: Reg))
7302 return hasSGPR102_SGPR103();
7303
7304 return true;
7305}
7306
7307ParseStatus AMDGPUAsmParser::parseOperand(OperandVector &Operands,
7308 StringRef Mnemonic,
7309 OperandMode Mode) {
7310 ParseStatus Res = parseVOPD(Operands);
7311 if (Res.isSuccess() || Res.isFailure() || isToken(Kind: AsmToken::EndOfStatement))
7312 return Res;
7313
7314 // Try to parse with a custom parser
7315 Res = MatchOperandParserImpl(Operands, Mnemonic);
7316
7317 // If we successfully parsed the operand or if there as an error parsing,
7318 // we are done.
7319 //
7320 // If we are parsing after we reach EndOfStatement then this means we
7321 // are appending default values to the Operands list. This is only done
7322 // by custom parser, so we shouldn't continue on to the generic parsing.
7323 if (Res.isSuccess() || Res.isFailure() || isToken(Kind: AsmToken::EndOfStatement))
7324 return Res;
7325
7326 SMLoc RBraceLoc;
7327 SMLoc LBraceLoc = getLoc();
7328 if (Mode == OperandMode_NSA && trySkipToken(Kind: AsmToken::LBrac)) {
7329 unsigned Prefix = Operands.size();
7330
7331 for (;;) {
7332 auto Loc = getLoc();
7333 Res = parseReg(Operands);
7334 if (Res.isNoMatch())
7335 Error(L: Loc, Msg: "expected a register");
7336 if (!Res.isSuccess())
7337 return ParseStatus::Failure;
7338
7339 RBraceLoc = getLoc();
7340 if (trySkipToken(Kind: AsmToken::RBrac))
7341 break;
7342
7343 if (!skipToken(Kind: AsmToken::Comma,
7344 ErrMsg: "expected a comma or a closing square bracket"))
7345 return ParseStatus::Failure;
7346 }
7347
7348 if (Operands.size() - Prefix > 1) {
7349 Operands.insert(I: Operands.begin() + Prefix,
7350 Elt: AMDGPUOperand::CreateToken(AsmParser: this, Str: "[", Loc: LBraceLoc));
7351 Operands.push_back(Elt: AMDGPUOperand::CreateToken(AsmParser: this, Str: "]", Loc: RBraceLoc));
7352 }
7353
7354 return ParseStatus::Success;
7355 }
7356
7357 return parseRegOrImm(Operands);
7358}
7359
7360StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
7361 // Clear any forced encodings from the previous instruction.
7362 setForcedEncodingSize(0);
7363 setForcedDPP(false);
7364 setForcedSDWA(false);
7365
7366 if (Name.consume_back(Suffix: "_e64_dpp")) {
7367 setForcedDPP(true);
7368 setForcedEncodingSize(64);
7369 return Name;
7370 }
7371 if (Name.consume_back(Suffix: "_e64")) {
7372 setForcedEncodingSize(64);
7373 return Name;
7374 }
7375 if (Name.consume_back(Suffix: "_e32")) {
7376 setForcedEncodingSize(32);
7377 return Name;
7378 }
7379 if (Name.consume_back(Suffix: "_dpp")) {
7380 setForcedDPP(true);
7381 return Name;
7382 }
7383 if (Name.consume_back(Suffix: "_sdwa")) {
7384 setForcedSDWA(true);
7385 return Name;
7386 }
7387 return Name;
7388}
7389
7390static void applyMnemonicAliases(StringRef &Mnemonic,
7391 const FeatureBitset &Features,
7392 unsigned VariantID);
7393
7394bool AMDGPUAsmParser::parseInstruction(ParseInstructionInfo &Info,
7395 StringRef Name, SMLoc NameLoc,
7396 OperandVector &Operands) {
7397 // Add the instruction mnemonic
7398 Name = parseMnemonicSuffix(Name);
7399
7400 // If the target architecture uses MnemonicAlias, call it here to parse
7401 // operands correctly.
7402 applyMnemonicAliases(Mnemonic&: Name, Features: getAvailableFeatures(), VariantID: 0);
7403
7404 Operands.push_back(Elt: AMDGPUOperand::CreateToken(AsmParser: this, Str: Name, Loc: NameLoc));
7405
7406 bool IsMIMG = Name.starts_with(Prefix: "image_");
7407
7408 while (!trySkipToken(Kind: AsmToken::EndOfStatement)) {
7409 OperandMode Mode = OperandMode_Default;
7410 if (IsMIMG && isGFX10Plus() && Operands.size() == 2)
7411 Mode = OperandMode_NSA;
7412 ParseStatus Res = parseOperand(Operands, Mnemonic: Name, Mode);
7413
7414 if (!Res.isSuccess()) {
7415 checkUnsupportedInstruction(Mnemo: Name, IDLoc: NameLoc);
7416 if (!Parser.hasPendingError()) {
7417 // FIXME: use real operand location rather than the current location.
7418 StringRef Msg = Res.isFailure() ? "failed parsing operand."
7419 : "not a valid operand.";
7420 Error(L: getLoc(), Msg);
7421 }
7422 while (!trySkipToken(Kind: AsmToken::EndOfStatement)) {
7423 lex();
7424 }
7425 return true;
7426 }
7427
7428 // Eat the comma or space if there is one.
7429 trySkipToken(Kind: AsmToken::Comma);
7430 }
7431
7432 return false;
7433}
7434
7435//===----------------------------------------------------------------------===//
7436// Utility functions
7437//===----------------------------------------------------------------------===//
7438
7439ParseStatus AMDGPUAsmParser::parseTokenOp(StringRef Name,
7440 OperandVector &Operands) {
7441 SMLoc S = getLoc();
7442 if (!trySkipId(Id: Name))
7443 return ParseStatus::NoMatch;
7444
7445 Operands.push_back(Elt: AMDGPUOperand::CreateToken(AsmParser: this, Str: Name, Loc: S));
7446 return ParseStatus::Success;
7447}
7448
7449ParseStatus AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix,
7450 int64_t &IntVal) {
7451
7452 if (!trySkipId(Id: Prefix, Kind: AsmToken::Colon))
7453 return ParseStatus::NoMatch;
7454
7455 return parseExpr(Imm&: IntVal) ? ParseStatus::Success : ParseStatus::Failure;
7456}
7457
7458ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
7459 const char *Prefix, OperandVector &Operands, AMDGPUOperand::ImmTy ImmTy,
7460 std::function<bool(int64_t &)> ConvertResult) {
7461 SMLoc S = getLoc();
7462 int64_t Value = 0;
7463
7464 ParseStatus Res = parseIntWithPrefix(Prefix, IntVal&: Value);
7465 if (!Res.isSuccess())
7466 return Res;
7467
7468 if (ConvertResult && !ConvertResult(Value)) {
7469 Error(L: S, Msg: "invalid " + StringRef(Prefix) + " value.");
7470 }
7471
7472 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Value, Loc: S, Type: ImmTy));
7473 return ParseStatus::Success;
7474}
7475
7476ParseStatus AMDGPUAsmParser::parseOperandArrayWithPrefix(
7477 const char *Prefix, OperandVector &Operands, AMDGPUOperand::ImmTy ImmTy,
7478 bool (*ConvertResult)(int64_t &)) {
7479 SMLoc S = getLoc();
7480 if (!trySkipId(Id: Prefix, Kind: AsmToken::Colon))
7481 return ParseStatus::NoMatch;
7482
7483 if (!skipToken(Kind: AsmToken::LBrac, ErrMsg: "expected a left square bracket"))
7484 return ParseStatus::Failure;
7485
7486 unsigned Val = 0;
7487 const unsigned MaxSize = 4;
7488
7489 // FIXME: How to verify the number of elements matches the number of src
7490 // operands?
7491 for (int I = 0;; ++I) {
7492 int64_t Op;
7493 SMLoc Loc = getLoc();
7494 if (!parseExpr(Imm&: Op))
7495 return ParseStatus::Failure;
7496
7497 if (Op != 0 && Op != 1)
7498 return Error(L: Loc, Msg: "invalid " + StringRef(Prefix) + " value.");
7499
7500 Val |= (Op << I);
7501
7502 if (trySkipToken(Kind: AsmToken::RBrac))
7503 break;
7504
7505 if (I + 1 == MaxSize)
7506 return Error(L: getLoc(), Msg: "expected a closing square bracket");
7507
7508 if (!skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma"))
7509 return ParseStatus::Failure;
7510 }
7511
7512 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val, Loc: S, Type: ImmTy));
7513 return ParseStatus::Success;
7514}
7515
7516ParseStatus AMDGPUAsmParser::parseNamedBit(StringRef Name,
7517 OperandVector &Operands,
7518 AMDGPUOperand::ImmTy ImmTy,
7519 bool IgnoreNegative) {
7520 int64_t Bit;
7521 SMLoc S = getLoc();
7522
7523 if (trySkipId(Id: Name)) {
7524 Bit = 1;
7525 } else if (trySkipId(Pref: "no", Id: Name)) {
7526 if (IgnoreNegative)
7527 return ParseStatus::Success;
7528 Bit = 0;
7529 } else {
7530 return ParseStatus::NoMatch;
7531 }
7532
7533 if (Name == "r128" && !hasMIMG_R128())
7534 return Error(L: S, Msg: "r128 modifier is not supported on this GPU");
7535 if (Name == "a16" && !hasA16())
7536 return Error(L: S, Msg: "a16 modifier is not supported on this GPU");
7537
7538 if (Bit == 0 && Name == "gds") {
7539 StringRef Mnemo = ((AMDGPUOperand &)*Operands[0]).getToken();
7540 if (Mnemo.starts_with(Prefix: "ds_gws"))
7541 return Error(L: S, Msg: "nogds is not allowed");
7542 }
7543
7544 if (isGFX9() && ImmTy == AMDGPUOperand::ImmTyA16)
7545 ImmTy = AMDGPUOperand::ImmTyR128A16;
7546
7547 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Bit, Loc: S, Type: ImmTy));
7548 return ParseStatus::Success;
7549}
7550
7551unsigned AMDGPUAsmParser::getCPolKind(StringRef Id, StringRef Mnemo,
7552 bool &Disabling) const {
7553 Disabling = Id.consume_front(Prefix: "no");
7554
7555 if (isGFX940() && !Mnemo.starts_with(Prefix: "s_")) {
7556 return StringSwitch<unsigned>(Id)
7557 .Case(S: "nt", Value: AMDGPU::CPol::NT)
7558 .Case(S: "sc0", Value: AMDGPU::CPol::SC0)
7559 .Case(S: "sc1", Value: AMDGPU::CPol::SC1)
7560 .Default(Value: 0);
7561 }
7562
7563 return StringSwitch<unsigned>(Id)
7564 .Case(S: "dlc", Value: AMDGPU::CPol::DLC)
7565 .Case(S: "glc", Value: AMDGPU::CPol::GLC)
7566 .Case(S: "scc", Value: AMDGPU::CPol::SCC)
7567 .Case(S: "slc", Value: AMDGPU::CPol::SLC)
7568 .Default(Value: 0);
7569}
7570
7571ParseStatus AMDGPUAsmParser::parseCPol(OperandVector &Operands) {
7572 if (isGFX12Plus()) {
7573 SMLoc StringLoc = getLoc();
7574
7575 int64_t CPolVal = 0;
7576 ParseStatus ResTH = ParseStatus::NoMatch;
7577 ParseStatus ResScope = ParseStatus::NoMatch;
7578 ParseStatus ResNV = ParseStatus::NoMatch;
7579 ParseStatus ResScal = ParseStatus::NoMatch;
7580
7581 for (;;) {
7582 if (ResTH.isNoMatch()) {
7583 int64_t TH;
7584 ResTH = parseTH(Operands, TH);
7585 if (ResTH.isFailure())
7586 return ResTH;
7587 if (ResTH.isSuccess()) {
7588 CPolVal |= TH;
7589 continue;
7590 }
7591 }
7592
7593 if (ResScope.isNoMatch()) {
7594 int64_t Scope;
7595 ResScope = parseScope(Operands, Scope);
7596 if (ResScope.isFailure())
7597 return ResScope;
7598 if (ResScope.isSuccess()) {
7599 CPolVal |= Scope;
7600 continue;
7601 }
7602 }
7603
7604 // NV bit exists on GFX12+, but does something starting from GFX1250.
7605 // Allow parsing on all GFX12 and fail on validation for better
7606 // diagnostics.
7607 if (ResNV.isNoMatch()) {
7608 if (trySkipId(Id: "nv")) {
7609 ResNV = ParseStatus::Success;
7610 CPolVal |= CPol::NV;
7611 continue;
7612 } else if (trySkipId(Pref: "no", Id: "nv")) {
7613 ResNV = ParseStatus::Success;
7614 continue;
7615 }
7616 }
7617
7618 if (ResScal.isNoMatch()) {
7619 if (trySkipId(Id: "scale_offset")) {
7620 ResScal = ParseStatus::Success;
7621 CPolVal |= CPol::SCAL;
7622 continue;
7623 } else if (trySkipId(Pref: "no", Id: "scale_offset")) {
7624 ResScal = ParseStatus::Success;
7625 continue;
7626 }
7627 }
7628
7629 break;
7630 }
7631
7632 if (ResTH.isNoMatch() && ResScope.isNoMatch() && ResNV.isNoMatch() &&
7633 ResScal.isNoMatch())
7634 return ParseStatus::NoMatch;
7635
7636 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: CPolVal, Loc: StringLoc,
7637 Type: AMDGPUOperand::ImmTyCPol));
7638 return ParseStatus::Success;
7639 }
7640
7641 StringRef Mnemo = ((AMDGPUOperand &)*Operands[0]).getToken();
7642 SMLoc OpLoc = getLoc();
7643 unsigned Enabled = 0, Seen = 0;
7644 for (;;) {
7645 SMLoc S = getLoc();
7646 bool Disabling;
7647 unsigned CPol = getCPolKind(Id: getId(), Mnemo, Disabling);
7648 if (!CPol)
7649 break;
7650
7651 lex();
7652
7653 if (!isGFX10Plus() && CPol == AMDGPU::CPol::DLC)
7654 return Error(L: S, Msg: "dlc modifier is not supported on this GPU");
7655
7656 if (!isGFX90A() && CPol == AMDGPU::CPol::SCC)
7657 return Error(L: S, Msg: "scc modifier is not supported on this GPU");
7658
7659 if (Seen & CPol)
7660 return Error(L: S, Msg: "duplicate cache policy modifier");
7661
7662 if (!Disabling)
7663 Enabled |= CPol;
7664
7665 Seen |= CPol;
7666 }
7667
7668 if (!Seen)
7669 return ParseStatus::NoMatch;
7670
7671 Operands.push_back(
7672 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Enabled, Loc: OpLoc, Type: AMDGPUOperand::ImmTyCPol));
7673 return ParseStatus::Success;
7674}
7675
7676ParseStatus AMDGPUAsmParser::parseScope(OperandVector &Operands,
7677 int64_t &Scope) {
7678 static const unsigned Scopes[] = {CPol::SCOPE_CU, CPol::SCOPE_SE,
7679 CPol::SCOPE_DEV, CPol::SCOPE_SYS};
7680
7681 ParseStatus Res = parseStringOrIntWithPrefix(
7682 Operands, Name: "scope", Ids: {"SCOPE_CU", "SCOPE_SE", "SCOPE_DEV", "SCOPE_SYS"},
7683 IntVal&: Scope);
7684
7685 if (Res.isSuccess())
7686 Scope = Scopes[Scope];
7687
7688 return Res;
7689}
7690
7691ParseStatus AMDGPUAsmParser::parseTH(OperandVector &Operands, int64_t &TH) {
7692 TH = AMDGPU::CPol::TH_RT; // default
7693
7694 StringRef Value;
7695 SMLoc StringLoc;
7696 ParseStatus Res = parseStringWithPrefix(Prefix: "th", Value, StringLoc);
7697 if (!Res.isSuccess())
7698 return Res;
7699
7700 if (Value == "TH_DEFAULT")
7701 TH = AMDGPU::CPol::TH_RT;
7702 else if (Value == "TH_STORE_LU" || Value == "TH_LOAD_WB" ||
7703 Value == "TH_LOAD_NT_WB") {
7704 return Error(L: StringLoc, Msg: "invalid th value");
7705 } else if (Value.consume_front(Prefix: "TH_ATOMIC_")) {
7706 TH = AMDGPU::CPol::TH_TYPE_ATOMIC;
7707 } else if (Value.consume_front(Prefix: "TH_LOAD_")) {
7708 TH = AMDGPU::CPol::TH_TYPE_LOAD;
7709 } else if (Value.consume_front(Prefix: "TH_STORE_")) {
7710 TH = AMDGPU::CPol::TH_TYPE_STORE;
7711 } else {
7712 return Error(L: StringLoc, Msg: "invalid th value");
7713 }
7714
7715 if (Value == "BYPASS")
7716 TH |= AMDGPU::CPol::TH_REAL_BYPASS;
7717
7718 if (TH != 0) {
7719 if (TH & AMDGPU::CPol::TH_TYPE_ATOMIC)
7720 TH |= StringSwitch<int64_t>(Value)
7721 .Case(S: "RETURN", Value: AMDGPU::CPol::TH_ATOMIC_RETURN)
7722 .Case(S: "RT", Value: AMDGPU::CPol::TH_RT)
7723 .Case(S: "RT_RETURN", Value: AMDGPU::CPol::TH_ATOMIC_RETURN)
7724 .Case(S: "NT", Value: AMDGPU::CPol::TH_ATOMIC_NT)
7725 .Case(S: "NT_RETURN", Value: AMDGPU::CPol::TH_ATOMIC_NT |
7726 AMDGPU::CPol::TH_ATOMIC_RETURN)
7727 .Case(S: "CASCADE_RT", Value: AMDGPU::CPol::TH_ATOMIC_CASCADE)
7728 .Case(S: "CASCADE_NT", Value: AMDGPU::CPol::TH_ATOMIC_CASCADE |
7729 AMDGPU::CPol::TH_ATOMIC_NT)
7730 .Default(Value: 0xffffffff);
7731 else
7732 TH |= StringSwitch<int64_t>(Value)
7733 .Case(S: "RT", Value: AMDGPU::CPol::TH_RT)
7734 .Case(S: "NT", Value: AMDGPU::CPol::TH_NT)
7735 .Case(S: "HT", Value: AMDGPU::CPol::TH_HT)
7736 .Case(S: "LU", Value: AMDGPU::CPol::TH_LU)
7737 .Case(S: "WB", Value: AMDGPU::CPol::TH_WB)
7738 .Case(S: "NT_RT", Value: AMDGPU::CPol::TH_NT_RT)
7739 .Case(S: "RT_NT", Value: AMDGPU::CPol::TH_RT_NT)
7740 .Case(S: "NT_HT", Value: AMDGPU::CPol::TH_NT_HT)
7741 .Case(S: "NT_WB", Value: AMDGPU::CPol::TH_NT_WB)
7742 .Case(S: "BYPASS", Value: AMDGPU::CPol::TH_BYPASS)
7743 .Default(Value: 0xffffffff);
7744 }
7745
7746 if (TH == 0xffffffff)
7747 return Error(L: StringLoc, Msg: "invalid th value");
7748
7749 return ParseStatus::Success;
7750}
7751
7752static void
7753addOptionalImmOperand(MCInst &Inst, const OperandVector &Operands,
7754 AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx,
7755 AMDGPUOperand::ImmTy ImmT, int64_t Default = 0,
7756 std::optional<unsigned> InsertAt = std::nullopt) {
7757 auto i = OptionalIdx.find(x: ImmT);
7758 if (i != OptionalIdx.end()) {
7759 unsigned Idx = i->second;
7760 const AMDGPUOperand &Op =
7761 static_cast<const AMDGPUOperand &>(*Operands[Idx]);
7762 if (InsertAt)
7763 Inst.insert(I: Inst.begin() + *InsertAt, Op: MCOperand::createImm(Val: Op.getImm()));
7764 else
7765 Op.addImmOperands(Inst, N: 1);
7766 } else {
7767 if (InsertAt.has_value())
7768 Inst.insert(I: Inst.begin() + *InsertAt, Op: MCOperand::createImm(Val: Default));
7769 else
7770 Inst.addOperand(Op: MCOperand::createImm(Val: Default));
7771 }
7772}
7773
7774ParseStatus AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix,
7775 StringRef &Value,
7776 SMLoc &StringLoc) {
7777 if (!trySkipId(Id: Prefix, Kind: AsmToken::Colon))
7778 return ParseStatus::NoMatch;
7779
7780 StringLoc = getLoc();
7781 return parseId(Val&: Value, ErrMsg: "expected an identifier") ? ParseStatus::Success
7782 : ParseStatus::Failure;
7783}
7784
7785ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7786 OperandVector &Operands, StringRef Name, ArrayRef<const char *> Ids,
7787 int64_t &IntVal) {
7788 if (!trySkipId(Id: Name, Kind: AsmToken::Colon))
7789 return ParseStatus::NoMatch;
7790
7791 SMLoc StringLoc = getLoc();
7792
7793 StringRef Value;
7794 if (isToken(Kind: AsmToken::Identifier)) {
7795 Value = getTokenStr();
7796 lex();
7797
7798 for (IntVal = 0; IntVal < (int64_t)Ids.size(); ++IntVal)
7799 if (Value == Ids[IntVal])
7800 break;
7801 } else if (!parseExpr(Imm&: IntVal))
7802 return ParseStatus::Failure;
7803
7804 if (IntVal < 0 || IntVal >= (int64_t)Ids.size())
7805 return Error(L: StringLoc, Msg: "invalid " + Twine(Name) + " value");
7806
7807 return ParseStatus::Success;
7808}
7809
7810ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7811 OperandVector &Operands, StringRef Name, ArrayRef<const char *> Ids,
7812 AMDGPUOperand::ImmTy Type) {
7813 SMLoc S = getLoc();
7814 int64_t IntVal;
7815
7816 ParseStatus Res = parseStringOrIntWithPrefix(Operands, Name, Ids, IntVal);
7817 if (Res.isSuccess())
7818 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: IntVal, Loc: S, Type));
7819
7820 return Res;
7821}
7822
7823//===----------------------------------------------------------------------===//
7824// MTBUF format
7825//===----------------------------------------------------------------------===//
7826
7827bool AMDGPUAsmParser::tryParseFmt(const char *Pref, int64_t MaxVal,
7828 int64_t &Fmt) {
7829 int64_t Val;
7830 SMLoc Loc = getLoc();
7831
7832 auto Res = parseIntWithPrefix(Prefix: Pref, IntVal&: Val);
7833 if (Res.isFailure())
7834 return false;
7835 if (Res.isNoMatch())
7836 return true;
7837
7838 if (Val < 0 || Val > MaxVal) {
7839 Error(L: Loc, Msg: Twine("out of range ", StringRef(Pref)));
7840 return false;
7841 }
7842
7843 Fmt = Val;
7844 return true;
7845}
7846
7847ParseStatus AMDGPUAsmParser::tryParseIndexKey(OperandVector &Operands,
7848 AMDGPUOperand::ImmTy ImmTy) {
7849 const char *Pref = "index_key";
7850 int64_t ImmVal = 0;
7851 SMLoc Loc = getLoc();
7852 auto Res = parseIntWithPrefix(Prefix: Pref, IntVal&: ImmVal);
7853 if (!Res.isSuccess())
7854 return Res;
7855
7856 if ((ImmTy == AMDGPUOperand::ImmTyIndexKey16bit ||
7857 ImmTy == AMDGPUOperand::ImmTyIndexKey32bit) &&
7858 (ImmVal < 0 || ImmVal > 1))
7859 return Error(L: Loc, Msg: Twine("out of range ", StringRef(Pref)));
7860
7861 if (ImmTy == AMDGPUOperand::ImmTyIndexKey8bit && (ImmVal < 0 || ImmVal > 3))
7862 return Error(L: Loc, Msg: Twine("out of range ", StringRef(Pref)));
7863
7864 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: ImmVal, Loc, Type: ImmTy));
7865 return ParseStatus::Success;
7866}
7867
7868ParseStatus AMDGPUAsmParser::parseIndexKey8bit(OperandVector &Operands) {
7869 return tryParseIndexKey(Operands, ImmTy: AMDGPUOperand::ImmTyIndexKey8bit);
7870}
7871
7872ParseStatus AMDGPUAsmParser::parseIndexKey16bit(OperandVector &Operands) {
7873 return tryParseIndexKey(Operands, ImmTy: AMDGPUOperand::ImmTyIndexKey16bit);
7874}
7875
7876ParseStatus AMDGPUAsmParser::parseIndexKey32bit(OperandVector &Operands) {
7877 return tryParseIndexKey(Operands, ImmTy: AMDGPUOperand::ImmTyIndexKey32bit);
7878}
7879
7880ParseStatus AMDGPUAsmParser::tryParseMatrixFMT(OperandVector &Operands,
7881 StringRef Name,
7882 AMDGPUOperand::ImmTy Type) {
7883 return parseStringOrIntWithPrefix(Operands, Name, Ids: WMMAMods::ModMatrixFmt,
7884 Type);
7885}
7886
7887ParseStatus AMDGPUAsmParser::parseMatrixAFMT(OperandVector &Operands) {
7888 return tryParseMatrixFMT(Operands, Name: "matrix_a_fmt",
7889 Type: AMDGPUOperand::ImmTyMatrixAFMT);
7890}
7891
7892ParseStatus AMDGPUAsmParser::parseMatrixBFMT(OperandVector &Operands) {
7893 return tryParseMatrixFMT(Operands, Name: "matrix_b_fmt",
7894 Type: AMDGPUOperand::ImmTyMatrixBFMT);
7895}
7896
7897ParseStatus AMDGPUAsmParser::tryParseMatrixScale(OperandVector &Operands,
7898 StringRef Name,
7899 AMDGPUOperand::ImmTy Type) {
7900 return parseStringOrIntWithPrefix(Operands, Name, Ids: WMMAMods::ModMatrixScale,
7901 Type);
7902}
7903
7904ParseStatus AMDGPUAsmParser::parseMatrixAScale(OperandVector &Operands) {
7905 return tryParseMatrixScale(Operands, Name: "matrix_a_scale",
7906 Type: AMDGPUOperand::ImmTyMatrixAScale);
7907}
7908
7909ParseStatus AMDGPUAsmParser::parseMatrixBScale(OperandVector &Operands) {
7910 return tryParseMatrixScale(Operands, Name: "matrix_b_scale",
7911 Type: AMDGPUOperand::ImmTyMatrixBScale);
7912}
7913
7914ParseStatus AMDGPUAsmParser::tryParseMatrixScaleFmt(OperandVector &Operands,
7915 StringRef Name,
7916 AMDGPUOperand::ImmTy Type) {
7917 return parseStringOrIntWithPrefix(Operands, Name, Ids: WMMAMods::ModMatrixScaleFmt,
7918 Type);
7919}
7920
7921ParseStatus AMDGPUAsmParser::parseMatrixAScaleFmt(OperandVector &Operands) {
7922 return tryParseMatrixScaleFmt(Operands, Name: "matrix_a_scale_fmt",
7923 Type: AMDGPUOperand::ImmTyMatrixAScaleFmt);
7924}
7925
7926ParseStatus AMDGPUAsmParser::parseMatrixBScaleFmt(OperandVector &Operands) {
7927 return tryParseMatrixScaleFmt(Operands, Name: "matrix_b_scale_fmt",
7928 Type: AMDGPUOperand::ImmTyMatrixBScaleFmt);
7929}
7930
7931// dfmt and nfmt (in a tbuffer instruction) are parsed as one to allow their
7932// values to live in a joint format operand in the MCInst encoding.
7933ParseStatus AMDGPUAsmParser::parseDfmtNfmt(int64_t &Format) {
7934 using namespace llvm::AMDGPU::MTBUFFormat;
7935
7936 int64_t Dfmt = DFMT_UNDEF;
7937 int64_t Nfmt = NFMT_UNDEF;
7938
7939 // dfmt and nfmt can appear in either order, and each is optional.
7940 for (int I = 0; I < 2; ++I) {
7941 if (Dfmt == DFMT_UNDEF && !tryParseFmt(Pref: "dfmt", MaxVal: DFMT_MAX, Fmt&: Dfmt))
7942 return ParseStatus::Failure;
7943
7944 if (Nfmt == NFMT_UNDEF && !tryParseFmt(Pref: "nfmt", MaxVal: NFMT_MAX, Fmt&: Nfmt))
7945 return ParseStatus::Failure;
7946
7947 // Skip optional comma between dfmt/nfmt
7948 // but guard against 2 commas following each other.
7949 if ((Dfmt == DFMT_UNDEF) != (Nfmt == NFMT_UNDEF) &&
7950 !peekToken().is(K: AsmToken::Comma)) {
7951 trySkipToken(Kind: AsmToken::Comma);
7952 }
7953 }
7954
7955 if (Dfmt == DFMT_UNDEF && Nfmt == NFMT_UNDEF)
7956 return ParseStatus::NoMatch;
7957
7958 Dfmt = (Dfmt == DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7959 Nfmt = (Nfmt == NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7960
7961 Format = encodeDfmtNfmt(Dfmt, Nfmt);
7962 return ParseStatus::Success;
7963}
7964
7965ParseStatus AMDGPUAsmParser::parseUfmt(int64_t &Format) {
7966 using namespace llvm::AMDGPU::MTBUFFormat;
7967
7968 int64_t Fmt = UFMT_UNDEF;
7969
7970 if (!tryParseFmt(Pref: "format", MaxVal: UFMT_MAX, Fmt))
7971 return ParseStatus::Failure;
7972
7973 if (Fmt == UFMT_UNDEF)
7974 return ParseStatus::NoMatch;
7975
7976 Format = Fmt;
7977 return ParseStatus::Success;
7978}
7979
7980bool AMDGPUAsmParser::matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt,
7981 StringRef FormatStr, SMLoc Loc) {
7982 using namespace llvm::AMDGPU::MTBUFFormat;
7983 int64_t Format;
7984
7985 Format = getDfmt(Name: FormatStr);
7986 if (Format != DFMT_UNDEF) {
7987 Dfmt = Format;
7988 return true;
7989 }
7990
7991 Format = getNfmt(Name: FormatStr, STI: getSTI());
7992 if (Format != NFMT_UNDEF) {
7993 Nfmt = Format;
7994 return true;
7995 }
7996
7997 Error(L: Loc, Msg: "unsupported format");
7998 return false;
7999}
8000
8001ParseStatus AMDGPUAsmParser::parseSymbolicSplitFormat(StringRef FormatStr,
8002 SMLoc FormatLoc,
8003 int64_t &Format) {
8004 using namespace llvm::AMDGPU::MTBUFFormat;
8005
8006 int64_t Dfmt = DFMT_UNDEF;
8007 int64_t Nfmt = NFMT_UNDEF;
8008 if (!matchDfmtNfmt(Dfmt, Nfmt, FormatStr, Loc: FormatLoc))
8009 return ParseStatus::Failure;
8010
8011 if (trySkipToken(Kind: AsmToken::Comma)) {
8012 StringRef Str;
8013 SMLoc Loc = getLoc();
8014 if (!parseId(Val&: Str, ErrMsg: "expected a format string") ||
8015 !matchDfmtNfmt(Dfmt, Nfmt, FormatStr: Str, Loc))
8016 return ParseStatus::Failure;
8017 if (Dfmt == DFMT_UNDEF)
8018 return Error(L: Loc, Msg: "duplicate numeric format");
8019 if (Nfmt == NFMT_UNDEF)
8020 return Error(L: Loc, Msg: "duplicate data format");
8021 }
8022
8023 Dfmt = (Dfmt == DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
8024 Nfmt = (Nfmt == NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
8025
8026 if (isGFX10Plus()) {
8027 auto Ufmt = convertDfmtNfmt2Ufmt(Dfmt, Nfmt, STI: getSTI());
8028 if (Ufmt == UFMT_UNDEF)
8029 return Error(L: FormatLoc, Msg: "unsupported format");
8030 Format = Ufmt;
8031 } else {
8032 Format = encodeDfmtNfmt(Dfmt, Nfmt);
8033 }
8034
8035 return ParseStatus::Success;
8036}
8037
8038ParseStatus AMDGPUAsmParser::parseSymbolicUnifiedFormat(StringRef FormatStr,
8039 SMLoc Loc,
8040 int64_t &Format) {
8041 using namespace llvm::AMDGPU::MTBUFFormat;
8042
8043 auto Id = getUnifiedFormat(Name: FormatStr, STI: getSTI());
8044 if (Id == UFMT_UNDEF)
8045 return ParseStatus::NoMatch;
8046
8047 if (!isGFX10Plus())
8048 return Error(L: Loc, Msg: "unified format is not supported on this GPU");
8049
8050 Format = Id;
8051 return ParseStatus::Success;
8052}
8053
8054ParseStatus AMDGPUAsmParser::parseNumericFormat(int64_t &Format) {
8055 using namespace llvm::AMDGPU::MTBUFFormat;
8056 SMLoc Loc = getLoc();
8057
8058 if (!parseExpr(Imm&: Format))
8059 return ParseStatus::Failure;
8060 if (!isValidFormatEncoding(Val: Format, STI: getSTI()))
8061 return Error(L: Loc, Msg: "out of range format");
8062
8063 return ParseStatus::Success;
8064}
8065
8066ParseStatus AMDGPUAsmParser::parseSymbolicOrNumericFormat(int64_t &Format) {
8067 using namespace llvm::AMDGPU::MTBUFFormat;
8068
8069 if (!trySkipId(Id: "format", Kind: AsmToken::Colon))
8070 return ParseStatus::NoMatch;
8071
8072 if (trySkipToken(Kind: AsmToken::LBrac)) {
8073 StringRef FormatStr;
8074 SMLoc Loc = getLoc();
8075 if (!parseId(Val&: FormatStr, ErrMsg: "expected a format string"))
8076 return ParseStatus::Failure;
8077
8078 auto Res = parseSymbolicUnifiedFormat(FormatStr, Loc, Format);
8079 if (Res.isNoMatch())
8080 Res = parseSymbolicSplitFormat(FormatStr, FormatLoc: Loc, Format);
8081 if (!Res.isSuccess())
8082 return Res;
8083
8084 if (!skipToken(Kind: AsmToken::RBrac, ErrMsg: "expected a closing square bracket"))
8085 return ParseStatus::Failure;
8086
8087 return ParseStatus::Success;
8088 }
8089
8090 return parseNumericFormat(Format);
8091}
8092
8093ParseStatus AMDGPUAsmParser::parseFORMAT(OperandVector &Operands) {
8094 using namespace llvm::AMDGPU::MTBUFFormat;
8095
8096 int64_t Format = getDefaultFormatEncoding(STI: getSTI());
8097 ParseStatus Res;
8098 SMLoc Loc = getLoc();
8099
8100 // Parse legacy format syntax.
8101 Res = isGFX10Plus() ? parseUfmt(Format) : parseDfmtNfmt(Format);
8102 if (Res.isFailure())
8103 return Res;
8104
8105 bool FormatFound = Res.isSuccess();
8106
8107 Operands.push_back(
8108 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Format, Loc, Type: AMDGPUOperand::ImmTyFORMAT));
8109
8110 if (FormatFound)
8111 trySkipToken(Kind: AsmToken::Comma);
8112
8113 if (isToken(Kind: AsmToken::EndOfStatement)) {
8114 // We are expecting an soffset operand,
8115 // but let matcher handle the error.
8116 return ParseStatus::Success;
8117 }
8118
8119 // Parse soffset.
8120 Res = parseRegOrImm(Operands);
8121 if (!Res.isSuccess())
8122 return Res;
8123
8124 trySkipToken(Kind: AsmToken::Comma);
8125
8126 if (!FormatFound) {
8127 Res = parseSymbolicOrNumericFormat(Format);
8128 if (Res.isFailure())
8129 return Res;
8130 if (Res.isSuccess()) {
8131 auto Size = Operands.size();
8132 AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands[Size - 2]);
8133 assert(Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyFORMAT);
8134 Op.setImm(Format);
8135 }
8136 return ParseStatus::Success;
8137 }
8138
8139 if (isId(Id: "format") && peekToken().is(K: AsmToken::Colon))
8140 return Error(L: getLoc(), Msg: "duplicate format");
8141 return ParseStatus::Success;
8142}
8143
8144ParseStatus AMDGPUAsmParser::parseFlatOffset(OperandVector &Operands) {
8145 ParseStatus Res =
8146 parseIntWithPrefix(Prefix: "offset", Operands, ImmTy: AMDGPUOperand::ImmTyOffset);
8147 if (Res.isNoMatch()) {
8148 Res = parseIntWithPrefix(Prefix: "inst_offset", Operands,
8149 ImmTy: AMDGPUOperand::ImmTyInstOffset);
8150 }
8151 return Res;
8152}
8153
8154ParseStatus AMDGPUAsmParser::parseR128A16(OperandVector &Operands) {
8155 ParseStatus Res =
8156 parseNamedBit(Name: "r128", Operands, ImmTy: AMDGPUOperand::ImmTyR128A16);
8157 if (Res.isNoMatch())
8158 Res = parseNamedBit(Name: "a16", Operands, ImmTy: AMDGPUOperand::ImmTyA16);
8159 return Res;
8160}
8161
8162ParseStatus AMDGPUAsmParser::parseBLGP(OperandVector &Operands) {
8163 ParseStatus Res =
8164 parseIntWithPrefix(Prefix: "blgp", Operands, ImmTy: AMDGPUOperand::ImmTyBLGP);
8165 if (Res.isNoMatch()) {
8166 Res =
8167 parseOperandArrayWithPrefix(Prefix: "neg", Operands, ImmTy: AMDGPUOperand::ImmTyBLGP);
8168 }
8169 return Res;
8170}
8171
8172//===----------------------------------------------------------------------===//
8173// Exp
8174//===----------------------------------------------------------------------===//
8175
8176void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
8177 OptionalImmIndexMap OptionalIdx;
8178
8179 unsigned OperandIdx[4];
8180 unsigned EnMask = 0;
8181 int SrcIdx = 0;
8182
8183 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
8184 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
8185
8186 // Add the register arguments
8187 if (Op.isReg()) {
8188 assert(SrcIdx < 4);
8189 OperandIdx[SrcIdx] = Inst.size();
8190 Op.addRegOperands(Inst, N: 1);
8191 ++SrcIdx;
8192 continue;
8193 }
8194
8195 if (Op.isOff()) {
8196 assert(SrcIdx < 4);
8197 OperandIdx[SrcIdx] = Inst.size();
8198 Inst.addOperand(Op: MCOperand::createReg(Reg: MCRegister()));
8199 ++SrcIdx;
8200 continue;
8201 }
8202
8203 if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
8204 Op.addImmOperands(Inst, N: 1);
8205 continue;
8206 }
8207
8208 if (Op.isToken() && (Op.getToken() == "done" || Op.getToken() == "row_en"))
8209 continue;
8210
8211 // Handle optional arguments
8212 OptionalIdx[Op.getImmTy()] = i;
8213 }
8214
8215 assert(SrcIdx == 4);
8216
8217 bool Compr = false;
8218 if (OptionalIdx.find(x: AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
8219 Compr = true;
8220 Inst.getOperand(i: OperandIdx[1]) = Inst.getOperand(i: OperandIdx[2]);
8221 Inst.getOperand(i: OperandIdx[2]).setReg(MCRegister());
8222 Inst.getOperand(i: OperandIdx[3]).setReg(MCRegister());
8223 }
8224
8225 for (auto i = 0; i < SrcIdx; ++i) {
8226 if (Inst.getOperand(i: OperandIdx[i]).getReg()) {
8227 EnMask |= Compr ? (0x3 << i * 2) : (0x1 << i);
8228 }
8229 }
8230
8231 addOptionalImmOperand(Inst, Operands, OptionalIdx, ImmT: AMDGPUOperand::ImmTyExpVM);
8232 addOptionalImmOperand(Inst, Operands, OptionalIdx,
8233 ImmT: AMDGPUOperand::ImmTyExpCompr);
8234
8235 Inst.addOperand(Op: MCOperand::createImm(Val: EnMask));
8236}
8237
8238//===----------------------------------------------------------------------===//
8239// s_waitcnt
8240//===----------------------------------------------------------------------===//
8241
8242static bool encodeCnt(const AMDGPU::IsaVersion ISA, int64_t &IntVal,
8243 int64_t CntVal, bool Saturate,
8244 unsigned (*encode)(const IsaVersion &Version, unsigned,
8245 unsigned),
8246 unsigned (*decode)(const IsaVersion &Version, unsigned)) {
8247 bool Failed = false;
8248
8249 IntVal = encode(ISA, IntVal, CntVal);
8250 if (CntVal != decode(ISA, IntVal)) {
8251 if (Saturate) {
8252 IntVal = encode(ISA, IntVal, -1);
8253 } else {
8254 Failed = true;
8255 }
8256 }
8257 return Failed;
8258}
8259
8260bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
8261
8262 SMLoc CntLoc = getLoc();
8263 StringRef CntName = getTokenStr();
8264
8265 if (!skipToken(Kind: AsmToken::Identifier, ErrMsg: "expected a counter name") ||
8266 !skipToken(Kind: AsmToken::LParen, ErrMsg: "expected a left parenthesis"))
8267 return false;
8268
8269 int64_t CntVal;
8270 SMLoc ValLoc = getLoc();
8271 if (!parseExpr(Imm&: CntVal))
8272 return false;
8273
8274 bool Failed = true;
8275 bool Sat = CntName.ends_with(Suffix: "_sat");
8276
8277 if (CntName == "vmcnt" || CntName == "vmcnt_sat") {
8278 Failed = encodeCnt(ISA, IntVal, CntVal, Saturate: Sat, encode: encodeVmcnt, decode: decodeVmcnt);
8279 } else if (CntName == "expcnt" || CntName == "expcnt_sat") {
8280 Failed = encodeCnt(ISA, IntVal, CntVal, Saturate: Sat, encode: encodeExpcnt, decode: decodeExpcnt);
8281 } else if (CntName == "lgkmcnt" || CntName == "lgkmcnt_sat") {
8282 Failed = encodeCnt(ISA, IntVal, CntVal, Saturate: Sat, encode: encodeLgkmcnt, decode: decodeLgkmcnt);
8283 } else {
8284 Error(L: CntLoc, Msg: "invalid counter name " + CntName);
8285 return false;
8286 }
8287
8288 if (Failed) {
8289 Error(L: ValLoc, Msg: "too large value for " + CntName);
8290 return false;
8291 }
8292
8293 if (!skipToken(Kind: AsmToken::RParen, ErrMsg: "expected a closing parenthesis"))
8294 return false;
8295
8296 if (trySkipToken(Kind: AsmToken::Amp) || trySkipToken(Kind: AsmToken::Comma)) {
8297 if (isToken(Kind: AsmToken::EndOfStatement)) {
8298 Error(L: getLoc(), Msg: "expected a counter name");
8299 return false;
8300 }
8301 }
8302
8303 return true;
8304}
8305
8306ParseStatus AMDGPUAsmParser::parseSWaitCnt(OperandVector &Operands) {
8307 int64_t Waitcnt = getWaitcntBitMask(Version: ISA);
8308 SMLoc S = getLoc();
8309
8310 if (isToken(Kind: AsmToken::Identifier) && peekToken().is(K: AsmToken::LParen)) {
8311 while (!isToken(Kind: AsmToken::EndOfStatement)) {
8312 if (!parseCnt(IntVal&: Waitcnt))
8313 return ParseStatus::Failure;
8314 }
8315 } else {
8316 if (!parseExpr(Imm&: Waitcnt))
8317 return ParseStatus::Failure;
8318 }
8319
8320 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Waitcnt, Loc: S));
8321 return ParseStatus::Success;
8322}
8323
8324bool AMDGPUAsmParser::parseDelay(int64_t &Delay) {
8325 SMLoc FieldLoc = getLoc();
8326 StringRef FieldName = getTokenStr();
8327 if (!skipToken(Kind: AsmToken::Identifier, ErrMsg: "expected a field name") ||
8328 !skipToken(Kind: AsmToken::LParen, ErrMsg: "expected a left parenthesis"))
8329 return false;
8330
8331 SMLoc ValueLoc = getLoc();
8332 StringRef ValueName = getTokenStr();
8333 if (!skipToken(Kind: AsmToken::Identifier, ErrMsg: "expected a value name") ||
8334 !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected a right parenthesis"))
8335 return false;
8336
8337 unsigned Shift;
8338 if (FieldName == "instid0") {
8339 Shift = 0;
8340 } else if (FieldName == "instskip") {
8341 Shift = 4;
8342 } else if (FieldName == "instid1") {
8343 Shift = 7;
8344 } else {
8345 Error(L: FieldLoc, Msg: "invalid field name " + FieldName);
8346 return false;
8347 }
8348
8349 int Value;
8350 if (Shift == 4) {
8351 // Parse values for instskip.
8352 Value = StringSwitch<int>(ValueName)
8353 .Case(S: "SAME", Value: 0)
8354 .Case(S: "NEXT", Value: 1)
8355 .Case(S: "SKIP_1", Value: 2)
8356 .Case(S: "SKIP_2", Value: 3)
8357 .Case(S: "SKIP_3", Value: 4)
8358 .Case(S: "SKIP_4", Value: 5)
8359 .Default(Value: -1);
8360 } else {
8361 // Parse values for instid0 and instid1.
8362 Value = StringSwitch<int>(ValueName)
8363 .Case(S: "NO_DEP", Value: 0)
8364 .Case(S: "VALU_DEP_1", Value: 1)
8365 .Case(S: "VALU_DEP_2", Value: 2)
8366 .Case(S: "VALU_DEP_3", Value: 3)
8367 .Case(S: "VALU_DEP_4", Value: 4)
8368 .Case(S: "TRANS32_DEP_1", Value: 5)
8369 .Case(S: "TRANS32_DEP_2", Value: 6)
8370 .Case(S: "TRANS32_DEP_3", Value: 7)
8371 .Case(S: "FMA_ACCUM_CYCLE_1", Value: 8)
8372 .Case(S: "SALU_CYCLE_1", Value: 9)
8373 .Case(S: "SALU_CYCLE_2", Value: 10)
8374 .Case(S: "SALU_CYCLE_3", Value: 11)
8375 .Default(Value: -1);
8376 }
8377 if (Value < 0) {
8378 Error(L: ValueLoc, Msg: "invalid value name " + ValueName);
8379 return false;
8380 }
8381
8382 Delay |= Value << Shift;
8383 return true;
8384}
8385
8386ParseStatus AMDGPUAsmParser::parseSDelayALU(OperandVector &Operands) {
8387 int64_t Delay = 0;
8388 SMLoc S = getLoc();
8389
8390 if (isToken(Kind: AsmToken::Identifier) && peekToken().is(K: AsmToken::LParen)) {
8391 do {
8392 if (!parseDelay(Delay))
8393 return ParseStatus::Failure;
8394 } while (trySkipToken(Kind: AsmToken::Pipe));
8395 } else {
8396 if (!parseExpr(Imm&: Delay))
8397 return ParseStatus::Failure;
8398 }
8399
8400 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Delay, Loc: S));
8401 return ParseStatus::Success;
8402}
8403
8404bool AMDGPUOperand::isSWaitCnt() const { return isImm(); }
8405
8406bool AMDGPUOperand::isSDelayALU() const { return isImm(); }
8407
8408//===----------------------------------------------------------------------===//
8409// DepCtr
8410//===----------------------------------------------------------------------===//
8411
8412void AMDGPUAsmParser::depCtrError(SMLoc Loc, int ErrorId,
8413 StringRef DepCtrName) {
8414 switch (ErrorId) {
8415 case OPR_ID_UNKNOWN:
8416 Error(L: Loc, Msg: Twine("invalid counter name ", DepCtrName));
8417 return;
8418 case OPR_ID_UNSUPPORTED:
8419 Error(L: Loc, Msg: Twine(DepCtrName, " is not supported on this GPU"));
8420 return;
8421 case OPR_ID_DUPLICATE:
8422 Error(L: Loc, Msg: Twine("duplicate counter name ", DepCtrName));
8423 return;
8424 case OPR_VAL_INVALID:
8425 Error(L: Loc, Msg: Twine("invalid value for ", DepCtrName));
8426 return;
8427 default:
8428 assert(false);
8429 }
8430}
8431
8432bool AMDGPUAsmParser::parseDepCtr(int64_t &DepCtr, unsigned &UsedOprMask) {
8433
8434 using namespace llvm::AMDGPU::DepCtr;
8435
8436 SMLoc DepCtrLoc = getLoc();
8437 StringRef DepCtrName = getTokenStr();
8438
8439 if (!skipToken(Kind: AsmToken::Identifier, ErrMsg: "expected a counter name") ||
8440 !skipToken(Kind: AsmToken::LParen, ErrMsg: "expected a left parenthesis"))
8441 return false;
8442
8443 int64_t ExprVal;
8444 if (!parseExpr(Imm&: ExprVal))
8445 return false;
8446
8447 unsigned PrevOprMask = UsedOprMask;
8448 int CntVal = encodeDepCtr(Name: DepCtrName, Val: ExprVal, UsedOprMask, STI: getSTI());
8449
8450 if (CntVal < 0) {
8451 depCtrError(Loc: DepCtrLoc, ErrorId: CntVal, DepCtrName);
8452 return false;
8453 }
8454
8455 if (!skipToken(Kind: AsmToken::RParen, ErrMsg: "expected a closing parenthesis"))
8456 return false;
8457
8458 if (trySkipToken(Kind: AsmToken::Amp) || trySkipToken(Kind: AsmToken::Comma)) {
8459 if (isToken(Kind: AsmToken::EndOfStatement)) {
8460 Error(L: getLoc(), Msg: "expected a counter name");
8461 return false;
8462 }
8463 }
8464
8465 int64_t CntValMask = PrevOprMask ^ UsedOprMask;
8466 DepCtr = (DepCtr & ~CntValMask) | CntVal;
8467 return true;
8468}
8469
8470ParseStatus AMDGPUAsmParser::parseDepCtr(OperandVector &Operands) {
8471 using namespace llvm::AMDGPU::DepCtr;
8472
8473 int64_t DepCtr = getDefaultDepCtrEncoding(STI: getSTI());
8474 SMLoc Loc = getLoc();
8475
8476 if (isToken(Kind: AsmToken::Identifier) && peekToken().is(K: AsmToken::LParen)) {
8477 unsigned UsedOprMask = 0;
8478 while (!isToken(Kind: AsmToken::EndOfStatement)) {
8479 if (!parseDepCtr(DepCtr, UsedOprMask))
8480 return ParseStatus::Failure;
8481 }
8482 } else {
8483 if (!parseExpr(Imm&: DepCtr))
8484 return ParseStatus::Failure;
8485 }
8486
8487 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: DepCtr, Loc));
8488 return ParseStatus::Success;
8489}
8490
8491bool AMDGPUOperand::isDepCtr() const { return isS16Imm(); }
8492
8493//===----------------------------------------------------------------------===//
8494// hwreg
8495//===----------------------------------------------------------------------===//
8496
8497ParseStatus AMDGPUAsmParser::parseHwregFunc(OperandInfoTy &HwReg,
8498 OperandInfoTy &Offset,
8499 OperandInfoTy &Width) {
8500 using namespace llvm::AMDGPU::Hwreg;
8501
8502 if (!trySkipId(Id: "hwreg", Kind: AsmToken::LParen))
8503 return ParseStatus::NoMatch;
8504
8505 // The register may be specified by name or using a numeric code
8506 HwReg.Loc = getLoc();
8507 if (isToken(Kind: AsmToken::Identifier) &&
8508 (HwReg.Val = getHwregId(Name: getTokenStr(), STI: getSTI())) != OPR_ID_UNKNOWN) {
8509 HwReg.IsSymbolic = true;
8510 lex(); // skip register name
8511 } else if (!parseExpr(Imm&: HwReg.Val, Expected: "a register name")) {
8512 return ParseStatus::Failure;
8513 }
8514
8515 if (trySkipToken(Kind: AsmToken::RParen))
8516 return ParseStatus::Success;
8517
8518 // parse optional params
8519 if (!skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma or a closing parenthesis"))
8520 return ParseStatus::Failure;
8521
8522 Offset.Loc = getLoc();
8523 if (!parseExpr(Imm&: Offset.Val))
8524 return ParseStatus::Failure;
8525
8526 if (!skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma"))
8527 return ParseStatus::Failure;
8528
8529 Width.Loc = getLoc();
8530 if (!parseExpr(Imm&: Width.Val) ||
8531 !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected a closing parenthesis"))
8532 return ParseStatus::Failure;
8533
8534 return ParseStatus::Success;
8535}
8536
8537ParseStatus AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
8538 using namespace llvm::AMDGPU::Hwreg;
8539
8540 int64_t ImmVal = 0;
8541 SMLoc Loc = getLoc();
8542
8543 StructuredOpField HwReg("id", "hardware register", HwregId::Width,
8544 HwregId::Default);
8545 StructuredOpField Offset("offset", "bit offset", HwregOffset::Width,
8546 HwregOffset::Default);
8547 struct : StructuredOpField {
8548 using StructuredOpField::StructuredOpField;
8549 bool validate(AMDGPUAsmParser &Parser) const override {
8550 if (!isUIntN(N: Width, x: Val - 1))
8551 return Error(Parser, Err: "only values from 1 to 32 are legal");
8552 return true;
8553 }
8554 } Width("size", "bitfield width", HwregSize::Width, HwregSize::Default);
8555 ParseStatus Res = parseStructuredOpFields(Fields: {&HwReg, &Offset, &Width});
8556
8557 if (Res.isNoMatch())
8558 Res = parseHwregFunc(HwReg, Offset, Width);
8559
8560 if (Res.isSuccess()) {
8561 if (!validateStructuredOpFields(Fields: {&HwReg, &Offset, &Width}))
8562 return ParseStatus::Failure;
8563 ImmVal = HwregEncoding::encode(Values: HwReg.Val, Values: Offset.Val, Values: Width.Val);
8564 }
8565
8566 if (Res.isNoMatch() &&
8567 parseExpr(Imm&: ImmVal, Expected: "a hwreg macro, structured immediate"))
8568 Res = ParseStatus::Success;
8569
8570 if (!Res.isSuccess())
8571 return ParseStatus::Failure;
8572
8573 if (!isUInt<16>(x: ImmVal))
8574 return Error(L: Loc, Msg: "invalid immediate: only 16-bit values are legal");
8575 Operands.push_back(
8576 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: ImmVal, Loc, Type: AMDGPUOperand::ImmTyHwreg));
8577 return ParseStatus::Success;
8578}
8579
8580bool AMDGPUOperand::isHwreg() const { return isImmTy(ImmT: ImmTyHwreg); }
8581
8582//===----------------------------------------------------------------------===//
8583// sendmsg
8584//===----------------------------------------------------------------------===//
8585
8586bool AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &Msg, OperandInfoTy &Op,
8587 OperandInfoTy &Stream) {
8588 using namespace llvm::AMDGPU::SendMsg;
8589
8590 Msg.Loc = getLoc();
8591 if (isToken(Kind: AsmToken::Identifier) &&
8592 (Msg.Val = getMsgId(Name: getTokenStr(), STI: getSTI())) != OPR_ID_UNKNOWN) {
8593 Msg.IsSymbolic = true;
8594 lex(); // skip message name
8595 } else if (!parseExpr(Imm&: Msg.Val, Expected: "a message name")) {
8596 return false;
8597 }
8598
8599 if (trySkipToken(Kind: AsmToken::Comma)) {
8600 Op.IsDefined = true;
8601 Op.Loc = getLoc();
8602 if (isToken(Kind: AsmToken::Identifier) &&
8603 (Op.Val = getMsgOpId(MsgId: Msg.Val, Name: getTokenStr(), STI: getSTI())) !=
8604 OPR_ID_UNKNOWN) {
8605 lex(); // skip operation name
8606 } else if (!parseExpr(Imm&: Op.Val, Expected: "an operation name")) {
8607 return false;
8608 }
8609
8610 if (trySkipToken(Kind: AsmToken::Comma)) {
8611 Stream.IsDefined = true;
8612 Stream.Loc = getLoc();
8613 if (!parseExpr(Imm&: Stream.Val))
8614 return false;
8615 }
8616 }
8617
8618 return skipToken(Kind: AsmToken::RParen, ErrMsg: "expected a closing parenthesis");
8619}
8620
8621bool AMDGPUAsmParser::validateSendMsg(const OperandInfoTy &Msg,
8622 const OperandInfoTy &Op,
8623 const OperandInfoTy &Stream) {
8624 using namespace llvm::AMDGPU::SendMsg;
8625
8626 // Validation strictness depends on whether message is specified
8627 // in a symbolic or in a numeric form. In the latter case
8628 // only encoding possibility is checked.
8629 bool Strict = Msg.IsSymbolic;
8630
8631 if (Strict) {
8632 if (Msg.Val == OPR_ID_UNSUPPORTED) {
8633 Error(L: Msg.Loc, Msg: "specified message id is not supported on this GPU");
8634 return false;
8635 }
8636 } else {
8637 if (!isValidMsgId(MsgId: Msg.Val, STI: getSTI())) {
8638 Error(L: Msg.Loc, Msg: "invalid message id");
8639 return false;
8640 }
8641 }
8642 if (Strict && (msgRequiresOp(MsgId: Msg.Val, STI: getSTI()) != Op.IsDefined)) {
8643 if (Op.IsDefined) {
8644 Error(L: Op.Loc, Msg: "message does not support operations");
8645 } else {
8646 Error(L: Msg.Loc, Msg: "missing message operation");
8647 }
8648 return false;
8649 }
8650 if (!isValidMsgOp(MsgId: Msg.Val, OpId: Op.Val, STI: getSTI(), Strict)) {
8651 if (Op.Val == OPR_ID_UNSUPPORTED)
8652 Error(L: Op.Loc, Msg: "specified operation id is not supported on this GPU");
8653 else
8654 Error(L: Op.Loc, Msg: "invalid operation id");
8655 return false;
8656 }
8657 if (Strict && !msgSupportsStream(MsgId: Msg.Val, OpId: Op.Val, STI: getSTI()) &&
8658 Stream.IsDefined) {
8659 Error(L: Stream.Loc, Msg: "message operation does not support streams");
8660 return false;
8661 }
8662 if (!isValidMsgStream(MsgId: Msg.Val, OpId: Op.Val, StreamId: Stream.Val, STI: getSTI(), Strict)) {
8663 Error(L: Stream.Loc, Msg: "invalid message stream id");
8664 return false;
8665 }
8666 return true;
8667}
8668
8669ParseStatus AMDGPUAsmParser::parseSendMsg(OperandVector &Operands) {
8670 using namespace llvm::AMDGPU::SendMsg;
8671
8672 int64_t ImmVal = 0;
8673 SMLoc Loc = getLoc();
8674
8675 if (trySkipId(Id: "sendmsg", Kind: AsmToken::LParen)) {
8676 OperandInfoTy Msg(OPR_ID_UNKNOWN);
8677 OperandInfoTy Op(OP_NONE_);
8678 OperandInfoTy Stream(STREAM_ID_NONE_);
8679 if (parseSendMsgBody(Msg, Op, Stream) && validateSendMsg(Msg, Op, Stream)) {
8680 ImmVal = encodeMsg(MsgId: Msg.Val, OpId: Op.Val, StreamId: Stream.Val);
8681 } else {
8682 return ParseStatus::Failure;
8683 }
8684 } else if (parseExpr(Imm&: ImmVal, Expected: "a sendmsg macro")) {
8685 if (ImmVal < 0 || !isUInt<16>(x: ImmVal))
8686 return Error(L: Loc, Msg: "invalid immediate: only 16-bit values are legal");
8687 } else {
8688 return ParseStatus::Failure;
8689 }
8690
8691 Operands.push_back(
8692 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: ImmVal, Loc, Type: AMDGPUOperand::ImmTySendMsg));
8693 return ParseStatus::Success;
8694}
8695
8696bool AMDGPUOperand::isSendMsg() const { return isImmTy(ImmT: ImmTySendMsg); }
8697
8698ParseStatus AMDGPUAsmParser::parseWaitEvent(OperandVector &Operands) {
8699 using namespace llvm::AMDGPU::WaitEvent;
8700
8701 SMLoc Loc = getLoc();
8702 int64_t ImmVal = 0;
8703
8704 StructuredOpField DontWaitExportReady("dont_wait_export_ready", "bit value",
8705 1, 0);
8706 StructuredOpField ExportReady("export_ready", "bit value", 1, 0);
8707
8708 StructuredOpField *TargetBitfield =
8709 isGFX11() ? &DontWaitExportReady : &ExportReady;
8710
8711 ParseStatus Res = parseStructuredOpFields(Fields: {TargetBitfield});
8712 if (Res.isNoMatch() && parseExpr(Imm&: ImmVal, Expected: "structured immediate"))
8713 Res = ParseStatus::Success;
8714 else if (Res.isSuccess()) {
8715 if (!validateStructuredOpFields(Fields: {TargetBitfield}))
8716 return ParseStatus::Failure;
8717 ImmVal = TargetBitfield->Val;
8718 }
8719
8720 if (!Res.isSuccess())
8721 return ParseStatus::Failure;
8722
8723 if (!isUInt<16>(x: ImmVal))
8724 return Error(L: Loc, Msg: "invalid immediate: only 16-bit values are legal");
8725
8726 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: ImmVal, Loc,
8727 Type: AMDGPUOperand::ImmTyWaitEvent));
8728 return ParseStatus::Success;
8729}
8730
8731bool AMDGPUOperand::isWaitEvent() const { return isImmTy(ImmT: ImmTyWaitEvent); }
8732
8733//===----------------------------------------------------------------------===//
8734// v_interp
8735//===----------------------------------------------------------------------===//
8736
8737ParseStatus AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
8738 StringRef Str;
8739 SMLoc S = getLoc();
8740
8741 if (!parseId(Val&: Str))
8742 return ParseStatus::NoMatch;
8743
8744 int Slot = StringSwitch<int>(Str)
8745 .Case(S: "p10", Value: 0)
8746 .Case(S: "p20", Value: 1)
8747 .Case(S: "p0", Value: 2)
8748 .Default(Value: -1);
8749
8750 if (Slot == -1)
8751 return Error(L: S, Msg: "invalid interpolation slot");
8752
8753 Operands.push_back(
8754 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Slot, Loc: S, Type: AMDGPUOperand::ImmTyInterpSlot));
8755 return ParseStatus::Success;
8756}
8757
8758ParseStatus AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
8759 StringRef Str;
8760 SMLoc S = getLoc();
8761
8762 if (!parseId(Val&: Str))
8763 return ParseStatus::NoMatch;
8764
8765 if (!Str.starts_with(Prefix: "attr"))
8766 return Error(L: S, Msg: "invalid interpolation attribute");
8767
8768 StringRef Chan = Str.take_back(N: 2);
8769 int AttrChan = StringSwitch<int>(Chan)
8770 .Case(S: ".x", Value: 0)
8771 .Case(S: ".y", Value: 1)
8772 .Case(S: ".z", Value: 2)
8773 .Case(S: ".w", Value: 3)
8774 .Default(Value: -1);
8775 if (AttrChan == -1)
8776 return Error(L: S, Msg: "invalid or missing interpolation attribute channel");
8777
8778 Str = Str.drop_back(N: 2).drop_front(N: 4);
8779
8780 uint8_t Attr;
8781 if (Str.getAsInteger(Radix: 10, Result&: Attr))
8782 return Error(L: S, Msg: "invalid or missing interpolation attribute number");
8783
8784 if (Attr > 32)
8785 return Error(L: S, Msg: "out of bounds interpolation attribute number");
8786
8787 SMLoc SChan = SMLoc::getFromPointer(Ptr: Chan.data());
8788
8789 Operands.push_back(
8790 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Attr, Loc: S, Type: AMDGPUOperand::ImmTyInterpAttr));
8791 Operands.push_back(Elt: AMDGPUOperand::CreateImm(
8792 AsmParser: this, Val: AttrChan, Loc: SChan, Type: AMDGPUOperand::ImmTyInterpAttrChan));
8793 return ParseStatus::Success;
8794}
8795
8796//===----------------------------------------------------------------------===//
8797// exp
8798//===----------------------------------------------------------------------===//
8799
8800ParseStatus AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
8801 using namespace llvm::AMDGPU::Exp;
8802
8803 StringRef Str;
8804 SMLoc S = getLoc();
8805
8806 if (!parseId(Val&: Str))
8807 return ParseStatus::NoMatch;
8808
8809 unsigned Id = getTgtId(Name: Str);
8810 if (Id == ET_INVALID || !isSupportedTgtId(Id, STI: getSTI()))
8811 return Error(L: S, Msg: (Id == ET_INVALID)
8812 ? "invalid exp target"
8813 : "exp target is not supported on this GPU");
8814
8815 Operands.push_back(
8816 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Id, Loc: S, Type: AMDGPUOperand::ImmTyExpTgt));
8817 return ParseStatus::Success;
8818}
8819
8820//===----------------------------------------------------------------------===//
8821// parser helpers
8822//===----------------------------------------------------------------------===//
8823
8824bool AMDGPUAsmParser::isId(const AsmToken &Token, const StringRef Id) const {
8825 return Token.is(K: AsmToken::Identifier) && Token.getString() == Id;
8826}
8827
8828bool AMDGPUAsmParser::isId(const StringRef Id) const {
8829 return isId(Token: getToken(), Id);
8830}
8831
8832bool AMDGPUAsmParser::isToken(const AsmToken::TokenKind Kind) const {
8833 return getTokenKind() == Kind;
8834}
8835
8836StringRef AMDGPUAsmParser::getId() const {
8837 return isToken(Kind: AsmToken::Identifier) ? getTokenStr() : StringRef();
8838}
8839
8840bool AMDGPUAsmParser::trySkipId(const StringRef Id) {
8841 if (isId(Id)) {
8842 lex();
8843 return true;
8844 }
8845 return false;
8846}
8847
8848bool AMDGPUAsmParser::trySkipId(const StringRef Pref, const StringRef Id) {
8849 if (isToken(Kind: AsmToken::Identifier)) {
8850 StringRef Tok = getTokenStr();
8851 if (Tok.starts_with(Prefix: Pref) && Tok.drop_front(N: Pref.size()) == Id) {
8852 lex();
8853 return true;
8854 }
8855 }
8856 return false;
8857}
8858
8859bool AMDGPUAsmParser::trySkipId(const StringRef Id,
8860 const AsmToken::TokenKind Kind) {
8861 if (isId(Id) && peekToken().is(K: Kind)) {
8862 lex();
8863 lex();
8864 return true;
8865 }
8866 return false;
8867}
8868
8869bool AMDGPUAsmParser::trySkipToken(const AsmToken::TokenKind Kind) {
8870 if (isToken(Kind)) {
8871 lex();
8872 return true;
8873 }
8874 return false;
8875}
8876
8877bool AMDGPUAsmParser::skipToken(const AsmToken::TokenKind Kind,
8878 const StringRef ErrMsg) {
8879 if (!trySkipToken(Kind)) {
8880 Error(L: getLoc(), Msg: ErrMsg);
8881 return false;
8882 }
8883 return true;
8884}
8885
8886bool AMDGPUAsmParser::parseExpr(int64_t &Imm, StringRef Expected) {
8887 SMLoc S = getLoc();
8888
8889 const MCExpr *Expr;
8890 if (Parser.parseExpression(Res&: Expr))
8891 return false;
8892
8893 if (Expr->evaluateAsAbsolute(Res&: Imm))
8894 return true;
8895
8896 if (Expected.empty()) {
8897 Error(L: S, Msg: "expected absolute expression");
8898 } else {
8899 Error(L: S,
8900 Msg: Twine("expected ", Expected) + Twine(" or an absolute expression"));
8901 }
8902 return false;
8903}
8904
8905bool AMDGPUAsmParser::parseExpr(OperandVector &Operands) {
8906 SMLoc S = getLoc();
8907
8908 const MCExpr *Expr;
8909 if (Parser.parseExpression(Res&: Expr))
8910 return false;
8911
8912 int64_t IntVal;
8913 if (Expr->evaluateAsAbsolute(Res&: IntVal)) {
8914 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: IntVal, Loc: S));
8915 } else {
8916 Operands.push_back(Elt: AMDGPUOperand::CreateExpr(AsmParser: this, Expr, S));
8917 }
8918 return true;
8919}
8920
8921bool AMDGPUAsmParser::parseString(StringRef &Val, const StringRef ErrMsg) {
8922 if (isToken(Kind: AsmToken::String)) {
8923 Val = getToken().getStringContents();
8924 lex();
8925 return true;
8926 }
8927 Error(L: getLoc(), Msg: ErrMsg);
8928 return false;
8929}
8930
8931bool AMDGPUAsmParser::parseId(StringRef &Val, const StringRef ErrMsg) {
8932 if (isToken(Kind: AsmToken::Identifier)) {
8933 Val = getTokenStr();
8934 lex();
8935 return true;
8936 }
8937 if (!ErrMsg.empty())
8938 Error(L: getLoc(), Msg: ErrMsg);
8939 return false;
8940}
8941
8942AsmToken AMDGPUAsmParser::getToken() const { return Parser.getTok(); }
8943
8944AsmToken AMDGPUAsmParser::peekToken(bool ShouldSkipSpace) {
8945 return isToken(Kind: AsmToken::EndOfStatement)
8946 ? getToken()
8947 : getLexer().peekTok(ShouldSkipSpace);
8948}
8949
8950void AMDGPUAsmParser::peekTokens(MutableArrayRef<AsmToken> Tokens) {
8951 auto TokCount = getLexer().peekTokens(Buf: Tokens);
8952
8953 for (auto Idx = TokCount; Idx < Tokens.size(); ++Idx)
8954 Tokens[Idx] = AsmToken(AsmToken::Error, "");
8955}
8956
8957AsmToken::TokenKind AMDGPUAsmParser::getTokenKind() const {
8958 return getLexer().getKind();
8959}
8960
8961SMLoc AMDGPUAsmParser::getLoc() const { return getToken().getLoc(); }
8962
8963StringRef AMDGPUAsmParser::getTokenStr() const {
8964 return getToken().getString();
8965}
8966
8967void AMDGPUAsmParser::lex() { Parser.Lex(); }
8968
8969const AMDGPUOperand &
8970AMDGPUAsmParser::findMCOperand(const OperandVector &Operands,
8971 int MCOpIdx) const {
8972 for (const auto &Op : Operands) {
8973 const AMDGPUOperand &TargetOp = static_cast<AMDGPUOperand &>(*Op);
8974 if (TargetOp.getMCOpIdx() == MCOpIdx)
8975 return TargetOp;
8976 }
8977 llvm_unreachable("no such MC operand!");
8978}
8979
8980SMLoc AMDGPUAsmParser::getInstLoc(const OperandVector &Operands) const {
8981 return ((AMDGPUOperand &)*Operands[0]).getStartLoc();
8982}
8983
8984// Returns one of the given locations that comes later in the source.
8985SMLoc AMDGPUAsmParser::getLaterLoc(SMLoc a, SMLoc b) {
8986 return a.getPointer() < b.getPointer() ? b : a;
8987}
8988
8989SMLoc AMDGPUAsmParser::getOperandLoc(const OperandVector &Operands,
8990 int MCOpIdx) const {
8991 return findMCOperand(Operands, MCOpIdx).getStartLoc();
8992}
8993
8994SMLoc AMDGPUAsmParser::getOperandLoc(
8995 std::function<bool(const AMDGPUOperand &)> Test,
8996 const OperandVector &Operands) const {
8997 for (unsigned i = Operands.size() - 1; i > 0; --i) {
8998 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
8999 if (Test(Op))
9000 return Op.getStartLoc();
9001 }
9002 return getInstLoc(Operands);
9003}
9004
9005SMLoc AMDGPUAsmParser::getImmLoc(AMDGPUOperand::ImmTy Type,
9006 const OperandVector &Operands) const {
9007 auto Test = [=](const AMDGPUOperand &Op) { return Op.isImmTy(ImmT: Type); };
9008 return getOperandLoc(Test, Operands);
9009}
9010
9011ParseStatus
9012AMDGPUAsmParser::parseStructuredOpFields(ArrayRef<StructuredOpField *> Fields) {
9013 if (!trySkipToken(Kind: AsmToken::LCurly))
9014 return ParseStatus::NoMatch;
9015
9016 bool First = true;
9017 while (!trySkipToken(Kind: AsmToken::RCurly)) {
9018 if (!First &&
9019 !skipToken(Kind: AsmToken::Comma, ErrMsg: "comma or closing brace expected"))
9020 return ParseStatus::Failure;
9021
9022 StringRef Id = getTokenStr();
9023 SMLoc IdLoc = getLoc();
9024 if (!skipToken(Kind: AsmToken::Identifier, ErrMsg: "field name expected") ||
9025 !skipToken(Kind: AsmToken::Colon, ErrMsg: "colon expected"))
9026 return ParseStatus::Failure;
9027
9028 const auto *I =
9029 find_if(Range&: Fields, P: [Id](StructuredOpField *F) { return F->Id == Id; });
9030 if (I == Fields.end())
9031 return Error(L: IdLoc, Msg: "unknown field");
9032 if ((*I)->IsDefined)
9033 return Error(L: IdLoc, Msg: "duplicate field");
9034
9035 // TODO: Support symbolic values.
9036 (*I)->Loc = getLoc();
9037 if (!parseExpr(Imm&: (*I)->Val))
9038 return ParseStatus::Failure;
9039 (*I)->IsDefined = true;
9040
9041 First = false;
9042 }
9043 return ParseStatus::Success;
9044}
9045
9046bool AMDGPUAsmParser::validateStructuredOpFields(
9047 ArrayRef<const StructuredOpField *> Fields) {
9048 return all_of(Range&: Fields, P: [this](const StructuredOpField *F) {
9049 return F->validate(Parser&: *this);
9050 });
9051}
9052
9053//===----------------------------------------------------------------------===//
9054// swizzle
9055//===----------------------------------------------------------------------===//
9056
9057LLVM_READNONE
9058static unsigned encodeBitmaskPerm(const unsigned AndMask, const unsigned OrMask,
9059 const unsigned XorMask) {
9060 using namespace llvm::AMDGPU::Swizzle;
9061
9062 return BITMASK_PERM_ENC | (AndMask << BITMASK_AND_SHIFT) |
9063 (OrMask << BITMASK_OR_SHIFT) | (XorMask << BITMASK_XOR_SHIFT);
9064}
9065
9066bool AMDGPUAsmParser::parseSwizzleOperand(int64_t &Op, const unsigned MinVal,
9067 const unsigned MaxVal,
9068 const Twine &ErrMsg, SMLoc &Loc) {
9069 if (!skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma")) {
9070 return false;
9071 }
9072 Loc = getLoc();
9073 if (!parseExpr(Imm&: Op)) {
9074 return false;
9075 }
9076 if (Op < MinVal || Op > MaxVal) {
9077 Error(L: Loc, Msg: ErrMsg);
9078 return false;
9079 }
9080
9081 return true;
9082}
9083
9084bool AMDGPUAsmParser::parseSwizzleOperands(const unsigned OpNum, int64_t *Op,
9085 const unsigned MinVal,
9086 const unsigned MaxVal,
9087 const StringRef ErrMsg) {
9088 SMLoc Loc;
9089 for (unsigned i = 0; i < OpNum; ++i) {
9090 if (!parseSwizzleOperand(Op&: Op[i], MinVal, MaxVal, ErrMsg, Loc))
9091 return false;
9092 }
9093
9094 return true;
9095}
9096
9097bool AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &Imm) {
9098 using namespace llvm::AMDGPU::Swizzle;
9099
9100 int64_t Lane[LANE_NUM];
9101 if (parseSwizzleOperands(OpNum: LANE_NUM, Op: Lane, MinVal: 0, MaxVal: LANE_MAX,
9102 ErrMsg: "expected a 2-bit lane id")) {
9103 Imm = QUAD_PERM_ENC;
9104 for (unsigned I = 0; I < LANE_NUM; ++I) {
9105 Imm |= Lane[I] << (LANE_SHIFT * I);
9106 }
9107 return true;
9108 }
9109 return false;
9110}
9111
9112bool AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &Imm) {
9113 using namespace llvm::AMDGPU::Swizzle;
9114
9115 SMLoc Loc;
9116 int64_t GroupSize;
9117 int64_t LaneIdx;
9118
9119 if (!parseSwizzleOperand(Op&: GroupSize, MinVal: 2, MaxVal: 32,
9120 ErrMsg: "group size must be in the interval [2,32]", Loc)) {
9121 return false;
9122 }
9123 if (!isPowerOf2_64(Value: GroupSize)) {
9124 Error(L: Loc, Msg: "group size must be a power of two");
9125 return false;
9126 }
9127 if (parseSwizzleOperand(Op&: LaneIdx, MinVal: 0, MaxVal: GroupSize - 1,
9128 ErrMsg: "lane id must be in the interval [0,group size - 1]",
9129 Loc)) {
9130 Imm = encodeBitmaskPerm(AndMask: BITMASK_MAX - GroupSize + 1, OrMask: LaneIdx, XorMask: 0);
9131 return true;
9132 }
9133 return false;
9134}
9135
9136bool AMDGPUAsmParser::parseSwizzleReverse(int64_t &Imm) {
9137 using namespace llvm::AMDGPU::Swizzle;
9138
9139 SMLoc Loc;
9140 int64_t GroupSize;
9141
9142 if (!parseSwizzleOperand(Op&: GroupSize, MinVal: 2, MaxVal: 32,
9143 ErrMsg: "group size must be in the interval [2,32]", Loc)) {
9144 return false;
9145 }
9146 if (!isPowerOf2_64(Value: GroupSize)) {
9147 Error(L: Loc, Msg: "group size must be a power of two");
9148 return false;
9149 }
9150
9151 Imm = encodeBitmaskPerm(AndMask: BITMASK_MAX, OrMask: 0, XorMask: GroupSize - 1);
9152 return true;
9153}
9154
9155bool AMDGPUAsmParser::parseSwizzleSwap(int64_t &Imm) {
9156 using namespace llvm::AMDGPU::Swizzle;
9157
9158 SMLoc Loc;
9159 int64_t GroupSize;
9160
9161 if (!parseSwizzleOperand(Op&: GroupSize, MinVal: 1, MaxVal: 16,
9162 ErrMsg: "group size must be in the interval [1,16]", Loc)) {
9163 return false;
9164 }
9165 if (!isPowerOf2_64(Value: GroupSize)) {
9166 Error(L: Loc, Msg: "group size must be a power of two");
9167 return false;
9168 }
9169
9170 Imm = encodeBitmaskPerm(AndMask: BITMASK_MAX, OrMask: 0, XorMask: GroupSize);
9171 return true;
9172}
9173
9174bool AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &Imm) {
9175 using namespace llvm::AMDGPU::Swizzle;
9176
9177 if (!skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma")) {
9178 return false;
9179 }
9180
9181 StringRef Ctl;
9182 SMLoc StrLoc = getLoc();
9183 if (!parseString(Val&: Ctl)) {
9184 return false;
9185 }
9186 if (Ctl.size() != BITMASK_WIDTH) {
9187 Error(L: StrLoc, Msg: "expected a 5-character mask");
9188 return false;
9189 }
9190
9191 unsigned AndMask = 0;
9192 unsigned OrMask = 0;
9193 unsigned XorMask = 0;
9194
9195 for (size_t i = 0; i < Ctl.size(); ++i) {
9196 unsigned Mask = 1 << (BITMASK_WIDTH - 1 - i);
9197 switch (Ctl[i]) {
9198 default:
9199 Error(L: StrLoc, Msg: "invalid mask");
9200 return false;
9201 case '0':
9202 break;
9203 case '1':
9204 OrMask |= Mask;
9205 break;
9206 case 'p':
9207 AndMask |= Mask;
9208 break;
9209 case 'i':
9210 AndMask |= Mask;
9211 XorMask |= Mask;
9212 break;
9213 }
9214 }
9215
9216 Imm = encodeBitmaskPerm(AndMask, OrMask, XorMask);
9217 return true;
9218}
9219
9220bool AMDGPUAsmParser::parseSwizzleFFT(int64_t &Imm) {
9221 using namespace llvm::AMDGPU::Swizzle;
9222
9223 if (!AMDGPU::isGFX9Plus(STI: getSTI())) {
9224 Error(L: getLoc(), Msg: "FFT mode swizzle not supported on this GPU");
9225 return false;
9226 }
9227
9228 int64_t Swizzle;
9229 SMLoc Loc;
9230 if (!parseSwizzleOperand(Op&: Swizzle, MinVal: 0, MaxVal: FFT_SWIZZLE_MAX,
9231 ErrMsg: "FFT swizzle must be in the interval [0," +
9232 Twine(FFT_SWIZZLE_MAX) + Twine(']'),
9233 Loc))
9234 return false;
9235
9236 Imm = FFT_MODE_ENC | Swizzle;
9237 return true;
9238}
9239
9240bool AMDGPUAsmParser::parseSwizzleRotate(int64_t &Imm) {
9241 using namespace llvm::AMDGPU::Swizzle;
9242
9243 if (!AMDGPU::isGFX9Plus(STI: getSTI())) {
9244 Error(L: getLoc(), Msg: "Rotate mode swizzle not supported on this GPU");
9245 return false;
9246 }
9247
9248 SMLoc Loc;
9249 int64_t Direction;
9250
9251 if (!parseSwizzleOperand(Op&: Direction, MinVal: 0, MaxVal: 1,
9252 ErrMsg: "direction must be 0 (left) or 1 (right)", Loc))
9253 return false;
9254
9255 int64_t RotateSize;
9256 if (!parseSwizzleOperand(
9257 Op&: RotateSize, MinVal: 0, MaxVal: ROTATE_MAX_SIZE,
9258 ErrMsg: "number of threads to rotate must be in the interval [0," +
9259 Twine(ROTATE_MAX_SIZE) + Twine(']'),
9260 Loc))
9261 return false;
9262
9263 Imm = ROTATE_MODE_ENC | (Direction << ROTATE_DIR_SHIFT) |
9264 (RotateSize << ROTATE_SIZE_SHIFT);
9265 return true;
9266}
9267
9268bool AMDGPUAsmParser::parseSwizzleOffset(int64_t &Imm) {
9269
9270 SMLoc OffsetLoc = getLoc();
9271
9272 if (!parseExpr(Imm, Expected: "a swizzle macro")) {
9273 return false;
9274 }
9275 if (!isUInt<16>(x: Imm)) {
9276 Error(L: OffsetLoc, Msg: "expected a 16-bit offset");
9277 return false;
9278 }
9279 return true;
9280}
9281
9282bool AMDGPUAsmParser::parseSwizzleMacro(int64_t &Imm) {
9283 using namespace llvm::AMDGPU::Swizzle;
9284
9285 if (skipToken(Kind: AsmToken::LParen, ErrMsg: "expected a left parentheses")) {
9286
9287 SMLoc ModeLoc = getLoc();
9288 bool Ok = false;
9289
9290 if (trySkipId(Id: IdSymbolic[ID_QUAD_PERM])) {
9291 Ok = parseSwizzleQuadPerm(Imm);
9292 } else if (trySkipId(Id: IdSymbolic[ID_BITMASK_PERM])) {
9293 Ok = parseSwizzleBitmaskPerm(Imm);
9294 } else if (trySkipId(Id: IdSymbolic[ID_BROADCAST])) {
9295 Ok = parseSwizzleBroadcast(Imm);
9296 } else if (trySkipId(Id: IdSymbolic[ID_SWAP])) {
9297 Ok = parseSwizzleSwap(Imm);
9298 } else if (trySkipId(Id: IdSymbolic[ID_REVERSE])) {
9299 Ok = parseSwizzleReverse(Imm);
9300 } else if (trySkipId(Id: IdSymbolic[ID_FFT])) {
9301 Ok = parseSwizzleFFT(Imm);
9302 } else if (trySkipId(Id: IdSymbolic[ID_ROTATE])) {
9303 Ok = parseSwizzleRotate(Imm);
9304 } else {
9305 Error(L: ModeLoc, Msg: "expected a swizzle mode");
9306 }
9307
9308 return Ok && skipToken(Kind: AsmToken::RParen, ErrMsg: "expected a closing parentheses");
9309 }
9310
9311 return false;
9312}
9313
9314ParseStatus AMDGPUAsmParser::parseSwizzle(OperandVector &Operands) {
9315 SMLoc S = getLoc();
9316 int64_t Imm = 0;
9317
9318 if (trySkipId(Id: "offset")) {
9319
9320 bool Ok = false;
9321 if (skipToken(Kind: AsmToken::Colon, ErrMsg: "expected a colon")) {
9322 if (trySkipId(Id: "swizzle")) {
9323 Ok = parseSwizzleMacro(Imm);
9324 } else {
9325 Ok = parseSwizzleOffset(Imm);
9326 }
9327 }
9328
9329 Operands.push_back(
9330 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Imm, Loc: S, Type: AMDGPUOperand::ImmTySwizzle));
9331
9332 return Ok ? ParseStatus::Success : ParseStatus::Failure;
9333 }
9334 return ParseStatus::NoMatch;
9335}
9336
9337bool AMDGPUOperand::isSwizzle() const { return isImmTy(ImmT: ImmTySwizzle); }
9338
9339//===----------------------------------------------------------------------===//
9340// VGPR Index Mode
9341//===----------------------------------------------------------------------===//
9342
9343int64_t AMDGPUAsmParser::parseGPRIdxMacro() {
9344
9345 using namespace llvm::AMDGPU::VGPRIndexMode;
9346
9347 if (trySkipToken(Kind: AsmToken::RParen)) {
9348 return OFF;
9349 }
9350
9351 int64_t Imm = 0;
9352
9353 while (true) {
9354 unsigned Mode = 0;
9355 SMLoc S = getLoc();
9356
9357 for (unsigned ModeId = ID_MIN; ModeId <= ID_MAX; ++ModeId) {
9358 if (trySkipId(Id: IdSymbolic[ModeId])) {
9359 Mode = 1 << ModeId;
9360 break;
9361 }
9362 }
9363
9364 if (Mode == 0) {
9365 Error(L: S, Msg: (Imm == 0)
9366 ? "expected a VGPR index mode or a closing parenthesis"
9367 : "expected a VGPR index mode");
9368 return UNDEF;
9369 }
9370
9371 if (Imm & Mode) {
9372 Error(L: S, Msg: "duplicate VGPR index mode");
9373 return UNDEF;
9374 }
9375 Imm |= Mode;
9376
9377 if (trySkipToken(Kind: AsmToken::RParen))
9378 break;
9379 if (!skipToken(Kind: AsmToken::Comma,
9380 ErrMsg: "expected a comma or a closing parenthesis"))
9381 return UNDEF;
9382 }
9383
9384 return Imm;
9385}
9386
9387ParseStatus AMDGPUAsmParser::parseGPRIdxMode(OperandVector &Operands) {
9388
9389 using namespace llvm::AMDGPU::VGPRIndexMode;
9390
9391 int64_t Imm = 0;
9392 SMLoc S = getLoc();
9393
9394 if (trySkipId(Id: "gpr_idx", Kind: AsmToken::LParen)) {
9395 Imm = parseGPRIdxMacro();
9396 if (Imm == UNDEF)
9397 return ParseStatus::Failure;
9398 } else {
9399 if (getParser().parseAbsoluteExpression(Res&: Imm))
9400 return ParseStatus::Failure;
9401 if (Imm < 0 || !isUInt<4>(x: Imm))
9402 return Error(L: S, Msg: "invalid immediate: only 4-bit values are legal");
9403 }
9404
9405 Operands.push_back(
9406 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Imm, Loc: S, Type: AMDGPUOperand::ImmTyGprIdxMode));
9407 return ParseStatus::Success;
9408}
9409
9410bool AMDGPUOperand::isGPRIdxMode() const { return isImmTy(ImmT: ImmTyGprIdxMode); }
9411
9412//===----------------------------------------------------------------------===//
9413// sopp branch targets
9414//===----------------------------------------------------------------------===//
9415
9416ParseStatus AMDGPUAsmParser::parseSOPPBrTarget(OperandVector &Operands) {
9417
9418 // Make sure we are not parsing something
9419 // that looks like a label or an expression but is not.
9420 // This will improve error messages.
9421 if (isRegister() || isModifier())
9422 return ParseStatus::NoMatch;
9423
9424 if (!parseExpr(Operands))
9425 return ParseStatus::Failure;
9426
9427 AMDGPUOperand &Opr = ((AMDGPUOperand &)*Operands[Operands.size() - 1]);
9428 assert(Opr.isImm() || Opr.isExpr());
9429 SMLoc Loc = Opr.getStartLoc();
9430
9431 // Currently we do not support arbitrary expressions as branch targets.
9432 // Only labels and absolute expressions are accepted.
9433 if (Opr.isExpr() && !Opr.isSymbolRefExpr()) {
9434 Error(L: Loc, Msg: "expected an absolute expression or a label");
9435 } else if (Opr.isImm() && !Opr.isS16Imm()) {
9436 Error(L: Loc, Msg: "expected a 16-bit signed jump offset");
9437 }
9438
9439 return ParseStatus::Success;
9440}
9441
9442//===----------------------------------------------------------------------===//
9443// Boolean holding registers
9444//===----------------------------------------------------------------------===//
9445
9446ParseStatus AMDGPUAsmParser::parseBoolReg(OperandVector &Operands) {
9447 return parseReg(Operands);
9448}
9449
9450//===----------------------------------------------------------------------===//
9451// mubuf
9452//===----------------------------------------------------------------------===//
9453
9454void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
9455 bool IsAtomic) {
9456 OptionalImmIndexMap OptionalIdx;
9457 unsigned FirstOperandIdx = 1;
9458 bool IsAtomicReturn = false;
9459
9460 if (IsAtomic) {
9461 IsAtomicReturn = SIInstrFlags::isAtomicRet(O: MII, O: Inst);
9462 }
9463
9464 for (unsigned i = FirstOperandIdx, e = Operands.size(); i != e; ++i) {
9465 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
9466
9467 // Add the register arguments
9468 if (Op.isReg()) {
9469 Op.addRegOperands(Inst, N: 1);
9470 // Insert a tied src for atomic return dst.
9471 // This cannot be postponed as subsequent calls to
9472 // addImmOperands rely on correct number of MC operands.
9473 if (IsAtomicReturn && i == FirstOperandIdx)
9474 Op.addRegOperands(Inst, N: 1);
9475 continue;
9476 }
9477
9478 // Handle the case where soffset is an immediate
9479 if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
9480 Op.addImmOperands(Inst, N: 1);
9481 continue;
9482 }
9483
9484 // Handle tokens like 'offen' which are sometimes hard-coded into the
9485 // asm string. There are no MCInst operands for these.
9486 if (Op.isToken()) {
9487 continue;
9488 }
9489 assert(Op.isImm());
9490
9491 // Handle optional arguments
9492 OptionalIdx[Op.getImmTy()] = i;
9493 }
9494
9495 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9496 ImmT: AMDGPUOperand::ImmTyOffset);
9497 addOptionalImmOperand(Inst, Operands, OptionalIdx, ImmT: AMDGPUOperand::ImmTyCPol,
9498 Default: 0);
9499 // Parse a dummy operand as a placeholder for the SWZ operand. This enforces
9500 // agreement between MCInstrDesc.getNumOperands and MCInst.getNumOperands.
9501 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
9502 // The LDS variants carry a trailing IsAsync operand. Parse a dummy the same
9503 // way as the SWZ operand.
9504 if (AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::IsAsync))
9505 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
9506}
9507
9508//===----------------------------------------------------------------------===//
9509// smrd
9510//===----------------------------------------------------------------------===//
9511
9512bool AMDGPUOperand::isSMRDOffset8() const {
9513 return isImmLiteral() && isUInt<8>(x: getImm());
9514}
9515
9516bool AMDGPUOperand::isSMEMOffset() const {
9517 // Offset range is checked later by validator.
9518 return isImmLiteral();
9519}
9520
9521bool AMDGPUOperand::isSMRDLiteralOffset() const {
9522 // 32-bit literals are only supported on CI and we only want to use them
9523 // when the offset is > 8-bits.
9524 return isImmLiteral() && !isUInt<8>(x: getImm()) && isUInt<32>(x: getImm());
9525}
9526
9527//===----------------------------------------------------------------------===//
9528// vop3
9529//===----------------------------------------------------------------------===//
9530
9531static bool ConvertOmodMul(int64_t &Mul) {
9532 if (Mul != 1 && Mul != 2 && Mul != 4)
9533 return false;
9534
9535 Mul >>= 1;
9536 return true;
9537}
9538
9539static bool ConvertOmodDiv(int64_t &Div) {
9540 if (Div == 1) {
9541 Div = 0;
9542 return true;
9543 }
9544
9545 if (Div == 2) {
9546 Div = 3;
9547 return true;
9548 }
9549
9550 return false;
9551}
9552
9553// For pre-gfx11 targets, both bound_ctrl:0 and bound_ctrl:1 are encoded as 1.
9554// This is intentional and ensures compatibility with sp3.
9555// See bug 35397 for details.
9556bool AMDGPUAsmParser::convertDppBoundCtrl(int64_t &BoundCtrl) {
9557 if (BoundCtrl == 0 || BoundCtrl == 1) {
9558 if (!isGFX11Plus())
9559 BoundCtrl = 1;
9560 return true;
9561 }
9562 return false;
9563}
9564
9565void AMDGPUAsmParser::onBeginOfFile() {
9566 if (!getParser().getStreamer().getTargetStreamer())
9567 return;
9568
9569 if (!getTargetStreamer().getTargetID())
9570 getTargetStreamer().initializeTargetID(STI: getSTI(),
9571 /*ApplyFeatureString=*/true);
9572}
9573
9574void AMDGPUAsmParser::emitTargetDirective() {
9575 if (TargetDirectiveEmitted)
9576 return;
9577 TargetDirectiveEmitted = true;
9578
9579 if (!getParser().getStreamer().getTargetStreamer() ||
9580 getSTI().getTargetTriple().getArch() == Triple::r600)
9581 return;
9582
9583 if (isHsaAbi(STI: getSTI()))
9584 getTargetStreamer().EmitDirectiveAMDGCNTarget();
9585}
9586
9587/// Parse AMDGPU specific expressions.
9588///
9589/// expr ::= or(expr, ...) |
9590/// max(expr, ...) |
9591/// min(expr, ...)
9592///
9593bool AMDGPUAsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) {
9594 using AGVK = AMDGPUMCExpr::VariantKind;
9595
9596 if (isToken(Kind: AsmToken::Identifier)) {
9597 StringRef TokenId = getTokenStr();
9598 AGVK VK = StringSwitch<AGVK>(TokenId)
9599 .Case(S: "max", Value: AGVK::AGVK_Max)
9600 .Case(S: "min", Value: AGVK::AGVK_Min)
9601 .Case(S: "or", Value: AGVK::AGVK_Or)
9602 .Case(S: "extrasgprs", Value: AGVK::AGVK_ExtraSGPRs)
9603 .Case(S: "totalnumvgprs", Value: AGVK::AGVK_TotalNumVGPRs)
9604 .Case(S: "alignto", Value: AGVK::AGVK_AlignTo)
9605 .Case(S: "occupancy", Value: AGVK::AGVK_Occupancy)
9606 .Case(S: "instprefsize", Value: AGVK::AGVK_InstPrefSize)
9607 .Default(Value: AGVK::AGVK_None);
9608
9609 if (VK != AGVK::AGVK_None && peekToken().is(K: AsmToken::LParen)) {
9610 SmallVector<const MCExpr *, 4> Exprs;
9611 uint64_t CommaCount = 0;
9612 lex(); // Eat Arg ('or', 'max', 'occupancy', etc.)
9613 lex(); // Eat '('
9614 while (true) {
9615 if (trySkipToken(Kind: AsmToken::RParen)) {
9616 if (Exprs.empty()) {
9617 Error(L: getToken().getLoc(),
9618 Msg: "empty " + Twine(TokenId) + " expression");
9619 return true;
9620 }
9621 if (CommaCount + 1 != Exprs.size()) {
9622 Error(L: getToken().getLoc(),
9623 Msg: "mismatch of commas in " + Twine(TokenId) + " expression");
9624 return true;
9625 }
9626 if (unsigned Expected = AMDGPUMCExpr::getNumExpectedArgs(Kind: VK);
9627 Expected && Exprs.size() != Expected) {
9628 Error(L: getToken().getLoc(), Msg: Twine(TokenId) + " expression expects " +
9629 Twine(Expected) + " operands");
9630 return true;
9631 }
9632 Res = AMDGPUMCExpr::create(Kind: VK, Args: Exprs, Ctx&: getContext());
9633 return false;
9634 }
9635 const MCExpr *Expr;
9636 if (getParser().parseExpression(Res&: Expr, EndLoc))
9637 return true;
9638 Exprs.push_back(Elt: Expr);
9639 bool LastTokenWasComma = trySkipToken(Kind: AsmToken::Comma);
9640 if (LastTokenWasComma)
9641 CommaCount++;
9642 if (!LastTokenWasComma && !isToken(Kind: AsmToken::RParen)) {
9643 Error(L: getToken().getLoc(),
9644 Msg: "unexpected token in " + Twine(TokenId) + " expression");
9645 return true;
9646 }
9647 }
9648 }
9649 }
9650 return getParser().parsePrimaryExpr(Res, EndLoc, TypeInfo: nullptr);
9651}
9652
9653ParseStatus AMDGPUAsmParser::parseOModSI(OperandVector &Operands) {
9654 StringRef Name = getTokenStr();
9655 if (Name == "mul") {
9656 return parseIntWithPrefix(Prefix: "mul", Operands, ImmTy: AMDGPUOperand::ImmTyOModSI,
9657 ConvertResult: ConvertOmodMul);
9658 }
9659
9660 if (Name == "div") {
9661 return parseIntWithPrefix(Prefix: "div", Operands, ImmTy: AMDGPUOperand::ImmTyOModSI,
9662 ConvertResult: ConvertOmodDiv);
9663 }
9664
9665 return ParseStatus::NoMatch;
9666}
9667
9668// Determines which bit DST_OP_SEL occupies in the op_sel operand according to
9669// the number of src operands present, then copies that bit into src0_modifiers.
9670static void cvtVOP3DstOpSelOnly(MCInst &Inst, const MCRegisterInfo &MRI) {
9671 int Opc = Inst.getOpcode();
9672 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel);
9673 if (OpSelIdx == -1)
9674 return;
9675
9676 int SrcNum;
9677 const AMDGPU::OpName Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9678 AMDGPU::OpName::src2};
9679 for (SrcNum = 0; SrcNum < 3 && AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: Ops[SrcNum]);
9680 ++SrcNum)
9681 ;
9682 assert(SrcNum > 0);
9683
9684 unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm();
9685
9686 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
9687 if (DstIdx == -1)
9688 return;
9689
9690 const MCOperand &DstOp = Inst.getOperand(i: DstIdx);
9691 int ModIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0_modifiers);
9692 uint32_t ModVal = Inst.getOperand(i: ModIdx).getImm();
9693 if (DstOp.isReg() &&
9694 MRI.getRegClass(i: AMDGPU::VGPR_16RegClassID).contains(Reg: DstOp.getReg())) {
9695 if (AMDGPU::isHi16Reg(Reg: DstOp.getReg(), MRI))
9696 ModVal |= SISrcMods::DST_OP_SEL;
9697 } else {
9698 if ((OpSel & (1 << SrcNum)) != 0)
9699 ModVal |= SISrcMods::DST_OP_SEL;
9700 }
9701 Inst.getOperand(i: ModIdx).setImm(ModVal);
9702}
9703
9704void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst,
9705 const OperandVector &Operands) {
9706 cvtVOP3P(Inst, Operands);
9707 cvtVOP3DstOpSelOnly(Inst, MRI: *getMRI());
9708}
9709
9710void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands,
9711 OptionalImmIndexMap &OptionalIdx) {
9712 cvtVOP3P(Inst, Operands, OptionalIdx);
9713 cvtVOP3DstOpSelOnly(Inst, MRI: *getMRI());
9714}
9715
9716static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
9717 return
9718 // 1. This operand is input modifiers
9719 Desc.operands()[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
9720 // 2. This is not last operand
9721 && Desc.NumOperands > (OpNum + 1)
9722 // 3. Next operand is register class
9723 && Desc.operands()[OpNum + 1].RegClass != -1
9724 // 4. Next register is not tied to any other operand
9725 && Desc.getOperandConstraint(OpNum: OpNum + 1,
9726 Constraint: MCOI::OperandConstraint::TIED_TO) == -1;
9727}
9728
9729void AMDGPUAsmParser::cvtOpSelHelper(MCInst &Inst, unsigned OpSel) {
9730 unsigned Opc = Inst.getOpcode();
9731 constexpr AMDGPU::OpName Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9732 AMDGPU::OpName::src2};
9733 constexpr AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9734 AMDGPU::OpName::src1_modifiers,
9735 AMDGPU::OpName::src2_modifiers};
9736 for (int J = 0; J < 3; ++J) {
9737 int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: Ops[J]);
9738 if (OpIdx == -1)
9739 // Some instructions, e.g. v_interp_p2_f16 in GFX9, have src0, src2, but
9740 // no src1. So continue instead of break.
9741 continue;
9742
9743 int ModIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: ModOps[J]);
9744 uint32_t ModVal = Inst.getOperand(i: ModIdx).getImm();
9745
9746 if ((OpSel & (1 << J)) != 0)
9747 ModVal |= SISrcMods::OP_SEL_0;
9748 // op_sel[3] is encoded in src0_modifiers.
9749 if (ModOps[J] == AMDGPU::OpName::src0_modifiers && (OpSel & (1 << 3)) != 0)
9750 ModVal |= SISrcMods::DST_OP_SEL;
9751
9752 Inst.getOperand(i: ModIdx).setImm(ModVal);
9753 }
9754}
9755
9756void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst,
9757 const OperandVector &Operands) {
9758 OptionalImmIndexMap OptionalIdx;
9759 unsigned Opc = Inst.getOpcode();
9760
9761 unsigned I = 1;
9762 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
9763 for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
9764 ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, N: 1);
9765 }
9766
9767 for (unsigned E = Operands.size(); I != E; ++I) {
9768 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
9769 if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) {
9770 Op.addRegOrImmWithFPInputModsOperands(Inst, N: 2);
9771 } else if (Op.isInterpSlot() || Op.isInterpAttr() ||
9772 Op.isInterpAttrChan()) {
9773 Inst.addOperand(Op: MCOperand::createImm(Val: Op.getImm()));
9774 } else if (Op.isImmModifier()) {
9775 OptionalIdx[Op.getImmTy()] = I;
9776 } else {
9777 llvm_unreachable("unhandled operand type");
9778 }
9779 }
9780
9781 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::high))
9782 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9783 ImmT: AMDGPUOperand::ImmTyHigh);
9784
9785 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::clamp))
9786 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9787 ImmT: AMDGPUOperand::ImmTyClamp);
9788
9789 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::omod))
9790 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9791 ImmT: AMDGPUOperand::ImmTyOModSI);
9792
9793 // Some v_interp instructions use op_sel[3] for dst.
9794 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::op_sel)) {
9795 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9796 ImmT: AMDGPUOperand::ImmTyOpSel);
9797 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel);
9798 unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm();
9799
9800 cvtOpSelHelper(Inst, OpSel);
9801 }
9802}
9803
9804void AMDGPUAsmParser::cvtVINTERP(MCInst &Inst, const OperandVector &Operands) {
9805 OptionalImmIndexMap OptionalIdx;
9806 unsigned Opc = Inst.getOpcode();
9807
9808 unsigned I = 1;
9809 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
9810 for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
9811 ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, N: 1);
9812 }
9813
9814 for (unsigned E = Operands.size(); I != E; ++I) {
9815 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
9816 if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) {
9817 Op.addRegOrImmWithFPInputModsOperands(Inst, N: 2);
9818 } else if (Op.isImmModifier()) {
9819 OptionalIdx[Op.getImmTy()] = I;
9820 } else {
9821 llvm_unreachable("unhandled operand type");
9822 }
9823 }
9824
9825 addOptionalImmOperand(Inst, Operands, OptionalIdx, ImmT: AMDGPUOperand::ImmTyClamp);
9826
9827 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel);
9828 if (OpSelIdx != -1)
9829 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9830 ImmT: AMDGPUOperand::ImmTyOpSel);
9831
9832 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9833 ImmT: AMDGPUOperand::ImmTyWaitEXP);
9834
9835 if (OpSelIdx == -1)
9836 return;
9837
9838 unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm();
9839 cvtOpSelHelper(Inst, OpSel);
9840}
9841
9842void AMDGPUAsmParser::cvtScaledMFMA(MCInst &Inst,
9843 const OperandVector &Operands) {
9844 OptionalImmIndexMap OptionalIdx;
9845 unsigned Opc = Inst.getOpcode();
9846 unsigned I = 1;
9847 int CbszOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::cbsz);
9848
9849 const MCInstrDesc &Desc = MII.get(Opcode: Opc);
9850
9851 for (unsigned J = 0; J < Desc.getNumDefs(); ++J)
9852 static_cast<AMDGPUOperand &>(*Operands[I++]).addRegOperands(Inst, N: 1);
9853
9854 for (unsigned E = Operands.size(); I != E; ++I) {
9855 AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands[I]);
9856 int NumOperands = Inst.getNumOperands();
9857 // The order of operands in MCInst and parsed operands are different.
9858 // Adding dummy cbsz and blgp operands at corresponding MCInst operand
9859 // indices for parsing scale values correctly.
9860 if (NumOperands == CbszOpIdx) {
9861 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
9862 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
9863 }
9864 if (isRegOrImmWithInputMods(Desc, OpNum: NumOperands)) {
9865 Op.addRegOrImmWithFPInputModsOperands(Inst, N: 2);
9866 } else if (Op.isImmModifier()) {
9867 OptionalIdx[Op.getImmTy()] = I;
9868 } else {
9869 Op.addRegOrImmOperands(Inst, N: 1);
9870 }
9871 }
9872
9873 // Insert CBSZ and BLGP operands for F8F6F4 variants
9874 auto CbszIdx = OptionalIdx.find(x: AMDGPUOperand::ImmTyCBSZ);
9875 if (CbszIdx != OptionalIdx.end()) {
9876 int CbszVal = ((AMDGPUOperand &)*Operands[CbszIdx->second]).getImm();
9877 Inst.getOperand(i: CbszOpIdx).setImm(CbszVal);
9878 }
9879
9880 int BlgpOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::blgp);
9881 auto BlgpIdx = OptionalIdx.find(x: AMDGPUOperand::ImmTyBLGP);
9882 if (BlgpIdx != OptionalIdx.end()) {
9883 int BlgpVal = ((AMDGPUOperand &)*Operands[BlgpIdx->second]).getImm();
9884 Inst.getOperand(i: BlgpOpIdx).setImm(BlgpVal);
9885 }
9886
9887 // Add dummy src_modifiers
9888 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
9889 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
9890
9891 // Handle op_sel fields
9892
9893 unsigned OpSel = 0;
9894 auto OpselIdx = OptionalIdx.find(x: AMDGPUOperand::ImmTyOpSel);
9895 if (OpselIdx != OptionalIdx.end()) {
9896 OpSel = static_cast<const AMDGPUOperand &>(*Operands[OpselIdx->second])
9897 .getImm();
9898 }
9899
9900 unsigned OpSelHi = 0;
9901 auto OpselHiIdx = OptionalIdx.find(x: AMDGPUOperand::ImmTyOpSelHi);
9902 if (OpselHiIdx != OptionalIdx.end()) {
9903 OpSelHi = static_cast<const AMDGPUOperand &>(*Operands[OpselHiIdx->second])
9904 .getImm();
9905 }
9906 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9907 AMDGPU::OpName::src1_modifiers};
9908
9909 for (unsigned J = 0; J < 2; ++J) {
9910 unsigned ModVal = 0;
9911 if (OpSel & (1 << J))
9912 ModVal |= SISrcMods::OP_SEL_0;
9913 if (OpSelHi & (1 << J))
9914 ModVal |= SISrcMods::OP_SEL_1;
9915
9916 const int ModIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: ModOps[J]);
9917 Inst.getOperand(i: ModIdx).setImm(ModVal);
9918 }
9919}
9920
9921void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands,
9922 OptionalImmIndexMap &OptionalIdx) {
9923 unsigned Opc = Inst.getOpcode();
9924
9925 unsigned I = 1;
9926 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
9927 for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
9928 ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, N: 1);
9929 }
9930
9931 for (unsigned E = Operands.size(); I != E; ++I) {
9932 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
9933 if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) {
9934 Op.addRegOrImmWithFPInputModsOperands(Inst, N: 2);
9935 } else if (Op.isImmModifier()) {
9936 OptionalIdx[Op.getImmTy()] = I;
9937 } else {
9938 Op.addRegOrImmOperands(Inst, N: 1);
9939 }
9940 }
9941
9942 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::scale_sel))
9943 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9944 ImmT: AMDGPUOperand::ImmTyScaleSel);
9945
9946 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::clamp))
9947 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9948 ImmT: AMDGPUOperand::ImmTyClamp);
9949
9950 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::byte_sel)) {
9951 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::vdst_in))
9952 Inst.addOperand(Op: Inst.getOperand(i: 0));
9953 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9954 ImmT: AMDGPUOperand::ImmTyByteSel);
9955 }
9956
9957 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::omod))
9958 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9959 ImmT: AMDGPUOperand::ImmTyOModSI);
9960
9961 // Special case v_mac_{f16, f32} and v_fmac_{f16, f32} (gfx906/gfx10+):
9962 // it has src2 register operand that is tied to dst operand
9963 // we don't allow modifiers for this operand in assembler so src2_modifiers
9964 // should be 0.
9965 if (isMAC(Opc)) {
9966 auto *it = Inst.begin();
9967 std::advance(
9968 i&: it, n: AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2_modifiers));
9969 it = Inst.insert(I: it, Op: MCOperand::createImm(Val: 0)); // no modifiers for src2
9970 ++it;
9971 // Copy the operand to ensure it's not invalidated when Inst grows.
9972 Inst.insert(I: it, Op: MCOperand(Inst.getOperand(i: 0))); // src2 = dst
9973 }
9974}
9975
9976void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
9977 OptionalImmIndexMap OptionalIdx;
9978 cvtVOP3(Inst, Operands, OptionalIdx);
9979}
9980
9981void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands,
9982 OptionalImmIndexMap &OptIdx) {
9983 const int Opc = Inst.getOpcode();
9984
9985 const bool IsPacked = SIInstrFlags::isPacked(O: MII, O: Inst);
9986
9987 if (Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_F16_vi ||
9988 Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_BF16_vi ||
9989 Opc == AMDGPU::V_CVT_SR_BF8_F32_vi ||
9990 Opc == AMDGPU::V_CVT_SR_FP8_F32_vi ||
9991 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx11 ||
9992 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx11 ||
9993 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx12 ||
9994 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx12 ||
9995 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx13 ||
9996 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx13) {
9997 Inst.addOperand(Op: MCOperand::createImm(Val: 0)); // Placeholder for src2_mods
9998 Inst.addOperand(Op: Inst.getOperand(i: 0));
9999 }
10000
10001 // Append vdst_in only if a previous converter (cvtVOP3DPP for DPP variants,
10002 // cvtVOP3 for byte_sel variants) hasn't already placed it. Use the position
10003 // of the named operand to detect that, the same way cvtVOP3DPP does
10004 // internally.
10005 int VdstInIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst_in);
10006 if (VdstInIdx != -1 && VdstInIdx == static_cast<int>(Inst.getNumOperands()))
10007 Inst.addOperand(Op: Inst.getOperand(i: 0));
10008
10009 int BitOp3Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::bitop3);
10010 if (BitOp3Idx != -1) {
10011 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyBitOp3);
10012 }
10013
10014 // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3
10015 // instruction, and then figure out where to actually put the modifiers
10016
10017 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel);
10018 if (OpSelIdx != -1) {
10019 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyOpSel);
10020 }
10021
10022 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel_hi);
10023 if (OpSelHiIdx != -1) {
10024 int DefaultVal = IsPacked ? -1 : 0;
10025 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyOpSelHi,
10026 Default: DefaultVal);
10027 }
10028
10029 int MatrixAFMTIdx =
10030 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_a_fmt);
10031 if (MatrixAFMTIdx != -1) {
10032 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
10033 ImmT: AMDGPUOperand::ImmTyMatrixAFMT, Default: 0);
10034 }
10035
10036 int MatrixBFMTIdx =
10037 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_b_fmt);
10038 if (MatrixBFMTIdx != -1) {
10039 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
10040 ImmT: AMDGPUOperand::ImmTyMatrixBFMT, Default: 0);
10041 }
10042
10043 int MatrixAScaleIdx =
10044 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_a_scale);
10045 if (MatrixAScaleIdx != -1) {
10046 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
10047 ImmT: AMDGPUOperand::ImmTyMatrixAScale, Default: 0);
10048 }
10049
10050 int MatrixBScaleIdx =
10051 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_b_scale);
10052 if (MatrixBScaleIdx != -1) {
10053 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
10054 ImmT: AMDGPUOperand::ImmTyMatrixBScale, Default: 0);
10055 }
10056
10057 int MatrixAScaleFmtIdx =
10058 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_a_scale_fmt);
10059 if (MatrixAScaleFmtIdx != -1) {
10060 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
10061 ImmT: AMDGPUOperand::ImmTyMatrixAScaleFmt, Default: 0);
10062 }
10063
10064 int MatrixBScaleFmtIdx =
10065 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_b_scale_fmt);
10066 if (MatrixBScaleFmtIdx != -1) {
10067 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
10068 ImmT: AMDGPUOperand::ImmTyMatrixBScaleFmt, Default: 0);
10069 }
10070
10071 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::matrix_a_reuse))
10072 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
10073 ImmT: AMDGPUOperand::ImmTyMatrixAReuse, Default: 0);
10074
10075 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::matrix_b_reuse))
10076 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
10077 ImmT: AMDGPUOperand::ImmTyMatrixBReuse, Default: 0);
10078
10079 int NegLoIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::neg_lo);
10080 if (NegLoIdx != -1)
10081 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyNegLo);
10082
10083 int NegHiIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::neg_hi);
10084 if (NegHiIdx != -1)
10085 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyNegHi);
10086
10087 const AMDGPU::OpName Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
10088 AMDGPU::OpName::src2};
10089 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
10090 AMDGPU::OpName::src1_modifiers,
10091 AMDGPU::OpName::src2_modifiers};
10092
10093 unsigned OpSel = 0;
10094 unsigned OpSelHi = 0;
10095 unsigned NegLo = 0;
10096 unsigned NegHi = 0;
10097
10098 if (OpSelIdx != -1)
10099 OpSel = Inst.getOperand(i: OpSelIdx).getImm();
10100
10101 if (OpSelHiIdx != -1)
10102 OpSelHi = Inst.getOperand(i: OpSelHiIdx).getImm();
10103
10104 if (NegLoIdx != -1)
10105 NegLo = Inst.getOperand(i: NegLoIdx).getImm();
10106
10107 if (NegHiIdx != -1)
10108 NegHi = Inst.getOperand(i: NegHiIdx).getImm();
10109
10110 for (int J = 0; J < 3; ++J) {
10111 int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: Ops[J]);
10112 if (OpIdx == -1)
10113 break;
10114
10115 int ModIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: ModOps[J]);
10116
10117 if (ModIdx == -1)
10118 continue;
10119
10120 // For MAC instructions, src2 is tied to vdst and its op_sel bit
10121 // is not encoded.
10122 if (AMDGPU::isMAC(Opc) && ModOps[J] == AMDGPU::OpName::src2_modifiers)
10123 continue;
10124
10125 uint32_t ModVal = 0;
10126
10127 const MCOperand &SrcOp = Inst.getOperand(i: OpIdx);
10128 if (SrcOp.isReg() && getMRI()
10129 ->getRegClass(i: AMDGPU::VGPR_16RegClassID)
10130 .contains(Reg: SrcOp.getReg())) {
10131 bool VGPRSuffixIsHi = AMDGPU::isHi16Reg(Reg: SrcOp.getReg(), MRI: *getMRI());
10132 if (VGPRSuffixIsHi)
10133 ModVal |= SISrcMods::OP_SEL_0;
10134 } else {
10135 if ((OpSel & (1 << J)) != 0)
10136 ModVal |= SISrcMods::OP_SEL_0;
10137 }
10138
10139 if ((OpSelHi & (1 << J)) != 0)
10140 ModVal |= SISrcMods::OP_SEL_1;
10141
10142 if ((NegLo & (1 << J)) != 0)
10143 ModVal |= SISrcMods::NEG;
10144
10145 if ((NegHi & (1 << J)) != 0)
10146 ModVal |= SISrcMods::NEG_HI;
10147
10148 Inst.getOperand(i: ModIdx).setImm(Inst.getOperand(i: ModIdx).getImm() | ModVal);
10149 }
10150}
10151
10152void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands) {
10153 OptionalImmIndexMap OptIdx;
10154 cvtVOP3(Inst, Operands, OptionalIdx&: OptIdx);
10155 cvtVOP3P(Inst, Operands, OptIdx);
10156}
10157
10158static void addSrcModifiersAndSrc(MCInst &Inst, const OperandVector &Operands,
10159 unsigned i, unsigned Opc,
10160 AMDGPU::OpName OpName) {
10161 if (AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: OpName) != -1)
10162 ((AMDGPUOperand &)*Operands[i]).addRegOrImmWithFPInputModsOperands(Inst, N: 2);
10163 else
10164 ((AMDGPUOperand &)*Operands[i]).addRegOperands(Inst, N: 1);
10165}
10166
10167void AMDGPUAsmParser::cvtSWMMAC(MCInst &Inst, const OperandVector &Operands) {
10168 unsigned Opc = Inst.getOpcode();
10169
10170 ((AMDGPUOperand &)*Operands[1]).addRegOperands(Inst, N: 1);
10171 addSrcModifiersAndSrc(Inst, Operands, i: 2, Opc, OpName: AMDGPU::OpName::src0_modifiers);
10172 addSrcModifiersAndSrc(Inst, Operands, i: 3, Opc, OpName: AMDGPU::OpName::src1_modifiers);
10173 ((AMDGPUOperand &)*Operands[1]).addRegOperands(Inst, N: 1); // srcTiedDef
10174 ((AMDGPUOperand &)*Operands[4]).addRegOperands(Inst, N: 1); // src2
10175
10176 OptionalImmIndexMap OptIdx;
10177 for (unsigned i = 5; i < Operands.size(); ++i) {
10178 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
10179 OptIdx[Op.getImmTy()] = i;
10180 }
10181
10182 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::index_key_8bit))
10183 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
10184 ImmT: AMDGPUOperand::ImmTyIndexKey8bit);
10185
10186 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::index_key_16bit))
10187 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
10188 ImmT: AMDGPUOperand::ImmTyIndexKey16bit);
10189
10190 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::index_key_32bit))
10191 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
10192 ImmT: AMDGPUOperand::ImmTyIndexKey32bit);
10193
10194 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::clamp))
10195 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyClamp);
10196
10197 cvtVOP3P(Inst, Operands, OptIdx);
10198}
10199
10200//===----------------------------------------------------------------------===//
10201// VOPD
10202//===----------------------------------------------------------------------===//
10203
10204ParseStatus AMDGPUAsmParser::parseVOPD(OperandVector &Operands) {
10205 if (!hasVOPD(STI: getSTI()))
10206 return ParseStatus::NoMatch;
10207
10208 if (isToken(Kind: AsmToken::Colon) && peekToken(ShouldSkipSpace: false).is(K: AsmToken::Colon)) {
10209 SMLoc S = getLoc();
10210 lex();
10211 lex();
10212 Operands.push_back(Elt: AMDGPUOperand::CreateToken(AsmParser: this, Str: "::", Loc: S));
10213 SMLoc OpYLoc = getLoc();
10214 StringRef OpYName;
10215 if (isToken(Kind: AsmToken::Identifier) && !Parser.parseIdentifier(Res&: OpYName)) {
10216 Operands.push_back(Elt: AMDGPUOperand::CreateToken(AsmParser: this, Str: OpYName, Loc: OpYLoc));
10217 return ParseStatus::Success;
10218 }
10219 return Error(L: OpYLoc, Msg: "expected a VOPDY instruction after ::");
10220 }
10221 return ParseStatus::NoMatch;
10222}
10223
10224// Create VOPD MCInst operands using parsed assembler operands.
10225void AMDGPUAsmParser::cvtVOPD(MCInst &Inst, const OperandVector &Operands) {
10226 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
10227
10228 auto addOp = [&](uint16_t ParsedOprIdx) { // NOLINT:function pointer
10229 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[ParsedOprIdx]);
10230 if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) {
10231 Op.addRegOrImmWithFPInputModsOperands(Inst, N: 2);
10232 return;
10233 }
10234 if (Op.isReg()) {
10235 Op.addRegOperands(Inst, N: 1);
10236 return;
10237 }
10238 if (Op.isImm()) {
10239 Op.addImmOperands(Inst, N: 1);
10240 return;
10241 }
10242 llvm_unreachable("Unhandled operand type in cvtVOPD");
10243 };
10244
10245 const auto &InstInfo = getVOPDInstInfo(VOPDOpcode: Inst.getOpcode(), InstrInfo: &MII);
10246
10247 // MCInst operands are ordered as follows:
10248 // dstX, dstY, src0X [, other OpX operands], src0Y [, other OpY operands]
10249
10250 for (auto CompIdx : VOPD::COMPONENTS) {
10251 addOp(InstInfo[CompIdx].getIndexOfDstInParsedOperands());
10252 }
10253
10254 for (auto CompIdx : VOPD::COMPONENTS) {
10255 const auto &CInfo = InstInfo[CompIdx];
10256 auto CompSrcOperandsNum = InstInfo[CompIdx].getCompParsedSrcOperandsNum();
10257 for (unsigned CompSrcIdx = 0; CompSrcIdx < CompSrcOperandsNum; ++CompSrcIdx)
10258 addOp(CInfo.getIndexOfSrcInParsedOperands(CompSrcIdx));
10259 if (CInfo.hasSrc2Acc())
10260 addOp(CInfo.getIndexOfDstInParsedOperands());
10261 }
10262
10263 int BitOp3Idx =
10264 AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::bitop3);
10265 if (BitOp3Idx != -1) {
10266 OptionalImmIndexMap OptIdx;
10267 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands.back());
10268 if (Op.isImm())
10269 OptIdx[Op.getImmTy()] = Operands.size() - 1;
10270
10271 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyBitOp3);
10272 }
10273}
10274
10275//===----------------------------------------------------------------------===//
10276// dpp
10277//===----------------------------------------------------------------------===//
10278
10279bool AMDGPUOperand::isDPP8() const { return isImmTy(ImmT: ImmTyDPP8); }
10280
10281bool AMDGPUOperand::isDPPCtrl() const {
10282 using namespace AMDGPU::DPP;
10283
10284 bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(x: getImm());
10285 if (result) {
10286 int64_t Imm = getImm();
10287 return (Imm >= DppCtrl::QUAD_PERM_FIRST &&
10288 Imm <= DppCtrl::QUAD_PERM_LAST) ||
10289 (Imm >= DppCtrl::ROW_SHL_FIRST && Imm <= DppCtrl::ROW_SHL_LAST) ||
10290 (Imm >= DppCtrl::ROW_SHR_FIRST && Imm <= DppCtrl::ROW_SHR_LAST) ||
10291 (Imm >= DppCtrl::ROW_ROR_FIRST && Imm <= DppCtrl::ROW_ROR_LAST) ||
10292 (Imm == DppCtrl::WAVE_SHL1) || (Imm == DppCtrl::WAVE_ROL1) ||
10293 (Imm == DppCtrl::WAVE_SHR1) || (Imm == DppCtrl::WAVE_ROR1) ||
10294 (Imm == DppCtrl::ROW_MIRROR) || (Imm == DppCtrl::ROW_HALF_MIRROR) ||
10295 (Imm == DppCtrl::BCAST15) || (Imm == DppCtrl::BCAST31) ||
10296 (Imm >= DppCtrl::ROW_SHARE_FIRST &&
10297 Imm <= DppCtrl::ROW_SHARE_LAST) ||
10298 (Imm >= DppCtrl::ROW_XMASK_FIRST && Imm <= DppCtrl::ROW_XMASK_LAST);
10299 }
10300 return false;
10301}
10302
10303//===----------------------------------------------------------------------===//
10304// mAI
10305//===----------------------------------------------------------------------===//
10306
10307bool AMDGPUOperand::isBLGP() const {
10308 return isImm() && getImmTy() == ImmTyBLGP && isUInt<3>(x: getImm());
10309}
10310
10311bool AMDGPUOperand::isS16Imm() const {
10312 return isImmLiteral() && (isInt<16>(x: getImm()) || isUInt<16>(x: getImm()));
10313}
10314
10315bool AMDGPUOperand::isU16Imm() const {
10316 return isImmLiteral() && isUInt<16>(x: getImm());
10317}
10318
10319//===----------------------------------------------------------------------===//
10320// dim
10321//===----------------------------------------------------------------------===//
10322
10323bool AMDGPUAsmParser::parseDimId(unsigned &Encoding) {
10324 // We want to allow "dim:1D" etc.,
10325 // but the initial 1 is tokenized as an integer.
10326 std::string Token;
10327 if (isToken(Kind: AsmToken::Integer)) {
10328 SMLoc Loc = getToken().getEndLoc();
10329 Token = std::string(getTokenStr());
10330 lex();
10331 if (getLoc() != Loc)
10332 return false;
10333 }
10334
10335 StringRef Suffix;
10336 if (!parseId(Val&: Suffix))
10337 return false;
10338 Token += Suffix;
10339
10340 StringRef DimId = Token;
10341 DimId.consume_front(Prefix: "SQ_RSRC_IMG_");
10342
10343 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfoByAsmSuffix(AsmSuffix: DimId);
10344 if (!DimInfo)
10345 return false;
10346
10347 Encoding = DimInfo->Encoding;
10348 return true;
10349}
10350
10351ParseStatus AMDGPUAsmParser::parseDim(OperandVector &Operands) {
10352 if (!isGFX10Plus())
10353 return ParseStatus::NoMatch;
10354
10355 SMLoc S = getLoc();
10356
10357 if (!trySkipId(Id: "dim", Kind: AsmToken::Colon))
10358 return ParseStatus::NoMatch;
10359
10360 unsigned Encoding;
10361 SMLoc Loc = getLoc();
10362 if (!parseDimId(Encoding))
10363 return Error(L: Loc, Msg: "invalid dim value");
10364
10365 Operands.push_back(
10366 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Encoding, Loc: S, Type: AMDGPUOperand::ImmTyDim));
10367 return ParseStatus::Success;
10368}
10369
10370//===----------------------------------------------------------------------===//
10371// dpp
10372//===----------------------------------------------------------------------===//
10373
10374ParseStatus AMDGPUAsmParser::parseDPP8(OperandVector &Operands) {
10375 SMLoc S = getLoc();
10376
10377 if (!isGFX10Plus() || !trySkipId(Id: "dpp8", Kind: AsmToken::Colon))
10378 return ParseStatus::NoMatch;
10379
10380 // dpp8:[%d,%d,%d,%d,%d,%d,%d,%d]
10381
10382 int64_t Sels[8];
10383
10384 if (!skipToken(Kind: AsmToken::LBrac, ErrMsg: "expected an opening square bracket"))
10385 return ParseStatus::Failure;
10386
10387 for (size_t i = 0; i < 8; ++i) {
10388 if (i > 0 && !skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma"))
10389 return ParseStatus::Failure;
10390
10391 SMLoc Loc = getLoc();
10392 if (getParser().parseAbsoluteExpression(Res&: Sels[i]))
10393 return ParseStatus::Failure;
10394 if (0 > Sels[i] || 7 < Sels[i])
10395 return Error(L: Loc, Msg: "expected a 3-bit value");
10396 }
10397
10398 if (!skipToken(Kind: AsmToken::RBrac, ErrMsg: "expected a closing square bracket"))
10399 return ParseStatus::Failure;
10400
10401 unsigned DPP8 = 0;
10402 for (size_t i = 0; i < 8; ++i)
10403 DPP8 |= (Sels[i] << (i * 3));
10404
10405 Operands.push_back(
10406 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: DPP8, Loc: S, Type: AMDGPUOperand::ImmTyDPP8));
10407 return ParseStatus::Success;
10408}
10409
10410bool AMDGPUAsmParser::isSupportedDPPCtrl(StringRef Ctrl,
10411 const OperandVector &Operands) {
10412 if (Ctrl == "row_newbcast")
10413 return isGFX90A();
10414
10415 if (Ctrl == "row_share" || Ctrl == "row_xmask")
10416 return isGFX10Plus();
10417
10418 if (Ctrl == "wave_shl" || Ctrl == "wave_shr" || Ctrl == "wave_rol" ||
10419 Ctrl == "wave_ror" || Ctrl == "row_bcast")
10420 return isVI() || isGFX9();
10421
10422 return Ctrl == "row_mirror" || Ctrl == "row_half_mirror" ||
10423 Ctrl == "quad_perm" || Ctrl == "row_shl" || Ctrl == "row_shr" ||
10424 Ctrl == "row_ror";
10425}
10426
10427int64_t AMDGPUAsmParser::parseDPPCtrlPerm() {
10428 // quad_perm:[%d,%d,%d,%d]
10429
10430 if (!skipToken(Kind: AsmToken::LBrac, ErrMsg: "expected an opening square bracket"))
10431 return -1;
10432
10433 int64_t Val = 0;
10434 for (int i = 0; i < 4; ++i) {
10435 if (i > 0 && !skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma"))
10436 return -1;
10437
10438 int64_t Temp;
10439 SMLoc Loc = getLoc();
10440 if (getParser().parseAbsoluteExpression(Res&: Temp))
10441 return -1;
10442 if (Temp < 0 || Temp > 3) {
10443 Error(L: Loc, Msg: "expected a 2-bit value");
10444 return -1;
10445 }
10446
10447 Val += (Temp << i * 2);
10448 }
10449
10450 if (!skipToken(Kind: AsmToken::RBrac, ErrMsg: "expected a closing square bracket"))
10451 return -1;
10452
10453 return Val;
10454}
10455
10456int64_t AMDGPUAsmParser::parseDPPCtrlSel(StringRef Ctrl) {
10457 using namespace AMDGPU::DPP;
10458
10459 // sel:%d
10460
10461 int64_t Val;
10462 SMLoc Loc = getLoc();
10463
10464 if (getParser().parseAbsoluteExpression(Res&: Val))
10465 return -1;
10466
10467 struct DppCtrlCheck {
10468 int64_t Ctrl;
10469 int Lo;
10470 int Hi;
10471 };
10472
10473 DppCtrlCheck Check =
10474 StringSwitch<DppCtrlCheck>(Ctrl)
10475 .Case(S: "wave_shl", Value: {.Ctrl: DppCtrl::WAVE_SHL1, .Lo: 1, .Hi: 1})
10476 .Case(S: "wave_rol", Value: {.Ctrl: DppCtrl::WAVE_ROL1, .Lo: 1, .Hi: 1})
10477 .Case(S: "wave_shr", Value: {.Ctrl: DppCtrl::WAVE_SHR1, .Lo: 1, .Hi: 1})
10478 .Case(S: "wave_ror", Value: {.Ctrl: DppCtrl::WAVE_ROR1, .Lo: 1, .Hi: 1})
10479 .Case(S: "row_shl", Value: {.Ctrl: DppCtrl::ROW_SHL0, .Lo: 1, .Hi: 15})
10480 .Case(S: "row_shr", Value: {.Ctrl: DppCtrl::ROW_SHR0, .Lo: 1, .Hi: 15})
10481 .Case(S: "row_ror", Value: {.Ctrl: DppCtrl::ROW_ROR0, .Lo: 1, .Hi: 15})
10482 .Case(S: "row_share", Value: {.Ctrl: DppCtrl::ROW_SHARE_FIRST, .Lo: 0, .Hi: 15})
10483 .Case(S: "row_xmask", Value: {.Ctrl: DppCtrl::ROW_XMASK_FIRST, .Lo: 0, .Hi: 15})
10484 .Case(S: "row_newbcast", Value: {.Ctrl: DppCtrl::ROW_NEWBCAST_FIRST, .Lo: 0, .Hi: 15})
10485 .Default(Value: {.Ctrl: -1, .Lo: 0, .Hi: 0});
10486
10487 bool Valid;
10488 if (Check.Ctrl == -1) {
10489 Valid = (Ctrl == "row_bcast" && (Val == 15 || Val == 31));
10490 Val = (Val == 15) ? DppCtrl::BCAST15 : DppCtrl::BCAST31;
10491 } else {
10492 Valid = Check.Lo <= Val && Val <= Check.Hi;
10493 Val = (Check.Lo == Check.Hi) ? Check.Ctrl : (Check.Ctrl | Val);
10494 }
10495
10496 if (!Valid) {
10497 Error(L: Loc, Msg: Twine("invalid ", Ctrl) + Twine(" value"));
10498 return -1;
10499 }
10500
10501 return Val;
10502}
10503
10504ParseStatus AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
10505 using namespace AMDGPU::DPP;
10506
10507 if (!isToken(Kind: AsmToken::Identifier) ||
10508 !isSupportedDPPCtrl(Ctrl: getTokenStr(), Operands))
10509 return ParseStatus::NoMatch;
10510
10511 SMLoc S = getLoc();
10512 int64_t Val = -1;
10513 StringRef Ctrl;
10514
10515 parseId(Val&: Ctrl);
10516
10517 if (Ctrl == "row_mirror") {
10518 Val = DppCtrl::ROW_MIRROR;
10519 } else if (Ctrl == "row_half_mirror") {
10520 Val = DppCtrl::ROW_HALF_MIRROR;
10521 } else {
10522 if (skipToken(Kind: AsmToken::Colon, ErrMsg: "expected a colon")) {
10523 if (Ctrl == "quad_perm") {
10524 Val = parseDPPCtrlPerm();
10525 } else {
10526 Val = parseDPPCtrlSel(Ctrl);
10527 }
10528 }
10529 }
10530
10531 if (Val == -1)
10532 return ParseStatus::Failure;
10533
10534 Operands.push_back(
10535 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val, Loc: S, Type: AMDGPUOperand::ImmTyDppCtrl));
10536 return ParseStatus::Success;
10537}
10538
10539void AMDGPUAsmParser::cvtVOP3DPP(MCInst &Inst, const OperandVector &Operands,
10540 bool IsDPP8) {
10541 OptionalImmIndexMap OptionalIdx;
10542 unsigned Opc = Inst.getOpcode();
10543 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
10544
10545 // MAC instructions are special because they have 'old'
10546 // operand which is not tied to dst (but assumed to be).
10547 // They also have dummy unused src2_modifiers.
10548 int OldIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::old);
10549 int Src2ModIdx =
10550 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2_modifiers);
10551 bool IsMAC = OldIdx != -1 && Src2ModIdx != -1 &&
10552 Desc.getOperandConstraint(OpNum: OldIdx, Constraint: MCOI::TIED_TO) == -1;
10553
10554 unsigned I = 1;
10555 for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
10556 ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, N: 1);
10557 }
10558
10559 int Fi = 0;
10560 int VdstInIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst_in);
10561 bool IsVOP3CvtSrDpp = Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx12 ||
10562 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx13 ||
10563 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx12 ||
10564 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx13 ||
10565 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx12 ||
10566 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx13 ||
10567 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx12 ||
10568 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx13;
10569
10570 for (unsigned E = Operands.size(); I != E; ++I) {
10571
10572 if (IsMAC) {
10573 int NumOperands = Inst.getNumOperands();
10574 if (OldIdx == NumOperands) {
10575 // Handle old operand
10576 constexpr int DST_IDX = 0;
10577 Inst.addOperand(Op: Inst.getOperand(i: DST_IDX));
10578 } else if (Src2ModIdx == NumOperands) {
10579 // Add unused dummy src2_modifiers
10580 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
10581 }
10582 }
10583
10584 if (VdstInIdx == static_cast<int>(Inst.getNumOperands())) {
10585 Inst.addOperand(Op: Inst.getOperand(i: 0));
10586 }
10587
10588 if (IsVOP3CvtSrDpp) {
10589 if (Src2ModIdx == static_cast<int>(Inst.getNumOperands())) {
10590 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
10591 Inst.addOperand(Op: MCOperand::createReg(Reg: MCRegister()));
10592 }
10593 }
10594
10595 auto TiedTo =
10596 Desc.getOperandConstraint(OpNum: Inst.getNumOperands(), Constraint: MCOI::TIED_TO);
10597 if (TiedTo != -1) {
10598 assert((unsigned)TiedTo < Inst.getNumOperands());
10599 // handle tied old or src2 for MAC instructions
10600 Inst.addOperand(Op: Inst.getOperand(i: TiedTo));
10601 }
10602 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
10603 // Add the register arguments
10604 if (IsDPP8 && Op.isDppFI()) {
10605 Fi = Op.getImm();
10606 } else if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) {
10607 Op.addRegOrImmWithFPInputModsOperands(Inst, N: 2);
10608 } else if (Op.isReg()) {
10609 Op.addRegOperands(Inst, N: 1);
10610 } else if (Op.isImm() &&
10611 Desc.operands()[Inst.getNumOperands()].RegClass != -1) {
10612 Op.addImmOperands(Inst, N: 1);
10613 } else if (Op.isImm()) {
10614 OptionalIdx[Op.getImmTy()] = I;
10615 } else {
10616 llvm_unreachable("unhandled operand type");
10617 }
10618 }
10619
10620 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::clamp) && !IsVOP3CvtSrDpp)
10621 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10622 ImmT: AMDGPUOperand::ImmTyClamp);
10623
10624 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::byte_sel)) {
10625 if (VdstInIdx == static_cast<int>(Inst.getNumOperands()))
10626 Inst.addOperand(Op: Inst.getOperand(i: 0));
10627 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10628 ImmT: AMDGPUOperand::ImmTyByteSel);
10629 }
10630
10631 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::omod))
10632 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10633 ImmT: AMDGPUOperand::ImmTyOModSI);
10634
10635 if (SIInstrFlags::isVOP3P(O: Desc))
10636 cvtVOP3P(Inst, Operands, OptIdx&: OptionalIdx);
10637 else if (SIInstrFlags::isVOP3(O: Desc))
10638 cvtVOP3OpSel(Inst, Operands, OptionalIdx);
10639 else if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::op_sel)) {
10640 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10641 ImmT: AMDGPUOperand::ImmTyOpSel);
10642 }
10643
10644 if (IsDPP8) {
10645 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10646 ImmT: AMDGPUOperand::ImmTyDPP8);
10647 using namespace llvm::AMDGPU::DPP;
10648 Inst.addOperand(Op: MCOperand::createImm(Val: Fi ? DPP8_FI_1 : DPP8_FI_0));
10649 } else {
10650 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10651 ImmT: AMDGPUOperand::ImmTyDppCtrl, Default: 0xe4);
10652 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10653 ImmT: AMDGPUOperand::ImmTyDppRowMask, Default: 0xf);
10654 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10655 ImmT: AMDGPUOperand::ImmTyDppBankMask, Default: 0xf);
10656 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10657 ImmT: AMDGPUOperand::ImmTyDppBoundCtrl);
10658
10659 if (AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::fi))
10660 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10661 ImmT: AMDGPUOperand::ImmTyDppFI);
10662 }
10663}
10664
10665void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands,
10666 bool IsDPP8) {
10667 OptionalImmIndexMap OptionalIdx;
10668
10669 unsigned I = 1;
10670 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
10671 for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
10672 ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, N: 1);
10673 }
10674
10675 int Fi = 0;
10676 for (unsigned E = Operands.size(); I != E; ++I) {
10677 auto TiedTo =
10678 Desc.getOperandConstraint(OpNum: Inst.getNumOperands(), Constraint: MCOI::TIED_TO);
10679 if (TiedTo != -1) {
10680 assert((unsigned)TiedTo < Inst.getNumOperands());
10681 // handle tied old or src2 for MAC instructions
10682 Inst.addOperand(Op: Inst.getOperand(i: TiedTo));
10683 }
10684 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
10685 // Add the register arguments
10686 if (Op.isReg() && validateVccOperand(Reg: Op.getReg())) {
10687 // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
10688 // Skip it.
10689 continue;
10690 }
10691
10692 if (IsDPP8) {
10693 if (Op.isDPP8()) {
10694 Op.addImmOperands(Inst, N: 1);
10695 } else if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) {
10696 Op.addRegWithFPInputModsOperands(Inst, N: 2);
10697 } else if (Op.isDppFI()) {
10698 Fi = Op.getImm();
10699 } else if (Op.isReg()) {
10700 Op.addRegOperands(Inst, N: 1);
10701 } else {
10702 llvm_unreachable("Invalid operand type");
10703 }
10704 } else {
10705 if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) {
10706 Op.addRegWithFPInputModsOperands(Inst, N: 2);
10707 } else if (Op.isReg()) {
10708 Op.addRegOperands(Inst, N: 1);
10709 } else if (Op.isDPPCtrl()) {
10710 Op.addImmOperands(Inst, N: 1);
10711 } else if (Op.isImm()) {
10712 // Handle optional arguments
10713 OptionalIdx[Op.getImmTy()] = I;
10714 } else {
10715 llvm_unreachable("Invalid operand type");
10716 }
10717 }
10718 }
10719
10720 if (IsDPP8) {
10721 using namespace llvm::AMDGPU::DPP;
10722 Inst.addOperand(Op: MCOperand::createImm(Val: Fi ? DPP8_FI_1 : DPP8_FI_0));
10723 } else {
10724 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10725 ImmT: AMDGPUOperand::ImmTyDppRowMask, Default: 0xf);
10726 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10727 ImmT: AMDGPUOperand::ImmTyDppBankMask, Default: 0xf);
10728 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10729 ImmT: AMDGPUOperand::ImmTyDppBoundCtrl);
10730 if (AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::fi)) {
10731 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10732 ImmT: AMDGPUOperand::ImmTyDppFI);
10733 }
10734 }
10735}
10736
10737//===----------------------------------------------------------------------===//
10738// sdwa
10739//===----------------------------------------------------------------------===//
10740
10741ParseStatus AMDGPUAsmParser::parseSDWASel(OperandVector &Operands,
10742 StringRef Prefix,
10743 AMDGPUOperand::ImmTy Type) {
10744 return parseStringOrIntWithPrefix(
10745 Operands, Name: Prefix,
10746 Ids: {"BYTE_0", "BYTE_1", "BYTE_2", "BYTE_3", "WORD_0", "WORD_1", "DWORD"},
10747 Type);
10748}
10749
10750ParseStatus AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
10751 return parseStringOrIntWithPrefix(
10752 Operands, Name: "dst_unused", Ids: {"UNUSED_PAD", "UNUSED_SEXT", "UNUSED_PRESERVE"},
10753 Type: AMDGPUOperand::ImmTySDWADstUnused);
10754}
10755
10756void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
10757 cvtSDWA(Inst, Operands, BasicInstType: SDWAInstType::VOP1);
10758}
10759
10760void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
10761 cvtSDWA(Inst, Operands, BasicInstType: SDWAInstType::VOP2);
10762}
10763
10764void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst,
10765 const OperandVector &Operands) {
10766 cvtSDWA(Inst, Operands, BasicInstType: SDWAInstType::VOP2, SkipDstVcc: true, SkipSrcVcc: true);
10767}
10768
10769void AMDGPUAsmParser::cvtSdwaVOP2e(MCInst &Inst,
10770 const OperandVector &Operands) {
10771 cvtSDWA(Inst, Operands, BasicInstType: SDWAInstType::VOP2, SkipDstVcc: false, SkipSrcVcc: true);
10772}
10773
10774void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
10775 cvtSDWA(Inst, Operands, BasicInstType: SDWAInstType::VOPC, SkipDstVcc: isVI());
10776}
10777
10778void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
10779 SDWAInstType BasicInstType, bool SkipDstVcc,
10780 bool SkipSrcVcc) {
10781 using namespace llvm::AMDGPU::SDWA;
10782
10783 OptionalImmIndexMap OptionalIdx;
10784 bool SkipVcc = SkipDstVcc || SkipSrcVcc;
10785 bool SkippedVcc = false;
10786
10787 unsigned I = 1;
10788 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
10789 for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
10790 ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, N: 1);
10791 }
10792
10793 for (unsigned E = Operands.size(); I != E; ++I) {
10794 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
10795 if (SkipVcc && !SkippedVcc && Op.isReg() &&
10796 (Op.getReg() == AMDGPU::VCC || Op.getReg() == AMDGPU::VCC_LO)) {
10797 // VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
10798 // Skip it if it's 2nd (e.g. v_add_i32_sdwa v1, vcc, v2, v3)
10799 // or 4th (v_addc_u32_sdwa v1, vcc, v2, v3, vcc) operand.
10800 // Skip VCC only if we didn't skip it on previous iteration.
10801 // Note that src0 and src1 occupy 2 slots each because of modifiers.
10802 if (BasicInstType == SDWAInstType::VOP2 &&
10803 ((SkipDstVcc && Inst.getNumOperands() == 1) ||
10804 (SkipSrcVcc && Inst.getNumOperands() == 5))) {
10805 SkippedVcc = true;
10806 continue;
10807 }
10808 if (BasicInstType == SDWAInstType::VOPC && Inst.getNumOperands() == 0) {
10809 SkippedVcc = true;
10810 continue;
10811 }
10812 }
10813 if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) {
10814 Op.addRegOrImmWithInputModsOperands(Inst, N: 2);
10815 } else if (Op.isImm()) {
10816 // Handle optional arguments
10817 OptionalIdx[Op.getImmTy()] = I;
10818 } else {
10819 llvm_unreachable("Invalid operand type");
10820 }
10821 SkippedVcc = false;
10822 }
10823
10824 const unsigned Opc = Inst.getOpcode();
10825 if (Opc != AMDGPU::V_NOP_sdwa_gfx10 && Opc != AMDGPU::V_NOP_sdwa_gfx9 &&
10826 Opc != AMDGPU::V_NOP_sdwa_vi) {
10827 // v_nop_sdwa_sdwa_vi/gfx9 has no optional sdwa arguments
10828 switch (BasicInstType) {
10829 case SDWAInstType::VOP1:
10830 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::clamp))
10831 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10832 ImmT: AMDGPUOperand::ImmTyClamp, Default: 0);
10833
10834 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::omod))
10835 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10836 ImmT: AMDGPUOperand::ImmTyOModSI, Default: 0);
10837
10838 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::dst_sel))
10839 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10840 ImmT: AMDGPUOperand::ImmTySDWADstSel, Default: SdwaSel::DWORD);
10841
10842 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::dst_unused))
10843 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10844 ImmT: AMDGPUOperand::ImmTySDWADstUnused,
10845 Default: DstUnused::UNUSED_PRESERVE);
10846
10847 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10848 ImmT: AMDGPUOperand::ImmTySDWASrc0Sel, Default: SdwaSel::DWORD);
10849 break;
10850
10851 case SDWAInstType::VOP2:
10852 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10853 ImmT: AMDGPUOperand::ImmTyClamp, Default: 0);
10854
10855 if (AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::omod))
10856 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10857 ImmT: AMDGPUOperand::ImmTyOModSI, Default: 0);
10858
10859 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10860 ImmT: AMDGPUOperand::ImmTySDWADstSel, Default: SdwaSel::DWORD);
10861 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10862 ImmT: AMDGPUOperand::ImmTySDWADstUnused,
10863 Default: DstUnused::UNUSED_PRESERVE);
10864 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10865 ImmT: AMDGPUOperand::ImmTySDWASrc0Sel, Default: SdwaSel::DWORD);
10866 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10867 ImmT: AMDGPUOperand::ImmTySDWASrc1Sel, Default: SdwaSel::DWORD);
10868 break;
10869
10870 case SDWAInstType::VOPC:
10871 if (AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::clamp))
10872 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10873 ImmT: AMDGPUOperand::ImmTyClamp, Default: 0);
10874 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10875 ImmT: AMDGPUOperand::ImmTySDWASrc0Sel, Default: SdwaSel::DWORD);
10876 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10877 ImmT: AMDGPUOperand::ImmTySDWASrc1Sel, Default: SdwaSel::DWORD);
10878 break;
10879 }
10880 }
10881
10882 // special case v_mac_{f16, f32}:
10883 // it has src2 register operand that is tied to dst operand
10884 if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
10885 Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
10886 auto *it = Inst.begin();
10887 std::advance(
10888 i&: it, n: AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::src2));
10889 Inst.insert(I: it, Op: Inst.getOperand(i: 0)); // src2 = dst
10890 }
10891}
10892
10893/// Force static initialization.
10894extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
10895LLVMInitializeAMDGPUAsmParser() {
10896 RegisterMCAsmParser<AMDGPUAsmParser> A(getTheR600Target());
10897 RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
10898 RegisterMCAsmParser<AMDGPUAsmParser> C(getTheGCNLegacyTarget());
10899}
10900
10901#define GET_MATCHER_IMPLEMENTATION
10902#define GET_MNEMONIC_SPELL_CHECKER
10903#define GET_MNEMONIC_CHECKER
10904#include "AMDGPUGenAsmMatcher.inc"
10905
10906ParseStatus AMDGPUAsmParser::parseCustomOperand(OperandVector &Operands,
10907 unsigned MCK) {
10908 switch (MCK) {
10909 case MCK_addr64:
10910 return parseTokenOp(Name: "addr64", Operands);
10911 case MCK_done:
10912 return parseNamedBit(Name: "done", Operands, ImmTy: AMDGPUOperand::ImmTyDone, IgnoreNegative: true);
10913 case MCK_idxen:
10914 return parseTokenOp(Name: "idxen", Operands);
10915 case MCK_lds:
10916 return parseNamedBit(Name: "lds", Operands, ImmTy: AMDGPUOperand::ImmTyLDS,
10917 /*IgnoreNegative=*/true);
10918 case MCK_offen:
10919 return parseTokenOp(Name: "offen", Operands);
10920 case MCK_off:
10921 return parseTokenOp(Name: "off", Operands);
10922 case MCK_row_95_en:
10923 return parseNamedBit(Name: "row_en", Operands, ImmTy: AMDGPUOperand::ImmTyRowEn, IgnoreNegative: true);
10924 case MCK_gds:
10925 return parseNamedBit(Name: "gds", Operands, ImmTy: AMDGPUOperand::ImmTyGDS);
10926 case MCK_tfe:
10927 return parseNamedBit(Name: "tfe", Operands, ImmTy: AMDGPUOperand::ImmTyTFE);
10928 }
10929 return tryCustomParseOperand(Operands, MCK);
10930}
10931
10932// This function should be defined after auto-generated include so that we have
10933// MatchClassKind enum defined
10934unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
10935 unsigned Kind) {
10936 // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
10937 // But MatchInstructionImpl() expects to meet token and fails to validate
10938 // operand. This method checks if we are given immediate operand but expect to
10939 // get corresponding token.
10940 AMDGPUOperand &Operand = (AMDGPUOperand &)Op;
10941 switch (Kind) {
10942 case MCK_addr64:
10943 return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
10944 case MCK_gds:
10945 return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
10946 case MCK_lds:
10947 return Operand.isLDS() ? Match_Success : Match_InvalidOperand;
10948 case MCK_idxen:
10949 return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
10950 case MCK_offen:
10951 return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
10952 case MCK_tfe:
10953 return Operand.isTFE() ? Match_Success : Match_InvalidOperand;
10954 case MCK_done:
10955 return Operand.isDone() ? Match_Success : Match_InvalidOperand;
10956 case MCK_row_95_en:
10957 return Operand.isRowEn() ? Match_Success : Match_InvalidOperand;
10958 case MCK_SSrc_b32:
10959 // When operands have expression values, they will return true for isToken,
10960 // because it is not possible to distinguish between a token and an
10961 // expression at parse time. MatchInstructionImpl() will always try to
10962 // match an operand as a token, when isToken returns true, and when the
10963 // name of the expression is not a valid token, the match will fail,
10964 // so we need to handle it here.
10965 return Operand.isSSrc_b32() ? Match_Success : Match_InvalidOperand;
10966 case MCK_SSrc_f32:
10967 return Operand.isSSrc_f32() ? Match_Success : Match_InvalidOperand;
10968 case MCK_SOPPBrTarget:
10969 return Operand.isSOPPBrTarget() ? Match_Success : Match_InvalidOperand;
10970 case MCK_VReg32OrOff:
10971 return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
10972 case MCK_InterpSlot:
10973 return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
10974 case MCK_InterpAttr:
10975 return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
10976 case MCK_InterpAttrChan:
10977 return Operand.isInterpAttrChan() ? Match_Success : Match_InvalidOperand;
10978 case MCK_SReg_64:
10979 case MCK_SReg_64_XEXEC:
10980 // Null is defined as a 32-bit register but
10981 // it should also be enabled with 64-bit operands or larger.
10982 // The following code enables it for SReg_64 and larger operands
10983 // used as source and destination. Remaining source
10984 // operands are handled in isInlinableImm.
10985 case MCK_SReg_96:
10986 case MCK_SReg_128:
10987 case MCK_SReg_256:
10988 case MCK_SReg_512:
10989 return Operand.isNull() ? Match_Success : Match_InvalidOperand;
10990 default:
10991 return Match_InvalidOperand;
10992 }
10993}
10994
10995//===----------------------------------------------------------------------===//
10996// endpgm
10997//===----------------------------------------------------------------------===//
10998
10999ParseStatus AMDGPUAsmParser::parseEndpgm(OperandVector &Operands) {
11000 SMLoc S = getLoc();
11001 int64_t Imm = 0;
11002
11003 if (!parseExpr(Imm)) {
11004 // The operand is optional, if not present default to 0
11005 Imm = 0;
11006 }
11007
11008 if (!isUInt<16>(x: Imm))
11009 return Error(L: S, Msg: "expected a 16-bit value");
11010
11011 Operands.push_back(
11012 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Imm, Loc: S, Type: AMDGPUOperand::ImmTyEndpgm));
11013 return ParseStatus::Success;
11014}
11015
11016bool AMDGPUOperand::isEndpgm() const { return isImmTy(ImmT: ImmTyEndpgm); }
11017
11018//===----------------------------------------------------------------------===//
11019// Split Barrier
11020//===----------------------------------------------------------------------===//
11021
11022bool AMDGPUOperand::isSplitBarrier() const {
11023 if (!isImm())
11024 return false;
11025
11026 int64_t Imm = getImm();
11027 return isUInt<5>(x: Imm) || (AMDGPU::Barrier::CLUSTER_TRAP <= Imm &&
11028 Imm <= AMDGPU::Barrier::WORKGROUP);
11029}
11030