1//===- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AMDKernelCodeT.h"
10#include "MCTargetDesc/AMDGPUInstPrinter.h"
11#include "MCTargetDesc/AMDGPUMCAsmInfo.h"
12#include "MCTargetDesc/AMDGPUMCExpr.h"
13#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h"
14#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
15#include "MCTargetDesc/AMDGPUTargetStreamer.h"
16#include "SIDefines.h"
17#include "SIInstrInfo.h"
18#include "TargetInfo/AMDGPUTargetInfo.h"
19#include "Utils/AMDGPUAsmUtils.h"
20#include "Utils/AMDGPUBaseInfo.h"
21#include "Utils/AMDKernelCodeTUtils.h"
22#include "llvm/ADT/APFloat.h"
23#include "llvm/ADT/SmallBitVector.h"
24#include "llvm/ADT/Twine.h"
25#include "llvm/BinaryFormat/ELF.h"
26#include "llvm/CodeGenTypes/MachineValueType.h"
27#include "llvm/MC/MCAsmInfo.h"
28#include "llvm/MC/MCContext.h"
29#include "llvm/MC/MCExpr.h"
30#include "llvm/MC/MCInst.h"
31#include "llvm/MC/MCInstrDesc.h"
32#include "llvm/MC/MCParser/AsmLexer.h"
33#include "llvm/MC/MCParser/MCAsmParser.h"
34#include "llvm/MC/MCParser/MCParsedAsmOperand.h"
35#include "llvm/MC/MCParser/MCTargetAsmParser.h"
36#include "llvm/MC/MCRegisterInfo.h"
37#include "llvm/MC/MCSymbol.h"
38#include "llvm/MC/TargetRegistry.h"
39#include "llvm/Support/AMDGPUMetadata.h"
40#include "llvm/Support/AMDGPUObjLinkingInfo.h"
41#include "llvm/Support/AMDHSAKernelDescriptor.h"
42#include "llvm/Support/Casting.h"
43#include "llvm/Support/Compiler.h"
44#include "llvm/Support/MathExtras.h"
45#include "llvm/TargetParser/AMDGPUTargetParser.h"
46#include <optional>
47
48using namespace llvm;
49using namespace llvm::AMDGPU;
50using namespace llvm::amdhsa;
51
52namespace {
53
54class AMDGPUAsmParser;
55
56enum RegisterKind {
57 IS_UNKNOWN,
58 IS_VGPR,
59 IS_SGPR,
60 IS_AGPR,
61 IS_TTMP,
62 IS_SPECIAL
63};
64
65//===----------------------------------------------------------------------===//
66// Operand
67//===----------------------------------------------------------------------===//
68
69class AMDGPUOperand : public MCParsedAsmOperand {
70 enum KindTy { Token, Immediate, Register, Expression } Kind;
71
72 SMLoc StartLoc, EndLoc;
73 const AMDGPUAsmParser *AsmParser;
74
75public:
76 AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
77 : Kind(Kind_), AsmParser(AsmParser_) {}
78
79 using Ptr = std::unique_ptr<AMDGPUOperand>;
80
81 struct Modifiers {
82 bool Abs = false;
83 bool Neg = false;
84 bool Sext = false;
85 LitModifier Lit = LitModifier::None;
86
87 bool hasFPModifiers() const { return Abs || Neg; }
88 bool hasIntModifiers() const { return Sext; }
89 bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
90 bool isForcedLit() const { return Lit == LitModifier::Lit; }
91 bool isForcedLit64() const { return Lit == LitModifier::Lit64; }
92
93 int64_t getFPModifiersOperand() const {
94 int64_t Operand = 0;
95 Operand |= Abs ? SISrcMods::ABS : 0u;
96 Operand |= Neg ? SISrcMods::NEG : 0u;
97 return Operand;
98 }
99
100 int64_t getIntModifiersOperand() const {
101 int64_t Operand = 0;
102 Operand |= Sext ? SISrcMods::SEXT : 0u;
103 return Operand;
104 }
105
106 int64_t getModifiersOperand() const {
107 assert(!(hasFPModifiers() && hasIntModifiers()) &&
108 "fp and int modifiers should not be used simultaneously");
109 if (hasFPModifiers())
110 return getFPModifiersOperand();
111 if (hasIntModifiers())
112 return getIntModifiersOperand();
113 return 0;
114 }
115
116 friend raw_ostream &operator<<(raw_ostream &OS,
117 AMDGPUOperand::Modifiers Mods);
118 };
119
120 enum ImmTy {
121 ImmTyNone,
122 ImmTyGDS,
123 ImmTyLDS,
124 ImmTyOffen,
125 ImmTyIdxen,
126 ImmTyAddr64,
127 ImmTyOffset,
128 ImmTyInstOffset,
129 ImmTyOffset0,
130 ImmTyOffset1,
131 ImmTySMEMOffsetMod,
132 ImmTyCPol,
133 ImmTyTFE,
134 ImmTyIsAsync,
135 ImmTyD16,
136 ImmTyClamp,
137 ImmTyOModSI,
138 ImmTySDWADstSel,
139 ImmTySDWASrc0Sel,
140 ImmTySDWASrc1Sel,
141 ImmTySDWADstUnused,
142 ImmTyDMask,
143 ImmTyDim,
144 ImmTyUNorm,
145 ImmTyDA,
146 ImmTyR128A16,
147 ImmTyA16,
148 ImmTyLWE,
149 ImmTyExpTgt,
150 ImmTyExpCompr,
151 ImmTyExpVM,
152 ImmTyDone,
153 ImmTyRowEn,
154 ImmTyFormat,
155 ImmTyFormatOp,
156 ImmTyHwreg,
157 ImmTyOff,
158 ImmTySendMsg,
159 ImmTyWaitEvent,
160 ImmTyInterpSlot,
161 ImmTyInterpAttr,
162 ImmTyInterpAttrChan,
163 ImmTyOpSel,
164 ImmTyOpSelHi,
165 ImmTyNegLo,
166 ImmTyNegHi,
167 ImmTyIndexKey8bit,
168 ImmTyIndexKey16bit,
169 ImmTyIndexKey32bit,
170 ImmTyDPP8,
171 ImmTyDppCtrl,
172 ImmTyDppRowMask,
173 ImmTyDppBankMask,
174 ImmTyDppBoundCtrl,
175 ImmTyDppFI,
176 ImmTySwizzle,
177 ImmTyGprIdxMode,
178 ImmTyHigh,
179 ImmTyBLGP,
180 ImmTyCBSZ,
181 ImmTyABID,
182 ImmTyEndpgm,
183 ImmTyWaitVDST,
184 ImmTyWaitEXP,
185 ImmTyWaitVAVDst,
186 ImmTyWaitVMVSrc,
187 ImmTyBitOp3,
188 ImmTyMatrixAFMT,
189 ImmTyMatrixBFMT,
190 ImmTyMatrixAScale,
191 ImmTyMatrixBScale,
192 ImmTyMatrixAScaleFmt,
193 ImmTyMatrixBScaleFmt,
194 ImmTyMatrixAReuse,
195 ImmTyMatrixBReuse,
196 ImmTyScaleSel,
197 ImmTyByteSel,
198 };
199
200private:
201 struct TokOp {
202 const char *Data;
203 unsigned Length;
204 };
205
206 struct ImmOp {
207 int64_t Val;
208 ImmTy Type;
209 bool IsFPImm;
210 Modifiers Mods;
211 };
212
213 struct RegOp {
214 MCRegister RegNo;
215 Modifiers Mods;
216 };
217
218 union {
219 TokOp Tok;
220 ImmOp Imm;
221 RegOp Reg;
222 const MCExpr *Expr;
223 };
224
225 // The index of the associated MCInst operand.
226 mutable int MCOpIdx = -1;
227
228public:
229 bool isToken() const override { return Kind == Token; }
230
231 bool isSymbolRefExpr() const {
232 return isExpr() && Expr && isa<MCSymbolRefExpr>(Val: Expr);
233 }
234
235 bool isImm() const override { return Kind == Immediate; }
236
237 bool isInlinableImm(MVT type) const;
238 bool isLiteralImm(MVT type) const;
239
240 bool isRegKind() const { return Kind == Register; }
241
242 bool isReg() const override { return isRegKind() && !hasModifiers(); }
243
244 bool isRegOrInline(unsigned RCID, MVT type) const {
245 return isRegClass(RCID) || isInlinableImm(type);
246 }
247
248 bool isRegOrInlineTarget(unsigned TargetRCIdx, MVT type) const {
249 return isRegClassTarget(TargetRCIdx) || isInlinableImm(type);
250 }
251
252 bool isRegOrImmWithInputMods(unsigned RCID, MVT type) const {
253 return isRegOrInline(RCID, type) || isLiteralImm(type);
254 }
255
256 bool isRegOrImmWithInputModsTarget(unsigned TargetRCIdx, MVT type) const {
257 return isRegOrInlineTarget(TargetRCIdx, type) || isLiteralImm(type);
258 }
259
260 template <bool IsFake16> bool isRegOrImmWithIntT16InputMods() const {
261 return isRegOrImmWithInputMods(
262 RCID: IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, type: MVT::i16);
263 }
264
265 bool isRegOrImmWithInt32InputMods() const {
266 return isRegOrImmWithInputMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::i32);
267 }
268
269 template <bool IsFake16> bool isRegOrInlineImmWithIntT16InputMods() const {
270 return isRegOrInline(
271 RCID: IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, type: MVT::i16);
272 }
273
274 bool isRegOrInlineImmWithInt32InputMods() const {
275 return isRegOrInline(RCID: AMDGPU::VS_32RegClassID, type: MVT::i32);
276 }
277
278 bool isRegOrImmWithInt64InputMods() const {
279 return isRegOrImmWithInputModsTarget(TargetRCIdx: AMDGPU::VS_64_AlignTarget, type: MVT::i64);
280 }
281
282 bool isRegOrImmWithFP16InputMods() const {
283 return isRegOrImmWithInputMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::f16);
284 }
285
286 template <bool IsFake16> bool isRegOrImmWithFPT16InputMods() const {
287 return isRegOrImmWithInputMods(
288 RCID: IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, type: MVT::f16);
289 }
290
291 bool isRegOrImmWithFPT16_LO16InputMods() const {
292 return isRegOrImmWithInputMods(RCID: AMDGPU::VS_16_LO16RegClassID, type: MVT::f16);
293 }
294
295 bool isRegOrImmWithFP32InputMods() const {
296 return isRegOrImmWithInputMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::f32);
297 }
298
299 bool isRegOrImmWithFP64InputMods() const {
300 return isRegOrImmWithInputModsTarget(TargetRCIdx: AMDGPU::VS_64_AlignTarget, type: MVT::f64);
301 }
302
303 template <bool IsFake16> bool isRegOrInlineImmWithFP16InputMods() const {
304 return isRegOrInline(
305 RCID: IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, type: MVT::f16);
306 }
307
308 bool isRegOrInlineImmWithFP32InputMods() const {
309 return isRegOrInline(RCID: AMDGPU::VS_32RegClassID, type: MVT::f32);
310 }
311
312 bool isRegOrInlineImmWithFP64InputMods() const {
313 return isRegOrInlineTarget(TargetRCIdx: AMDGPU::VS_64_AlignTarget, type: MVT::f64);
314 }
315
316 bool isVRegWithInputMods(unsigned RCID) const { return isRegClass(RCID); }
317
318 bool isVRegWithFP32InputMods() const {
319 return isVRegWithInputMods(RCID: AMDGPU::VGPR_32RegClassID);
320 }
321
322 bool isVRegWithFP64InputMods() const {
323 return isRegClassTarget(TargetRCIdx: AMDGPU::VReg_64_AlignTarget);
324 }
325
326 bool isPackedFP16InputMods() const {
327 return isRegOrImmWithInputMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::v2f16);
328 }
329
330 bool isPackedVGPRFP32InputMods() const {
331 return isRegOrImmWithInputMods(RCID: AMDGPU::VReg_64RegClassID, type: MVT::v2f32);
332 }
333
334 bool isVReg32() const { return isRegClass(RCID: AMDGPU::VGPR_32RegClassID); }
335
336 bool isVReg32OrOff() const { return isOff() || isVReg32(); }
337
338 bool isRsrcReg32() const { return isRegClass(RCID: AMDGPU::RsrcReg32RegClassID); }
339
340 bool isNull() const { return isRegKind() && getReg() == AMDGPU::SGPR_NULL; }
341
342 bool isAV_LdSt_32_Align2_RegOp() const {
343 return isRegClass(RCID: AMDGPU::VGPR_32RegClassID) ||
344 isRegClass(RCID: AMDGPU::AGPR_32RegClassID);
345 }
346
347 bool isVRegWithInputMods() const;
348 template <bool IsFake16> bool isT16_Lo128VRegWithInputMods() const;
349 template <bool IsFake16> bool isT16VRegWithInputMods() const;
350 bool isT16_LO16VRegWithInputMods() const;
351
352 bool isSDWAOperand(MVT type) const;
353 bool isSDWAFP16Operand() const;
354 bool isSDWAFP32Operand() const;
355 bool isSDWAInt16Operand() const;
356 bool isSDWAInt32Operand() const;
357
358 bool isImmTy(ImmTy ImmT) const { return isImm() && Imm.Type == ImmT; }
359
360 template <ImmTy Ty> bool isImmTy() const { return isImmTy(ImmT: Ty); }
361
362 bool isImmLiteral() const { return isImmTy(ImmT: ImmTyNone); }
363
364 bool isImmModifier() const { return isImm() && Imm.Type != ImmTyNone; }
365
366 bool isOModSI() const { return isImmTy(ImmT: ImmTyOModSI); }
367 bool isDim() const { return isImmTy(ImmT: ImmTyDim); }
368 bool isR128A16() const { return isImmTy(ImmT: ImmTyR128A16); }
369 bool isOff() const { return isImmTy(ImmT: ImmTyOff); }
370 bool isExpTgt() const { return isImmTy(ImmT: ImmTyExpTgt); }
371 bool isOffen() const { return isImmTy(ImmT: ImmTyOffen); }
372 bool isIdxen() const { return isImmTy(ImmT: ImmTyIdxen); }
373 bool isAddr64() const { return isImmTy(ImmT: ImmTyAddr64); }
374 bool isSMEMOffsetMod() const { return isImmTy(ImmT: ImmTySMEMOffsetMod); }
375 bool isFlatOffset() const {
376 return isImmTy(ImmT: ImmTyOffset) || isImmTy(ImmT: ImmTyInstOffset);
377 }
378 bool isGDS() const { return isImmTy(ImmT: ImmTyGDS); }
379 bool isLDS() const { return isImmTy(ImmT: ImmTyLDS); }
380 bool isCPol() const { return isImmTy(ImmT: ImmTyCPol); }
381 bool isIndexKey8bit() const { return isImmTy(ImmT: ImmTyIndexKey8bit); }
382 bool isIndexKey16bit() const { return isImmTy(ImmT: ImmTyIndexKey16bit); }
383 bool isIndexKey32bit() const { return isImmTy(ImmT: ImmTyIndexKey32bit); }
384 bool isMatrixAFMT() const { return isImmTy(ImmT: ImmTyMatrixAFMT); }
385 bool isMatrixBFMT() const { return isImmTy(ImmT: ImmTyMatrixBFMT); }
386 bool isMatrixAScale() const { return isImmTy(ImmT: ImmTyMatrixAScale); }
387 bool isMatrixBScale() const { return isImmTy(ImmT: ImmTyMatrixBScale); }
388 bool isMatrixAScaleFmt() const { return isImmTy(ImmT: ImmTyMatrixAScaleFmt); }
389 bool isMatrixBScaleFmt() const { return isImmTy(ImmT: ImmTyMatrixBScaleFmt); }
390 bool isTFE() const { return isImmTy(ImmT: ImmTyTFE); }
391 bool isFormat() const { return isImmTy(ImmT: ImmTyFormat) && isUInt<7>(x: getImm()); }
392 bool isFormatOp() const {
393 return isImmTy(ImmT: ImmTyFormatOp) && isUInt<7>(x: getImm());
394 }
395 bool isDppFI() const { return isImmTy(ImmT: ImmTyDppFI); }
396 bool isSDWADstSel() const { return isImmTy(ImmT: ImmTySDWADstSel); }
397 bool isSDWASrc0Sel() const { return isImmTy(ImmT: ImmTySDWASrc0Sel); }
398 bool isSDWASrc1Sel() const { return isImmTy(ImmT: ImmTySDWASrc1Sel); }
399 bool isSDWADstUnused() const { return isImmTy(ImmT: ImmTySDWADstUnused); }
400 bool isInterpSlot() const { return isImmTy(ImmT: ImmTyInterpSlot); }
401 bool isInterpAttr() const { return isImmTy(ImmT: ImmTyInterpAttr); }
402 bool isInterpAttrChan() const { return isImmTy(ImmT: ImmTyInterpAttrChan); }
403 bool isOpSel() const { return isImmTy(ImmT: ImmTyOpSel); }
404 bool isOpSelHi() const { return isImmTy(ImmT: ImmTyOpSelHi); }
405 bool isNegLo() const { return isImmTy(ImmT: ImmTyNegLo); }
406 bool isNegHi() const { return isImmTy(ImmT: ImmTyNegHi); }
407 bool isDone() const { return isImmTy(ImmT: ImmTyDone); }
408 bool isRowEn() const { return isImmTy(ImmT: ImmTyRowEn); }
409
410 bool isRegOrImm() const { return isReg() || isImm(); }
411
412 bool isRegClass(unsigned RCID) const;
413
414 // Check the register against the HwMode-resolved operand class.
415 bool isRegClassTarget(unsigned TargetRCIdx) const;
416
417 bool isInlineValue() const;
418
419 bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
420 return isRegOrInline(RCID, type) && !hasModifiers();
421 }
422
423 bool isRegOrInlineNoModsTarget(unsigned TargetRCIdx, MVT type) const {
424 return isRegOrInlineTarget(TargetRCIdx, type) && !hasModifiers();
425 }
426
427 bool isSCSrcB16() const {
428 return isRegOrInlineNoMods(RCID: AMDGPU::SReg_32RegClassID, type: MVT::i16);
429 }
430
431 bool isSCSrc_b32() const {
432 return isRegOrInlineNoMods(RCID: AMDGPU::SReg_32RegClassID, type: MVT::i32);
433 }
434
435 bool isSCSrc_b64() const {
436 return isRegOrInlineNoMods(RCID: AMDGPU::SReg_64RegClassID, type: MVT::i64);
437 }
438
439 bool isBoolReg() const;
440
441 bool isSSrc_b32() const {
442 return isSCSrc_b32() || isLiteralImm(type: MVT::i32) || isExpr();
443 }
444
445 bool isSSrc_b16() const { return isSCSrcB16() || isLiteralImm(type: MVT::i16); }
446
447 bool isSSrc_b64() const {
448 // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
449 // See isVSrc64().
450 return isSCSrc_b64() || isLiteralImm(type: MVT::i64) ||
451 (((const MCTargetAsmParser *)AsmParser)
452 ->getAvailableFeatures()[AMDGPU::Feature64BitLiterals] &&
453 isExpr());
454 }
455
456 bool isSSrc_f32() const {
457 return isSCSrc_b32() || isLiteralImm(type: MVT::f32) || isExpr();
458 }
459
460 bool isSSrc_bf16() const { return isSCSrcB16() || isLiteralImm(type: MVT::bf16); }
461
462 bool isSSrc_f16() const { return isSCSrcB16() || isLiteralImm(type: MVT::f16); }
463
464 bool isSSrc_NoInline_f16() const { return isSSrc_f16(); }
465
466 bool isSSrcOrLds_b32() const {
467 return isRegOrInlineNoMods(RCID: AMDGPU::SRegOrLds_32RegClassID, type: MVT::i32) ||
468 isLiteralImm(type: MVT::i32) || isExpr();
469 }
470
471 bool isVCSrc_b32() const {
472 return isRegOrInlineNoMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::i32);
473 }
474
475 bool isVCSrc_b32_Lo256() const {
476 return isRegOrInlineNoMods(RCID: AMDGPU::VS_32_Lo256RegClassID, type: MVT::i32);
477 }
478
479 bool isVCSrc_b64_Lo256() const {
480 return isRegOrInlineNoMods(RCID: AMDGPU::VS_64_Lo256RegClassID, type: MVT::i64);
481 }
482
483 bool isVCSrc_b64() const {
484 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VS_64_AlignTarget, type: MVT::i64);
485 }
486
487 bool isVCSrcT_b16() const {
488 return isRegOrInlineNoMods(RCID: AMDGPU::VS_16RegClassID, type: MVT::i16);
489 }
490
491 bool isVCSrcTB16_Lo128() const {
492 return isRegOrInlineNoMods(RCID: AMDGPU::VS_16_Lo128RegClassID, type: MVT::i16);
493 }
494
495 bool isVCSrcFake16B16_Lo128() const {
496 return isRegOrInlineNoMods(RCID: AMDGPU::VS_32_Lo128RegClassID, type: MVT::i16);
497 }
498
499 bool isVCSrc_b16() const {
500 return isRegOrInlineNoMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::i16);
501 }
502
503 bool isVCSrc_v2b16() const { return isVCSrc_b16(); }
504
505 bool isVCSrc_f32() const {
506 return isRegOrInlineNoMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::f32);
507 }
508
509 bool isVCSrc_f64() const {
510 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VS_64_AlignTarget, type: MVT::f64);
511 }
512
513 bool isVCSrcTBF16() const {
514 return isRegOrInlineNoMods(RCID: AMDGPU::VS_16RegClassID, type: MVT::bf16);
515 }
516
517 bool isVCSrcT_f16() const {
518 return isRegOrInlineNoMods(RCID: AMDGPU::VS_16RegClassID, type: MVT::f16);
519 }
520
521 bool isVCSrcT_bf16() const {
522 return isRegOrInlineNoMods(RCID: AMDGPU::VS_16RegClassID, type: MVT::f16);
523 }
524
525 bool isVCSrcTF16_LO16() const {
526 return isRegOrInlineNoMods(RCID: AMDGPU::VS_16_LO16RegClassID, type: MVT::f16);
527 }
528
529 bool isVCSrcTBF16_Lo128() const {
530 return isRegOrInlineNoMods(RCID: AMDGPU::VS_16_Lo128RegClassID, type: MVT::bf16);
531 }
532
533 bool isVCSrcTF16_Lo128() const {
534 return isRegOrInlineNoMods(RCID: AMDGPU::VS_16_Lo128RegClassID, type: MVT::f16);
535 }
536
537 bool isVCSrcFake16BF16_Lo128() const {
538 return isRegOrInlineNoMods(RCID: AMDGPU::VS_32_Lo128RegClassID, type: MVT::bf16);
539 }
540
541 bool isVCSrcFake16F16_Lo128() const {
542 return isRegOrInlineNoMods(RCID: AMDGPU::VS_32_Lo128RegClassID, type: MVT::f16);
543 }
544
545 bool isVCSrc_bf16() const {
546 return isRegOrInlineNoMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::bf16);
547 }
548
549 bool isVCSrc_f16() const {
550 return isRegOrInlineNoMods(RCID: AMDGPU::VS_32RegClassID, type: MVT::f16);
551 }
552
553 bool isVCSrc_v2bf16() const { return isVCSrc_bf16(); }
554
555 bool isVCSrc_v2f16() const { return isVCSrc_f16(); }
556
557 bool isVSrc_b32() const {
558 return isVCSrc_f32() || isLiteralImm(type: MVT::i32) || isExpr();
559 }
560
561 bool isVSrc_b64() const { return isVCSrc_f64() || isLiteralImm(type: MVT::i64); }
562
563 bool isVSrc_v2b64() const {
564 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VS_128_AlignTarget, type: MVT::i64) ||
565 isLiteralImm(type: MVT::i64);
566 }
567
568 bool isVSrc_v2f64() const {
569 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VS_128_AlignTarget, type: MVT::f64) ||
570 isLiteralImm(type: MVT::f64);
571 }
572
573 bool isVSrcT_b16() const { return isVCSrcT_b16() || isLiteralImm(type: MVT::i16); }
574
575 bool isVSrcT_b16_Lo128() const {
576 return isVCSrcTB16_Lo128() || isLiteralImm(type: MVT::i16);
577 }
578
579 bool isVSrcFake16_b16_Lo128() const {
580 return isVCSrcFake16B16_Lo128() || isLiteralImm(type: MVT::i16);
581 }
582
583 bool isVSrc_b16() const { return isVCSrc_b16() || isLiteralImm(type: MVT::i16); }
584
585 bool isVSrc_v2b16() const { return isVSrc_b16() || isLiteralImm(type: MVT::v2i16); }
586
587 bool isVSrc_v2f32() const { return isVSrc_f64() || isLiteralImm(type: MVT::v2f32); }
588
589 bool isVCSrc_v2b32() const { return isVCSrc_b64(); }
590
591 bool isVSrc_v2b32() const { return isVSrc_b64() || isLiteralImm(type: MVT::v2i32); }
592
593 bool isVSrc_f32() const {
594 return isVCSrc_f32() || isLiteralImm(type: MVT::f32) || isExpr();
595 }
596
597 bool isVSrc_f64() const {
598 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VS_64_AlignTarget, type: MVT::f64) ||
599 isLiteralImm(type: MVT::f64);
600 }
601
602 bool isVSrcT_bf16() const {
603 return isVCSrcTBF16() || isLiteralImm(type: MVT::bf16);
604 }
605
606 bool isVSrcT_f16() const { return isVCSrcT_f16() || isLiteralImm(type: MVT::f16); }
607
608 bool isVSrcT_f16_LO16() const {
609 return isVCSrcTF16_LO16() || isLiteralImm(type: MVT::f16);
610 }
611
612 bool isVSrcT_bf16_Lo128() const {
613 return isVCSrcTBF16_Lo128() || isLiteralImm(type: MVT::bf16);
614 }
615
616 bool isVSrcT_f16_Lo128() const {
617 return isVCSrcTF16_Lo128() || isLiteralImm(type: MVT::f16);
618 }
619
620 bool isVSrcFake16_bf16_Lo128() const {
621 return isVCSrcFake16BF16_Lo128() || isLiteralImm(type: MVT::bf16);
622 }
623
624 bool isVSrcFake16_f16_Lo128() const {
625 return isVCSrcFake16F16_Lo128() || isLiteralImm(type: MVT::f16);
626 }
627
628 bool isVSrc_bf16() const { return isVCSrc_bf16() || isLiteralImm(type: MVT::bf16); }
629
630 bool isVSrc_f16() const { return isVCSrc_f16() || isLiteralImm(type: MVT::f16); }
631
632 bool isVSrc_v2bf16() const {
633 return isVSrc_bf16() || isLiteralImm(type: MVT::v2bf16);
634 }
635
636 bool isVSrc_v2f16() const { return isVSrc_f16() || isLiteralImm(type: MVT::v2f16); }
637
638 bool isVSrc_v2f16_splat() const { return isVSrc_v2f16(); }
639
640 bool isVSrc_NoInline_v2f16() const { return isVSrc_v2f16(); }
641
642 bool isVISrc_64_bf16() const {
643 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_64_AlignTarget, type: MVT::bf16);
644 }
645
646 bool isVISrc_64_f16() const {
647 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_64_AlignTarget, type: MVT::f16);
648 }
649
650 bool isVISrc_64_b32() const {
651 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_64_AlignTarget, type: MVT::i32);
652 }
653
654 bool isVISrc_64_f64() const {
655 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_64_AlignTarget, type: MVT::f64);
656 }
657
658 bool isVISrc_256_b32() const {
659 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_256_AlignTarget, type: MVT::i32);
660 }
661
662 bool isVISrc_256_f32() const {
663 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_256_AlignTarget, type: MVT::f32);
664 }
665
666 bool isVISrc_256_f64() const {
667 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_256_AlignTarget, type: MVT::f64);
668 }
669
670 bool isVISrc_512_f64() const {
671 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_512_AlignTarget, type: MVT::f64);
672 }
673
674 bool isVISrc_128_b32() const {
675 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_128_AlignTarget, type: MVT::i32);
676 }
677
678 bool isVISrc_128_f32() const {
679 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_128_AlignTarget, type: MVT::f32);
680 }
681
682 bool isVISrc_512_b32() const {
683 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_512_AlignTarget, type: MVT::i32);
684 }
685
686 bool isVISrc_512_f32() const {
687 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_512_AlignTarget, type: MVT::f32);
688 }
689
690 bool isVISrc_1024_b32() const {
691 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_1024_AlignTarget, type: MVT::i32);
692 }
693
694 bool isVISrc_1024_f32() const {
695 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_1024_AlignTarget, type: MVT::f32);
696 }
697
698 bool isAISrc_64_f64() const {
699 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::AReg_64_AlignTarget, type: MVT::f64);
700 }
701
702 bool isAISrc_128_b32() const {
703 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::AReg_128_AlignTarget, type: MVT::i32);
704 }
705
706 bool isAISrc_128_f32() const {
707 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::AReg_128_AlignTarget, type: MVT::f32);
708 }
709
710 bool isVISrc_128_bf16() const {
711 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_128_AlignTarget, type: MVT::bf16);
712 }
713
714 bool isVISrc_128_f16() const {
715 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::VReg_128_AlignTarget, type: MVT::f16);
716 }
717
718 bool isAISrc_256_f64() const {
719 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::AReg_256_AlignTarget, type: MVT::f64);
720 }
721
722 bool isAISrc_512_b32() const {
723 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::AReg_512_AlignTarget, type: MVT::i32);
724 }
725
726 bool isAISrc_512_f32() const {
727 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::AReg_512_AlignTarget, type: MVT::f32);
728 }
729
730 bool isAISrc_1024_b32() const {
731 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::AReg_1024_AlignTarget, type: MVT::i32);
732 }
733
734 bool isAISrc_1024_f32() const {
735 return isRegOrInlineNoModsTarget(TargetRCIdx: AMDGPU::AReg_1024_AlignTarget, type: MVT::f32);
736 }
737
738 bool isKImmFP32() const { return isLiteralImm(type: MVT::f32); }
739
740 bool isKImmFP16() const { return isLiteralImm(type: MVT::f16); }
741
742 bool isKImmFP64() const { return isLiteralImm(type: MVT::f64); }
743
744 bool isMem() const override { return false; }
745
746 bool isExpr() const { return Kind == Expression; }
747
748 bool isSOPPBrTarget() const { return isExpr() || isImm(); }
749
750 bool isSWaitCnt() const;
751 bool isDepCtr() const;
752 bool isSDelayALU() const;
753 bool isHwreg() const;
754 bool isSendMsg() const;
755 bool isWaitEvent() const;
756 bool isSplitBarrier() const;
757 bool isSwizzle() const;
758 bool isSMRDOffset8() const;
759 bool isSMEMOffset() const;
760 bool isSMRDLiteralOffset() const;
761 bool isDPP8() const;
762 bool isDPPCtrl() const;
763 bool isBLGP() const;
764 bool isGPRIdxMode() const;
765 bool isS16Imm() const;
766 bool isU16Imm() const;
767 bool isEndpgm() const;
768
769 auto getPredicate(std::function<bool(const AMDGPUOperand &Op)> P) const {
770 return [this, P]() { return P(*this); };
771 }
772
773 StringRef getToken() const {
774 assert(isToken());
775 return StringRef(Tok.Data, Tok.Length);
776 }
777
778 int64_t getImm() const {
779 assert(isImm());
780 return Imm.Val;
781 }
782
783 ImmTy getImmTy() const {
784 assert(isImm());
785 return Imm.Type;
786 }
787
788 MCRegister getReg() const override {
789 assert(isRegKind());
790 return Reg.RegNo;
791 }
792
793 SMLoc getStartLoc() const override { return StartLoc; }
794
795 SMLoc getEndLoc() const override { return EndLoc; }
796
797 int getMCOpIdx() const { return MCOpIdx; }
798
799 Modifiers getModifiers() const {
800 assert(isRegKind() || isImmTy(ImmTyNone));
801 return isRegKind() ? Reg.Mods : Imm.Mods;
802 }
803
804 void setModifiers(Modifiers Mods) {
805 assert(isRegKind() || isImmTy(ImmTyNone));
806 if (isRegKind())
807 Reg.Mods = Mods;
808 else
809 Imm.Mods = Mods;
810 }
811
812 bool hasModifiers() const { return getModifiers().hasModifiers(); }
813
814 bool hasFPModifiers() const { return getModifiers().hasFPModifiers(); }
815
816 bool hasIntModifiers() const { return getModifiers().hasIntModifiers(); }
817
818 bool isForcedLit() const {
819 return isImmLiteral() && getModifiers().isForcedLit();
820 }
821
822 bool isForcedLit64() const {
823 return isImmLiteral() && getModifiers().isForcedLit64();
824 }
825
826 uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const;
827
828 void addImmOperands(MCInst &Inst, unsigned N,
829 bool ApplyModifiers = true) const;
830
831 void addLiteralImmOperand(MCInst &Inst, int64_t Val,
832 bool ApplyModifiers) const;
833
834 void addRegOperands(MCInst &Inst, unsigned N) const;
835
836 void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
837 if (isRegKind())
838 addRegOperands(Inst, N);
839 else
840 addImmOperands(Inst, N);
841 }
842
843 void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
844 Modifiers Mods = getModifiers();
845 Inst.addOperand(Op: MCOperand::createImm(Val: Mods.getModifiersOperand()));
846 if (isRegKind()) {
847 addRegOperands(Inst, N);
848 } else {
849 addImmOperands(Inst, N, ApplyModifiers: false);
850 }
851 }
852
853 void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
854 assert(!hasIntModifiers());
855 addRegOrImmWithInputModsOperands(Inst, N);
856 }
857
858 void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
859 Modifiers Mods = getModifiers();
860 Inst.addOperand(Op: MCOperand::createImm(Val: Mods.getModifiersOperand()));
861 assert(isRegKind());
862 addRegOperands(Inst, N);
863 }
864
865 void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
866 assert(!hasIntModifiers());
867 addRegWithInputModsOperands(Inst, N);
868 }
869
870 static void printImmTy(raw_ostream &OS, ImmTy Type) {
871 // clang-format off
872 switch (Type) {
873 case ImmTyNone: OS << "None"; break;
874 case ImmTyGDS: OS << "GDS"; break;
875 case ImmTyLDS: OS << "LDS"; break;
876 case ImmTyOffen: OS << "Offen"; break;
877 case ImmTyIdxen: OS << "Idxen"; break;
878 case ImmTyAddr64: OS << "Addr64"; break;
879 case ImmTyOffset: OS << "Offset"; break;
880 case ImmTyInstOffset: OS << "InstOffset"; break;
881 case ImmTyOffset0: OS << "Offset0"; break;
882 case ImmTyOffset1: OS << "Offset1"; break;
883 case ImmTySMEMOffsetMod: OS << "SMEMOffsetMod"; break;
884 case ImmTyCPol: OS << "CPol"; break;
885 case ImmTyIndexKey8bit: OS << "index_key"; break;
886 case ImmTyIndexKey16bit: OS << "index_key"; break;
887 case ImmTyIndexKey32bit: OS << "index_key"; break;
888 case ImmTyTFE: OS << "TFE"; break;
889 case ImmTyIsAsync: OS << "IsAsync"; break;
890 case ImmTyD16: OS << "D16"; break;
891 case ImmTyFormat: OS << "Format"; break;
892 case ImmTyFormatOp: OS << "FormatOp"; break;
893 case ImmTyClamp: OS << "Clamp"; break;
894 case ImmTyOModSI: OS << "OModSI"; break;
895 case ImmTyDPP8: OS << "DPP8"; break;
896 case ImmTyDppCtrl: OS << "DppCtrl"; break;
897 case ImmTyDppRowMask: OS << "DppRowMask"; break;
898 case ImmTyDppBankMask: OS << "DppBankMask"; break;
899 case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
900 case ImmTyDppFI: OS << "DppFI"; break;
901 case ImmTySDWADstSel: OS << "SDWADstSel"; break;
902 case ImmTySDWASrc0Sel: OS << "SDWASrc0Sel"; break;
903 case ImmTySDWASrc1Sel: OS << "SDWASrc1Sel"; break;
904 case ImmTySDWADstUnused: OS << "SDWADstUnused"; break;
905 case ImmTyDMask: OS << "DMask"; break;
906 case ImmTyDim: OS << "Dim"; break;
907 case ImmTyUNorm: OS << "UNorm"; break;
908 case ImmTyDA: OS << "DA"; break;
909 case ImmTyR128A16: OS << "R128A16"; break;
910 case ImmTyA16: OS << "A16"; break;
911 case ImmTyLWE: OS << "LWE"; break;
912 case ImmTyOff: OS << "Off"; break;
913 case ImmTyExpTgt: OS << "ExpTgt"; break;
914 case ImmTyExpCompr: OS << "ExpCompr"; break;
915 case ImmTyExpVM: OS << "ExpVM"; break;
916 case ImmTyDone: OS << "Done"; break;
917 case ImmTyRowEn: OS << "RowEn"; break;
918 case ImmTyHwreg: OS << "Hwreg"; break;
919 case ImmTySendMsg: OS << "SendMsg"; break;
920 case ImmTyWaitEvent: OS << "WaitEvent"; break;
921 case ImmTyInterpSlot: OS << "InterpSlot"; break;
922 case ImmTyInterpAttr: OS << "InterpAttr"; break;
923 case ImmTyInterpAttrChan: OS << "InterpAttrChan"; break;
924 case ImmTyOpSel: OS << "OpSel"; break;
925 case ImmTyOpSelHi: OS << "OpSelHi"; break;
926 case ImmTyNegLo: OS << "NegLo"; break;
927 case ImmTyNegHi: OS << "NegHi"; break;
928 case ImmTySwizzle: OS << "Swizzle"; break;
929 case ImmTyGprIdxMode: OS << "GprIdxMode"; break;
930 case ImmTyHigh: OS << "High"; break;
931 case ImmTyBLGP: OS << "BLGP"; break;
932 case ImmTyCBSZ: OS << "CBSZ"; break;
933 case ImmTyABID: OS << "ABID"; break;
934 case ImmTyEndpgm: OS << "Endpgm"; break;
935 case ImmTyWaitVDST: OS << "WaitVDST"; break;
936 case ImmTyWaitEXP: OS << "WaitEXP"; break;
937 case ImmTyWaitVAVDst: OS << "WaitVAVDst"; break;
938 case ImmTyWaitVMVSrc: OS << "WaitVMVSrc"; break;
939 case ImmTyBitOp3: OS << "BitOp3"; break;
940 case ImmTyMatrixAFMT: OS << "ImmTyMatrixAFMT"; break;
941 case ImmTyMatrixBFMT: OS << "ImmTyMatrixBFMT"; break;
942 case ImmTyMatrixAScale: OS << "ImmTyMatrixAScale"; break;
943 case ImmTyMatrixBScale: OS << "ImmTyMatrixBScale"; break;
944 case ImmTyMatrixAScaleFmt: OS << "ImmTyMatrixAScaleFmt"; break;
945 case ImmTyMatrixBScaleFmt: OS << "ImmTyMatrixBScaleFmt"; break;
946 case ImmTyMatrixAReuse: OS << "ImmTyMatrixAReuse"; break;
947 case ImmTyMatrixBReuse: OS << "ImmTyMatrixBReuse"; break;
948 case ImmTyScaleSel: OS << "ScaleSel" ; break;
949 case ImmTyByteSel: OS << "ByteSel" ; break;
950 }
951 // clang-format on
952 }
953
954 void print(raw_ostream &OS, const MCAsmInfo &MAI) const override {
955 switch (Kind) {
956 case Register:
957 OS << "<register " << AMDGPUInstPrinter::getRegisterName(Reg: getReg())
958 << " mods: " << Reg.Mods << '>';
959 break;
960 case Immediate:
961 OS << '<' << getImm();
962 if (getImmTy() != ImmTyNone) {
963 OS << " type: ";
964 printImmTy(OS, Type: getImmTy());
965 }
966 OS << " mods: " << Imm.Mods << '>';
967 break;
968 case Token:
969 OS << '\'' << getToken() << '\'';
970 break;
971 case Expression:
972 OS << "<expr ";
973 MAI.printExpr(OS, *Expr);
974 OS << '>';
975 break;
976 }
977 }
978
979 static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
980 int64_t Val, SMLoc Loc,
981 ImmTy Type = ImmTyNone,
982 bool IsFPImm = false) {
983 auto Op = std::make_unique<AMDGPUOperand>(args: Immediate, args&: AsmParser);
984 Op->Imm.Val = Val;
985 Op->Imm.IsFPImm = IsFPImm;
986 Op->Imm.Type = Type;
987 Op->Imm.Mods = Modifiers();
988 Op->StartLoc = Loc;
989 Op->EndLoc = Loc;
990 return Op;
991 }
992
993 static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
994 StringRef Str, SMLoc Loc,
995 bool HasExplicitEncodingSize = true) {
996 auto Res = std::make_unique<AMDGPUOperand>(args: Token, args&: AsmParser);
997 Res->Tok.Data = Str.data();
998 Res->Tok.Length = Str.size();
999 Res->StartLoc = Loc;
1000 Res->EndLoc = Loc;
1001 return Res;
1002 }
1003
1004 static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
1005 MCRegister Reg, SMLoc S, SMLoc E) {
1006 auto Op = std::make_unique<AMDGPUOperand>(args: Register, args&: AsmParser);
1007 Op->Reg.RegNo = Reg;
1008 Op->Reg.Mods = Modifiers();
1009 Op->StartLoc = S;
1010 Op->EndLoc = E;
1011 return Op;
1012 }
1013
1014 static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
1015 const class MCExpr *Expr, SMLoc S) {
1016 auto Op = std::make_unique<AMDGPUOperand>(args: Expression, args&: AsmParser);
1017 Op->Expr = Expr;
1018 Op->StartLoc = S;
1019 Op->EndLoc = S;
1020 return Op;
1021 }
1022};
1023
1024raw_ostream &operator<<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
1025 OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
1026 return OS;
1027}
1028
1029//===----------------------------------------------------------------------===//
1030// AsmParser
1031//===----------------------------------------------------------------------===//
1032
1033// TODO: define GET_SUBTARGET_FEATURE_NAME
1034#define GET_REGISTER_MATCHER
1035#include "AMDGPUGenAsmMatcher.inc"
1036#undef GET_REGISTER_MATCHER
1037#undef GET_SUBTARGET_FEATURE_NAME
1038
1039// Holds info related to the current kernel, e.g. count of SGPRs used.
1040// Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
1041// .amdgpu_hsa_kernel or at EOF.
1042class KernelScopeInfo {
1043 int SgprIndexUnusedMin = -1;
1044 int VgprIndexUnusedMin = -1;
1045 int AgprIndexUnusedMin = -1;
1046 MCContext *Ctx = nullptr;
1047 MCSubtargetInfo const *MSTI = nullptr;
1048
1049 void usesSgprAt(int i) {
1050 if (i >= SgprIndexUnusedMin) {
1051 SgprIndexUnusedMin = ++i;
1052 if (Ctx) {
1053 MCSymbol *const Sym =
1054 Ctx->getOrCreateSymbol(Name: Twine(".kernel.sgpr_count"));
1055 Sym->setVariableValue(MCConstantExpr::create(Value: SgprIndexUnusedMin, Ctx&: *Ctx));
1056 }
1057 }
1058 }
1059
1060 void usesVgprAt(int i) {
1061 if (i >= VgprIndexUnusedMin) {
1062 VgprIndexUnusedMin = ++i;
1063 if (Ctx) {
1064 MCSymbol *const Sym =
1065 Ctx->getOrCreateSymbol(Name: Twine(".kernel.vgpr_count"));
1066 int totalVGPR =
1067 getTotalNumVGPRs(has90AInsts: isGFX90A(STI: *MSTI), ArgNumAGPR: std::max(a: AgprIndexUnusedMin, b: 0),
1068 ArgNumVGPR: VgprIndexUnusedMin);
1069 Sym->setVariableValue(MCConstantExpr::create(Value: totalVGPR, Ctx&: *Ctx));
1070 }
1071 }
1072 }
1073
1074 void usesAgprAt(int i) {
1075 // Instruction will error in AMDGPUAsmParser::matchAndEmitInstruction
1076 if (!hasMAIInsts(STI: *MSTI))
1077 return;
1078
1079 if (i >= AgprIndexUnusedMin) {
1080 AgprIndexUnusedMin = ++i;
1081 if (Ctx) {
1082 MCSymbol *const Sym =
1083 Ctx->getOrCreateSymbol(Name: Twine(".kernel.agpr_count"));
1084 Sym->setVariableValue(MCConstantExpr::create(Value: AgprIndexUnusedMin, Ctx&: *Ctx));
1085
1086 // Also update vgpr_count (dependent on agpr_count for gfx908/gfx90a)
1087 MCSymbol *const vSym =
1088 Ctx->getOrCreateSymbol(Name: Twine(".kernel.vgpr_count"));
1089 int totalVGPR =
1090 getTotalNumVGPRs(has90AInsts: isGFX90A(STI: *MSTI), ArgNumAGPR: std::max(a: AgprIndexUnusedMin, b: 0),
1091 ArgNumVGPR: VgprIndexUnusedMin);
1092 vSym->setVariableValue(MCConstantExpr::create(Value: totalVGPR, Ctx&: *Ctx));
1093 }
1094 }
1095 }
1096
1097public:
1098 KernelScopeInfo() = default;
1099
1100 void initialize(MCContext &Context) {
1101 Ctx = &Context;
1102 MSTI = Ctx->getSubtargetInfo();
1103
1104 usesSgprAt(i: SgprIndexUnusedMin = -1);
1105 usesVgprAt(i: VgprIndexUnusedMin = -1);
1106 if (hasMAIInsts(STI: *MSTI)) {
1107 usesAgprAt(i: AgprIndexUnusedMin = -1);
1108 }
1109 }
1110
1111 void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex,
1112 unsigned RegWidth) {
1113 switch (RegKind) {
1114 case IS_SGPR:
1115 usesSgprAt(i: DwordRegIndex + divideCeil(Numerator: RegWidth, Denominator: 32) - 1);
1116 break;
1117 case IS_AGPR:
1118 usesAgprAt(i: DwordRegIndex + divideCeil(Numerator: RegWidth, Denominator: 32) - 1);
1119 break;
1120 case IS_VGPR:
1121 usesVgprAt(i: DwordRegIndex + divideCeil(Numerator: RegWidth, Denominator: 32) - 1);
1122 break;
1123 default:
1124 break;
1125 }
1126 }
1127};
1128
1129class AMDGPUAsmParser : public MCTargetAsmParser {
1130 MCAsmParser &Parser;
1131
1132 unsigned ForcedEncodingSize = 0;
1133 bool ForcedDPP = false;
1134 bool ForcedSDWA = false;
1135 KernelScopeInfo KernelScope;
1136 const unsigned HwMode;
1137 const AMDGPU::GPUKind Gfx;
1138 const AMDGPU::IsaVersion ISA;
1139
1140 /// @name Auto-generated Match Functions
1141 /// {
1142
1143#define GET_ASSEMBLER_HEADER
1144#include "AMDGPUGenAsmMatcher.inc"
1145
1146 /// }
1147
1148 /// Get size of register operand
1149 unsigned getRegOperandSize(const MCInstrDesc &Desc, unsigned OpNo) const {
1150 assert(OpNo < Desc.NumOperands);
1151 int16_t RCID = MII.getOpRegClassID(OpInfo: Desc.operands()[OpNo], HwModeId: HwMode);
1152 return getRegBitWidth(RCID) / 8;
1153 }
1154
1155 std::optional<AMDGPU::InfoSectionData> InfoData;
1156
1157 /// State for checking that every kernel named in a .amdhsa_kernel directive
1158 /// begins with the required prologue instruction sequence. Because the
1159 /// directive may appear either before or after the kernel's label (it is
1160 /// normally emitted after the function body, in .rodata), validation is
1161 /// deferred to onEndOfFile(). We record an order-independent timeline of
1162 /// parsed labels and emitted instruction opcodes, plus the set of symbols
1163 /// named by .amdhsa_kernel directives, and match them up at end of file.
1164 SmallVector<unsigned> OpcodeStream;
1165 SmallVector<std::tuple<const MCSymbol *, SMLoc, unsigned>>
1166 OpcodeStreamSymbols;
1167 SmallPtrSet<const MCSymbol *, 8> AMDHSAKernelSymbols;
1168
1169 /// Verify recorded kernel prologues.
1170 void checkKernelPrologues();
1171
1172private:
1173 void createConstantSymbol(StringRef Id, int64_t Val);
1174
1175 bool ParseAsAbsoluteExpression(uint32_t &Ret);
1176 bool OutOfRangeError(SMRange Range);
1177 /// Calculate VGPR/SGPR blocks required for given target, reserved
1178 /// registers, and user-specified NextFreeXGPR values.
1179 ///
1180 /// \param Features [in] Target features, used for bug corrections.
1181 /// \param VCCUsed [in] Whether VCC special SGPR is reserved.
1182 /// \param FlatScrUsed [in] Whether FLAT_SCRATCH special SGPR is reserved.
1183 /// \param XNACKUsed [in] Whether XNACK_MASK special SGPR is reserved.
1184 /// \param EnableWavefrontSize32 [in] Value of ENABLE_WAVEFRONT_SIZE32 kernel
1185 /// descriptor field, if valid.
1186 /// \param NextFreeVGPR [in] Max VGPR number referenced, plus one.
1187 /// \param VGPRRange [in] Token range, used for VGPR diagnostics.
1188 /// \param NextFreeSGPR [in] Max SGPR number referenced, plus one.
1189 /// \param SGPRRange [in] Token range, used for SGPR diagnostics.
1190 /// \param VGPRBlocks [out] Result VGPR block count.
1191 /// \param SGPRBlocks [out] Result SGPR block count.
1192 bool calculateGPRBlocks(const FeatureBitset &Features, const MCExpr *VCCUsed,
1193 const MCExpr *FlatScrUsed, bool XNACKUsed,
1194 std::optional<bool> EnableWavefrontSize32,
1195 const MCExpr *NextFreeVGPR, SMRange VGPRRange,
1196 const MCExpr *NextFreeSGPR, SMRange SGPRRange,
1197 const MCExpr *&VGPRBlocks, const MCExpr *&SGPRBlocks);
1198 bool ParseDirectiveAMDGCNTarget();
1199 bool ParseDirectiveAMDHSACodeObjectVersion();
1200 bool ParseDirectiveAMDHSAKernel();
1201 bool ParseAMDKernelCodeTValue(StringRef ID, AMDGPUMCKernelCodeT &Header);
1202 bool ParseDirectiveAMDKernelCodeT();
1203 // TODO: Possibly make subtargetHasRegister const.
1204 bool subtargetHasRegister(const MCRegisterInfo &MRI, MCRegister Reg);
1205 bool ParseDirectiveAMDGPUHsaKernel();
1206
1207 bool ParseDirectiveISAVersion();
1208 bool ParseDirectiveHSAMetadata();
1209 bool ParseDirectivePALMetadataBegin();
1210 bool ParseDirectivePALMetadata();
1211 bool ParseDirectiveAMDGPULDS();
1212 bool ParseDirectiveAMDGPUInfo();
1213
1214 /// Common code to parse out a block of text (typically YAML) between start
1215 /// and end directives.
1216 bool ParseToEndDirective(const char *AssemblerDirectiveBegin,
1217 const char *AssemblerDirectiveEnd,
1218 std::string &CollectString);
1219
1220 bool AddNextRegisterToList(MCRegister &Reg, unsigned &RegWidth,
1221 RegisterKind RegKind, MCRegister Reg1,
1222 RegisterKind RegKind1, SMLoc Loc);
1223 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &Reg,
1224 unsigned &RegNum, unsigned &RegWidth,
1225 bool RestoreOnFailure = false);
1226 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &Reg,
1227 unsigned &RegNum, unsigned &RegWidth,
1228 SmallVectorImpl<AsmToken> &Tokens);
1229 MCRegister ParseRegularReg(RegisterKind &RegKind, unsigned &RegNum,
1230 unsigned &RegWidth,
1231 SmallVectorImpl<AsmToken> &Tokens);
1232 MCRegister ParseSpecialReg(RegisterKind &RegKind, unsigned &RegNum,
1233 unsigned &RegWidth,
1234 SmallVectorImpl<AsmToken> &Tokens);
1235 MCRegister ParseRegList(RegisterKind &RegKind, unsigned &RegNum,
1236 unsigned &RegWidth,
1237 SmallVectorImpl<AsmToken> &Tokens);
1238 bool ParseRegRange(unsigned &Num, unsigned &Width, unsigned &SubReg);
1239 MCRegister getRegularReg(RegisterKind RegKind, unsigned RegNum,
1240 unsigned SubReg, unsigned RegWidth, SMLoc Loc);
1241
1242 bool isRegister();
1243 bool isRegister(const AsmToken &Token, const AsmToken &NextToken) const;
1244 std::optional<StringRef> getGprCountSymbolName(RegisterKind RegKind);
1245 void initializeGprCountSymbol(RegisterKind RegKind);
1246 bool updateGprCountSymbols(RegisterKind RegKind, unsigned DwordRegIndex,
1247 unsigned RegWidth);
1248 void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands, bool IsAtomic);
1249
1250public:
1251 enum OperandMode {
1252 OperandMode_Default,
1253 OperandMode_NSA,
1254 };
1255
1256 using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
1257
1258 AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
1259 const MCInstrInfo &MII)
1260 : MCTargetAsmParser(STI, MII), Parser(_Parser),
1261 HwMode(STI.getHwMode(type: MCSubtargetInfo::HwMode_RegInfo)),
1262 Gfx(AMDGPU::parseArchAMDGCN(CPU: STI.getCPU())),
1263 ISA(AMDGPU::getIsaVersion(GPU: STI.getCPU())) {
1264 MCAsmParserExtension::Initialize(Parser);
1265
1266 setAvailableFeatures(ComputeAvailableFeatures(FB: getFeatureBits()));
1267
1268 if (ISA.Major >= 6 && isHsaAbi(STI: getSTI())) {
1269 createConstantSymbol(Id: ".amdgcn.gfx_generation_number", Val: ISA.Major);
1270 createConstantSymbol(Id: ".amdgcn.gfx_generation_minor", Val: ISA.Minor);
1271 createConstantSymbol(Id: ".amdgcn.gfx_generation_stepping", Val: ISA.Stepping);
1272 } else {
1273 createConstantSymbol(Id: ".option.machine_version_major", Val: ISA.Major);
1274 createConstantSymbol(Id: ".option.machine_version_minor", Val: ISA.Minor);
1275 createConstantSymbol(Id: ".option.machine_version_stepping", Val: ISA.Stepping);
1276 }
1277 if (ISA.Major >= 6 && isHsaAbi(STI: getSTI())) {
1278 initializeGprCountSymbol(RegKind: IS_VGPR);
1279 initializeGprCountSymbol(RegKind: IS_SGPR);
1280 } else
1281 KernelScope.initialize(Context&: getContext());
1282
1283 for (auto [Symbol, Code] : AMDGPU::UCVersion::getGFXVersions())
1284 createConstantSymbol(Id: Symbol, Val: Code);
1285
1286 createConstantSymbol(Id: "UC_VERSION_W64_BIT", Val: 0x2000);
1287 createConstantSymbol(Id: "UC_VERSION_W32_BIT", Val: 0x4000);
1288 createConstantSymbol(Id: "UC_VERSION_MDP_BIT", Val: 0x8000);
1289 }
1290
1291 bool hasMIMG_R128() const { return AMDGPU::hasMIMG_R128(STI: getSTI()); }
1292
1293 bool hasPackedD16() const { return AMDGPU::hasPackedD16(STI: getSTI()); }
1294
1295 bool hasA16() const { return AMDGPU::hasA16(STI: getSTI()); }
1296
1297 bool hasG16() const { return AMDGPU::hasG16(STI: getSTI()); }
1298
1299 bool hasGDS() const { return AMDGPU::hasGDS(STI: getSTI()); }
1300
1301 bool isSI() const { return AMDGPU::isSI(STI: getSTI()); }
1302
1303 bool isCI() const { return AMDGPU::isCI(STI: getSTI()); }
1304
1305 bool isVI() const { return AMDGPU::isVI(STI: getSTI()); }
1306
1307 bool isGFX9() const { return AMDGPU::isGFX9(STI: getSTI()); }
1308
1309 // TODO: isGFX90A is also true for GFX940. We need to clean it.
1310 bool isGFX90A() const { return AMDGPU::isGFX90A(STI: getSTI()); }
1311
1312 bool isGFX940() const { return AMDGPU::isGFX940(STI: getSTI()); }
1313
1314 bool isGFX9Plus() const { return AMDGPU::isGFX9Plus(STI: getSTI()); }
1315
1316 bool isGFX10Plus() const { return AMDGPU::isGFX10Plus(STI: getSTI()); }
1317
1318 bool isGFX11() const { return AMDGPU::isGFX11(STI: getSTI()); }
1319
1320 bool isGFX11Plus() const { return AMDGPU::isGFX11Plus(STI: getSTI()); }
1321
1322 bool isGFX12() const { return AMDGPU::isGFX12(STI: getSTI()); }
1323
1324 bool isGFX12Plus() const { return AMDGPU::isGFX12Plus(STI: getSTI()); }
1325
1326 bool isGFX1250Plus() const { return AMDGPU::isGFX1250Plus(STI: getSTI()); }
1327
1328 bool hasBVHRayTracingInsts() const {
1329 return getFeatureBits()[AMDGPU::FeatureBVHRayTracingInsts];
1330 }
1331
1332 bool isWave32() const { return getAvailableFeatures()[Feature_isWave32Bit]; }
1333
1334 bool isWave64() const { return getAvailableFeatures()[Feature_isWave64Bit]; }
1335
1336 bool hasInv2PiInlineImm() const {
1337 return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
1338 }
1339
1340 bool has64BitLiterals() const {
1341 return getFeatureBits()[AMDGPU::Feature64BitLiterals];
1342 }
1343
1344 bool hasFlatOffsets() const {
1345 return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
1346 }
1347
1348 bool hasTrue16Insts() const {
1349 return getFeatureBits()[AMDGPU::FeatureTrue16BitInsts];
1350 }
1351
1352 bool hasArchitectedFlatScratch() const {
1353 return getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch];
1354 }
1355
1356 bool hasSGPR102_SGPR103() const { return !isVI() && !isGFX9(); }
1357
1358 bool hasSGPR104_SGPR105() const { return isGFX10Plus(); }
1359
1360 bool hasIntClamp() const { return getFeatureBits()[AMDGPU::FeatureIntClamp]; }
1361
1362 bool hasPartialNSAEncoding() const {
1363 return getFeatureBits()[AMDGPU::FeaturePartialNSAEncoding];
1364 }
1365
1366 bool hasGloballyAddressableScratch() const {
1367 return getFeatureBits()[AMDGPU::FeatureGloballyAddressableScratch];
1368 }
1369
1370 bool hasScaleOffset() const {
1371 return getFeatureBits()[AMDGPU::FeatureScaleOffset];
1372 }
1373
1374 bool hasCacheFillSize() const {
1375 return getFeatureBits()[AMDGPU::FeatureCacheFillSize];
1376 }
1377
1378 unsigned getNSAMaxSize(bool HasSampler = false) const {
1379 return AMDGPU::getNSAMaxSize(STI: getSTI(), HasSampler);
1380 }
1381
1382 unsigned getMaxNumUserSGPRs() const {
1383 return AMDGPU::getMaxNumUserSGPRs(STI: getSTI());
1384 }
1385
1386 bool hasKernargPreload() const { return AMDGPU::hasKernargPreload(STI: getSTI()); }
1387
1388 AMDGPUTargetStreamer &getTargetStreamer() {
1389 MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
1390 return static_cast<AMDGPUTargetStreamer &>(TS);
1391 }
1392
1393 MCContext &getContext() const {
1394 // We need this const_cast because for some reason getContext() is not const
1395 // in MCAsmParser.
1396 return const_cast<AMDGPUAsmParser *>(this)->MCTargetAsmParser::getContext();
1397 }
1398
1399 const MCRegisterInfo *getMRI() const {
1400 return getContext().getRegisterInfo();
1401 }
1402
1403 const MCInstrInfo *getMII() const { return &MII; }
1404
1405 // Resolve a RegClassByHwModeUses index to a register class id for the active
1406 // HwMode; -1 if the mode has no entry.
1407 int16_t getTargetRegClass(unsigned TargetRCIdx) const {
1408 return MII.getRegClassByHwModeTable(ModeId: HwMode)[TargetRCIdx];
1409 }
1410
1411 // FIXME: This should not be used. Instead, should use queries derived from
1412 // getAvailableFeatures().
1413 const FeatureBitset &getFeatureBits() const {
1414 return getSTI().getFeatureBits();
1415 }
1416
1417 void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
1418 void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
1419 void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
1420
1421 unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
1422 bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
1423 bool isForcedDPP() const { return ForcedDPP; }
1424 bool isForcedSDWA() const { return ForcedSDWA; }
1425 ArrayRef<unsigned> getMatchedVariants() const;
1426 StringRef getMatchedVariantName() const;
1427
1428 std::unique_ptr<AMDGPUOperand> parseRegister(bool RestoreOnFailure = false);
1429 bool ParseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
1430 bool RestoreOnFailure);
1431 bool parseRegister(MCRegister &Reg, SMLoc &StartLoc, SMLoc &EndLoc) override;
1432 ParseStatus tryParseRegister(MCRegister &Reg, SMLoc &StartLoc,
1433 SMLoc &EndLoc) override;
1434 unsigned checkTargetMatchPredicate(MCInst &Inst) override;
1435 unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
1436 unsigned Kind) override;
1437 bool matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1438 OperandVector &Operands, MCStreamer &Out,
1439 uint64_t &ErrorInfo,
1440 bool MatchingInlineAsm) override;
1441 bool ParseDirective(AsmToken DirectiveID) override;
1442 void doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) override;
1443 void onEndOfFile() override;
1444 ParseStatus parseOperand(OperandVector &Operands, StringRef Mnemonic,
1445 OperandMode Mode = OperandMode_Default);
1446 StringRef parseMnemonicSuffix(StringRef Name);
1447 bool parseInstruction(ParseInstructionInfo &Info, StringRef Name,
1448 SMLoc NameLoc, OperandVector &Operands) override;
1449 // bool ProcessInstruction(MCInst &Inst);
1450
1451 ParseStatus parseTokenOp(StringRef Name, OperandVector &Operands);
1452
1453 ParseStatus parseIntWithPrefix(const char *Prefix, int64_t &Int);
1454
1455 ParseStatus
1456 parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1457 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1458 std::function<bool(int64_t &)> ConvertResult = nullptr);
1459
1460 ParseStatus parseOperandArrayWithPrefix(
1461 const char *Prefix, OperandVector &Operands,
1462 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1463 bool (*ConvertResult)(int64_t &) = nullptr);
1464
1465 ParseStatus
1466 parseNamedBit(StringRef Name, OperandVector &Operands,
1467 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1468 bool IgnoreNegative = false);
1469 unsigned getCPolKind(StringRef Id, StringRef Mnemo, bool &Disabling) const;
1470 ParseStatus parseCPol(OperandVector &Operands);
1471 ParseStatus parseScope(OperandVector &Operands, int64_t &Scope);
1472 ParseStatus parseTH(OperandVector &Operands, int64_t &TH);
1473 ParseStatus parseCFS(OperandVector &Operands, int64_t &CFS);
1474 ParseStatus parseStringWithPrefix(StringRef Prefix, StringRef &Value,
1475 SMLoc &StringLoc);
1476 ParseStatus parseStringOrIntWithPrefix(OperandVector &Operands,
1477 StringRef Name,
1478 ArrayRef<const char *> Ids,
1479 int64_t &IntVal);
1480 ParseStatus parseStringOrIntWithPrefix(OperandVector &Operands,
1481 StringRef Name,
1482 ArrayRef<const char *> Ids,
1483 AMDGPUOperand::ImmTy Type);
1484
1485 bool isModifier();
1486 bool isOperandModifier(const AsmToken &Token,
1487 const AsmToken &NextToken) const;
1488 bool isRegOrOperandModifier(const AsmToken &Token,
1489 const AsmToken &NextToken) const;
1490 bool isNamedOperandModifier(const AsmToken &Token,
1491 const AsmToken &NextToken) const;
1492 bool isOpcodeModifierWithVal(const AsmToken &Token,
1493 const AsmToken &NextToken) const;
1494 bool parseSP3NegModifier();
1495 ParseStatus parseImm(OperandVector &Operands, bool HasSP3AbsModifier = false,
1496 LitModifier Lit = LitModifier::None);
1497 ParseStatus parseReg(OperandVector &Operands);
1498 ParseStatus parseRegOrImm(OperandVector &Operands, bool HasSP3AbsMod = false,
1499 LitModifier Lit = LitModifier::None);
1500 ParseStatus parseRegOrImmWithFPInputMods(OperandVector &Operands,
1501 bool AllowImm = true);
1502 ParseStatus parseRegOrImmWithIntInputMods(OperandVector &Operands,
1503 bool AllowImm = true);
1504 ParseStatus parseRegWithFPInputMods(OperandVector &Operands);
1505 ParseStatus parseRegWithIntInputMods(OperandVector &Operands);
1506 ParseStatus parseRsrcReg(OperandVector &Operands);
1507 ParseStatus parseVReg32OrOff(OperandVector &Operands);
1508 ParseStatus tryParseIndexKey(OperandVector &Operands,
1509 AMDGPUOperand::ImmTy ImmTy);
1510 ParseStatus parseIndexKey8bit(OperandVector &Operands);
1511 ParseStatus parseIndexKey16bit(OperandVector &Operands);
1512 ParseStatus parseIndexKey32bit(OperandVector &Operands);
1513 ParseStatus tryParseMatrixFMT(OperandVector &Operands, StringRef Name,
1514 AMDGPUOperand::ImmTy Type);
1515 ParseStatus parseMatrixAFMT(OperandVector &Operands);
1516 ParseStatus parseMatrixBFMT(OperandVector &Operands);
1517 ParseStatus tryParseMatrixScale(OperandVector &Operands, StringRef Name,
1518 AMDGPUOperand::ImmTy Type);
1519 ParseStatus parseMatrixAScale(OperandVector &Operands);
1520 ParseStatus parseMatrixBScale(OperandVector &Operands);
1521 ParseStatus tryParseMatrixScaleFmt(OperandVector &Operands, StringRef Name,
1522 AMDGPUOperand::ImmTy Type);
1523 ParseStatus parseMatrixAScaleFmt(OperandVector &Operands);
1524 ParseStatus parseMatrixBScaleFmt(OperandVector &Operands);
1525
1526 ParseStatus parseDfmtNfmt(int64_t &Format);
1527 ParseStatus parseUfmt(int64_t &Format);
1528 ParseStatus parseSymbolicSplitFormat(StringRef FormatStr, SMLoc Loc,
1529 int64_t &Format);
1530 ParseStatus parseSymbolicUnifiedFormat(StringRef FormatStr, SMLoc Loc,
1531 int64_t &Format);
1532 ParseStatus parseFormat(OperandVector &Operands);
1533 ParseStatus parseFormatOp(OperandVector &Operands);
1534 ParseStatus parseSymbolicOrNumericFormat(int64_t &Format);
1535 ParseStatus parseNumericFormat(int64_t &Format);
1536 ParseStatus parseFlatOffset(OperandVector &Operands);
1537 ParseStatus parseR128A16(OperandVector &Operands);
1538 ParseStatus parseBLGP(OperandVector &Operands);
1539 ParseStatus tryParseFmt(const char *Pref, int64_t MaxVal, int64_t &Fmt);
1540 bool matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt, StringRef FormatStr,
1541 SMLoc Loc);
1542
1543 void cvtExp(MCInst &Inst, const OperandVector &Operands);
1544
1545 bool parseCnt(int64_t &IntVal);
1546 ParseStatus parseSWaitCnt(OperandVector &Operands);
1547
1548 bool parseDepCtr(int64_t &IntVal, unsigned &Mask);
1549 void depCtrError(SMLoc Loc, int ErrorId, StringRef DepCtrName);
1550 ParseStatus parseDepCtr(OperandVector &Operands);
1551
1552 bool parseDelay(int64_t &Delay);
1553 ParseStatus parseSDelayALU(OperandVector &Operands);
1554
1555 ParseStatus parseHwreg(OperandVector &Operands);
1556
1557private:
1558 struct OperandInfoTy {
1559 SMLoc Loc;
1560 int64_t Val;
1561 bool IsSymbolic = false;
1562 bool IsDefined = false;
1563
1564 constexpr OperandInfoTy(int64_t Val) : Val(Val) {}
1565 };
1566
1567 struct StructuredOpField : OperandInfoTy {
1568 StringLiteral Id;
1569 StringLiteral Desc;
1570 unsigned Width;
1571 bool IsDefined = false;
1572
1573 constexpr StructuredOpField(StringLiteral Id, StringLiteral Desc,
1574 unsigned Width, int64_t Default)
1575 : OperandInfoTy(Default), Id(Id), Desc(Desc), Width(Width) {}
1576 virtual ~StructuredOpField() = default;
1577
1578 bool Error(AMDGPUAsmParser &Parser, const Twine &Err) const {
1579 Parser.Error(L: Loc, Msg: "invalid " + Desc + ": " + Err);
1580 return false;
1581 }
1582
1583 virtual bool validate(AMDGPUAsmParser &Parser) const {
1584 if (IsSymbolic && Val == OPR_ID_UNSUPPORTED)
1585 return Error(Parser, Err: "not supported on this GPU");
1586 if (!isUIntN(N: Width, x: Val))
1587 return Error(Parser, Err: "only " + Twine(Width) + "-bit values are legal");
1588 return true;
1589 }
1590 };
1591
1592 ParseStatus parseStructuredOpFields(ArrayRef<StructuredOpField *> Fields);
1593 bool validateStructuredOpFields(ArrayRef<const StructuredOpField *> Fields);
1594
1595 bool parseSendMsgBody(OperandInfoTy &Msg, OperandInfoTy &Op,
1596 OperandInfoTy &Stream);
1597 bool validateSendMsg(const OperandInfoTy &Msg, const OperandInfoTy &Op,
1598 const OperandInfoTy &Stream);
1599
1600 ParseStatus parseHwregFunc(OperandInfoTy &HwReg, OperandInfoTy &Offset,
1601 OperandInfoTy &Width);
1602
1603 const AMDGPUOperand &findMCOperand(const OperandVector &Operands,
1604 int MCOpIdx) const;
1605
1606 static SMLoc getLaterLoc(SMLoc a, SMLoc b);
1607
1608 SMLoc getFlatOffsetLoc(const OperandVector &Operands) const;
1609 SMLoc getSMEMOffsetLoc(const OperandVector &Operands) const;
1610 SMLoc getBLGPLoc(const OperandVector &Operands) const;
1611
1612 SMLoc getOperandLoc(const OperandVector &Operands, int MCOpIdx) const;
1613 SMLoc getOperandLoc(std::function<bool(const AMDGPUOperand &)> Test,
1614 const OperandVector &Operands) const;
1615 SMLoc getImmLoc(AMDGPUOperand::ImmTy Type,
1616 const OperandVector &Operands) const;
1617 SMLoc getInstLoc(const OperandVector &Operands) const;
1618
1619 bool validateInstruction(const MCInst &Inst, SMLoc IDLoc,
1620 const OperandVector &Operands);
1621 bool validateOffset(const MCInst &Inst, const OperandVector &Operands);
1622 bool validateFlatOffset(const MCInst &Inst, const OperandVector &Operands);
1623 bool validateSMEMOffset(const MCInst &Inst, const OperandVector &Operands);
1624 bool validateSMEMSData(const MCInst &Inst, const OperandVector &Operands);
1625 bool validateBF16InlineConst(const MCInst &Inst,
1626 const OperandVector &Operands);
1627 bool validateSOPLiteral(const MCInst &Inst, const OperandVector &Operands);
1628 bool validateConstantBusLimitations(const MCInst &Inst,
1629 const OperandVector &Operands);
1630 std::optional<unsigned> checkVOPDRegBankConstraints(const MCInst &Inst,
1631 bool AsVOPD3);
1632 bool validateVOPD(const MCInst &Inst, const OperandVector &Operands);
1633 bool tryVOPD(const MCInst &Inst);
1634 bool tryVOPD3(const MCInst &Inst);
1635 bool tryAnotherVOPDEncoding(const MCInst &Inst);
1636
1637 bool validateIntClampSupported(const MCInst &Inst);
1638 bool validateMIMGAtomicDMask(const MCInst &Inst);
1639 bool validateMIMGGatherDMask(const MCInst &Inst);
1640 bool validateMovrels(const MCInst &Inst, const OperandVector &Operands);
1641 bool validateMIMGDataSize(const MCInst &Inst, SMLoc IDLoc);
1642 bool validateMIMGAddrSize(const MCInst &Inst, SMLoc IDLoc);
1643 bool validateMIMGD16(const MCInst &Inst);
1644 bool validateMIMGDim(const MCInst &Inst, const OperandVector &Operands);
1645 bool validateTensorR128(const MCInst &Inst);
1646 bool validateMIMGMSAA(const MCInst &Inst);
1647 bool validateOpSel(const MCInst &Inst);
1648 bool validateTrue16OpSel(const MCInst &Inst);
1649 bool validateNeg(const MCInst &Inst, AMDGPU::OpName OpName);
1650 bool validateDPP(const MCInst &Inst, const OperandVector &Operands);
1651 bool validateVccOperand(MCRegister Reg) const;
1652 bool validateVOPLiteral(const MCInst &Inst, const OperandVector &Operands);
1653 bool validateMAIAccWrite(const MCInst &Inst, const OperandVector &Operands);
1654 bool validateMAISrc2(const MCInst &Inst, const OperandVector &Operands);
1655 bool validateMFMA(const MCInst &Inst, const OperandVector &Operands);
1656 bool validateAGPRLdSt(const MCInst &Inst) const;
1657 bool validateVGPRAlign(const MCInst &Inst) const;
1658 bool validateBLGP(const MCInst &Inst, const OperandVector &Operands);
1659 bool validateDS(const MCInst &Inst, const OperandVector &Operands);
1660 bool validateGWS(const MCInst &Inst, const OperandVector &Operands);
1661 bool validateDivScale(const MCInst &Inst);
1662 bool validateWaitCnt(const MCInst &Inst, const OperandVector &Operands);
1663 bool validateCoherencyBits(const MCInst &Inst, const OperandVector &Operands,
1664 SMLoc IDLoc);
1665 bool validateTHAndScopeBits(const MCInst &Inst, const OperandVector &Operands,
1666 const unsigned CPol);
1667 bool validateCFSBits(const MCInst &Inst, const OperandVector &Operands);
1668 bool validateTFE(const MCInst &Inst, const OperandVector &Operands);
1669 bool validateLdsDirect(const MCInst &Inst, const OperandVector &Operands);
1670 bool validateWMMA(const MCInst &Inst, const OperandVector &Operands);
1671 bool validateMonitorSleep(const MCInst &Inst, const OperandVector &Operands);
1672 bool validateClusterBarrierIsFirst(const MCInst &Inst,
1673 const OperandVector &Operands);
1674 bool validateScaleSel(const MCInst &Inst, const OperandVector &Operands);
1675 unsigned getConstantBusLimit(unsigned Opcode) const;
1676 bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
1677 bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
1678 MCRegister findImplicitSGPRReadInVOP(const MCInst &Inst) const;
1679
1680 bool isSupportedMnemo(StringRef Mnemo, const FeatureBitset &FBS);
1681 bool isSupportedMnemo(StringRef Mnemo, const FeatureBitset &FBS,
1682 ArrayRef<unsigned> Variants);
1683 bool checkUnsupportedInstruction(StringRef Name, SMLoc IDLoc);
1684
1685 bool isId(const StringRef Id) const;
1686 bool isId(const AsmToken &Token, const StringRef Id) const;
1687 bool isToken(const AsmToken::TokenKind Kind) const;
1688 StringRef getId() const;
1689 bool trySkipId(const StringRef Id);
1690 bool trySkipId(const StringRef Pref, const StringRef Id);
1691 bool trySkipId(const StringRef Id, const AsmToken::TokenKind Kind);
1692 bool trySkipToken(const AsmToken::TokenKind Kind);
1693 bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
1694 bool parseString(StringRef &Val,
1695 const StringRef ErrMsg = "expected a string");
1696 bool parseId(StringRef &Val, const StringRef ErrMsg = "");
1697
1698 void peekTokens(MutableArrayRef<AsmToken> Tokens);
1699 AsmToken::TokenKind getTokenKind() const;
1700 bool parseExpr(int64_t &Imm, StringRef Expected = "");
1701 bool parseExpr(OperandVector &Operands);
1702 StringRef getTokenStr() const;
1703 AsmToken peekToken(bool ShouldSkipSpace = true);
1704 AsmToken getToken() const;
1705 SMLoc getLoc() const;
1706 void lex();
1707
1708public:
1709 void onBeginOfFile() override;
1710 /// Emit the deferred leading .amdgcn_target directive if it has not been
1711 /// emitted yet. Called before emitting the first instruction or kernel
1712 /// descriptor, so that any leading .amdgcn_target/.amd_amdgpu_isa directive
1713 /// in the source has had a chance to update the target ID first.
1714 void emitTargetDirective();
1715 bool parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) override;
1716
1717 ParseStatus parseCustomOperand(OperandVector &Operands, unsigned MCK);
1718
1719 ParseStatus parseExpTgt(OperandVector &Operands);
1720 ParseStatus parseSendMsg(OperandVector &Operands);
1721 ParseStatus parseWaitEvent(OperandVector &Operands);
1722 ParseStatus parseInterpSlot(OperandVector &Operands);
1723 ParseStatus parseInterpAttr(OperandVector &Operands);
1724 ParseStatus parseSOPPBrTarget(OperandVector &Operands);
1725 ParseStatus parseBoolReg(OperandVector &Operands);
1726
1727 bool parseSwizzleOperand(int64_t &Op, const unsigned MinVal,
1728 const unsigned MaxVal, const Twine &ErrMsg,
1729 SMLoc &Loc);
1730 bool parseSwizzleOperands(const unsigned OpNum, int64_t *Op,
1731 const unsigned MinVal, const unsigned MaxVal,
1732 const StringRef ErrMsg);
1733 ParseStatus parseSwizzle(OperandVector &Operands);
1734 bool parseSwizzleOffset(int64_t &Imm);
1735 bool parseSwizzleMacro(int64_t &Imm);
1736 bool parseSwizzleQuadPerm(int64_t &Imm);
1737 bool parseSwizzleBitmaskPerm(int64_t &Imm);
1738 bool parseSwizzleBroadcast(int64_t &Imm);
1739 bool parseSwizzleSwap(int64_t &Imm);
1740 bool parseSwizzleReverse(int64_t &Imm);
1741 bool parseSwizzleFFT(int64_t &Imm);
1742 bool parseSwizzleRotate(int64_t &Imm);
1743
1744 ParseStatus parseGPRIdxMode(OperandVector &Operands);
1745 int64_t parseGPRIdxMacro();
1746
1747 void cvtMubuf(MCInst &Inst, const OperandVector &Operands) {
1748 cvtMubufImpl(Inst, Operands, IsAtomic: false);
1749 }
1750 void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) {
1751 cvtMubufImpl(Inst, Operands, IsAtomic: true);
1752 }
1753
1754 ParseStatus parseOModSI(OperandVector &Operands);
1755
1756 void cvtVOP3(MCInst &Inst, const OperandVector &Operands,
1757 OptionalImmIndexMap &OptionalIdx);
1758 void cvtScaledMFMA(MCInst &Inst, const OperandVector &Operands);
1759 void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands);
1760 void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1761 void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1762 void cvtSWMMAC(MCInst &Inst, const OperandVector &Operands);
1763
1764 void cvtVOPD(MCInst &Inst, const OperandVector &Operands);
1765 void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands,
1766 OptionalImmIndexMap &OptionalIdx);
1767 void cvtVOP3P(MCInst &Inst, const OperandVector &Operands,
1768 OptionalImmIndexMap &OptionalIdx);
1769
1770 void cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands);
1771 void cvtVINTERP(MCInst &Inst, const OperandVector &Operands);
1772 void cvtOpSelHelper(MCInst &Inst, unsigned OpSel);
1773
1774 bool parseDimId(unsigned &Encoding);
1775 ParseStatus parseDim(OperandVector &Operands);
1776 bool convertDppBoundCtrl(int64_t &BoundCtrl);
1777 ParseStatus parseDPP8(OperandVector &Operands);
1778 ParseStatus parseDPPCtrl(OperandVector &Operands);
1779 bool isSupportedDPPCtrl(StringRef Ctrl, const OperandVector &Operands);
1780 int64_t parseDPPCtrlSel(StringRef Ctrl);
1781 int64_t parseDPPCtrlPerm();
1782 void cvtDPP(MCInst &Inst, const OperandVector &Operands, bool IsDPP8 = false);
1783 void cvtDPP8(MCInst &Inst, const OperandVector &Operands) {
1784 cvtDPP(Inst, Operands, IsDPP8: true);
1785 }
1786 void cvtVOP3DPP(MCInst &Inst, const OperandVector &Operands,
1787 bool IsDPP8 = false);
1788 void cvtVOP3DPP8(MCInst &Inst, const OperandVector &Operands) {
1789 cvtVOP3DPP(Inst, Operands, IsDPP8: true);
1790 }
1791
1792 ParseStatus parseSDWASel(OperandVector &Operands, StringRef Prefix,
1793 AMDGPUOperand::ImmTy Type);
1794 ParseStatus parseSDWADstUnused(OperandVector &Operands);
1795 void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
1796 void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
1797 void cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands);
1798 void cvtSdwaVOP2e(MCInst &Inst, const OperandVector &Operands);
1799 void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
1800
1801 enum class SDWAInstType : unsigned { VOP1 = 0, VOP2 = 1, VOPC = 2 };
1802
1803 void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
1804 SDWAInstType BasicInstType, bool SkipDstVcc = false,
1805 bool SkipSrcVcc = false);
1806
1807 ParseStatus parseEndpgm(OperandVector &Operands);
1808
1809 ParseStatus parseVOPD(OperandVector &Operands);
1810};
1811
1812} // end anonymous namespace
1813
1814// May be called with integer type with equivalent bitwidth.
1815static const fltSemantics *getFltSemantics(unsigned Size) {
1816 switch (Size) {
1817 case 4:
1818 return &APFloat::IEEEsingle();
1819 case 8:
1820 return &APFloat::IEEEdouble();
1821 case 2:
1822 return &APFloat::IEEEhalf();
1823 default:
1824 llvm_unreachable("unsupported fp type");
1825 }
1826}
1827
1828static const fltSemantics *getFltSemantics(MVT VT) {
1829 return getFltSemantics(Size: VT.getScalarSizeInBits() / 8);
1830}
1831
1832static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
1833 switch (OperandType) {
1834 // When floating-point immediate is used as operand of type i16, the 32-bit
1835 // representation of the constant truncated to the 16 LSBs should be used.
1836 case AMDGPU::OPERAND_REG_IMM_INT16:
1837 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1838 case AMDGPU::OPERAND_REG_IMM_INT32:
1839 case AMDGPU::OPERAND_REG_IMM_FP32:
1840 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1841 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1842 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
1843 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
1844 case AMDGPU::OPERAND_REG_IMM_V2FP32:
1845 case AMDGPU::OPERAND_REG_IMM_V2INT32:
1846 case AMDGPU::OPERAND_REG_IMM_V2INT16:
1847 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1848 case AMDGPU::OPERAND_KIMM32:
1849 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
1850 return &APFloat::IEEEsingle();
1851 case AMDGPU::OPERAND_REG_IMM_INT64:
1852 case AMDGPU::OPERAND_REG_IMM_FP64:
1853 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1854 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1855 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
1856 case AMDGPU::OPERAND_REG_IMM_V2FP64:
1857 case AMDGPU::OPERAND_REG_IMM_V2INT64:
1858 case AMDGPU::OPERAND_KIMM64:
1859 return &APFloat::IEEEdouble();
1860 case AMDGPU::OPERAND_REG_IMM_FP16:
1861 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
1862 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1863 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1864 case AMDGPU::OPERAND_REG_IMM_V2FP16:
1865 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
1866 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
1867 case AMDGPU::OPERAND_KIMM16:
1868 return &APFloat::IEEEhalf();
1869 case AMDGPU::OPERAND_REG_IMM_BF16:
1870 case AMDGPU::OPERAND_REG_INLINE_C_BF16:
1871 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
1872 case AMDGPU::OPERAND_REG_IMM_V2BF16:
1873 return &APFloat::BFloat();
1874 default:
1875 llvm_unreachable("unsupported fp type");
1876 }
1877}
1878
1879//===----------------------------------------------------------------------===//
1880// Operand
1881//===----------------------------------------------------------------------===//
1882
1883static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1884 bool Lost;
1885
1886 // Convert literal to single precision
1887 APFloat::opStatus Status = FPLiteral.convert(
1888 ToSemantics: *getFltSemantics(VT), RM: APFloat::rmNearestTiesToEven, losesInfo: &Lost);
1889 // We allow precision lost but not overflow or underflow
1890 if (Status != APFloat::opOK && Lost &&
1891 ((Status & APFloat::opOverflow) != 0 ||
1892 (Status & APFloat::opUnderflow) != 0)) {
1893 return false;
1894 }
1895
1896 return true;
1897}
1898
1899static bool isSafeTruncation(int64_t Val, unsigned Size) {
1900 return isUIntN(N: Size, x: Val) || isIntN(N: Size, x: Val);
1901}
1902
1903static bool isInlineableLiteralOp16(int64_t Val, MVT VT, bool HasInv2Pi) {
1904 if (VT.getScalarType() == MVT::i16)
1905 return isInlinableLiteral32(Literal: Val, HasInv2Pi);
1906
1907 if (VT.getScalarType() == MVT::f16)
1908 return AMDGPU::isInlinableLiteralFP16(Literal: Val, HasInv2Pi);
1909
1910 assert(VT.getScalarType() == MVT::bf16);
1911
1912 return AMDGPU::isInlinableLiteralBF16(Literal: Val, HasInv2Pi);
1913}
1914
1915bool AMDGPUOperand::isInlinableImm(MVT type) const {
1916
1917 // This is a hack to enable named inline values like
1918 // shared_base with both 32-bit and 64-bit operands.
1919 // Note that these values are defined as
1920 // 32-bit operands only.
1921 if (isInlineValue()) {
1922 return true;
1923 }
1924
1925 if (!isImmTy(ImmT: ImmTyNone)) {
1926 // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1927 return false;
1928 }
1929
1930 if (getModifiers().Lit != LitModifier::None)
1931 return false;
1932
1933 // TODO: We should avoid using host float here. It would be better to
1934 // check the float bit values which is what a few other places do.
1935 // We've had bot failures before due to weird NaN support on mips hosts.
1936
1937 APInt Literal(64, Imm.Val);
1938
1939 if (Imm.IsFPImm) { // We got fp literal token
1940 if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1941 return AMDGPU::isInlinableLiteral64(Literal: Imm.Val,
1942 HasInv2Pi: AsmParser->hasInv2PiInlineImm());
1943 }
1944
1945 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1946 if (!canLosslesslyConvertToFPType(FPLiteral, VT: type))
1947 return false;
1948
1949 if (type.getScalarSizeInBits() == 16) {
1950 bool Lost = false;
1951 switch (type.getScalarType().SimpleTy) {
1952 default:
1953 llvm_unreachable("unknown 16-bit type");
1954 case MVT::bf16:
1955 FPLiteral.convert(ToSemantics: APFloatBase::BFloat(), RM: APFloat::rmNearestTiesToEven,
1956 losesInfo: &Lost);
1957 break;
1958 case MVT::f16:
1959 FPLiteral.convert(ToSemantics: APFloatBase::IEEEhalf(), RM: APFloat::rmNearestTiesToEven,
1960 losesInfo: &Lost);
1961 break;
1962 case MVT::i16:
1963 FPLiteral.convert(ToSemantics: APFloatBase::IEEEsingle(),
1964 RM: APFloat::rmNearestTiesToEven, losesInfo: &Lost);
1965 break;
1966 }
1967 // We need to use 32-bit representation here because when a floating-point
1968 // inline constant is used as an i16 operand, its 32-bit representation
1969 // representation will be used. We will need the 32-bit value to check if
1970 // it is FP inline constant.
1971 uint32_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1972 return isInlineableLiteralOp16(Val: ImmVal, VT: type,
1973 HasInv2Pi: AsmParser->hasInv2PiInlineImm());
1974 }
1975
1976 // Check if single precision literal is inlinable
1977 return AMDGPU::isInlinableLiteral32(
1978 Literal: static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1979 HasInv2Pi: AsmParser->hasInv2PiInlineImm());
1980 }
1981
1982 // We got int literal token.
1983 if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1984 return AMDGPU::isInlinableLiteral64(Literal: Imm.Val,
1985 HasInv2Pi: AsmParser->hasInv2PiInlineImm());
1986 }
1987
1988 if (!isSafeTruncation(Val: Imm.Val, Size: type.getScalarSizeInBits())) {
1989 return false;
1990 }
1991
1992 if (type.getScalarSizeInBits() == 16) {
1993 return isInlineableLiteralOp16(
1994 Val: static_cast<int16_t>(Literal.getLoBits(numBits: 16).getSExtValue()), VT: type,
1995 HasInv2Pi: AsmParser->hasInv2PiInlineImm());
1996 }
1997
1998 return AMDGPU::isInlinableLiteral32(
1999 Literal: static_cast<int32_t>(Literal.getLoBits(numBits: 32).getZExtValue()),
2000 HasInv2Pi: AsmParser->hasInv2PiInlineImm());
2001}
2002
2003bool AMDGPUOperand::isLiteralImm(MVT type) const {
2004 // Check that this immediate can be added as literal
2005 if (!isImmTy(ImmT: ImmTyNone)) {
2006 return false;
2007 }
2008
2009 bool Allow64Bit =
2010 (type == MVT::i64 || type == MVT::f64) && AsmParser->has64BitLiterals();
2011
2012 if (!Imm.IsFPImm) {
2013 // We got int literal token.
2014
2015 if (type == MVT::f64 && hasFPModifiers()) {
2016 // Cannot apply fp modifiers to int literals preserving the same semantics
2017 // for VOP1/2/C and VOP3 because of integer truncation. To avoid
2018 // ambiguity, disable these cases.
2019 return false;
2020 }
2021
2022 unsigned Size = type.getSizeInBits();
2023 if (Size == 64) {
2024 if (Allow64Bit && !AMDGPU::isValid32BitLiteral(Val: Imm.Val, IsFP64: false))
2025 return true;
2026 Size = 32;
2027 }
2028
2029 // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
2030 // types.
2031 return isSafeTruncation(Val: Imm.Val, Size);
2032 }
2033
2034 // We got fp literal token
2035 if (type == MVT::f64) { // Expected 64-bit fp operand
2036 // We would set low 64-bits of literal to zeroes but we accept this literals
2037 return true;
2038 }
2039
2040 if (type == MVT::i64) { // Expected 64-bit int operand
2041 // We don't allow fp literals in 64-bit integer instructions. It is
2042 // unclear how we should encode them.
2043 return false;
2044 }
2045
2046 // We allow fp literals with f16x2 operands assuming that the specified
2047 // literal goes into the lower half and the upper half is zero. We also
2048 // require that the literal may be losslessly converted to f16.
2049 //
2050 // For i16x2 operands, we assume that the specified literal is encoded as a
2051 // single-precision float. This is pretty odd, but it matches SP3 and what
2052 // happens in hardware.
2053 MVT ExpectedType = (type == MVT::v2f16) ? MVT::f16
2054 : (type == MVT::v2i16) ? MVT::f32
2055 : (type == MVT::v2f32) ? MVT::f32
2056 : type;
2057
2058 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
2059 return canLosslesslyConvertToFPType(FPLiteral, VT: ExpectedType);
2060}
2061
2062bool AMDGPUOperand::isRegClassTarget(unsigned TargetRCIdx) const {
2063 if (!isRegKind())
2064 return false;
2065 int16_t RCID = AsmParser->getTargetRegClass(TargetRCIdx);
2066 return RCID >= 0 && isRegClass(RCID);
2067}
2068
2069bool AMDGPUOperand::isRegClass(unsigned RCID) const {
2070 return isRegKind() &&
2071 AsmParser->getMRI()->getRegClass(i: RCID).contains(Reg: getReg());
2072}
2073
2074bool AMDGPUOperand::isVRegWithInputMods() const {
2075 return isRegClass(RCID: AMDGPU::VGPR_32RegClassID) ||
2076 // GFX90A allows DPP on 64-bit operands.
2077 (AsmParser->getFeatureBits()[AMDGPU::FeatureDPALU_DPP] &&
2078 isRegClassTarget(TargetRCIdx: AMDGPU::VReg_64_AlignTarget));
2079}
2080
2081template <bool IsFake16>
2082bool AMDGPUOperand::isT16_Lo128VRegWithInputMods() const {
2083 return isRegClass(RCID: IsFake16 ? AMDGPU::VGPR_32_Lo128RegClassID
2084 : AMDGPU::VGPR_16_Lo128RegClassID);
2085}
2086
2087template <bool IsFake16> bool AMDGPUOperand::isT16VRegWithInputMods() const {
2088 return isRegClass(RCID: IsFake16 ? AMDGPU::VGPR_32RegClassID
2089 : AMDGPU::VGPR_16RegClassID);
2090}
2091
2092bool AMDGPUOperand::isT16_LO16VRegWithInputMods() const {
2093 return isRegClass(RCID: AMDGPU::VGPR_16_LO16RegClassID);
2094}
2095
2096bool AMDGPUOperand::isSDWAOperand(MVT type) const {
2097 if (AsmParser->isVI())
2098 return isVReg32();
2099 if (AsmParser->isGFX9Plus())
2100 return isRegClass(RCID: AMDGPU::VS_32RegClassID) || isInlinableImm(type);
2101 return false;
2102}
2103
2104bool AMDGPUOperand::isSDWAFP16Operand() const {
2105 return isSDWAOperand(type: MVT::f16);
2106}
2107
2108bool AMDGPUOperand::isSDWAFP32Operand() const {
2109 return isSDWAOperand(type: MVT::f32);
2110}
2111
2112bool AMDGPUOperand::isSDWAInt16Operand() const {
2113 return isSDWAOperand(type: MVT::i16);
2114}
2115
2116bool AMDGPUOperand::isSDWAInt32Operand() const {
2117 return isSDWAOperand(type: MVT::i32);
2118}
2119
2120bool AMDGPUOperand::isBoolReg() const {
2121 return isReg() && ((AsmParser->isWave64() && isSCSrc_b64()) ||
2122 (AsmParser->isWave32() && isSCSrc_b32()));
2123}
2124
2125uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val,
2126 unsigned Size) const {
2127 assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
2128 assert(Size == 2 || Size == 4 || Size == 8);
2129
2130 const uint64_t FpSignMask = (1ULL << (Size * 8 - 1));
2131
2132 if (Imm.Mods.Abs) {
2133 Val &= ~FpSignMask;
2134 }
2135 if (Imm.Mods.Neg) {
2136 Val ^= FpSignMask;
2137 }
2138
2139 return Val;
2140}
2141
2142void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N,
2143 bool ApplyModifiers) const {
2144 MCOpIdx = Inst.getNumOperands();
2145
2146 if (isExpr()) {
2147 Inst.addOperand(Op: MCOperand::createExpr(Val: Expr));
2148 return;
2149 }
2150
2151 if (AMDGPU::isSISrcOperand(Desc: AsmParser->getMII()->get(Opcode: Inst.getOpcode()),
2152 OpNo: Inst.getNumOperands())) {
2153 addLiteralImmOperand(Inst, Val: Imm.Val,
2154 ApplyModifiers: ApplyModifiers & isImmTy(ImmT: ImmTyNone) &&
2155 Imm.Mods.hasFPModifiers());
2156 } else {
2157 assert(!isImmTy(ImmTyNone) || !hasModifiers());
2158 Inst.addOperand(Op: MCOperand::createImm(Val: Imm.Val));
2159 }
2160}
2161
2162void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val,
2163 bool ApplyModifiers) const {
2164 const auto &InstDesc = AsmParser->getMII()->get(Opcode: Inst.getOpcode());
2165 auto OpNum = Inst.getNumOperands();
2166 // Check that this operand accepts literals
2167 assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
2168
2169 if (ApplyModifiers) {
2170 assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum));
2171 const unsigned Size =
2172 Imm.IsFPImm ? sizeof(double) : getOperandSize(Desc: InstDesc, OpNo: OpNum);
2173 Val = applyInputFPModifiers(Val, Size);
2174 }
2175
2176 APInt Literal(64, Val);
2177 uint8_t OpTy = InstDesc.operands()[OpNum].OperandType;
2178
2179 bool CanUse64BitLiterals =
2180 AsmParser->has64BitLiterals() && !SIInstrFlags::isVOP3Like(O: InstDesc);
2181 LitModifier Lit = getModifiers().Lit;
2182 MCContext &Ctx = AsmParser->getContext();
2183
2184 if (Imm.IsFPImm) { // We got fp literal token
2185 switch (OpTy) {
2186 case AMDGPU::OPERAND_REG_IMM_INT64:
2187 case AMDGPU::OPERAND_REG_IMM_FP64:
2188 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
2189 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
2190 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
2191 case AMDGPU::OPERAND_REG_IMM_V2FP64:
2192 case AMDGPU::OPERAND_REG_IMM_V2INT64:
2193 if (Lit == LitModifier::None &&
2194 AMDGPU::isInlinableLiteral64(Literal: Literal.getZExtValue(),
2195 HasInv2Pi: AsmParser->hasInv2PiInlineImm())) {
2196 Inst.addOperand(Op: MCOperand::createImm(Val: Literal.getZExtValue()));
2197 return;
2198 }
2199
2200 // Non-inlineable
2201 if (AMDGPU::isSISrcFPOperand(Desc: InstDesc,
2202 OpNo: OpNum)) { // Expected 64-bit fp operand
2203 bool HasMandatoryLiteral =
2204 AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::imm);
2205 // For fp operands we check if low 32 bits are zeros
2206 if (Literal.getLoBits(numBits: 32) != 0 &&
2207 (InstDesc.getSize() != 4 || !AsmParser->has64BitLiterals()) &&
2208 !HasMandatoryLiteral) {
2209 const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(
2210 L: Inst.getLoc(),
2211 Msg: "Can't encode literal as exact 64-bit floating-point operand. "
2212 "Low 32-bits will be set to zero");
2213 Val &= 0xffffffff00000000u;
2214 }
2215
2216 if ((OpTy == AMDGPU::OPERAND_REG_IMM_FP64 ||
2217 OpTy == AMDGPU::OPERAND_REG_INLINE_C_FP64 ||
2218 OpTy == AMDGPU::OPERAND_REG_INLINE_AC_FP64)) {
2219 if (CanUse64BitLiterals && Lit == LitModifier::None &&
2220 (isInt<32>(x: Val) || isUInt<32>(x: Val))) {
2221 // The floating-point operand will be verbalized as an
2222 // integer one. If that integer happens to fit 32 bits, on
2223 // re-assembling it will be intepreted as the high half of
2224 // the actual value, so we have to wrap it into lit64().
2225 Lit = LitModifier::Lit64;
2226 } else if (Lit == LitModifier::Lit) {
2227 // For FP64 operands lit() specifies the high half of the value.
2228 Val = Hi_32(Value: Val);
2229 }
2230 }
2231 break;
2232 }
2233
2234 // We don't allow fp literals in 64-bit integer instructions. It is
2235 // unclear how we should encode them. This case should be checked earlier
2236 // in predicate methods (isLiteralImm())
2237 llvm_unreachable("fp literal in 64-bit integer instruction.");
2238
2239 case AMDGPU::OPERAND_KIMM64:
2240 if (CanUse64BitLiterals && Lit == LitModifier::None &&
2241 (isInt<32>(x: Val) || isUInt<32>(x: Val)))
2242 Lit = LitModifier::Lit64;
2243 break;
2244
2245 case AMDGPU::OPERAND_REG_IMM_BF16:
2246 case AMDGPU::OPERAND_REG_INLINE_C_BF16:
2247 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
2248 case AMDGPU::OPERAND_REG_IMM_V2BF16:
2249 if (Lit == LitModifier::None && AsmParser->hasInv2PiInlineImm() &&
2250 Literal == 0x3fc45f306725feed) {
2251 // This is the 1/(2*pi) which is going to be truncated to bf16 with the
2252 // loss of precision. The constant represents idiomatic fp32 value of
2253 // 1/(2*pi) = 0.15915494 since bf16 is in fact fp32 with cleared low 16
2254 // bits. Prevent rounding below.
2255 Inst.addOperand(Op: MCOperand::createImm(Val: 0x3e22));
2256 return;
2257 }
2258 [[fallthrough]];
2259
2260 case AMDGPU::OPERAND_REG_IMM_INT32:
2261 case AMDGPU::OPERAND_REG_IMM_FP32:
2262 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
2263 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
2264 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
2265 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
2266 case AMDGPU::OPERAND_REG_IMM_INT16:
2267 case AMDGPU::OPERAND_REG_IMM_FP16:
2268 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
2269 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
2270 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
2271 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
2272 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
2273 case AMDGPU::OPERAND_REG_IMM_V2INT16:
2274 case AMDGPU::OPERAND_REG_IMM_V2FP16:
2275 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
2276 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
2277 case AMDGPU::OPERAND_REG_IMM_V2FP32:
2278 case AMDGPU::OPERAND_REG_IMM_V2INT32:
2279 case AMDGPU::OPERAND_KIMM32:
2280 case AMDGPU::OPERAND_KIMM16:
2281 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: {
2282 bool lost;
2283 APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
2284 // Convert literal to single precision
2285 FPLiteral.convert(ToSemantics: *getOpFltSemantics(OperandType: OpTy), RM: APFloat::rmNearestTiesToEven,
2286 losesInfo: &lost);
2287 // We allow precision lost but not overflow or underflow. This should be
2288 // checked earlier in isLiteralImm()
2289
2290 Val = FPLiteral.bitcastToAPInt().getZExtValue();
2291 break;
2292 }
2293 default:
2294 llvm_unreachable("invalid operand size");
2295 }
2296
2297 if (Lit != LitModifier::None) {
2298 Inst.addOperand(
2299 Op: MCOperand::createExpr(Val: AMDGPUMCExpr::createLit(Lit, Value: Val, Ctx)));
2300 } else {
2301 Inst.addOperand(Op: MCOperand::createImm(Val));
2302 }
2303 return;
2304 }
2305
2306 // We got int literal token.
2307 // Only sign extend inline immediates.
2308 switch (OpTy) {
2309 case AMDGPU::OPERAND_REG_IMM_INT32:
2310 case AMDGPU::OPERAND_REG_IMM_FP32:
2311 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
2312 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
2313 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
2314 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
2315 case AMDGPU::OPERAND_REG_IMM_V2INT16:
2316 case AMDGPU::OPERAND_REG_IMM_V2BF16:
2317 case AMDGPU::OPERAND_REG_IMM_V2FP16:
2318 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
2319 case AMDGPU::OPERAND_REG_IMM_V2FP32:
2320 case AMDGPU::OPERAND_REG_IMM_V2INT32:
2321 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
2322 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
2323 break;
2324
2325 case AMDGPU::OPERAND_REG_IMM_INT64:
2326 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
2327 case AMDGPU::OPERAND_REG_IMM_V2INT64:
2328 if (Lit == LitModifier::None &&
2329 AMDGPU::isInlinableLiteral64(Literal: Val, HasInv2Pi: AsmParser->hasInv2PiInlineImm())) {
2330 Inst.addOperand(Op: MCOperand::createImm(Val));
2331 return;
2332 }
2333
2334 // When the 32 MSBs are not zero (effectively means it can't be safely
2335 // truncated to uint32_t), if the target doesn't support 64-bit literals, or
2336 // the lit modifier is explicitly used, we need to truncate it to the 32
2337 // LSBs.
2338 if (!AsmParser->has64BitLiterals() || Lit == LitModifier::Lit)
2339 Val = Lo_32(Value: Val);
2340 break;
2341
2342 case AMDGPU::OPERAND_REG_IMM_FP64:
2343 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
2344 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
2345 case AMDGPU::OPERAND_REG_IMM_V2FP64:
2346 if (Lit == LitModifier::None &&
2347 AMDGPU::isInlinableLiteral64(Literal: Val, HasInv2Pi: AsmParser->hasInv2PiInlineImm())) {
2348 Inst.addOperand(Op: MCOperand::createImm(Val));
2349 return;
2350 }
2351
2352 // If the target doesn't support 64-bit literals, we need to use the
2353 // constant as the high 32 MSBs of a double-precision floating point value.
2354 if (!AsmParser->has64BitLiterals()) {
2355 Val = static_cast<uint64_t>(Val) << 32;
2356 } else {
2357 // Now the target does support 64-bit literals, there are two cases
2358 // where we still want to use src_literal encoding:
2359 // 1) explicitly forced by using lit modifier;
2360 // 2) the value is a valid 32-bit representation (signed or unsigned),
2361 // meanwhile not forced by lit64 modifier.
2362 if (Lit == LitModifier::Lit ||
2363 (Lit != LitModifier::Lit64 && (isInt<32>(x: Val) || isUInt<32>(x: Val))))
2364 Val = static_cast<uint64_t>(Val) << 32;
2365 }
2366
2367 // For FP64 operands lit() specifies the high half of the value.
2368 if (Lit == LitModifier::Lit)
2369 Val = Hi_32(Value: Val);
2370 break;
2371
2372 case AMDGPU::OPERAND_REG_IMM_INT16:
2373 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
2374 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
2375 case AMDGPU::OPERAND_REG_IMM_FP16:
2376 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
2377 case AMDGPU::OPERAND_REG_IMM_BF16:
2378 case AMDGPU::OPERAND_REG_INLINE_C_BF16:
2379 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
2380 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
2381 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
2382 case AMDGPU::OPERAND_KIMM32:
2383 case AMDGPU::OPERAND_KIMM16:
2384 break;
2385
2386 case AMDGPU::OPERAND_KIMM64:
2387 if ((isInt<32>(x: Val) || isUInt<32>(x: Val)) && Lit != LitModifier::Lit64)
2388 Val <<= 32;
2389 break;
2390
2391 default:
2392 llvm_unreachable("invalid operand type");
2393 }
2394
2395 if (Lit != LitModifier::None) {
2396 Inst.addOperand(
2397 Op: MCOperand::createExpr(Val: AMDGPUMCExpr::createLit(Lit, Value: Val, Ctx)));
2398 } else {
2399 Inst.addOperand(Op: MCOperand::createImm(Val));
2400 }
2401}
2402
2403void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
2404 MCOpIdx = Inst.getNumOperands();
2405 Inst.addOperand(
2406 Op: MCOperand::createReg(Reg: AMDGPU::getMCReg(Reg: getReg(), STI: AsmParser->getSTI())));
2407}
2408
2409bool AMDGPUOperand::isInlineValue() const {
2410 return isRegKind() && ::isInlineValue(Reg: getReg());
2411}
2412
2413//===----------------------------------------------------------------------===//
2414// AsmParser
2415//===----------------------------------------------------------------------===//
2416
2417void AMDGPUAsmParser::createConstantSymbol(StringRef Id, int64_t Val) {
2418 // TODO: make those pre-defined variables read-only.
2419 // Currently there is none suitable machinery in the core llvm-mc for this.
2420 // MCSymbol::isRedefinable is intended for another purpose, and
2421 // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
2422 MCContext &Ctx = getContext();
2423 MCSymbol *Sym = Ctx.getOrCreateSymbol(Name: Id);
2424 Sym->setVariableValue(MCConstantExpr::create(Value: Val, Ctx));
2425}
2426
2427static int getRegClass(RegisterKind Is, unsigned RegWidth) {
2428 if (Is == IS_VGPR) {
2429 switch (RegWidth) {
2430 default:
2431 return -1;
2432 case 32:
2433 return AMDGPU::VGPR_32RegClassID;
2434 case 64:
2435 return AMDGPU::VReg_64RegClassID;
2436 case 96:
2437 return AMDGPU::VReg_96RegClassID;
2438 case 128:
2439 return AMDGPU::VReg_128RegClassID;
2440 case 160:
2441 return AMDGPU::VReg_160RegClassID;
2442 case 192:
2443 return AMDGPU::VReg_192RegClassID;
2444 case 224:
2445 return AMDGPU::VReg_224RegClassID;
2446 case 256:
2447 return AMDGPU::VReg_256RegClassID;
2448 case 288:
2449 return AMDGPU::VReg_288RegClassID;
2450 case 320:
2451 return AMDGPU::VReg_320RegClassID;
2452 case 352:
2453 return AMDGPU::VReg_352RegClassID;
2454 case 384:
2455 return AMDGPU::VReg_384RegClassID;
2456 case 512:
2457 return AMDGPU::VReg_512RegClassID;
2458 case 1024:
2459 return AMDGPU::VReg_1024RegClassID;
2460 }
2461 } else if (Is == IS_TTMP) {
2462 switch (RegWidth) {
2463 default:
2464 return -1;
2465 case 32:
2466 return AMDGPU::TTMP_32RegClassID;
2467 case 64:
2468 return AMDGPU::TTMP_64RegClassID;
2469 case 128:
2470 return AMDGPU::TTMP_128RegClassID;
2471 case 256:
2472 return AMDGPU::TTMP_256RegClassID;
2473 case 512:
2474 return AMDGPU::TTMP_512RegClassID;
2475 }
2476 } else if (Is == IS_SGPR) {
2477 switch (RegWidth) {
2478 default:
2479 return -1;
2480 case 32:
2481 return AMDGPU::SGPR_32RegClassID;
2482 case 64:
2483 return AMDGPU::SGPR_64RegClassID;
2484 case 96:
2485 return AMDGPU::SGPR_96RegClassID;
2486 case 128:
2487 return AMDGPU::SGPR_128RegClassID;
2488 case 160:
2489 return AMDGPU::SGPR_160RegClassID;
2490 case 192:
2491 return AMDGPU::SGPR_192RegClassID;
2492 case 224:
2493 return AMDGPU::SGPR_224RegClassID;
2494 case 256:
2495 return AMDGPU::SGPR_256RegClassID;
2496 case 288:
2497 return AMDGPU::SGPR_288RegClassID;
2498 case 320:
2499 return AMDGPU::SGPR_320RegClassID;
2500 case 352:
2501 return AMDGPU::SGPR_352RegClassID;
2502 case 384:
2503 return AMDGPU::SGPR_384RegClassID;
2504 case 512:
2505 return AMDGPU::SGPR_512RegClassID;
2506 }
2507 } else if (Is == IS_AGPR) {
2508 switch (RegWidth) {
2509 default:
2510 return -1;
2511 case 32:
2512 return AMDGPU::AGPR_32RegClassID;
2513 case 64:
2514 return AMDGPU::AReg_64RegClassID;
2515 case 96:
2516 return AMDGPU::AReg_96RegClassID;
2517 case 128:
2518 return AMDGPU::AReg_128RegClassID;
2519 case 160:
2520 return AMDGPU::AReg_160RegClassID;
2521 case 192:
2522 return AMDGPU::AReg_192RegClassID;
2523 case 224:
2524 return AMDGPU::AReg_224RegClassID;
2525 case 256:
2526 return AMDGPU::AReg_256RegClassID;
2527 case 288:
2528 return AMDGPU::AReg_288RegClassID;
2529 case 320:
2530 return AMDGPU::AReg_320RegClassID;
2531 case 352:
2532 return AMDGPU::AReg_352RegClassID;
2533 case 384:
2534 return AMDGPU::AReg_384RegClassID;
2535 case 512:
2536 return AMDGPU::AReg_512RegClassID;
2537 case 1024:
2538 return AMDGPU::AReg_1024RegClassID;
2539 }
2540 }
2541 return -1;
2542}
2543
2544static MCRegister getSpecialRegForName(StringRef RegName) {
2545 return StringSwitch<unsigned>(RegName)
2546 .Case(S: "exec", Value: AMDGPU::EXEC)
2547 .Case(S: "vcc", Value: AMDGPU::VCC)
2548 .Case(S: "flat_scratch", Value: AMDGPU::FLAT_SCR)
2549 .Case(S: "xnack_mask", Value: AMDGPU::XNACK_MASK)
2550 .Case(S: "shared_base", Value: AMDGPU::SRC_SHARED_BASE)
2551 .Case(S: "src_shared_base", Value: AMDGPU::SRC_SHARED_BASE)
2552 .Case(S: "shared_limit", Value: AMDGPU::SRC_SHARED_LIMIT)
2553 .Case(S: "src_shared_limit", Value: AMDGPU::SRC_SHARED_LIMIT)
2554 .Case(S: "private_base", Value: AMDGPU::SRC_PRIVATE_BASE)
2555 .Case(S: "src_private_base", Value: AMDGPU::SRC_PRIVATE_BASE)
2556 .Case(S: "private_limit", Value: AMDGPU::SRC_PRIVATE_LIMIT)
2557 .Case(S: "src_private_limit", Value: AMDGPU::SRC_PRIVATE_LIMIT)
2558 .Case(S: "src_flat_scratch_base_lo", Value: AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)
2559 .Case(S: "src_flat_scratch_base_hi", Value: AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)
2560 .Case(S: "pops_exiting_wave_id", Value: AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2561 .Case(S: "src_pops_exiting_wave_id", Value: AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2562 .Case(S: "lds_direct", Value: AMDGPU::LDS_DIRECT)
2563 .Case(S: "src_lds_direct", Value: AMDGPU::LDS_DIRECT)
2564 .Case(S: "m0", Value: AMDGPU::M0)
2565 .Case(S: "vccz", Value: AMDGPU::SRC_VCCZ)
2566 .Case(S: "src_vccz", Value: AMDGPU::SRC_VCCZ)
2567 .Case(S: "execz", Value: AMDGPU::SRC_EXECZ)
2568 .Case(S: "src_execz", Value: AMDGPU::SRC_EXECZ)
2569 .Case(S: "scc", Value: AMDGPU::SRC_SCC)
2570 .Case(S: "src_scc", Value: AMDGPU::SRC_SCC)
2571 .Case(S: "tba", Value: AMDGPU::TBA)
2572 .Case(S: "tma", Value: AMDGPU::TMA)
2573 .Case(S: "flat_scratch_lo", Value: AMDGPU::FLAT_SCR_LO)
2574 .Case(S: "flat_scratch_hi", Value: AMDGPU::FLAT_SCR_HI)
2575 .Case(S: "xnack_mask_lo", Value: AMDGPU::XNACK_MASK_LO)
2576 .Case(S: "xnack_mask_hi", Value: AMDGPU::XNACK_MASK_HI)
2577 .Case(S: "vcc_lo", Value: AMDGPU::VCC_LO)
2578 .Case(S: "vcc_hi", Value: AMDGPU::VCC_HI)
2579 .Case(S: "exec_lo", Value: AMDGPU::EXEC_LO)
2580 .Case(S: "exec_hi", Value: AMDGPU::EXEC_HI)
2581 .Case(S: "tma_lo", Value: AMDGPU::TMA_LO)
2582 .Case(S: "tma_hi", Value: AMDGPU::TMA_HI)
2583 .Case(S: "tba_lo", Value: AMDGPU::TBA_LO)
2584 .Case(S: "tba_hi", Value: AMDGPU::TBA_HI)
2585 .Case(S: "pc", Value: AMDGPU::PC_REG)
2586 .Case(S: "null", Value: AMDGPU::SGPR_NULL)
2587 .Default(Value: AMDGPU::NoRegister);
2588}
2589
2590bool AMDGPUAsmParser::ParseRegister(MCRegister &RegNo, SMLoc &StartLoc,
2591 SMLoc &EndLoc, bool RestoreOnFailure) {
2592 auto R = parseRegister();
2593 if (!R)
2594 return true;
2595 assert(R->isReg());
2596 RegNo = R->getReg();
2597 StartLoc = R->getStartLoc();
2598 EndLoc = R->getEndLoc();
2599 return false;
2600}
2601
2602bool AMDGPUAsmParser::parseRegister(MCRegister &Reg, SMLoc &StartLoc,
2603 SMLoc &EndLoc) {
2604 return ParseRegister(RegNo&: Reg, StartLoc, EndLoc, /*RestoreOnFailure=*/false);
2605}
2606
2607ParseStatus AMDGPUAsmParser::tryParseRegister(MCRegister &Reg, SMLoc &StartLoc,
2608 SMLoc &EndLoc) {
2609 bool Result = ParseRegister(RegNo&: Reg, StartLoc, EndLoc, /*RestoreOnFailure=*/true);
2610 bool PendingErrors = getParser().hasPendingError();
2611 getParser().clearPendingErrors();
2612 if (PendingErrors)
2613 return ParseStatus::Failure;
2614 if (Result)
2615 return ParseStatus::NoMatch;
2616 return ParseStatus::Success;
2617}
2618
2619bool AMDGPUAsmParser::AddNextRegisterToList(MCRegister &Reg, unsigned &RegWidth,
2620 RegisterKind RegKind,
2621 MCRegister Reg1,
2622 RegisterKind RegKind1, SMLoc Loc) {
2623 // Allow VCC_LO/HI at the end of SGPR lists.
2624 if (RegKind == IS_SGPR) {
2625 unsigned RegIdx = (Reg - AMDGPU::SGPR0) + RegWidth / 32;
2626 if ((RegIdx == 106 && Reg1 == AMDGPU::VCC_LO) ||
2627 (RegIdx == 107 && Reg1 == AMDGPU::VCC_HI)) {
2628 RegWidth += 32;
2629 return true;
2630 }
2631 }
2632
2633 if (RegKind != RegKind1) {
2634 Error(L: Loc, Msg: "registers in a list must be of the same kind");
2635 return false;
2636 }
2637
2638 switch (RegKind) {
2639 case IS_SPECIAL:
2640 if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
2641 Reg = AMDGPU::EXEC;
2642 RegWidth = 64;
2643 return true;
2644 }
2645 if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
2646 Reg = AMDGPU::FLAT_SCR;
2647 RegWidth = 64;
2648 return true;
2649 }
2650 if (Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
2651 Reg = AMDGPU::XNACK_MASK;
2652 RegWidth = 64;
2653 return true;
2654 }
2655 if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
2656 Reg = AMDGPU::VCC;
2657 RegWidth = 64;
2658 return true;
2659 }
2660 if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
2661 Reg = AMDGPU::TBA;
2662 RegWidth = 64;
2663 return true;
2664 }
2665 if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
2666 Reg = AMDGPU::TMA;
2667 RegWidth = 64;
2668 return true;
2669 }
2670 Error(L: Loc, Msg: "register does not fit in the list");
2671 return false;
2672 case IS_VGPR:
2673 case IS_SGPR:
2674 case IS_AGPR:
2675 case IS_TTMP:
2676 if (Reg1 != Reg + RegWidth / 32) {
2677 Error(L: Loc, Msg: "registers in a list must have consecutive indices");
2678 return false;
2679 }
2680 RegWidth += 32;
2681 return true;
2682 default:
2683 llvm_unreachable("unexpected register kind");
2684 }
2685}
2686
2687struct RegInfo {
2688 StringLiteral Name;
2689 RegisterKind Kind;
2690};
2691
2692static constexpr RegInfo RegularRegisters[] = {
2693 {.Name: {"v"}, .Kind: IS_VGPR}, {.Name: {"s"}, .Kind: IS_SGPR}, {.Name: {"ttmp"}, .Kind: IS_TTMP},
2694 {.Name: {"acc"}, .Kind: IS_AGPR}, {.Name: {"a"}, .Kind: IS_AGPR},
2695};
2696
2697static bool isRegularReg(RegisterKind Kind) {
2698 return Kind == IS_VGPR || Kind == IS_SGPR || Kind == IS_TTMP ||
2699 Kind == IS_AGPR;
2700}
2701
2702static const RegInfo *getRegularRegInfo(StringRef Str) {
2703 for (const RegInfo &Reg : RegularRegisters)
2704 if (Str.starts_with(Prefix: Reg.Name))
2705 return &Reg;
2706 return nullptr;
2707}
2708
2709static bool getRegNum(StringRef Str, unsigned &Num) {
2710 return !Str.getAsInteger(Radix: 10, Result&: Num);
2711}
2712
2713bool AMDGPUAsmParser::isRegister(const AsmToken &Token,
2714 const AsmToken &NextToken) const {
2715
2716 // A list of consecutive registers: [s0,s1,s2,s3]
2717 if (Token.is(K: AsmToken::LBrac))
2718 return true;
2719
2720 if (!Token.is(K: AsmToken::Identifier))
2721 return false;
2722
2723 // A single register like s0 or a range of registers like s[0:1]
2724
2725 StringRef Str = Token.getString();
2726 const RegInfo *Reg = getRegularRegInfo(Str);
2727 if (Reg) {
2728 StringRef RegName = Reg->Name;
2729 StringRef RegSuffix = Str.substr(Start: RegName.size());
2730 if (!RegSuffix.empty()) {
2731 RegSuffix.consume_back(Suffix: ".l");
2732 RegSuffix.consume_back(Suffix: ".h");
2733 unsigned Num;
2734 // A single register with an index: rXX
2735 if (getRegNum(Str: RegSuffix, Num))
2736 return true;
2737 } else {
2738 // A range of registers: r[XX:YY].
2739 if (NextToken.is(K: AsmToken::LBrac))
2740 return true;
2741 }
2742 }
2743
2744 return getSpecialRegForName(RegName: Str).isValid();
2745}
2746
2747bool AMDGPUAsmParser::isRegister() {
2748 return isRegister(Token: getToken(), NextToken: peekToken());
2749}
2750
2751MCRegister AMDGPUAsmParser::getRegularReg(RegisterKind RegKind, unsigned RegNum,
2752 unsigned SubReg, unsigned RegWidth,
2753 SMLoc Loc) {
2754 assert(isRegularReg(RegKind));
2755
2756 unsigned AlignSize = 1;
2757 if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
2758 // SGPR and TTMP registers must be aligned.
2759 // Max required alignment is 4 dwords.
2760 AlignSize = std::min(a: llvm::bit_ceil(Value: RegWidth / 32), b: 4u);
2761 }
2762
2763 if (RegNum % AlignSize != 0) {
2764 Error(L: Loc, Msg: "invalid register alignment");
2765 return MCRegister();
2766 }
2767
2768 unsigned RegIdx = RegNum / AlignSize;
2769 int RCID = getRegClass(Is: RegKind, RegWidth);
2770 if (RCID == -1) {
2771 Error(L: Loc, Msg: "invalid or unsupported register size");
2772 return MCRegister();
2773 }
2774
2775 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2776 const MCRegisterClass &RC = TRI->getRegClass(i: RCID);
2777 if (RegIdx >= RC.getNumRegs() || (RegKind == IS_VGPR && RegIdx > 255)) {
2778 Error(L: Loc, Msg: "register index is out of range");
2779 return AMDGPU::NoRegister;
2780 }
2781
2782 if (RegKind == IS_VGPR && !isGFX1250Plus() && RegIdx + RegWidth / 32 > 256) {
2783 Error(L: Loc, Msg: "register index is out of range");
2784 return MCRegister();
2785 }
2786
2787 MCRegister Reg = RC.getRegister(i: RegIdx);
2788
2789 if (SubReg) {
2790 Reg = TRI->getSubReg(Reg, Idx: SubReg);
2791
2792 if (!Reg)
2793 Error(L: Loc, Msg: "invalid subregister");
2794 }
2795
2796 return Reg;
2797}
2798
2799bool AMDGPUAsmParser::ParseRegRange(unsigned &Num, unsigned &RegWidth,
2800 unsigned &SubReg) {
2801 int64_t RegLo, RegHi;
2802 if (!skipToken(Kind: AsmToken::LBrac, ErrMsg: "missing register index"))
2803 return false;
2804
2805 SMLoc FirstIdxLoc = getLoc();
2806 SMLoc SecondIdxLoc;
2807
2808 if (!parseExpr(Imm&: RegLo))
2809 return false;
2810
2811 if (trySkipToken(Kind: AsmToken::Colon)) {
2812 SecondIdxLoc = getLoc();
2813 if (!parseExpr(Imm&: RegHi))
2814 return false;
2815 } else {
2816 RegHi = RegLo;
2817 }
2818
2819 if (!skipToken(Kind: AsmToken::RBrac, ErrMsg: "expected a closing square bracket"))
2820 return false;
2821
2822 if (!isUInt<32>(x: RegLo)) {
2823 Error(L: FirstIdxLoc, Msg: "invalid register index");
2824 return false;
2825 }
2826
2827 if (!isUInt<32>(x: RegHi)) {
2828 Error(L: SecondIdxLoc, Msg: "invalid register index");
2829 return false;
2830 }
2831
2832 if (RegLo > RegHi) {
2833 Error(L: FirstIdxLoc, Msg: "first register index should not exceed second index");
2834 return false;
2835 }
2836
2837 if (RegHi == RegLo) {
2838 StringRef RegSuffix = getTokenStr();
2839 if (RegSuffix == ".l") {
2840 SubReg = AMDGPU::lo16;
2841 lex();
2842 } else if (RegSuffix == ".h") {
2843 SubReg = AMDGPU::hi16;
2844 lex();
2845 }
2846 }
2847
2848 Num = static_cast<unsigned>(RegLo);
2849 RegWidth = 32 * ((RegHi - RegLo) + 1);
2850
2851 return true;
2852}
2853
2854MCRegister AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind,
2855 unsigned &RegNum,
2856 unsigned &RegWidth,
2857 SmallVectorImpl<AsmToken> &Tokens) {
2858 assert(isToken(AsmToken::Identifier));
2859 MCRegister Reg = getSpecialRegForName(RegName: getTokenStr());
2860 if (Reg) {
2861 RegNum = 0;
2862 RegWidth = 32;
2863 RegKind = IS_SPECIAL;
2864 Tokens.push_back(Elt: getToken());
2865 lex(); // skip register name
2866 }
2867 return Reg;
2868}
2869
2870MCRegister AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
2871 unsigned &RegNum,
2872 unsigned &RegWidth,
2873 SmallVectorImpl<AsmToken> &Tokens) {
2874 assert(isToken(AsmToken::Identifier));
2875 StringRef RegName = getTokenStr();
2876 auto Loc = getLoc();
2877
2878 const RegInfo *RI = getRegularRegInfo(Str: RegName);
2879 if (!RI) {
2880 Error(L: Loc, Msg: "invalid register name");
2881 return MCRegister();
2882 }
2883
2884 Tokens.push_back(Elt: getToken());
2885 lex(); // skip register name
2886
2887 RegKind = RI->Kind;
2888 StringRef RegSuffix = RegName.substr(Start: RI->Name.size());
2889 unsigned SubReg = NoSubRegister;
2890 bool IsRange = false;
2891 if (!RegSuffix.empty()) {
2892 if (RegSuffix.consume_back(Suffix: ".l"))
2893 SubReg = AMDGPU::lo16;
2894 else if (RegSuffix.consume_back(Suffix: ".h"))
2895 SubReg = AMDGPU::hi16;
2896
2897 // Single 32-bit register: vXX.
2898 if (!getRegNum(Str: RegSuffix, Num&: RegNum)) {
2899 Error(L: Loc, Msg: "invalid register index");
2900 return MCRegister();
2901 }
2902 RegWidth = 32;
2903 } else {
2904 // Range of registers: v[XX:YY]. ":YY" is optional.
2905 IsRange = true;
2906 if (!ParseRegRange(Num&: RegNum, RegWidth, SubReg))
2907 return MCRegister();
2908 }
2909
2910 // Do not allow vcc_lo/hi be referred as s106/107.
2911 MCRegister Reg = getRegularReg(RegKind, RegNum, SubReg, RegWidth, Loc);
2912 const MCRegisterInfo &TRI = *getContext().getRegisterInfo();
2913 if (RegKind == IS_SGPR && IsRange
2914 ? (TRI.isSubRegister(RegA: Reg, RegB: VCC_LO) || TRI.isSubRegister(RegA: Reg, RegB: VCC_HI))
2915 : (Reg == VCC_LO || Reg == VCC_HI)) {
2916 Error(L: Loc, Msg: "register index is out of range");
2917 return MCRegister();
2918 }
2919
2920 return Reg;
2921}
2922
2923MCRegister AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind,
2924 unsigned &RegNum, unsigned &RegWidth,
2925 SmallVectorImpl<AsmToken> &Tokens) {
2926 MCRegister Reg;
2927 auto ListLoc = getLoc();
2928
2929 if (!skipToken(Kind: AsmToken::LBrac,
2930 ErrMsg: "expected a register or a list of registers")) {
2931 return MCRegister();
2932 }
2933
2934 // List of consecutive registers, e.g.: [s0,s1,s2,s3]
2935
2936 auto Loc = getLoc();
2937 if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth))
2938 return MCRegister();
2939 if (RegWidth != 32) {
2940 Error(L: Loc, Msg: "expected a single 32-bit register");
2941 return MCRegister();
2942 }
2943
2944 for (; trySkipToken(Kind: AsmToken::Comma);) {
2945 RegisterKind NextRegKind;
2946 MCRegister NextReg;
2947 unsigned NextRegNum, NextRegWidth;
2948 Loc = getLoc();
2949
2950 if (!ParseAMDGPURegister(RegKind&: NextRegKind, Reg&: NextReg, RegNum&: NextRegNum, RegWidth&: NextRegWidth,
2951 Tokens)) {
2952 return MCRegister();
2953 }
2954 if (NextRegWidth != 32) {
2955 Error(L: Loc, Msg: "expected a single 32-bit register");
2956 return MCRegister();
2957 }
2958 if (!AddNextRegisterToList(Reg, RegWidth, RegKind, Reg1: NextReg, RegKind1: NextRegKind,
2959 Loc))
2960 return MCRegister();
2961 }
2962
2963 if (!skipToken(Kind: AsmToken::RBrac,
2964 ErrMsg: "expected a comma or a closing square bracket")) {
2965 return MCRegister();
2966 }
2967
2968 if (isRegularReg(Kind: RegKind))
2969 Reg = getRegularReg(RegKind, RegNum, SubReg: NoSubRegister, RegWidth, Loc: ListLoc);
2970
2971 return Reg;
2972}
2973
2974bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
2975 MCRegister &Reg, unsigned &RegNum,
2976 unsigned &RegWidth,
2977 SmallVectorImpl<AsmToken> &Tokens) {
2978 auto Loc = getLoc();
2979 Reg = MCRegister();
2980
2981 if (isToken(Kind: AsmToken::Identifier)) {
2982 Reg = ParseSpecialReg(RegKind, RegNum, RegWidth, Tokens);
2983 if (!Reg)
2984 Reg = ParseRegularReg(RegKind, RegNum, RegWidth, Tokens);
2985 } else {
2986 Reg = ParseRegList(RegKind, RegNum, RegWidth, Tokens);
2987 }
2988
2989 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2990 if (!Reg) {
2991 assert(Parser.hasPendingError());
2992 return false;
2993 }
2994
2995 if (!subtargetHasRegister(MRI: *TRI, Reg)) {
2996 if (Reg == AMDGPU::SGPR_NULL) {
2997 Error(L: Loc, Msg: "'null' operand is not supported on this GPU");
2998 } else {
2999 Error(L: Loc, Msg: Twine(AMDGPUInstPrinter::getRegisterName(Reg)) +
3000 " register not available on this GPU");
3001 }
3002 return false;
3003 }
3004
3005 return true;
3006}
3007
3008bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3009 MCRegister &Reg, unsigned &RegNum,
3010 unsigned &RegWidth,
3011 bool RestoreOnFailure /*=false*/) {
3012 Reg = MCRegister();
3013
3014 SmallVector<AsmToken, 1> Tokens;
3015 if (ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, Tokens)) {
3016 if (RestoreOnFailure) {
3017 while (!Tokens.empty()) {
3018 getLexer().UnLex(Token: Tokens.pop_back_val());
3019 }
3020 }
3021 return true;
3022 }
3023 return false;
3024}
3025
3026std::optional<StringRef>
3027AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) {
3028 switch (RegKind) {
3029 case IS_VGPR:
3030 return StringRef(".amdgcn.next_free_vgpr");
3031 case IS_SGPR:
3032 return StringRef(".amdgcn.next_free_sgpr");
3033 default:
3034 return std::nullopt;
3035 }
3036}
3037
3038void AMDGPUAsmParser::initializeGprCountSymbol(RegisterKind RegKind) {
3039 auto SymbolName = getGprCountSymbolName(RegKind);
3040 assert(SymbolName && "initializing invalid register kind");
3041 MCSymbol *Sym = getContext().getOrCreateSymbol(Name: *SymbolName);
3042 Sym->setVariableValue(MCConstantExpr::create(Value: 0, Ctx&: getContext()));
3043 Sym->setRedefinable(true);
3044}
3045
3046bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind,
3047 unsigned DwordRegIndex,
3048 unsigned RegWidth) {
3049 // Symbols are only defined for GCN targets
3050 if (ISA.Major < 6)
3051 return true;
3052
3053 auto SymbolName = getGprCountSymbolName(RegKind);
3054 if (!SymbolName)
3055 return true;
3056 MCSymbol *Sym = getContext().getOrCreateSymbol(Name: *SymbolName);
3057
3058 int64_t NewMax = DwordRegIndex + divideCeil(Numerator: RegWidth, Denominator: 32) - 1;
3059 int64_t OldCount;
3060
3061 if (!Sym->isVariable())
3062 return !Error(L: getLoc(),
3063 Msg: ".amdgcn.next_free_{v,s}gpr symbols must be variable");
3064 if (!Sym->getVariableValue()->evaluateAsAbsolute(Res&: OldCount))
3065 return !Error(
3066 L: getLoc(),
3067 Msg: ".amdgcn.next_free_{v,s}gpr symbols must be absolute expressions");
3068
3069 if (OldCount <= NewMax)
3070 Sym->setVariableValue(MCConstantExpr::create(Value: NewMax + 1, Ctx&: getContext()));
3071
3072 return true;
3073}
3074
3075std::unique_ptr<AMDGPUOperand>
3076AMDGPUAsmParser::parseRegister(bool RestoreOnFailure) {
3077 const auto &Tok = getToken();
3078 SMLoc StartLoc = Tok.getLoc();
3079 SMLoc EndLoc = Tok.getEndLoc();
3080 RegisterKind RegKind;
3081 MCRegister Reg;
3082 unsigned RegNum, RegWidth;
3083
3084 if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) {
3085 return nullptr;
3086 }
3087 if (isHsaAbi(STI: getSTI())) {
3088 if (!updateGprCountSymbols(RegKind, DwordRegIndex: RegNum, RegWidth))
3089 return nullptr;
3090 } else
3091 KernelScope.usesRegister(RegKind, DwordRegIndex: RegNum, RegWidth);
3092 return AMDGPUOperand::CreateReg(AsmParser: this, Reg, S: StartLoc, E: EndLoc);
3093}
3094
3095ParseStatus AMDGPUAsmParser::parseImm(OperandVector &Operands,
3096 bool HasSP3AbsModifier, LitModifier Lit) {
3097 // TODO: add syntactic sugar for 1/(2*PI)
3098
3099 if (isRegister() || isModifier())
3100 return ParseStatus::NoMatch;
3101
3102 if (Lit == LitModifier::None) {
3103 if (trySkipId(Id: "lit"))
3104 Lit = LitModifier::Lit;
3105 else if (trySkipId(Id: "lit64"))
3106 Lit = LitModifier::Lit64;
3107
3108 if (Lit != LitModifier::None) {
3109 if (!skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after lit"))
3110 return ParseStatus::Failure;
3111 ParseStatus S = parseImm(Operands, HasSP3AbsModifier, Lit);
3112 if (S.isSuccess() &&
3113 !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected closing parentheses"))
3114 return ParseStatus::Failure;
3115 return S;
3116 }
3117 }
3118
3119 const auto &Tok = getToken();
3120 const auto &NextTok = peekToken();
3121 bool IsReal = Tok.is(K: AsmToken::Real);
3122 SMLoc S = getLoc();
3123 bool Negate = false;
3124
3125 if (!IsReal && Tok.is(K: AsmToken::Minus) && NextTok.is(K: AsmToken::Real)) {
3126 lex();
3127 IsReal = true;
3128 Negate = true;
3129 }
3130
3131 AMDGPUOperand::Modifiers Mods;
3132 Mods.Lit = Lit;
3133
3134 if (IsReal) {
3135 // Floating-point expressions are not supported.
3136 // Can only allow floating-point literals with an
3137 // optional sign.
3138
3139 StringRef Num = getTokenStr();
3140 lex();
3141
3142 APFloat RealVal(APFloat::IEEEdouble());
3143 auto roundMode = APFloat::rmNearestTiesToEven;
3144 if (errorToBool(Err: RealVal.convertFromString(Num, roundMode).takeError()))
3145 return ParseStatus::Failure;
3146 if (Negate)
3147 RealVal.changeSign();
3148
3149 Operands.push_back(
3150 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: RealVal.bitcastToAPInt().getZExtValue(),
3151 Loc: S, Type: AMDGPUOperand::ImmTyNone, IsFPImm: true));
3152 AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
3153 Op.setModifiers(Mods);
3154
3155 return ParseStatus::Success;
3156
3157 } else {
3158 int64_t IntVal;
3159 const MCExpr *Expr;
3160 SMLoc S = getLoc();
3161
3162 if (HasSP3AbsModifier) {
3163 // This is a workaround for handling expressions
3164 // as arguments of SP3 'abs' modifier, for example:
3165 // |1.0|
3166 // |-1|
3167 // |1+x|
3168 // This syntax is not compatible with syntax of standard
3169 // MC expressions (due to the trailing '|').
3170 SMLoc EndLoc;
3171 if (getParser().parsePrimaryExpr(Res&: Expr, EndLoc, TypeInfo: nullptr))
3172 return ParseStatus::Failure;
3173 } else {
3174 if (Parser.parseExpression(Res&: Expr))
3175 return ParseStatus::Failure;
3176 }
3177
3178 if (Expr->evaluateAsAbsolute(Res&: IntVal)) {
3179 if (Lit == LitModifier::Lit && !isInt<32>(x: IntVal) && !isUInt<32>(x: IntVal))
3180 return Error(L: S, Msg: "literal value out of range");
3181 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: IntVal, Loc: S));
3182 AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
3183 Op.setModifiers(Mods);
3184 } else {
3185 if (Lit != LitModifier::None)
3186 return ParseStatus::NoMatch;
3187 Operands.push_back(Elt: AMDGPUOperand::CreateExpr(AsmParser: this, Expr, S));
3188 }
3189
3190 return ParseStatus::Success;
3191 }
3192
3193 return ParseStatus::NoMatch;
3194}
3195
3196ParseStatus AMDGPUAsmParser::parseReg(OperandVector &Operands) {
3197 if (!isRegister())
3198 return ParseStatus::NoMatch;
3199
3200 if (auto R = parseRegister()) {
3201 assert(R->isReg());
3202 Operands.push_back(Elt: std::move(R));
3203 return ParseStatus::Success;
3204 }
3205 return ParseStatus::Failure;
3206}
3207
3208ParseStatus AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands,
3209 bool HasSP3AbsMod, LitModifier Lit) {
3210 ParseStatus Res = parseReg(Operands);
3211 if (!Res.isNoMatch())
3212 return Res;
3213 if (isModifier())
3214 return ParseStatus::NoMatch;
3215 return parseImm(Operands, HasSP3AbsModifier: HasSP3AbsMod, Lit);
3216}
3217
3218bool AMDGPUAsmParser::isNamedOperandModifier(const AsmToken &Token,
3219 const AsmToken &NextToken) const {
3220 if (Token.is(K: AsmToken::Identifier) && NextToken.is(K: AsmToken::LParen)) {
3221 const auto &str = Token.getString();
3222 return str == "abs" || str == "neg" || str == "sext";
3223 }
3224 return false;
3225}
3226
3227bool AMDGPUAsmParser::isOpcodeModifierWithVal(const AsmToken &Token,
3228 const AsmToken &NextToken) const {
3229 return Token.is(K: AsmToken::Identifier) && NextToken.is(K: AsmToken::Colon);
3230}
3231
3232bool AMDGPUAsmParser::isOperandModifier(const AsmToken &Token,
3233 const AsmToken &NextToken) const {
3234 return isNamedOperandModifier(Token, NextToken) || Token.is(K: AsmToken::Pipe);
3235}
3236
3237bool AMDGPUAsmParser::isRegOrOperandModifier(const AsmToken &Token,
3238 const AsmToken &NextToken) const {
3239 return isRegister(Token, NextToken) || isOperandModifier(Token, NextToken);
3240}
3241
3242// Check if this is an operand modifier or an opcode modifier
3243// which may look like an expression but it is not. We should
3244// avoid parsing these modifiers as expressions. Currently
3245// recognized sequences are:
3246// |...|
3247// abs(...)
3248// neg(...)
3249// sext(...)
3250// -reg
3251// -|...|
3252// -abs(...)
3253// name:...
3254// "name ::" is the VOPD separator, not an opcode modifier.
3255//
3256bool AMDGPUAsmParser::isModifier() {
3257
3258 AsmToken Tok = getToken();
3259 AsmToken NextToken[2];
3260 peekTokens(Tokens: NextToken);
3261
3262 // "name:value" is an opcode modifier. The second colon of "::" is the
3263 // VOPD separator, so a symbol written immediately before "::" is a literal.
3264 bool IsOpcodeModifier = isOpcodeModifierWithVal(Token: Tok, NextToken: NextToken[0]) &&
3265 !NextToken[1].is(K: AsmToken::Colon);
3266
3267 return isOperandModifier(Token: Tok, NextToken: NextToken[0]) ||
3268 (Tok.is(K: AsmToken::Minus) &&
3269 isRegOrOperandModifier(Token: NextToken[0], NextToken: NextToken[1])) ||
3270 IsOpcodeModifier;
3271}
3272
3273// Check if the current token is an SP3 'neg' modifier.
3274// Currently this modifier is allowed in the following context:
3275//
3276// 1. Before a register, e.g. "-v0", "-v[...]" or "-[v0,v1]".
3277// 2. Before an 'abs' modifier: -abs(...)
3278// 3. Before an SP3 'abs' modifier: -|...|
3279//
3280// In all other cases "-" is handled as a part
3281// of an expression that follows the sign.
3282//
3283// Note: When "-" is followed by an integer literal,
3284// this is interpreted as integer negation rather
3285// than a floating-point NEG modifier applied to N.
3286// Beside being contr-intuitive, such use of floating-point
3287// NEG modifier would have resulted in different meaning
3288// of integer literals used with VOP1/2/C and VOP3,
3289// for example:
3290// v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF
3291// v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001
3292// Negative fp literals with preceding "-" are
3293// handled likewise for uniformity
3294//
3295bool AMDGPUAsmParser::parseSP3NegModifier() {
3296
3297 AsmToken NextToken[2];
3298 peekTokens(Tokens: NextToken);
3299
3300 if (isToken(Kind: AsmToken::Minus) &&
3301 (isRegister(Token: NextToken[0], NextToken: NextToken[1]) ||
3302 NextToken[0].is(K: AsmToken::Pipe) || isId(Token: NextToken[0], Id: "abs"))) {
3303 lex();
3304 return true;
3305 }
3306
3307 return false;
3308}
3309
3310ParseStatus
3311AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
3312 bool AllowImm) {
3313 bool Neg, SP3Neg;
3314 bool Abs, SP3Abs;
3315 SMLoc Loc;
3316
3317 // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
3318 if (isToken(Kind: AsmToken::Minus) && peekToken().is(K: AsmToken::Minus))
3319 return Error(L: getLoc(), Msg: "invalid syntax, expected 'neg' modifier");
3320
3321 SP3Neg = parseSP3NegModifier();
3322
3323 Loc = getLoc();
3324 Neg = trySkipId(Id: "neg");
3325 if (Neg && SP3Neg)
3326 return Error(L: Loc, Msg: "expected register or immediate");
3327 if (Neg && !skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after neg"))
3328 return ParseStatus::Failure;
3329
3330 Abs = trySkipId(Id: "abs");
3331 if (Abs && !skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after abs"))
3332 return ParseStatus::Failure;
3333
3334 LitModifier Lit = LitModifier::None;
3335 if (trySkipId(Id: "lit")) {
3336 Lit = LitModifier::Lit;
3337 if (!skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after lit"))
3338 return ParseStatus::Failure;
3339 } else if (trySkipId(Id: "lit64")) {
3340 Lit = LitModifier::Lit64;
3341 if (!skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after lit64"))
3342 return ParseStatus::Failure;
3343 if (!has64BitLiterals())
3344 return Error(L: Loc, Msg: "lit64 is not supported on this GPU");
3345 }
3346
3347 Loc = getLoc();
3348 SP3Abs = trySkipToken(Kind: AsmToken::Pipe);
3349 if (Abs && SP3Abs)
3350 return Error(L: Loc, Msg: "expected register or immediate");
3351
3352 ParseStatus Res;
3353 if (AllowImm) {
3354 Res = parseRegOrImm(Operands, HasSP3AbsMod: SP3Abs, Lit);
3355 } else {
3356 Res = parseReg(Operands);
3357 }
3358 if (!Res.isSuccess())
3359 return (SP3Neg || Neg || SP3Abs || Abs || Lit != LitModifier::None)
3360 ? ParseStatus::Failure
3361 : Res;
3362
3363 if (Lit != LitModifier::None && !Operands.back()->isImm())
3364 Error(L: Loc, Msg: "expected immediate with lit modifier");
3365
3366 if (SP3Abs && !skipToken(Kind: AsmToken::Pipe, ErrMsg: "expected vertical bar"))
3367 return ParseStatus::Failure;
3368 if (Abs && !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected closing parentheses"))
3369 return ParseStatus::Failure;
3370 if (Neg && !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected closing parentheses"))
3371 return ParseStatus::Failure;
3372 if (Lit != LitModifier::None &&
3373 !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected closing parentheses"))
3374 return ParseStatus::Failure;
3375
3376 AMDGPUOperand::Modifiers Mods;
3377 Mods.Abs = Abs || SP3Abs;
3378 Mods.Neg = Neg || SP3Neg;
3379 Mods.Lit = Lit;
3380
3381 if (Mods.hasFPModifiers() || Lit != LitModifier::None) {
3382 AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
3383 if (Op.isExpr())
3384 return Error(L: Op.getStartLoc(), Msg: "expected an absolute expression");
3385 Op.setModifiers(Mods);
3386 }
3387 return ParseStatus::Success;
3388}
3389
3390ParseStatus
3391AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
3392 bool AllowImm) {
3393 bool Sext = trySkipId(Id: "sext");
3394 if (Sext && !skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after sext"))
3395 return ParseStatus::Failure;
3396
3397 ParseStatus Res;
3398 if (AllowImm) {
3399 Res = parseRegOrImm(Operands);
3400 } else {
3401 Res = parseReg(Operands);
3402 }
3403 if (!Res.isSuccess())
3404 return Sext ? ParseStatus::Failure : Res;
3405
3406 if (Sext && !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected closing parentheses"))
3407 return ParseStatus::Failure;
3408
3409 AMDGPUOperand::Modifiers Mods;
3410 Mods.Sext = Sext;
3411
3412 if (Mods.hasIntModifiers()) {
3413 AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
3414 if (Op.isExpr())
3415 return Error(L: Op.getStartLoc(), Msg: "expected an absolute expression");
3416 Op.setModifiers(Mods);
3417 }
3418
3419 return ParseStatus::Success;
3420}
3421
3422ParseStatus AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
3423 return parseRegOrImmWithFPInputMods(Operands, AllowImm: false);
3424}
3425
3426ParseStatus AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
3427 return parseRegOrImmWithIntInputMods(Operands, AllowImm: false);
3428}
3429
3430ParseStatus AMDGPUAsmParser::parseRsrcReg(OperandVector &Operands) {
3431 // Without the marker, fall back to plain register parsing so the legacy
3432 // bare-register form (e.g. `s8`, `v8`) still assembles for indexed
3433 // buffer/image instructions.
3434 if (!trySkipId(Id: "rsrcidx"))
3435 return parseReg(Operands);
3436
3437 if (!skipToken(Kind: AsmToken::LParen, ErrMsg: "expected left paren after rsrcidx"))
3438 return ParseStatus::Failure;
3439
3440 SMLoc RegLoc = getLoc();
3441 std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
3442 if (!Reg)
3443 return ParseStatus::Failure;
3444
3445 // Enforce that the inner register is a valid index register. The matcher
3446 // predicate alone is not sufficient: if it fails, the matcher will fall back
3447 // to a non-indexed instruction variant whose resource operand happens to
3448 // accept the same register, silently dropping the `rsrcidx` intent.
3449 if (!Reg->isRsrcReg32())
3450 return Error(L: RegLoc, Msg: "rsrcidx operand must be a 32-bit SGPR or VGPR");
3451
3452 if (!skipToken(Kind: AsmToken::RParen, ErrMsg: "expected closing parenthesis"))
3453 return ParseStatus::Failure;
3454
3455 Operands.push_back(Elt: std::move(Reg));
3456 return ParseStatus::Success;
3457}
3458
3459ParseStatus AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
3460 auto Loc = getLoc();
3461 if (trySkipId(Id: "off")) {
3462 Operands.push_back(
3463 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: 0, Loc, Type: AMDGPUOperand::ImmTyOff, IsFPImm: false));
3464 return ParseStatus::Success;
3465 }
3466
3467 if (!isRegister())
3468 return ParseStatus::NoMatch;
3469
3470 std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
3471 if (Reg) {
3472 Operands.push_back(Elt: std::move(Reg));
3473 return ParseStatus::Success;
3474 }
3475
3476 return ParseStatus::Failure;
3477}
3478
3479unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
3480 if ((getForcedEncodingSize() == 32 && SIInstrFlags::isVOP3(O: MII, O: Inst)) ||
3481 (getForcedEncodingSize() == 64 && !SIInstrFlags::isVOP3(O: MII, O: Inst)) ||
3482 (isForcedDPP() && !SIInstrFlags::isDPP(O: MII, O: Inst)) ||
3483 (isForcedSDWA() && !SIInstrFlags::isSDWA(O: MII, O: Inst)))
3484 return Match_InvalidOperand;
3485
3486 if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3487 Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
3488 // v_mac_f32/16 allow only dst_sel == DWORD;
3489 auto OpNum =
3490 AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::dst_sel);
3491 const auto &Op = Inst.getOperand(i: OpNum);
3492 if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
3493 return Match_InvalidOperand;
3494 }
3495 }
3496
3497 // Asm can first try to match VOPD or VOPD3. By failing early here with
3498 // Match_InvalidOperand, the parser will retry parsing as VOPD3 or VOPD.
3499 // Checking later during validateInstruction does not give a chance to retry
3500 // parsing as a different encoding.
3501 if (tryAnotherVOPDEncoding(Inst))
3502 return Match_InvalidOperand;
3503
3504 return Match_Success;
3505}
3506
3507static ArrayRef<unsigned> getAllVariants() {
3508 static const unsigned Variants[] = {
3509 AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
3510 AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::SDWA9,
3511 AMDGPUAsmVariants::DPP, AMDGPUAsmVariants::VOP3_DPP};
3512
3513 return ArrayRef(Variants);
3514}
3515
3516// What asm variants we should check
3517ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
3518 if (isForcedDPP() && isForcedVOP3()) {
3519 static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3_DPP};
3520 return ArrayRef(Variants);
3521 }
3522 if (getForcedEncodingSize() == 32) {
3523 static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
3524 return ArrayRef(Variants);
3525 }
3526
3527 if (isForcedVOP3()) {
3528 static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
3529 return ArrayRef(Variants);
3530 }
3531
3532 if (isForcedSDWA()) {
3533 static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA,
3534 AMDGPUAsmVariants::SDWA9};
3535 return ArrayRef(Variants);
3536 }
3537
3538 if (isForcedDPP()) {
3539 static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
3540 return ArrayRef(Variants);
3541 }
3542
3543 return getAllVariants();
3544}
3545
3546StringRef AMDGPUAsmParser::getMatchedVariantName() const {
3547 if (isForcedDPP() && isForcedVOP3())
3548 return "e64_dpp";
3549
3550 if (getForcedEncodingSize() == 32)
3551 return "e32";
3552
3553 if (isForcedVOP3())
3554 return "e64";
3555
3556 if (isForcedSDWA())
3557 return "sdwa";
3558
3559 if (isForcedDPP())
3560 return "dpp";
3561
3562 return "";
3563}
3564
3565MCRegister
3566AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
3567 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
3568 for (MCPhysReg Reg : Desc.implicit_uses()) {
3569 switch (Reg) {
3570 case AMDGPU::FLAT_SCR:
3571 case AMDGPU::VCC:
3572 case AMDGPU::VCC_LO:
3573 case AMDGPU::VCC_HI:
3574 case AMDGPU::M0:
3575 return Reg;
3576 default:
3577 break;
3578 }
3579 }
3580 return MCRegister();
3581}
3582
3583// NB: This code is correct only when used to check constant
3584// bus limitations because GFX7 support no f16 inline constants.
3585// Note that there are no cases when a GFX7 opcode violates
3586// constant bus limitations due to the use of an f16 constant.
3587bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
3588 unsigned OpIdx) const {
3589 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
3590
3591 if (!AMDGPU::isSISrcOperand(Desc, OpNo: OpIdx) ||
3592 AMDGPU::isKImmOperand(Desc, OpNo: OpIdx)) {
3593 return false;
3594 }
3595
3596 const MCOperand &MO = Inst.getOperand(i: OpIdx);
3597
3598 int64_t Val = MO.isImm() ? MO.getImm() : getLitValue(Expr: MO.getExpr());
3599 auto OpSize = AMDGPU::getOperandSize(Desc, OpNo: OpIdx);
3600
3601 switch (OpSize) { // expected operand size
3602 case 8:
3603 return AMDGPU::isInlinableLiteral64(Literal: Val, HasInv2Pi: hasInv2PiInlineImm());
3604 case 4:
3605 return AMDGPU::isInlinableLiteral32(Literal: Val, HasInv2Pi: hasInv2PiInlineImm());
3606 case 2: {
3607 const unsigned OperandType = Desc.operands()[OpIdx].OperandType;
3608 if (OperandType == AMDGPU::OPERAND_REG_IMM_INT16 ||
3609 OperandType == AMDGPU::OPERAND_REG_INLINE_C_INT16)
3610 return AMDGPU::isInlinableLiteralI16(Literal: Val, HasInv2Pi: hasInv2PiInlineImm());
3611
3612 if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
3613 OperandType == AMDGPU::OPERAND_REG_IMM_V2INT16)
3614 return AMDGPU::isInlinableLiteralV2I16(Literal: Val);
3615
3616 if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16 ||
3617 OperandType == AMDGPU::OPERAND_REG_IMM_V2FP16)
3618 return AMDGPU::isInlinableLiteralV2F16(Literal: Val);
3619
3620 if (OperandType == AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT)
3621 return AMDGPU::isPKFMACF16InlineConstant(Literal: Val, IsGFX11Plus: isGFX11Plus());
3622
3623 if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2BF16 ||
3624 OperandType == AMDGPU::OPERAND_REG_IMM_V2BF16)
3625 return AMDGPU::isInlinableLiteralV2BF16(Literal: Val);
3626
3627 if (OperandType == AMDGPU::OPERAND_REG_IMM_FP16 ||
3628 OperandType == AMDGPU::OPERAND_REG_INLINE_C_FP16)
3629 return AMDGPU::isInlinableLiteralFP16(Literal: Val, HasInv2Pi: hasInv2PiInlineImm());
3630
3631 if (OperandType == AMDGPU::OPERAND_REG_IMM_BF16 ||
3632 OperandType == AMDGPU::OPERAND_REG_INLINE_C_BF16)
3633 return AMDGPU::isInlinableLiteralBF16(Literal: Val, HasInv2Pi: hasInv2PiInlineImm());
3634
3635 if (OperandType == AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16 ||
3636 OperandType == AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16)
3637 return false;
3638
3639 llvm_unreachable("invalid operand type");
3640 }
3641 default:
3642 llvm_unreachable("invalid operand size");
3643 }
3644}
3645
3646unsigned AMDGPUAsmParser::getConstantBusLimit(unsigned Opcode) const {
3647 if (!isGFX10Plus())
3648 return 1;
3649
3650 switch (Opcode) {
3651 // 64-bit shift instructions can use only one scalar value input
3652 case AMDGPU::V_LSHLREV_B64_e64:
3653 case AMDGPU::V_LSHLREV_B64_gfx10:
3654 case AMDGPU::V_LSHLREV_B64_e64_gfx11:
3655 case AMDGPU::V_LSHLREV_B64_e32_gfx12:
3656 case AMDGPU::V_LSHLREV_B64_e64_gfx12:
3657 case AMDGPU::V_LSHRREV_B64_e64:
3658 case AMDGPU::V_LSHRREV_B64_gfx10:
3659 case AMDGPU::V_LSHRREV_B64_e64_gfx11:
3660 case AMDGPU::V_LSHRREV_B64_e64_gfx12:
3661 case AMDGPU::V_ASHRREV_I64_e64:
3662 case AMDGPU::V_ASHRREV_I64_gfx10:
3663 case AMDGPU::V_ASHRREV_I64_e64_gfx11:
3664 case AMDGPU::V_ASHRREV_I64_e64_gfx12:
3665 case AMDGPU::V_LSHL_B64_e64:
3666 case AMDGPU::V_LSHR_B64_e64:
3667 case AMDGPU::V_ASHR_I64_e64:
3668 return 1;
3669 default:
3670 return 2;
3671 }
3672}
3673
3674constexpr unsigned MAX_SRC_OPERANDS_NUM = 6;
3675using OperandIndices = SmallVector<int16_t, MAX_SRC_OPERANDS_NUM>;
3676
3677// Get regular operand indices in the same order as specified
3678// in the instruction (but append mandatory literals to the end).
3679static OperandIndices getSrcOperandIndices(unsigned Opcode,
3680 bool AddMandatoryLiterals = false) {
3681
3682 int16_t ImmIdx =
3683 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, Name: OpName::imm) : -1;
3684
3685 if (isVOPD(Opc: Opcode)) {
3686 int16_t ImmXIdx =
3687 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, Name: OpName::immX) : -1;
3688
3689 return {getNamedOperandIdx(Opcode, Name: OpName::src0X),
3690 getNamedOperandIdx(Opcode, Name: OpName::vsrc1X),
3691 getNamedOperandIdx(Opcode, Name: OpName::vsrc2X),
3692 getNamedOperandIdx(Opcode, Name: OpName::src0Y),
3693 getNamedOperandIdx(Opcode, Name: OpName::vsrc1Y),
3694 getNamedOperandIdx(Opcode, Name: OpName::vsrc2Y),
3695 ImmXIdx,
3696 ImmIdx};
3697 }
3698
3699 return {getNamedOperandIdx(Opcode, Name: OpName::src0),
3700 getNamedOperandIdx(Opcode, Name: OpName::src1),
3701 getNamedOperandIdx(Opcode, Name: OpName::src2), ImmIdx};
3702}
3703
3704bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
3705 const MCOperand &MO = Inst.getOperand(i: OpIdx);
3706 if (MO.isImm())
3707 return !isInlineConstant(Inst, OpIdx);
3708 if (MO.isReg()) {
3709 auto Reg = MO.getReg();
3710 if (!Reg)
3711 return false;
3712 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
3713 auto PReg = mc2PseudoReg(Reg);
3714 return isSGPR(Reg: PReg, TRI) && PReg != SGPR_NULL;
3715 }
3716 return true;
3717}
3718
3719// Based on the comment for `AMDGPUInstructionSelector::selectWritelane`:
3720// Writelane is special in that it can use SGPR and M0 (which would normally
3721// count as using the constant bus twice - but in this case it is allowed since
3722// the lane selector doesn't count as a use of the constant bus). However, it is
3723// still required to abide by the 1 SGPR rule.
3724static bool checkWriteLane(const MCInst &Inst) {
3725 const unsigned Opcode = Inst.getOpcode();
3726 if (Opcode != V_WRITELANE_B32_gfx6_gfx7 && Opcode != V_WRITELANE_B32_vi)
3727 return false;
3728 const MCOperand &LaneSelOp = Inst.getOperand(i: 2);
3729 if (!LaneSelOp.isReg())
3730 return false;
3731 auto LaneSelReg = mc2PseudoReg(Reg: LaneSelOp.getReg());
3732 return LaneSelReg == M0 || LaneSelReg == M0_gfxpre11;
3733}
3734
3735bool AMDGPUAsmParser::validateConstantBusLimitations(
3736 const MCInst &Inst, const OperandVector &Operands) {
3737 const unsigned Opcode = Inst.getOpcode();
3738 const MCInstrDesc &Desc = MII.get(Opcode);
3739 MCRegister LastSGPR;
3740 unsigned ConstantBusUseCount = 0;
3741 unsigned NumLiterals = 0;
3742 unsigned LiteralSize;
3743
3744 if (!SIInstrFlags::isVOPC(O: Desc) && !SIInstrFlags::isVOP1(O: Desc) &&
3745 !SIInstrFlags::isVOP2(O: Desc) && !SIInstrFlags::isVOP3Like(O: Desc) &&
3746 !SIInstrFlags::isSDWA(O: Desc) && !isVOPD(Opc: Opcode))
3747 return true;
3748
3749 if (checkWriteLane(Inst))
3750 return true;
3751
3752 // Check special imm operands (used by madmk, etc)
3753 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::imm)) {
3754 ++NumLiterals;
3755 LiteralSize = 4;
3756 }
3757
3758 SmallDenseSet<MCRegister> SGPRsUsed;
3759 MCRegister SGPRUsed = findImplicitSGPRReadInVOP(Inst);
3760 if (SGPRUsed) {
3761 SGPRsUsed.insert(V: SGPRUsed);
3762 ++ConstantBusUseCount;
3763 }
3764
3765 OperandIndices OpIndices = getSrcOperandIndices(Opcode);
3766
3767 unsigned ConstantBusLimit = getConstantBusLimit(Opcode);
3768
3769 for (int OpIdx : OpIndices) {
3770 if (OpIdx == -1)
3771 continue;
3772
3773 const MCOperand &MO = Inst.getOperand(i: OpIdx);
3774 if (usesConstantBus(Inst, OpIdx)) {
3775 if (MO.isReg()) {
3776 LastSGPR = mc2PseudoReg(Reg: MO.getReg());
3777 // Pairs of registers with a partial intersections like these
3778 // s0, s[0:1]
3779 // flat_scratch_lo, flat_scratch
3780 // flat_scratch_lo, flat_scratch_hi
3781 // are theoretically valid but they are disabled anyway.
3782 // Note that this code mimics SIInstrInfo::verifyInstruction
3783 if (SGPRsUsed.insert(V: LastSGPR).second) {
3784 ++ConstantBusUseCount;
3785 }
3786 } else { // Expression or a literal
3787
3788 if (Desc.operands()[OpIdx].OperandType == MCOI::OPERAND_IMMEDIATE)
3789 continue; // special operand like VINTERP attr_chan
3790
3791 // An instruction may use only one literal.
3792 // This has been validated on the previous step.
3793 // See validateVOPLiteral.
3794 // This literal may be used as more than one operand.
3795 // If all these operands are of the same size,
3796 // this literal counts as one scalar value.
3797 // Otherwise it counts as 2 scalar values.
3798 // See "GFX10 Shader Programming", section 3.6.2.3.
3799
3800 unsigned Size = AMDGPU::getOperandSize(Desc, OpNo: OpIdx);
3801 if (Size < 4)
3802 Size = 4;
3803
3804 if (NumLiterals == 0) {
3805 NumLiterals = 1;
3806 LiteralSize = Size;
3807 } else if (LiteralSize != Size) {
3808 NumLiterals = 2;
3809 }
3810 }
3811 }
3812
3813 if (ConstantBusUseCount + NumLiterals > ConstantBusLimit) {
3814 Error(L: getOperandLoc(Operands, MCOpIdx: OpIdx),
3815 Msg: "invalid operand (violates constant bus restrictions)");
3816 return false;
3817 }
3818 }
3819 return true;
3820}
3821
3822std::optional<unsigned>
3823AMDGPUAsmParser::checkVOPDRegBankConstraints(const MCInst &Inst, bool AsVOPD3) {
3824
3825 const unsigned Opcode = Inst.getOpcode();
3826 if (!isVOPD(Opc: Opcode))
3827 return {};
3828
3829 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
3830
3831 auto getVRegIdx = [&](unsigned, unsigned OperandIdx) {
3832 const MCOperand &Opr = Inst.getOperand(i: OperandIdx);
3833 return (Opr.isReg() && !isSGPR(Reg: mc2PseudoReg(Reg: Opr.getReg()), TRI))
3834 ? Opr.getReg()
3835 : MCRegister();
3836 };
3837
3838 // On GFX1170+ if both OpX and OpY are V_MOV_B32 then OPY uses SRC2
3839 // source-cache.
3840 bool SkipSrc =
3841 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1170 ||
3842 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 ||
3843 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 ||
3844 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx13 ||
3845 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 ||
3846 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx13;
3847 bool AllowSameVGPR = isGFX12Plus();
3848
3849 if (AsVOPD3) { // Literal constants are not allowed with VOPD3.
3850 for (auto OpName : {OpName::src0X, OpName::src0Y}) {
3851 int I = getNamedOperandIdx(Opcode, Name: OpName);
3852 const MCOperand &Op = Inst.getOperand(i: I);
3853 if (!Op.isImm())
3854 continue;
3855 int64_t Imm = Op.getImm();
3856 if (!AMDGPU::isInlinableLiteral32(Literal: Imm, HasInv2Pi: hasInv2PiInlineImm()) &&
3857 !AMDGPU::isInlinableLiteral64(Literal: Imm, HasInv2Pi: hasInv2PiInlineImm()))
3858 return (unsigned)I;
3859 }
3860
3861 for (auto OpName : {OpName::vsrc1X, OpName::vsrc1Y, OpName::vsrc2X,
3862 OpName::vsrc2Y, OpName::imm}) {
3863 int I = getNamedOperandIdx(Opcode, Name: OpName);
3864 if (I == -1)
3865 continue;
3866 const MCOperand &Op = Inst.getOperand(i: I);
3867 if (Op.isImm())
3868 return (unsigned)I;
3869 }
3870 }
3871
3872 const auto &InstInfo = getVOPDInstInfo(VOPDOpcode: Opcode, InstrInfo: &MII);
3873 auto InvalidCompOprIdx = InstInfo.getInvalidCompOperandIndex(
3874 GetRegIdx: getVRegIdx, MRI: *TRI, SkipSrc, AllowSameVGPR, VOPD3: AsVOPD3);
3875
3876 return InvalidCompOprIdx;
3877}
3878
3879bool AMDGPUAsmParser::validateVOPD(const MCInst &Inst,
3880 const OperandVector &Operands) {
3881
3882 unsigned Opcode = Inst.getOpcode();
3883 bool AsVOPD3 = SIInstrFlags::isVOPD3(O: MII, O: Inst);
3884
3885 if (AsVOPD3) {
3886 for (const std::unique_ptr<MCParsedAsmOperand> &Operand : Operands) {
3887 AMDGPUOperand &Op = (AMDGPUOperand &)*Operand;
3888 if ((Op.isRegKind() || Op.isImmTy(ImmT: AMDGPUOperand::ImmTyNone)) &&
3889 (Op.getModifiers().getFPModifiersOperand() & SISrcMods::ABS))
3890 Error(L: Op.getStartLoc(), Msg: "ABS not allowed in VOPD3 instructions");
3891 }
3892 }
3893
3894 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3);
3895 if (!InvalidCompOprIdx.has_value())
3896 return true;
3897
3898 auto CompOprIdx = *InvalidCompOprIdx;
3899 const auto &InstInfo = getVOPDInstInfo(VOPDOpcode: Opcode, InstrInfo: &MII);
3900 auto ParsedIdx =
3901 std::max(a: InstInfo[VOPD::X].getIndexInParsedOperands(CompOprIdx),
3902 b: InstInfo[VOPD::Y].getIndexInParsedOperands(CompOprIdx));
3903 assert(ParsedIdx > 0 && ParsedIdx < Operands.size());
3904
3905 auto Loc = ((AMDGPUOperand &)*Operands[ParsedIdx]).getStartLoc();
3906 if (CompOprIdx == VOPD::Component::DST) {
3907 if (AsVOPD3)
3908 Error(L: Loc, Msg: "dst registers must be distinct");
3909 else
3910 Error(L: Loc, Msg: "one dst register must be even and the other odd");
3911 } else {
3912 auto CompSrcIdx = CompOprIdx - VOPD::Component::DST_NUM;
3913 Error(L: Loc, Msg: Twine("src") + Twine(CompSrcIdx) +
3914 " operands must use different VGPR banks");
3915 }
3916
3917 return false;
3918}
3919
3920// \returns true if \p Inst does not satisfy VOPD constraints, but can be
3921// potentially used as VOPD3 with the same operands.
3922bool AMDGPUAsmParser::tryVOPD3(const MCInst &Inst) {
3923 // First check if it fits VOPD
3924 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3: false);
3925 if (!InvalidCompOprIdx.has_value())
3926 return false;
3927
3928 // Then if it fits VOPD3
3929 InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3: true);
3930 if (InvalidCompOprIdx.has_value()) {
3931 // If failed operand is dst it is better to show error about VOPD3
3932 // instruction as it has more capabilities and error message will be
3933 // more informative. If the dst is not legal for VOPD3, then it is not
3934 // legal for VOPD either.
3935 if (*InvalidCompOprIdx == VOPD::Component::DST)
3936 return true;
3937
3938 // Otherwise prefer VOPD as we may find ourselves in an awkward situation
3939 // with a conflict in tied implicit src2 of fmac and no asm operand to
3940 // to point to.
3941 return false;
3942 }
3943 return true;
3944}
3945
3946// \returns true is a VOPD3 instruction can be also represented as a shorter
3947// VOPD encoding.
3948bool AMDGPUAsmParser::tryVOPD(const MCInst &Inst) {
3949 const unsigned Opcode = Inst.getOpcode();
3950 const auto &II = getVOPDInstInfo(VOPDOpcode: Opcode, InstrInfo: &MII);
3951 unsigned EncodingFamily = AMDGPU::getVOPDEncodingFamily(ST: getSTI());
3952 if (!getCanBeVOPD(Opc: II[VOPD::X].getOpcode(), EncodingFamily, VOPD3: false).X ||
3953 !getCanBeVOPD(Opc: II[VOPD::Y].getOpcode(), EncodingFamily, VOPD3: false).Y)
3954 return false;
3955
3956 // This is an awkward exception, VOPD3 variant of V_DUAL_CNDMASK_B32 has
3957 // explicit src2 even if it is vcc_lo. If it was parsed as VOPD3 it cannot
3958 // be parsed as VOPD which does not accept src2.
3959 if (II[VOPD::X].getOpcode() == AMDGPU::V_CNDMASK_B32_e32 ||
3960 II[VOPD::Y].getOpcode() == AMDGPU::V_CNDMASK_B32_e32)
3961 return false;
3962
3963 // If any modifiers are set this cannot be VOPD.
3964 for (auto OpName : {OpName::src0X_modifiers, OpName::src0Y_modifiers,
3965 OpName::vsrc1X_modifiers, OpName::vsrc1Y_modifiers,
3966 OpName::vsrc2X_modifiers, OpName::vsrc2Y_modifiers}) {
3967 int I = getNamedOperandIdx(Opcode, Name: OpName);
3968 if (I == -1)
3969 continue;
3970 if (Inst.getOperand(i: I).getImm())
3971 return false;
3972 }
3973
3974 return !tryVOPD3(Inst);
3975}
3976
3977// VOPD3 has more relaxed register constraints than VOPD. We prefer shorter VOPD
3978// form but switch to VOPD3 otherwise.
3979bool AMDGPUAsmParser::tryAnotherVOPDEncoding(const MCInst &Inst) {
3980 if (!isGFX1250Plus() || !isVOPD(Opc: Inst.getOpcode()))
3981 return false;
3982
3983 if (SIInstrFlags::isVOPD3(O: MII, O: Inst))
3984 return tryVOPD(Inst);
3985 return tryVOPD3(Inst);
3986}
3987
3988bool AMDGPUAsmParser::validateIntClampSupported(const MCInst &Inst) {
3989
3990 const unsigned Opc = Inst.getOpcode();
3991
3992 if (SIInstrFlags::hasIntClamp(O: MII, O: Inst) && !hasIntClamp()) {
3993 int ClampIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::clamp);
3994 assert(ClampIdx != -1);
3995 return Inst.getOperand(i: ClampIdx).getImm() == 0;
3996 }
3997
3998 return true;
3999}
4000
4001bool AMDGPUAsmParser::validateMIMGDataSize(const MCInst &Inst, SMLoc IDLoc) {
4002
4003 const unsigned Opc = Inst.getOpcode();
4004 const MCInstrDesc &Desc = MII.get(Opcode: Opc);
4005
4006 if ((SIInstrFlags::isImage(O: Desc)) == 0)
4007 return true;
4008
4009 int VDataIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdata);
4010 int DMaskIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dmask);
4011 int TFEIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::tfe);
4012
4013 if (VDataIdx == -1 && isGFX10Plus()) // no return image_sample
4014 return true;
4015
4016 if ((DMaskIdx == -1 || TFEIdx == -1) &&
4017 hasBVHRayTracingInsts()) // intersect_ray
4018 return true;
4019
4020 unsigned VDataSize = getRegOperandSize(Desc, OpNo: VDataIdx);
4021 unsigned TFESize = (TFEIdx != -1 && Inst.getOperand(i: TFEIdx).getImm()) ? 1 : 0;
4022 unsigned DMask = Inst.getOperand(i: DMaskIdx).getImm() & 0xf;
4023 if (DMask == 0)
4024 DMask = 1;
4025
4026 bool IsPackedD16 = false;
4027 unsigned DataSize = SIInstrFlags::isGather4(O: Desc) ? 4 : llvm::popcount(Value: DMask);
4028 if (hasPackedD16()) {
4029 int D16Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::d16);
4030 IsPackedD16 = D16Idx >= 0;
4031 if (IsPackedD16 && Inst.getOperand(i: D16Idx).getImm())
4032 DataSize = (DataSize + 1) / 2;
4033 }
4034
4035 if ((VDataSize / 4) == DataSize + TFESize)
4036 return true;
4037
4038 StringRef Modifiers;
4039 if (isGFX90A())
4040 Modifiers = IsPackedD16 ? "dmask and d16" : "dmask";
4041 else
4042 Modifiers = IsPackedD16 ? "dmask, d16 and tfe" : "dmask and tfe";
4043
4044 Error(L: IDLoc, Msg: Twine("image data size does not match ") + Modifiers);
4045 return false;
4046}
4047
4048bool AMDGPUAsmParser::validateMIMGAddrSize(const MCInst &Inst, SMLoc IDLoc) {
4049 const unsigned Opc = Inst.getOpcode();
4050 const MCInstrDesc &Desc = MII.get(Opcode: Opc);
4051
4052 if (!SIInstrFlags::isImage(O: Desc) || !isGFX10Plus())
4053 return true;
4054
4055 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc);
4056
4057 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4058 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
4059 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr0);
4060 AMDGPU::OpName RSrcOpName =
4061 SIInstrFlags::isMIMG(O: Desc) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
4062 int SrsrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: RSrcOpName);
4063 int DimIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dim);
4064 int A16Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::a16);
4065
4066 assert(VAddr0Idx != -1);
4067 assert(SrsrcIdx != -1);
4068 assert(SrsrcIdx > VAddr0Idx);
4069
4070 bool IsA16 = (A16Idx != -1 && Inst.getOperand(i: A16Idx).getImm());
4071 if (BaseOpcode->BVH) {
4072 if (IsA16 == BaseOpcode->A16)
4073 return true;
4074 Error(L: IDLoc, Msg: "image address size does not match a16");
4075 return false;
4076 }
4077
4078 unsigned Dim = Inst.getOperand(i: DimIdx).getImm();
4079 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfoByEncoding(DimEnc: Dim);
4080 bool IsNSA = SrsrcIdx - VAddr0Idx > 1;
4081 unsigned ActualAddrSize =
4082 IsNSA ? SrsrcIdx - VAddr0Idx : getRegOperandSize(Desc, OpNo: VAddr0Idx) / 4;
4083
4084 unsigned ExpectedAddrSize =
4085 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, Dim: DimInfo, IsA16, IsG16Supported: hasG16());
4086
4087 if (IsNSA) {
4088 if (hasPartialNSAEncoding() &&
4089 ExpectedAddrSize > getNSAMaxSize(HasSampler: SIInstrFlags::isVSAMPLE(O: Desc))) {
4090 int VAddrLastIdx = SrsrcIdx - 1;
4091 unsigned VAddrLastSize = getRegOperandSize(Desc, OpNo: VAddrLastIdx) / 4;
4092
4093 ActualAddrSize = VAddrLastIdx - VAddr0Idx + VAddrLastSize;
4094 }
4095 } else {
4096 if (ExpectedAddrSize > 12)
4097 ExpectedAddrSize = 16;
4098
4099 // Allow oversized 8 VGPR vaddr when only 5/6/7 VGPRs are required.
4100 // This provides backward compatibility for assembly created
4101 // before 160b/192b/224b types were directly supported.
4102 if (ActualAddrSize == 8 && (ExpectedAddrSize >= 5 && ExpectedAddrSize <= 7))
4103 return true;
4104 }
4105
4106 if (ActualAddrSize == ExpectedAddrSize)
4107 return true;
4108
4109 Error(L: IDLoc, Msg: "image address size does not match dim and a16");
4110 return false;
4111}
4112
4113bool AMDGPUAsmParser::validateMIMGAtomicDMask(const MCInst &Inst) {
4114
4115 const unsigned Opc = Inst.getOpcode();
4116 const MCInstrDesc &Desc = MII.get(Opcode: Opc);
4117
4118 if ((SIInstrFlags::isImage(O: Desc)) == 0)
4119 return true;
4120 if (!Desc.mayLoad() || !Desc.mayStore())
4121 return true; // Not atomic
4122
4123 int DMaskIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dmask);
4124 unsigned DMask = Inst.getOperand(i: DMaskIdx).getImm() & 0xf;
4125
4126 // This is an incomplete check because image_atomic_cmpswap
4127 // may only use 0x3 and 0xf while other atomic operations
4128 // may use 0x1 and 0x3. However these limitations are
4129 // verified when we check that dmask matches dst size.
4130 return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
4131}
4132
4133bool AMDGPUAsmParser::validateMIMGGatherDMask(const MCInst &Inst) {
4134
4135 const unsigned Opc = Inst.getOpcode();
4136
4137 if (!SIInstrFlags::isGather4(O: MII, O: Inst))
4138 return true;
4139
4140 int DMaskIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dmask);
4141 unsigned DMask = Inst.getOperand(i: DMaskIdx).getImm() & 0xf;
4142
4143 // GATHER4 instructions use dmask in a different fashion compared to
4144 // other MIMG instructions. The only useful DMASK values are
4145 // 1=red, 2=green, 4=blue, 8=alpha. (e.g. 1 returns
4146 // (red,red,red,red) etc.) The ISA document doesn't mention
4147 // this.
4148 return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8;
4149}
4150
4151bool AMDGPUAsmParser::validateMIMGDim(const MCInst &Inst,
4152 const OperandVector &Operands) {
4153 if (!isGFX10Plus())
4154 return true;
4155
4156 const unsigned Opc = Inst.getOpcode();
4157
4158 if ((SIInstrFlags::isImage(O: MII, O: Inst)) == 0)
4159 return true;
4160
4161 // image_bvh_intersect_ray instructions do not have dim
4162 if (AMDGPU::getMIMGBaseOpcode(Opc)->BVH)
4163 return true;
4164
4165 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4166 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4167 if (Op.isDim())
4168 return true;
4169 }
4170 return false;
4171}
4172
4173bool AMDGPUAsmParser::validateMIMGMSAA(const MCInst &Inst) {
4174 const unsigned Opc = Inst.getOpcode();
4175
4176 if ((SIInstrFlags::isImage(O: MII, O: Inst)) == 0)
4177 return true;
4178
4179 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc);
4180 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4181 AMDGPU::getMIMGBaseOpcodeInfo(BaseOpcode: Info->BaseOpcode);
4182
4183 if (!BaseOpcode->MSAA)
4184 return true;
4185
4186 int DimIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dim);
4187 assert(DimIdx != -1);
4188
4189 unsigned Dim = Inst.getOperand(i: DimIdx).getImm();
4190 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfoByEncoding(DimEnc: Dim);
4191
4192 return DimInfo->MSAA;
4193}
4194
4195static bool IsMovrelsSDWAOpcode(const unsigned Opcode) {
4196 switch (Opcode) {
4197 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
4198 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
4199 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
4200 return true;
4201 default:
4202 return false;
4203 }
4204}
4205
4206// movrels* opcodes should only allow VGPRS as src0.
4207// This is specified in .td description for vop1/vop3,
4208// but sdwa is handled differently. See isSDWAOperand.
4209bool AMDGPUAsmParser::validateMovrels(const MCInst &Inst,
4210 const OperandVector &Operands) {
4211
4212 const unsigned Opc = Inst.getOpcode();
4213
4214 if (!SIInstrFlags::isSDWA(O: MII, O: Inst) || !IsMovrelsSDWAOpcode(Opcode: Opc))
4215 return true;
4216
4217 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
4218 assert(Src0Idx != -1);
4219
4220 const MCOperand &Src0 = Inst.getOperand(i: Src0Idx);
4221 if (Src0.isReg()) {
4222 auto Reg = mc2PseudoReg(Reg: Src0.getReg());
4223 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
4224 if (!isSGPR(Reg, TRI))
4225 return true;
4226 }
4227
4228 Error(L: getOperandLoc(Operands, MCOpIdx: Src0Idx), Msg: "source operand must be a VGPR");
4229 return false;
4230}
4231
4232bool AMDGPUAsmParser::validateMAIAccWrite(const MCInst &Inst,
4233 const OperandVector &Operands) {
4234
4235 const unsigned Opc = Inst.getOpcode();
4236
4237 if (Opc != AMDGPU::V_ACCVGPR_WRITE_B32_vi)
4238 return true;
4239
4240 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
4241 assert(Src0Idx != -1);
4242
4243 const MCOperand &Src0 = Inst.getOperand(i: Src0Idx);
4244 if (!Src0.isReg())
4245 return true;
4246
4247 auto Reg = mc2PseudoReg(Reg: Src0.getReg());
4248 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
4249 if (!isGFX90A() && isSGPR(Reg, TRI)) {
4250 Error(L: getOperandLoc(Operands, MCOpIdx: Src0Idx),
4251 Msg: "source operand must be either a VGPR or an inline constant");
4252 return false;
4253 }
4254
4255 return true;
4256}
4257
4258bool AMDGPUAsmParser::validateMAISrc2(const MCInst &Inst,
4259 const OperandVector &Operands) {
4260 unsigned Opcode = Inst.getOpcode();
4261
4262 if (!SIInstrFlags::isMAI(O: MII, O: Inst) ||
4263 !getFeatureBits()[FeatureMFMAInlineLiteralBug])
4264 return true;
4265
4266 const int Src2Idx = getNamedOperandIdx(Opcode, Name: OpName::src2);
4267 if (Src2Idx == -1)
4268 return true;
4269
4270 if (Inst.getOperand(i: Src2Idx).isImm() && isInlineConstant(Inst, OpIdx: Src2Idx)) {
4271 Error(L: getOperandLoc(Operands, MCOpIdx: Src2Idx),
4272 Msg: "inline constants are not allowed for this operand");
4273 return false;
4274 }
4275
4276 return true;
4277}
4278
4279bool AMDGPUAsmParser::validateMFMA(const MCInst &Inst,
4280 const OperandVector &Operands) {
4281 const unsigned Opc = Inst.getOpcode();
4282 const MCInstrDesc &Desc = MII.get(Opcode: Opc);
4283
4284 if (!SIInstrFlags::isMAI(O: Desc))
4285 return true;
4286
4287 int BlgpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::blgp);
4288 if (BlgpIdx != -1) {
4289 if (const MFMA_F8F6F4_Info *Info = AMDGPU::isMFMA_F8F6F4(Opcode: Opc)) {
4290 int CbszIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::cbsz);
4291
4292 unsigned CBSZ = Inst.getOperand(i: CbszIdx).getImm();
4293 unsigned BLGP = Inst.getOperand(i: BlgpIdx).getImm();
4294
4295 // Validate the correct register size was used for the floating point
4296 // format operands
4297
4298 bool Success = true;
4299 if (Info->NumRegsSrcA != mfmaScaleF8F6F4FormatToNumRegs(EncodingVal: CBSZ)) {
4300 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
4301 Error(L: getOperandLoc(Operands, MCOpIdx: Src0Idx),
4302 Msg: "wrong register tuple size for cbsz value " + Twine(CBSZ));
4303 Success = false;
4304 }
4305
4306 if (Info->NumRegsSrcB != mfmaScaleF8F6F4FormatToNumRegs(EncodingVal: BLGP)) {
4307 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
4308 Error(L: getOperandLoc(Operands, MCOpIdx: Src1Idx),
4309 Msg: "wrong register tuple size for blgp value " + Twine(BLGP));
4310 Success = false;
4311 }
4312
4313 return Success;
4314 }
4315 }
4316
4317 const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
4318 if (Src2Idx == -1)
4319 return true;
4320
4321 const MCOperand &Src2 = Inst.getOperand(i: Src2Idx);
4322 if (!Src2.isReg())
4323 return true;
4324
4325 MCRegister Src2Reg = Src2.getReg();
4326 MCRegister DstReg = Inst.getOperand(i: 0).getReg();
4327 if (Src2Reg == DstReg)
4328 return true;
4329
4330 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
4331 if (TRI->getRegClass(i: MII.getOpRegClassID(OpInfo: Desc.operands()[0], HwModeId: HwMode))
4332 .getSizeInBits() <= 128)
4333 return true;
4334
4335 if (TRI->regsOverlap(RegA: Src2Reg, RegB: DstReg)) {
4336 Error(L: getOperandLoc(Operands, MCOpIdx: Src2Idx),
4337 Msg: "source 2 operand must not partially overlap with dst");
4338 return false;
4339 }
4340
4341 return true;
4342}
4343
4344bool AMDGPUAsmParser::validateDivScale(const MCInst &Inst) {
4345 switch (Inst.getOpcode()) {
4346 default:
4347 return true;
4348 case V_DIV_SCALE_F32_gfx6_gfx7:
4349 case V_DIV_SCALE_F32_vi:
4350 case V_DIV_SCALE_F32_gfx10:
4351 case V_DIV_SCALE_F64_gfx6_gfx7:
4352 case V_DIV_SCALE_F64_vi:
4353 case V_DIV_SCALE_F64_gfx10:
4354 break;
4355 }
4356
4357 // TODO: Check that src0 = src1 or src2.
4358
4359 for (auto Name :
4360 {AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src2_modifiers,
4361 AMDGPU::OpName::src2_modifiers}) {
4362 if (Inst.getOperand(i: AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name))
4363 .getImm() &
4364 SISrcMods::ABS) {
4365 return false;
4366 }
4367 }
4368
4369 return true;
4370}
4371
4372bool AMDGPUAsmParser::validateMIMGD16(const MCInst &Inst) {
4373
4374 const unsigned Opc = Inst.getOpcode();
4375
4376 if ((SIInstrFlags::isImage(O: MII, O: Inst)) == 0)
4377 return true;
4378
4379 int D16Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::d16);
4380 if (D16Idx >= 0 && Inst.getOperand(i: D16Idx).getImm()) {
4381 if (isCI() || isSI())
4382 return false;
4383 }
4384
4385 return true;
4386}
4387
4388bool AMDGPUAsmParser::validateTensorR128(const MCInst &Inst) {
4389 const unsigned Opc = Inst.getOpcode();
4390
4391 if (!SIInstrFlags::usesTENSOR_CNT(O: MII, O: Inst))
4392 return true;
4393
4394 int R128Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::r128);
4395
4396 return R128Idx < 0 || !Inst.getOperand(i: R128Idx).getImm();
4397}
4398
4399static bool IsRevOpcode(const unsigned Opcode) {
4400 switch (Opcode) {
4401 case AMDGPU::V_SUBREV_F32_e32:
4402 case AMDGPU::V_SUBREV_F32_e64:
4403 case AMDGPU::V_SUBREV_F32_e32_gfx10:
4404 case AMDGPU::V_SUBREV_F32_e32_gfx6_gfx7:
4405 case AMDGPU::V_SUBREV_F32_e32_vi:
4406 case AMDGPU::V_SUBREV_F32_e64_gfx10:
4407 case AMDGPU::V_SUBREV_F32_e64_gfx6_gfx7:
4408 case AMDGPU::V_SUBREV_F32_e64_vi:
4409
4410 case AMDGPU::V_SUBREV_CO_U32_e32:
4411 case AMDGPU::V_SUBREV_CO_U32_e64:
4412 case AMDGPU::V_SUBREV_I32_e32_gfx6_gfx7:
4413 case AMDGPU::V_SUBREV_I32_e64_gfx6_gfx7:
4414
4415 case AMDGPU::V_SUBBREV_U32_e32:
4416 case AMDGPU::V_SUBBREV_U32_e64:
4417 case AMDGPU::V_SUBBREV_U32_e32_gfx6_gfx7:
4418 case AMDGPU::V_SUBBREV_U32_e32_vi:
4419 case AMDGPU::V_SUBBREV_U32_e64_gfx6_gfx7:
4420 case AMDGPU::V_SUBBREV_U32_e64_vi:
4421
4422 case AMDGPU::V_SUBREV_U32_e32:
4423 case AMDGPU::V_SUBREV_U32_e64:
4424 case AMDGPU::V_SUBREV_U32_e32_gfx9:
4425 case AMDGPU::V_SUBREV_U32_e32_vi:
4426 case AMDGPU::V_SUBREV_U32_e64_gfx9:
4427 case AMDGPU::V_SUBREV_U32_e64_vi:
4428
4429 case AMDGPU::V_SUBREV_F16_e32:
4430 case AMDGPU::V_SUBREV_F16_e64:
4431 case AMDGPU::V_SUBREV_F16_e32_gfx10:
4432 case AMDGPU::V_SUBREV_F16_e32_vi:
4433 case AMDGPU::V_SUBREV_F16_e64_gfx10:
4434 case AMDGPU::V_SUBREV_F16_e64_vi:
4435
4436 case AMDGPU::V_SUBREV_U16_e32:
4437 case AMDGPU::V_SUBREV_U16_e64:
4438 case AMDGPU::V_SUBREV_U16_e32_vi:
4439 case AMDGPU::V_SUBREV_U16_e64_vi:
4440
4441 case AMDGPU::V_SUBREV_CO_U32_e32_gfx9:
4442 case AMDGPU::V_SUBREV_CO_U32_e64_gfx10:
4443 case AMDGPU::V_SUBREV_CO_U32_e64_gfx9:
4444
4445 case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9:
4446 case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9:
4447
4448 case AMDGPU::V_SUBREV_NC_U32_e32_gfx10:
4449 case AMDGPU::V_SUBREV_NC_U32_e64_gfx10:
4450
4451 case AMDGPU::V_SUBREV_CO_CI_U32_e32_gfx10:
4452 case AMDGPU::V_SUBREV_CO_CI_U32_e64_gfx10:
4453
4454 case AMDGPU::V_LSHRREV_B32_e32:
4455 case AMDGPU::V_LSHRREV_B32_e64:
4456 case AMDGPU::V_LSHRREV_B32_e32_gfx6_gfx7:
4457 case AMDGPU::V_LSHRREV_B32_e64_gfx6_gfx7:
4458 case AMDGPU::V_LSHRREV_B32_e32_vi:
4459 case AMDGPU::V_LSHRREV_B32_e64_vi:
4460 case AMDGPU::V_LSHRREV_B32_e32_gfx10:
4461 case AMDGPU::V_LSHRREV_B32_e64_gfx10:
4462
4463 case AMDGPU::V_ASHRREV_I32_e32:
4464 case AMDGPU::V_ASHRREV_I32_e64:
4465 case AMDGPU::V_ASHRREV_I32_e32_gfx10:
4466 case AMDGPU::V_ASHRREV_I32_e32_gfx6_gfx7:
4467 case AMDGPU::V_ASHRREV_I32_e32_vi:
4468 case AMDGPU::V_ASHRREV_I32_e64_gfx10:
4469 case AMDGPU::V_ASHRREV_I32_e64_gfx6_gfx7:
4470 case AMDGPU::V_ASHRREV_I32_e64_vi:
4471
4472 case AMDGPU::V_LSHLREV_B32_e32:
4473 case AMDGPU::V_LSHLREV_B32_e64:
4474 case AMDGPU::V_LSHLREV_B32_e32_gfx10:
4475 case AMDGPU::V_LSHLREV_B32_e32_gfx6_gfx7:
4476 case AMDGPU::V_LSHLREV_B32_e32_vi:
4477 case AMDGPU::V_LSHLREV_B32_e64_gfx10:
4478 case AMDGPU::V_LSHLREV_B32_e64_gfx6_gfx7:
4479 case AMDGPU::V_LSHLREV_B32_e64_vi:
4480
4481 case AMDGPU::V_LSHLREV_B16_e32:
4482 case AMDGPU::V_LSHLREV_B16_e64:
4483 case AMDGPU::V_LSHLREV_B16_e32_vi:
4484 case AMDGPU::V_LSHLREV_B16_e64_vi:
4485 case AMDGPU::V_LSHLREV_B16_gfx10:
4486
4487 case AMDGPU::V_LSHRREV_B16_e32:
4488 case AMDGPU::V_LSHRREV_B16_e64:
4489 case AMDGPU::V_LSHRREV_B16_e32_vi:
4490 case AMDGPU::V_LSHRREV_B16_e64_vi:
4491 case AMDGPU::V_LSHRREV_B16_gfx10:
4492
4493 case AMDGPU::V_ASHRREV_I16_e32:
4494 case AMDGPU::V_ASHRREV_I16_e64:
4495 case AMDGPU::V_ASHRREV_I16_e32_vi:
4496 case AMDGPU::V_ASHRREV_I16_e64_vi:
4497 case AMDGPU::V_ASHRREV_I16_gfx10:
4498
4499 case AMDGPU::V_LSHLREV_B64_e64:
4500 case AMDGPU::V_LSHLREV_B64_gfx10:
4501 case AMDGPU::V_LSHLREV_B64_vi:
4502
4503 case AMDGPU::V_LSHRREV_B64_e64:
4504 case AMDGPU::V_LSHRREV_B64_gfx10:
4505 case AMDGPU::V_LSHRREV_B64_vi:
4506
4507 case AMDGPU::V_ASHRREV_I64_e64:
4508 case AMDGPU::V_ASHRREV_I64_gfx10:
4509 case AMDGPU::V_ASHRREV_I64_vi:
4510
4511 case AMDGPU::V_PK_LSHLREV_B16:
4512 case AMDGPU::V_PK_LSHLREV_B16_gfx10:
4513 case AMDGPU::V_PK_LSHLREV_B16_vi:
4514
4515 case AMDGPU::V_PK_LSHRREV_B16:
4516 case AMDGPU::V_PK_LSHRREV_B16_gfx10:
4517 case AMDGPU::V_PK_LSHRREV_B16_vi:
4518 case AMDGPU::V_PK_ASHRREV_I16:
4519 case AMDGPU::V_PK_ASHRREV_I16_gfx10:
4520 case AMDGPU::V_PK_ASHRREV_I16_vi:
4521 return true;
4522 default:
4523 return false;
4524 }
4525}
4526
4527bool AMDGPUAsmParser::validateLdsDirect(const MCInst &Inst,
4528 const OperandVector &Operands) {
4529 const unsigned Opcode = Inst.getOpcode();
4530
4531 // lds_direct register is defined so that it can be used
4532 // with 9-bit operands only. Ignore encodings which do not accept these.
4533 if (!SIInstrFlags::isVOP1(O: MII, O: Inst) && !SIInstrFlags::isVOP2(O: MII, O: Inst) &&
4534 !SIInstrFlags::isVOP3Like(O: MII, O: Inst) &&
4535 !SIInstrFlags::isVOPC(O: MII, O: Inst) && !SIInstrFlags::isSDWA(O: MII, O: Inst))
4536 return true;
4537
4538 for (auto SrcName : {OpName::src0, OpName::src1, OpName::src2}) {
4539 auto SrcIdx = getNamedOperandIdx(Opcode, Name: SrcName);
4540 if (SrcIdx == -1)
4541 break;
4542 const auto &Src = Inst.getOperand(i: SrcIdx);
4543 if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
4544
4545 if (isGFX90A() || isGFX11Plus()) {
4546 Error(L: getOperandLoc(Operands, MCOpIdx: SrcIdx),
4547 Msg: "lds_direct is not supported on this GPU");
4548 return false;
4549 }
4550
4551 if (IsRevOpcode(Opcode) || SIInstrFlags::isSDWA(O: MII, O: Inst)) {
4552 Error(L: getOperandLoc(Operands, MCOpIdx: SrcIdx),
4553 Msg: "lds_direct cannot be used with this instruction");
4554 return false;
4555 }
4556
4557 if (SrcName != OpName::src0) {
4558 Error(L: getOperandLoc(Operands, MCOpIdx: SrcIdx),
4559 Msg: "lds_direct may be used as src0 only");
4560 return false;
4561 }
4562 }
4563 }
4564
4565 return true;
4566}
4567
4568SMLoc AMDGPUAsmParser::getFlatOffsetLoc(const OperandVector &Operands) const {
4569 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4570 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4571 if (Op.isFlatOffset())
4572 return Op.getStartLoc();
4573 }
4574 return getLoc();
4575}
4576
4577bool AMDGPUAsmParser::validateOffset(const MCInst &Inst,
4578 const OperandVector &Operands) {
4579 auto Opcode = Inst.getOpcode();
4580 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::offset);
4581 if (OpNum == -1)
4582 return true;
4583
4584 if (SIInstrFlags::isFLAT(O: MII, O: Inst))
4585 return validateFlatOffset(Inst, Operands);
4586
4587 if (SIInstrFlags::isSMRD(O: MII, O: Inst))
4588 return validateSMEMOffset(Inst, Operands);
4589
4590 const auto &Op = Inst.getOperand(i: OpNum);
4591 // GFX12+ buffer ops: InstOffset is signed 24, but must not be a negative.
4592 if (isGFX12Plus() && SIInstrFlags::isBuffer(O: MII, O: Inst)) {
4593 const unsigned OffsetSize = 24;
4594 if (!isUIntN(N: OffsetSize - 1, x: Op.getImm())) {
4595 Error(L: getFlatOffsetLoc(Operands),
4596 Msg: Twine("expected a ") + Twine(OffsetSize - 1) +
4597 "-bit unsigned offset for buffer ops");
4598 return false;
4599 }
4600 } else {
4601 const unsigned OffsetSize = 16;
4602 if (!isUIntN(N: OffsetSize, x: Op.getImm())) {
4603 Error(L: getFlatOffsetLoc(Operands),
4604 Msg: Twine("expected a ") + Twine(OffsetSize) + "-bit unsigned offset");
4605 return false;
4606 }
4607 }
4608 return true;
4609}
4610
4611bool AMDGPUAsmParser::validateFlatOffset(const MCInst &Inst,
4612 const OperandVector &Operands) {
4613 if (!SIInstrFlags::isFLAT(O: MII, O: Inst))
4614 return true;
4615
4616 auto Opcode = Inst.getOpcode();
4617 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::offset);
4618 assert(OpNum != -1);
4619
4620 const auto &Op = Inst.getOperand(i: OpNum);
4621 if (!hasFlatOffsets() && Op.getImm() != 0) {
4622 Error(L: getFlatOffsetLoc(Operands),
4623 Msg: "flat offset modifier is not supported on this GPU");
4624 return false;
4625 }
4626
4627 // For pre-GFX12 FLAT instructions the offset must be positive;
4628 // MSB is ignored and forced to zero.
4629 unsigned OffsetSize = AMDGPU::getNumFlatOffsetBits(ST: getSTI());
4630 bool AllowNegative =
4631 SIInstrFlags::isSegmentSpecificFLAT(O: MII, O: Inst) || isGFX12Plus();
4632 if (!isIntN(N: OffsetSize, x: Op.getImm()) || (!AllowNegative && Op.getImm() < 0)) {
4633 Error(L: getFlatOffsetLoc(Operands),
4634 Msg: Twine("expected a ") +
4635 (AllowNegative ? Twine(OffsetSize) + "-bit signed offset"
4636 : Twine(OffsetSize - 1) + "-bit unsigned offset"));
4637 return false;
4638 }
4639
4640 return true;
4641}
4642
4643SMLoc AMDGPUAsmParser::getSMEMOffsetLoc(const OperandVector &Operands) const {
4644 // Start with second operand because SMEM Offset cannot be dst or src0.
4645 for (unsigned i = 2, e = Operands.size(); i != e; ++i) {
4646 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4647 if (Op.isSMEMOffset() || Op.isSMEMOffsetMod())
4648 return Op.getStartLoc();
4649 }
4650 return getLoc();
4651}
4652
4653bool AMDGPUAsmParser::validateSMEMOffset(const MCInst &Inst,
4654 const OperandVector &Operands) {
4655 if (isCI() || isSI())
4656 return true;
4657
4658 if (!SIInstrFlags::isSMRD(O: MII, O: Inst))
4659 return true;
4660
4661 auto Opcode = Inst.getOpcode();
4662 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::offset);
4663 if (OpNum == -1)
4664 return true;
4665
4666 const auto &Op = Inst.getOperand(i: OpNum);
4667 if (!Op.isImm())
4668 return true;
4669
4670 uint64_t Offset = Op.getImm();
4671 bool IsBuffer = AMDGPU::getSMEMIsBuffer(Opc: Opcode);
4672 if (AMDGPU::isLegalSMRDEncodedUnsignedOffset(ST: getSTI(), EncodedOffset: Offset) ||
4673 AMDGPU::isLegalSMRDEncodedSignedOffset(ST: getSTI(), EncodedOffset: Offset, IsBuffer))
4674 return true;
4675
4676 Error(L: getSMEMOffsetLoc(Operands),
4677 Msg: isGFX12Plus() && IsBuffer
4678 ? "expected a 23-bit unsigned offset for buffer ops"
4679 : isGFX12Plus() ? "expected a 24-bit signed offset"
4680 : (isVI() || IsBuffer) ? "expected a 20-bit unsigned offset"
4681 : "expected a 21-bit signed offset");
4682
4683 return false;
4684}
4685
4686// GFX12+ SMEM prefetches and probes encode an immediate sdata in 5 bits.
4687bool AMDGPUAsmParser::validateSMEMSData(const MCInst &Inst,
4688 const OperandVector &Operands) {
4689 if (!isGFX12Plus() || !SIInstrFlags::isSMRD(O: MII, O: Inst))
4690 return true;
4691
4692 int OpNum =
4693 AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::sdata);
4694 if (OpNum == -1 || !Inst.getOperand(i: OpNum).isImm() ||
4695 isUInt<5>(x: Inst.getOperand(i: OpNum).getImm()))
4696 return true;
4697
4698 Error(L: getOperandLoc(Operands, MCOpIdx: OpNum), Msg: "expected a 5-bit unsigned immediate");
4699 return false;
4700}
4701
4702// On subtargets with FeatureBF16InlineConstFromUpperFP32 the hardware generates
4703// a bf16 inline constant in the high half of the corresponding fp32 inline
4704// constant. A VOP1 bf16 opcode (v_cvt_f32_bf16 and the bf16 transcendentals)
4705// reads the low half of its source, so it must use the VOP3 encoding with
4706// op_sel[0] set in order to see the constant at all.
4707bool AMDGPUAsmParser::validateBF16InlineConst(const MCInst &Inst,
4708 const OperandVector &Operands) {
4709 if (!getFeatureBits()[AMDGPU::FeatureBF16InlineConstFromUpperFP32])
4710 return true;
4711
4712 const unsigned Opc = Inst.getOpcode();
4713 const MCInstrDesc &Desc = MII.get(Opcode: Opc);
4714 const bool IsVOP3 =
4715 SIInstrFlags::isVOP3(O: Desc) && !SIInstrFlags::isVOP3P(O: Desc);
4716 if (!SIInstrFlags::isVOP1(O: Desc) && !IsVOP3)
4717 return true;
4718
4719 // Only the single-source VOP1 bf16 opcodes are affected. Multi-source and
4720 // packed bf16 instructions such as v_fma_mix*_bf16 are not.
4721 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::src1))
4722 return true;
4723
4724 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
4725 if (Src0Idx == -1)
4726 return true;
4727
4728 const MCOperandInfo &Src0Info = Desc.operands()[Src0Idx];
4729 if (!AMDGPU::isBF16SrcOperand(OpInfo: Src0Info))
4730 return true;
4731
4732 const MCOperand &Src0 = Inst.getOperand(i: Src0Idx);
4733 if (!Src0.isImm() ||
4734 !AMDGPU::isInlinableLiteralBF16(Literal: static_cast<int16_t>(Src0.getImm()),
4735 HasInv2Pi: hasInv2PiInlineImm()))
4736 return true;
4737
4738 if (IsVOP3) {
4739 const int ModsIdx =
4740 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0_modifiers);
4741 if (ModsIdx != -1 &&
4742 (Inst.getOperand(i: ModsIdx).getImm() & SISrcMods::OP_SEL_0))
4743 return true;
4744 }
4745
4746 Error(L: getOperandLoc(Operands, MCOpIdx: Src0Idx),
4747 Msg: "bf16 inline constant is read from the high half of the fp32 inline "
4748 "constant on this GPU; use the e64 encoding with op_sel:[1,0]");
4749 return false;
4750}
4751
4752bool AMDGPUAsmParser::validateSOPLiteral(const MCInst &Inst,
4753 const OperandVector &Operands) {
4754 unsigned Opcode = Inst.getOpcode();
4755 const MCInstrDesc &Desc = MII.get(Opcode);
4756 if (!SIInstrFlags::isSOP2(O: Desc) && !SIInstrFlags::isSOPC(O: Desc))
4757 return true;
4758
4759 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src0);
4760 const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, Name: AMDGPU::OpName::src1);
4761
4762 const int OpIndices[] = {Src0Idx, Src1Idx};
4763
4764 unsigned NumExprs = 0;
4765 unsigned NumLiterals = 0;
4766 int64_t LiteralValue;
4767
4768 for (int OpIdx : OpIndices) {
4769 if (OpIdx == -1)
4770 break;
4771
4772 const MCOperand &MO = Inst.getOperand(i: OpIdx);
4773 // Exclude special imm operands (like that used by s_set_gpr_idx_on)
4774 if (AMDGPU::isSISrcOperand(Desc, OpNo: OpIdx)) {
4775 bool IsLit = false;
4776 std::optional<int64_t> Imm;
4777 if (MO.isImm()) {
4778 Imm = MO.getImm();
4779 } else if (MO.isExpr()) {
4780 if (isLitExpr(Expr: MO.getExpr())) {
4781 IsLit = true;
4782 Imm = getLitValue(Expr: MO.getExpr());
4783 }
4784 } else {
4785 continue;
4786 }
4787
4788 if (!Imm.has_value()) {
4789 ++NumExprs;
4790 } else if (!isInlineConstant(Inst, OpIdx)) {
4791 auto OpType = static_cast<AMDGPU::OperandType>(
4792 Desc.operands()[OpIdx].OperandType);
4793 int64_t Value = encode32BitLiteral(Imm: *Imm, Type: OpType, IsLit);
4794 if (NumLiterals == 0 || LiteralValue != Value) {
4795 LiteralValue = Value;
4796 ++NumLiterals;
4797 }
4798 }
4799 }
4800 }
4801
4802 if (NumLiterals + NumExprs <= 1)
4803 return true;
4804
4805 Error(L: getOperandLoc(Operands, MCOpIdx: Src1Idx),
4806 Msg: "only one unique literal operand is allowed");
4807 return false;
4808}
4809
4810bool AMDGPUAsmParser::validateOpSel(const MCInst &Inst) {
4811 const unsigned Opc = Inst.getOpcode();
4812 if (isPermlane16(Opc)) {
4813 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel);
4814 unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm();
4815
4816 if (OpSel & ~3)
4817 return false;
4818 }
4819
4820 if (isGFX940() && SIInstrFlags::isDOT(O: MII, O: Inst)) {
4821 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel);
4822 if (OpSelIdx != -1) {
4823 if (Inst.getOperand(i: OpSelIdx).getImm() != 0)
4824 return false;
4825 }
4826 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel_hi);
4827 if (OpSelHiIdx != -1) {
4828 if (Inst.getOperand(i: OpSelHiIdx).getImm() != -1)
4829 return false;
4830 }
4831 }
4832
4833 // op_sel[0:1] must be 0 for v_dot2_bf16_bf16 and v_dot2_f16_f16 (VOP3 Dot).
4834 if (isGFX11Plus() && SIInstrFlags::isDOT(O: MII, O: Inst) &&
4835 SIInstrFlags::isVOP3(O: MII, O: Inst) && !SIInstrFlags::isVOP3P(O: MII, O: Inst)) {
4836 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel);
4837 unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm();
4838 if (OpSel & 3)
4839 return false;
4840 }
4841
4842 // gfx950 v_pk_add/mul/fma_f32 have several invalid op_sel encodings
4843 if (getFeatureBits()[AMDGPU::FeatureGFX950Insts] &&
4844 (Opc == AMDGPU::V_PK_ADD_F32_vi || Opc == AMDGPU::V_PK_MUL_F32_vi ||
4845 Opc == AMDGPU::V_PK_FMA_F32_vi)) {
4846 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel);
4847 unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm();
4848
4849 // op_sel[2:1] == 2'b01, 2'b10, or 2'b11 for v_pk_fma.
4850 if (Opc == AMDGPU::V_PK_FMA_F32_vi &&
4851 (((OpSel >> 1) & 1) || ((OpSel >> 2) & 1)))
4852 return false;
4853 // op_sel[1:0] == 2'b10 for v_pk_add/mul_f32.
4854 else if ((OpSel & 3) == 2)
4855 return false;
4856 }
4857
4858 // Packed math FP32 instructions typically accept SGPRs or VGPRs as source
4859 // operands. On gfx12+, if a source operand uses SGPRs, the HW can only read
4860 // the first SGPR and use it for both the low and high operations.
4861 if (isPackedSingleSGPRFP32Inst(Opc)) {
4862 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
4863 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
4864 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel);
4865 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel_hi);
4866
4867 const MCOperand &Src0 = Inst.getOperand(i: Src0Idx);
4868 const MCOperand &Src1 = Inst.getOperand(i: Src1Idx);
4869 unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm();
4870 unsigned OpSelHi = Inst.getOperand(i: OpSelHiIdx).getImm();
4871
4872 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
4873
4874 auto VerifyOneSGPR = [OpSel, OpSelHi](unsigned Index) -> bool {
4875 unsigned Mask = 1U << Index;
4876 return ((OpSel & Mask) == 0) && ((OpSelHi & Mask) == 0);
4877 };
4878
4879 if (Src0.isReg() && isSGPR(Reg: Src0.getReg(), TRI) &&
4880 !VerifyOneSGPR(/*Index=*/0))
4881 return false;
4882 if (Src1.isReg() && isSGPR(Reg: Src1.getReg(), TRI) &&
4883 !VerifyOneSGPR(/*Index=*/1))
4884 return false;
4885
4886 int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2);
4887 if (Src2Idx != -1) {
4888 const MCOperand &Src2 = Inst.getOperand(i: Src2Idx);
4889 if (Src2.isReg() && isSGPR(Reg: Src2.getReg(), TRI) &&
4890 !VerifyOneSGPR(/*Index=*/2))
4891 return false;
4892 }
4893 }
4894
4895 return true;
4896}
4897
4898bool AMDGPUAsmParser::validateTrue16OpSel(const MCInst &Inst) {
4899 if (!hasTrue16Insts())
4900 return true;
4901 const MCRegisterInfo *MRI = getMRI();
4902 const unsigned Opc = Inst.getOpcode();
4903 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel);
4904 if (OpSelIdx == -1)
4905 return true;
4906 unsigned OpSelOpValue = Inst.getOperand(i: OpSelIdx).getImm();
4907 // If the value is 0 we could have a default OpSel Operand, so conservatively
4908 // allow it.
4909 if (OpSelOpValue == 0)
4910 return true;
4911 unsigned OpCount = 0;
4912 for (AMDGPU::OpName OpName : {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
4913 AMDGPU::OpName::src2, AMDGPU::OpName::vdst}) {
4914 int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: OpName);
4915 if (OpIdx == -1)
4916 continue;
4917 const MCOperand &Op = Inst.getOperand(i: OpIdx);
4918 if (Op.isReg() &&
4919 MRI->getRegClass(i: AMDGPU::VGPR_16RegClassID).contains(Reg: Op.getReg())) {
4920 bool VGPRSuffixIsHi = AMDGPU::isHi16Reg(Reg: Op.getReg(), MRI: *MRI);
4921 bool OpSelOpIsHi = ((OpSelOpValue & (1 << OpCount)) != 0);
4922 if (OpSelOpIsHi != VGPRSuffixIsHi)
4923 return false;
4924 }
4925 ++OpCount;
4926 }
4927
4928 return true;
4929}
4930
4931bool AMDGPUAsmParser::validateNeg(const MCInst &Inst, AMDGPU::OpName OpName) {
4932 assert(OpName == AMDGPU::OpName::neg_lo || OpName == AMDGPU::OpName::neg_hi);
4933
4934 const unsigned Opc = Inst.getOpcode();
4935
4936 // v_dot4 fp8/bf8 neg_lo/neg_hi not allowed on src0 and src1 (allowed on src2)
4937 // v_wmma iu4/iu8 neg_lo not allowed on src2 (allowed on src0, src1)
4938 // v_swmmac f16/bf16 neg_lo/neg_hi not allowed on src2 (allowed on src0, src1)
4939 // other wmma/swmmac instructions don't have neg_lo/neg_hi operand.
4940 if (!SIInstrFlags::isDOT(O: MII, O: Inst) && !SIInstrFlags::isWMMA(O: MII, O: Inst) &&
4941 !SIInstrFlags::isSWMMAC(O: MII, O: Inst))
4942 return true;
4943
4944 int NegIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: OpName);
4945 if (NegIdx == -1)
4946 return true;
4947
4948 unsigned Neg = Inst.getOperand(i: NegIdx).getImm();
4949
4950 // Instructions that have neg_lo or neg_hi operand but neg modifier is allowed
4951 // on some src operands but not allowed on other.
4952 // It is convenient that such instructions don't have src_modifiers operand
4953 // for src operands that don't allow neg because they also don't allow opsel.
4954
4955 const AMDGPU::OpName SrcMods[3] = {AMDGPU::OpName::src0_modifiers,
4956 AMDGPU::OpName::src1_modifiers,
4957 AMDGPU::OpName::src2_modifiers};
4958
4959 for (unsigned i = 0; i < 3; ++i) {
4960 if (!AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: SrcMods[i])) {
4961 if (Neg & (1 << i))
4962 return false;
4963 }
4964 }
4965
4966 return true;
4967}
4968
4969bool AMDGPUAsmParser::validateDPP(const MCInst &Inst,
4970 const OperandVector &Operands) {
4971 const unsigned Opc = Inst.getOpcode();
4972 int DppCtrlIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dpp_ctrl);
4973 if (DppCtrlIdx >= 0) {
4974 unsigned DppCtrl = Inst.getOperand(i: DppCtrlIdx).getImm();
4975
4976 if (!AMDGPU::isLegalDPALU_DPPControl(ST: getSTI(), DC: DppCtrl) &&
4977 getSTI().hasFeature(Feature: AMDGPU::FeatureDPALU_DPP) &&
4978 AMDGPU::isDPALU_DPP(OpDesc: MII.get(Opcode: Opc), MII, ST: getSTI())) {
4979 // DP ALU DPP is supported for row_newbcast only on GFX9* and row_share
4980 // only on GFX12.
4981 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyDppCtrl, Operands);
4982 Error(L: S, Msg: isGFX12() ? "DP ALU dpp only supports row_share"
4983 : "DP ALU dpp only supports row_newbcast");
4984 return false;
4985 }
4986 }
4987
4988 int Dpp8Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::dpp8);
4989 bool IsDPP = DppCtrlIdx >= 0 || Dpp8Idx >= 0;
4990
4991 if (IsDPP && !hasDPPSrc1SGPR(STI: getSTI())) {
4992 int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src1);
4993 if (Src1Idx >= 0) {
4994 const MCOperand &Src1 = Inst.getOperand(i: Src1Idx);
4995 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
4996 if (Src1.isReg() && isSGPR(Reg: mc2PseudoReg(Reg: Src1.getReg()), TRI)) {
4997 Error(L: getOperandLoc(Operands, MCOpIdx: Src1Idx),
4998 Msg: "invalid operand for instruction");
4999 return false;
5000 }
5001 if (Src1.isImm()) {
5002 Error(L: getInstLoc(Operands),
5003 Msg: "src1 immediate operand invalid for instruction");
5004 return false;
5005 }
5006 }
5007 }
5008
5009 return true;
5010}
5011
5012// Check if VCC register matches wavefront size
5013bool AMDGPUAsmParser::validateVccOperand(MCRegister Reg) const {
5014 return (Reg == AMDGPU::VCC && isWave64()) ||
5015 (Reg == AMDGPU::VCC_LO && isWave32());
5016}
5017
5018// One unique literal can be used. VOP3 literal is only allowed in GFX10+
5019bool AMDGPUAsmParser::validateVOPLiteral(const MCInst &Inst,
5020 const OperandVector &Operands) {
5021 unsigned Opcode = Inst.getOpcode();
5022 const MCInstrDesc &Desc = MII.get(Opcode);
5023 bool HasMandatoryLiteral = getNamedOperandIdx(Opcode, Name: OpName::imm) != -1;
5024 if (!SIInstrFlags::isVOP3Like(O: Desc) && !HasMandatoryLiteral &&
5025 !isVOPD(Opc: Opcode))
5026 return true;
5027
5028 OperandIndices OpIndices = getSrcOperandIndices(Opcode, AddMandatoryLiterals: HasMandatoryLiteral);
5029
5030 std::optional<unsigned> LiteralOpIdx;
5031 std::optional<uint64_t> LiteralValue;
5032
5033 for (int OpIdx : OpIndices) {
5034 if (OpIdx == -1)
5035 continue;
5036
5037 const MCOperand &MO = Inst.getOperand(i: OpIdx);
5038 if (!MO.isImm() && !MO.isExpr())
5039 continue;
5040 if (!isSISrcOperand(Desc, OpNo: OpIdx))
5041 continue;
5042
5043 std::optional<int64_t> Imm;
5044 if (MO.isImm())
5045 Imm = MO.getImm();
5046 else if (MO.isExpr() && isLitExpr(Expr: MO.getExpr()))
5047 Imm = getLitValue(Expr: MO.getExpr());
5048
5049 bool IsAnotherLiteral = false;
5050 bool IsForcedLit = findMCOperand(Operands, MCOpIdx: OpIdx).isForcedLit();
5051 bool IsForcedLit64 = findMCOperand(Operands, MCOpIdx: OpIdx).isForcedLit64();
5052 if (!Imm.has_value()) {
5053 // Literal value not known, so we conservately assume it's different.
5054 IsAnotherLiteral = true;
5055 } else if (IsForcedLit || IsForcedLit64 || !isInlineConstant(Inst, OpIdx)) {
5056 uint64_t Value = *Imm;
5057 bool IsForcedFP64 =
5058 Desc.operands()[OpIdx].OperandType == AMDGPU::OPERAND_KIMM64 ||
5059 (Desc.operands()[OpIdx].OperandType == AMDGPU::OPERAND_REG_IMM_FP64 &&
5060 HasMandatoryLiteral);
5061 AMDGPU::OperandType OpTy =
5062 static_cast<AMDGPU::OperandType>(Desc.operands()[OpIdx].OperandType);
5063 bool IsFP64 =
5064 (IsForcedFP64 || (AMDGPU::isSISrcFPOperand(Desc, OpNo: OpIdx) &&
5065 OpTy != AMDGPU::OPERAND_REG_IMM_V2INT64)) &&
5066 AMDGPU::getOperandSize(OpInfo: Desc.operands()[OpIdx]) == 8;
5067 bool IsValid32Op =
5068 IsForcedLit || AMDGPU::isValid32BitLiteral(Val: Value, IsFP64);
5069
5070 if (((!IsValid32Op && !isInt<32>(x: Value) && !isUInt<32>(x: Value) &&
5071 !IsForcedFP64) ||
5072 (IsForcedLit64 && !HasMandatoryLiteral)) &&
5073 (!has64BitLiterals() || Desc.getSize() != 4)) {
5074 Error(L: getOperandLoc(Operands, MCOpIdx: OpIdx),
5075 Msg: "invalid operand for instruction");
5076 return false;
5077 }
5078
5079 // Only src0 can use lit64 in VOP* encoding.
5080 if (!IsForcedFP64 && (IsForcedLit64 || !IsValid32Op) &&
5081 OpIdx != getNamedOperandIdx(Opcode, Name: OpName::src0)) {
5082 Error(L: getOperandLoc(Operands, MCOpIdx: OpIdx),
5083 Msg: "invalid operand for instruction");
5084 return false;
5085 }
5086
5087 // Compare values using the word encoded by a 32-bit literal.
5088 if (IsValid32Op && !IsForcedFP64 && !IsForcedLit64) {
5089 Value = static_cast<uint32_t>(
5090 AMDGPU::encode32BitLiteral(Imm: Value, Type: OpTy, IsLit: IsForcedLit));
5091 }
5092
5093 IsAnotherLiteral = !LiteralValue || *LiteralValue != Value;
5094 LiteralValue = Value;
5095 }
5096
5097 if (IsAnotherLiteral && !HasMandatoryLiteral &&
5098 !getFeatureBits()[FeatureVOP3Literal]) {
5099 Error(L: getOperandLoc(Operands, MCOpIdx: OpIdx),
5100 Msg: "literal operands are not supported");
5101 return false;
5102 }
5103
5104 if (LiteralOpIdx && IsAnotherLiteral) {
5105 Error(L: getLaterLoc(a: getOperandLoc(Operands, MCOpIdx: OpIdx),
5106 b: getOperandLoc(Operands, MCOpIdx: *LiteralOpIdx)),
5107 Msg: "only one unique literal operand is allowed");
5108 return false;
5109 }
5110
5111 if (IsAnotherLiteral)
5112 LiteralOpIdx = OpIdx;
5113 }
5114
5115 return true;
5116}
5117
5118// Returns -1 if not a register, 0 if VGPR and 1 if AGPR.
5119static int IsAGPROperand(const MCInst &Inst, AMDGPU::OpName Name,
5120 const MCRegisterInfo *MRI) {
5121 int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name);
5122 if (OpIdx < 0)
5123 return -1;
5124
5125 const MCOperand &Op = Inst.getOperand(i: OpIdx);
5126 if (!Op.isReg())
5127 return -1;
5128
5129 MCRegister Sub = MRI->getSubReg(Reg: Op.getReg(), Idx: AMDGPU::sub0);
5130 auto Reg = Sub ? Sub : Op.getReg();
5131 const MCRegisterClass &AGPR32 = MRI->getRegClass(i: AMDGPU::AGPR_32RegClassID);
5132 return AGPR32.contains(Reg) ? 1 : 0;
5133}
5134
5135bool AMDGPUAsmParser::validateAGPRLdSt(const MCInst &Inst) const {
5136 if (!SIInstrFlags::isFLAT(O: MII, O: Inst) && !SIInstrFlags::isBuffer(O: MII, O: Inst) &&
5137 !SIInstrFlags::isMIMG(O: MII, O: Inst) && !SIInstrFlags::isDS(O: MII, O: Inst))
5138 return true;
5139
5140 AMDGPU::OpName DataName = SIInstrFlags::isDS(O: MII, O: Inst)
5141 ? AMDGPU::OpName::data0
5142 : AMDGPU::OpName::vdata;
5143
5144 const MCRegisterInfo *MRI = getMRI();
5145 int DstAreg = IsAGPROperand(Inst, Name: AMDGPU::OpName::vdst, MRI);
5146 int DataAreg = IsAGPROperand(Inst, Name: DataName, MRI);
5147
5148 if (SIInstrFlags::isDS(O: MII, O: Inst) && DataAreg >= 0) {
5149 int Data2Areg = IsAGPROperand(Inst, Name: AMDGPU::OpName::data1, MRI);
5150 if (Data2Areg >= 0 && Data2Areg != DataAreg)
5151 return false;
5152 }
5153
5154 auto FB = getFeatureBits();
5155 if (FB[AMDGPU::FeatureGFX90AInsts]) {
5156 if (DataAreg < 0 || DstAreg < 0)
5157 return true;
5158 return DstAreg == DataAreg;
5159 }
5160
5161 return DstAreg < 1 && DataAreg < 1;
5162}
5163
5164bool AMDGPUAsmParser::validateVGPRAlign(const MCInst &Inst) const {
5165 auto FB = getFeatureBits();
5166 if (!FB[AMDGPU::FeatureRequiresAlignedVGPRs])
5167 return true;
5168
5169 unsigned Opc = Inst.getOpcode();
5170 const MCRegisterInfo *MRI = getMRI();
5171 // DS_READ_B96_TR_B6 is the only DS instruction in GFX950, that allows
5172 // unaligned VGPR. All others only allow even aligned VGPRs.
5173 if (FB[AMDGPU::FeatureGFX90AInsts] && Opc == AMDGPU::DS_READ_B96_TR_B6_vi)
5174 return true;
5175
5176 if (FB[AMDGPU::FeatureGFX1250Insts]) {
5177 switch (Opc) {
5178 default:
5179 break;
5180 case AMDGPU::DS_LOAD_TR6_B96:
5181 case AMDGPU::DS_LOAD_TR6_B96_gfx12:
5182 // DS_LOAD_TR6_B96 is the only DS instruction in GFX1250, that
5183 // allows unaligned VGPR. All others only allow even aligned VGPRs.
5184 return true;
5185 case AMDGPU::GLOBAL_LOAD_TR6_B96:
5186 case AMDGPU::GLOBAL_LOAD_TR6_B96_gfx1250: {
5187 // GLOBAL_LOAD_TR6_B96 is the only GLOBAL instruction in GFX1250, that
5188 // allows unaligned VGPR for vdst, but other operands still only allow
5189 // even aligned VGPRs.
5190 int VAddrIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vaddr);
5191 if (VAddrIdx != -1) {
5192 const MCOperand &Op = Inst.getOperand(i: VAddrIdx);
5193 MCRegister Sub = MRI->getSubReg(Reg: Op.getReg(), Idx: AMDGPU::sub0);
5194 if ((Sub - AMDGPU::VGPR0) & 1)
5195 return false;
5196 }
5197 return true;
5198 }
5199 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR:
5200 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR_gfx1250:
5201 return true;
5202 }
5203 }
5204
5205 const MCRegisterClass &VGPR32 = MRI->getRegClass(i: AMDGPU::VGPR_32RegClassID);
5206 const MCRegisterClass &AGPR32 = MRI->getRegClass(i: AMDGPU::AGPR_32RegClassID);
5207 for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) {
5208 const MCOperand &Op = Inst.getOperand(i: I);
5209 if (!Op.isReg())
5210 continue;
5211
5212 MCRegister Sub = MRI->getSubReg(Reg: Op.getReg(), Idx: AMDGPU::sub0);
5213 if (!Sub)
5214 continue;
5215
5216 if (VGPR32.contains(Reg: Sub) && ((Sub - AMDGPU::VGPR0) & 1))
5217 return false;
5218 if (AGPR32.contains(Reg: Sub) && ((Sub - AMDGPU::AGPR0) & 1))
5219 return false;
5220 }
5221
5222 return true;
5223}
5224
5225SMLoc AMDGPUAsmParser::getBLGPLoc(const OperandVector &Operands) const {
5226 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
5227 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
5228 if (Op.isBLGP())
5229 return Op.getStartLoc();
5230 }
5231 return SMLoc();
5232}
5233
5234bool AMDGPUAsmParser::validateBLGP(const MCInst &Inst,
5235 const OperandVector &Operands) {
5236 unsigned Opc = Inst.getOpcode();
5237 int BlgpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::blgp);
5238 if (BlgpIdx == -1)
5239 return true;
5240 SMLoc BLGPLoc = getBLGPLoc(Operands);
5241 if (!BLGPLoc.isValid())
5242 return true;
5243 bool IsNeg = StringRef(BLGPLoc.getPointer()).starts_with(Prefix: "neg:");
5244 auto FB = getFeatureBits();
5245 bool UsesNeg = false;
5246 if (FB[AMDGPU::FeatureGFX940Insts]) {
5247 switch (Opc) {
5248 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_acd:
5249 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_vcd:
5250 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_acd:
5251 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_vcd:
5252 UsesNeg = true;
5253 }
5254 }
5255
5256 if (IsNeg == UsesNeg)
5257 return true;
5258
5259 Error(L: BLGPLoc, Msg: UsesNeg ? "invalid modifier: blgp is not supported"
5260 : "invalid modifier: neg is not supported");
5261
5262 return false;
5263}
5264
5265bool AMDGPUAsmParser::validateWaitCnt(const MCInst &Inst,
5266 const OperandVector &Operands) {
5267 if (!isGFX11Plus())
5268 return true;
5269
5270 unsigned Opc = Inst.getOpcode();
5271 if (Opc != AMDGPU::S_WAITCNT_EXPCNT_gfx11 &&
5272 Opc != AMDGPU::S_WAITCNT_LGKMCNT_gfx11 &&
5273 Opc != AMDGPU::S_WAITCNT_VMCNT_gfx11 &&
5274 Opc != AMDGPU::S_WAITCNT_VSCNT_gfx11)
5275 return true;
5276
5277 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::sdst);
5278 assert(Src0Idx >= 0 && Inst.getOperand(Src0Idx).isReg());
5279 auto Reg = mc2PseudoReg(Reg: Inst.getOperand(i: Src0Idx).getReg());
5280 if (Reg == AMDGPU::SGPR_NULL)
5281 return true;
5282
5283 Error(L: getOperandLoc(Operands, MCOpIdx: Src0Idx), Msg: "src0 must be null");
5284 return false;
5285}
5286
5287bool AMDGPUAsmParser::validateDS(const MCInst &Inst,
5288 const OperandVector &Operands) {
5289 if (!SIInstrFlags::isDS(O: MII, O: Inst))
5290 return true;
5291 if (SIInstrFlags::isGWS(O: MII, O: Inst))
5292 return validateGWS(Inst, Operands);
5293 // Only validate GDS for non-GWS instructions.
5294 if (hasGDS())
5295 return true;
5296 int GDSIdx =
5297 AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::gds);
5298 if (GDSIdx < 0)
5299 return true;
5300 unsigned GDS = Inst.getOperand(i: GDSIdx).getImm();
5301 if (GDS) {
5302 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyGDS, Operands);
5303 Error(L: S, Msg: "gds modifier is not supported on this GPU");
5304 return false;
5305 }
5306 return true;
5307}
5308
5309// gfx90a has an undocumented limitation:
5310// DS_GWS opcodes must use even aligned registers.
5311bool AMDGPUAsmParser::validateGWS(const MCInst &Inst,
5312 const OperandVector &Operands) {
5313 if (!getFeatureBits()[AMDGPU::FeatureGFX90AInsts])
5314 return true;
5315
5316 int Opc = Inst.getOpcode();
5317 if (Opc != AMDGPU::DS_GWS_INIT_vi && Opc != AMDGPU::DS_GWS_BARRIER_vi &&
5318 Opc != AMDGPU::DS_GWS_SEMA_BR_vi)
5319 return true;
5320
5321 const MCRegisterInfo *MRI = getMRI();
5322 const MCRegisterClass &VGPR32 = MRI->getRegClass(i: AMDGPU::VGPR_32RegClassID);
5323 int Data0Pos =
5324 AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::data0);
5325 assert(Data0Pos != -1);
5326 auto Reg = Inst.getOperand(i: Data0Pos).getReg();
5327 auto RegIdx = Reg - (VGPR32.contains(Reg) ? AMDGPU::VGPR0 : AMDGPU::AGPR0);
5328 if (RegIdx & 1) {
5329 Error(L: getOperandLoc(Operands, MCOpIdx: Data0Pos), Msg: "vgpr must be even aligned");
5330 return false;
5331 }
5332
5333 return true;
5334}
5335
5336bool AMDGPUAsmParser::validateCoherencyBits(const MCInst &Inst,
5337 const OperandVector &Operands,
5338 SMLoc IDLoc) {
5339 int CPolPos =
5340 AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::cpol);
5341 if (CPolPos == -1)
5342 return true;
5343
5344 unsigned CPol = Inst.getOperand(i: CPolPos).getImm();
5345
5346 if (CPol & CPol::CFS) {
5347 if (!hasCacheFillSize()) {
5348 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands);
5349 StringRef CStr(S.getPointer());
5350 S = SMLoc::getFromPointer(Ptr: &CStr.data()[CStr.find(Str: "cfs")]);
5351 Error(L: S, Msg: "Cache fill size is not supported on this GPU");
5352 } else if (!validateCFSBits(Inst, Operands)) {
5353 return false;
5354 }
5355 }
5356
5357 if (!hasScaleOffset() && (CPol & CPol::SCAL)) {
5358 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands);
5359 StringRef CStr(S.getPointer());
5360 S = SMLoc::getFromPointer(Ptr: &CStr.data()[CStr.find(Str: "scale_offset")]);
5361 Error(L: S, Msg: "scale_offset is not supported on this GPU");
5362 }
5363
5364 if (!isGFX1250Plus() && (CPol & CPol::NV)) {
5365 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands);
5366 StringRef CStr(S.getPointer());
5367 S = SMLoc::getFromPointer(Ptr: &CStr.data()[CStr.find(Str: "nv")]);
5368 Error(L: S, Msg: "nv is not supported on this GPU");
5369 }
5370
5371 if ((CPol & CPol::SCAL) && !supportsScaleOffset(MII, Opcode: Inst.getOpcode())) {
5372 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands);
5373 StringRef CStr(S.getPointer());
5374 S = SMLoc::getFromPointer(Ptr: &CStr.data()[CStr.find(Str: "scale_offset")]);
5375 Error(L: S, Msg: "scale_offset is not supported for this instruction");
5376 }
5377
5378 if (isGFX12Plus())
5379 return validateTHAndScopeBits(Inst, Operands, CPol);
5380
5381 if (SIInstrFlags::isSMRD(O: MII, O: Inst)) {
5382 if (CPol && (isSI() || isCI())) {
5383 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands);
5384 Error(L: S, Msg: "cache policy is not supported for SMRD instructions");
5385 return false;
5386 }
5387 if (CPol & ~(AMDGPU::CPol::GLC | AMDGPU::CPol::DLC)) {
5388 Error(L: IDLoc, Msg: "invalid cache policy for SMEM instruction");
5389 return false;
5390 }
5391 }
5392
5393 if (isGFX90A() && !isGFX940() && (CPol & CPol::SCC)) {
5394 if (!SIInstrFlags::isVMEM(O: MII, O: Inst)) {
5395 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands);
5396 StringRef CStr(S.getPointer());
5397 S = SMLoc::getFromPointer(Ptr: &CStr.data()[CStr.find(Str: "scc")]);
5398 Error(L: S,
5399 Msg: "scc modifier is not supported for this instruction on this GPU");
5400 return false;
5401 }
5402 }
5403
5404 if (!SIInstrFlags::isAtomic(O: MII, O: Inst))
5405 return true;
5406
5407 if (SIInstrFlags::isAtomicRet(O: MII, O: Inst)) {
5408 if (!SIInstrFlags::isMIMG(O: MII, O: Inst) && !(CPol & CPol::GLC)) {
5409 Error(L: IDLoc, Msg: isGFX940() ? "instruction must use sc0"
5410 : "instruction must use glc");
5411 return false;
5412 }
5413 } else {
5414 if (CPol & CPol::GLC) {
5415 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands);
5416 StringRef CStr(S.getPointer());
5417 S = SMLoc::getFromPointer(
5418 Ptr: &CStr.data()[CStr.find(Str: isGFX940() ? "sc0" : "glc")]);
5419 Error(L: S, Msg: isGFX940() ? "instruction must not use sc0"
5420 : "instruction must not use glc");
5421 return false;
5422 }
5423 }
5424
5425 return true;
5426}
5427
5428bool AMDGPUAsmParser::validateTHAndScopeBits(const MCInst &Inst,
5429 const OperandVector &Operands,
5430 const unsigned CPol) {
5431 const unsigned TH = CPol & AMDGPU::CPol::TH;
5432 const unsigned Scope = CPol & AMDGPU::CPol::SCOPE;
5433
5434 auto PrintError = [&](StringRef Msg) {
5435 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands);
5436 Error(L: S, Msg);
5437 return false;
5438 };
5439
5440 if ((TH & AMDGPU::CPol::TH_ATOMIC_RETURN) &&
5441 SIInstrFlags::isAtomicNoRet(O: MII, O: Inst))
5442 return PrintError("th:TH_ATOMIC_RETURN requires a destination operand");
5443
5444 if (SIInstrFlags::isAtomicRet(O: MII, O: Inst) &&
5445 (SIInstrFlags::isFLAT(O: MII, O: Inst) || SIInstrFlags::isMUBUF(O: MII, O: Inst)) &&
5446 (!(TH & AMDGPU::CPol::TH_ATOMIC_RETURN)))
5447 return PrintError("instruction must use th:TH_ATOMIC_RETURN");
5448
5449 if (TH == 0)
5450 return true;
5451
5452 if (SIInstrFlags::isSMRD(O: MII, O: Inst) &&
5453 ((TH == AMDGPU::CPol::TH_NT_RT) || (TH == AMDGPU::CPol::TH_RT_NT) ||
5454 (TH == AMDGPU::CPol::TH_NT_HT)))
5455 return PrintError("invalid th value for SMEM instruction");
5456
5457 if (TH == AMDGPU::CPol::TH_BYPASS) {
5458 if ((Scope != AMDGPU::CPol::SCOPE_SYS &&
5459 CPol & AMDGPU::CPol::TH_REAL_BYPASS) ||
5460 (Scope == AMDGPU::CPol::SCOPE_SYS &&
5461 !(CPol & AMDGPU::CPol::TH_REAL_BYPASS)))
5462 return PrintError("scope and th combination is not valid");
5463 }
5464
5465 unsigned THType = AMDGPU::getTemporalHintType(TID: MII.get(Opcode: Inst.getOpcode()));
5466 if (THType == AMDGPU::CPol::TH_TYPE_ATOMIC) {
5467 if (!(CPol & AMDGPU::CPol::TH_TYPE_ATOMIC))
5468 return PrintError("invalid th value for atomic instructions");
5469 } else if (THType == AMDGPU::CPol::TH_TYPE_STORE) {
5470 if (!(CPol & AMDGPU::CPol::TH_TYPE_STORE))
5471 return PrintError("invalid th value for store instructions");
5472 } else {
5473 if (!(CPol & AMDGPU::CPol::TH_TYPE_LOAD))
5474 return PrintError("invalid th value for load instructions");
5475 }
5476
5477 return true;
5478}
5479
5480bool AMDGPUAsmParser::validateCFSBits(const MCInst &Inst,
5481 const OperandVector &Operands) {
5482 const unsigned Opcode = Inst.getOpcode();
5483 const MCInstrDesc &Desc = MII.get(Opcode);
5484
5485 auto PrintError = [&](StringRef Msg) {
5486 SMLoc S = getImmLoc(Type: AMDGPUOperand::ImmTyCPol, Operands);
5487 Error(L: S, Msg);
5488 return false;
5489 };
5490
5491 if (SIInstrFlags::isVMEM(O: Desc))
5492 return true;
5493
5494 return PrintError("Cache fill size is only supported for VC operations");
5495}
5496
5497bool AMDGPUAsmParser::validateTFE(const MCInst &Inst,
5498 const OperandVector &Operands) {
5499 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
5500 if (Desc.mayStore() && SIInstrFlags::isBuffer(O: Desc)) {
5501 SMLoc Loc = getImmLoc(Type: AMDGPUOperand::ImmTyTFE, Operands);
5502 if (Loc != getInstLoc(Operands)) {
5503 Error(L: Loc, Msg: "TFE modifier has no meaning for store instructions");
5504 return false;
5505 }
5506 }
5507
5508 return true;
5509}
5510
5511bool AMDGPUAsmParser::validateWMMA(const MCInst &Inst,
5512 const OperandVector &Operands) {
5513 unsigned Opc = Inst.getOpcode();
5514 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
5515 const MCInstrDesc &Desc = MII.get(Opcode: Opc);
5516
5517 int AFmtIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_a_fmt);
5518 if (AFmtIdx == -1)
5519 return true;
5520 unsigned AFmt = Inst.getOperand(i: AFmtIdx).getImm();
5521 int BFmtIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_b_fmt);
5522 unsigned BFmt = Inst.getOperand(i: BFmtIdx).getImm();
5523
5524 auto validateFmt = [&](unsigned Fmt, AMDGPU::OpName SrcOp) -> bool {
5525 int SrcIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: SrcOp);
5526 unsigned RegSize =
5527 TRI->getRegClass(i: MII.getOpRegClassID(OpInfo: Desc.operands()[SrcIdx], HwModeId: HwMode))
5528 .getSizeInBits();
5529
5530 if (RegSize == AMDGPU::wmmaScaleF8F6F4FormatToNumRegs(Fmt) * 32)
5531 return true;
5532
5533 Error(L: getOperandLoc(Operands, MCOpIdx: SrcIdx),
5534 Msg: "wrong register tuple size for " +
5535 Twine(WMMAMods::ModMatrixFmt[Fmt]));
5536 return false;
5537 };
5538
5539 if (!validateFmt(AFmt, AMDGPU::OpName::src0) ||
5540 !validateFmt(BFmt, AMDGPU::OpName::src1))
5541 return false;
5542
5543 int AScaleIdx =
5544 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_a_scale_fmt);
5545 if (AScaleIdx == -1)
5546 return true;
5547 unsigned AScale = Inst.getOperand(i: AScaleIdx).getImm();
5548 int BScaleIdx =
5549 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_b_scale_fmt);
5550 unsigned BScale = Inst.getOperand(i: BScaleIdx).getImm();
5551 if (!isValidWMMAScaleFmtCombination(AFmt, AScale, BFmt, BScale)) {
5552 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyMatrixAFMT, Operands),
5553 Msg: "invalid matrix and scale format combination");
5554 return false;
5555 }
5556
5557 return true;
5558}
5559
5560bool AMDGPUAsmParser::validateMonitorSleep(const MCInst &Inst,
5561 const OperandVector &Operands) {
5562 unsigned Opc = Inst.getOpcode();
5563 if (Opc != AMDGPU::S_MONITOR_SLEEP_gfx12 ||
5564 !getSTI().hasFeature(Feature: AMDGPU::FeatureNoSleepForever))
5565 return true;
5566
5567 int ImmIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::simm16);
5568 if (Inst.getOperand(i: ImmIdx).getImm() & 0x8000) {
5569 Error(L: getOperandLoc(Operands, MCOpIdx: ImmIdx),
5570 Msg: "sleep forever is unsupported on the target");
5571 return false;
5572 }
5573
5574 return true;
5575}
5576
5577bool AMDGPUAsmParser::validateClusterBarrierIsFirst(
5578 const MCInst &Inst, const OperandVector &Operands) {
5579 unsigned Opc = Inst.getOpcode();
5580 if (Opc != AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM_gfx12 &&
5581 Opc != AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM_gfx13)
5582 return true;
5583
5584 int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0);
5585 int BarrierID = Inst.getOperand(i: Src0Idx).getImm();
5586 if (BarrierID != AMDGPU::Barrier::CLUSTER)
5587 return true;
5588
5589 Error(
5590 L: getOperandLoc(Operands, MCOpIdx: Src0Idx),
5591 Msg: "s_barrier_signal_isfirst does not support user_cluster_barrier_id (-3)");
5592 return false;
5593}
5594
5595bool AMDGPUAsmParser::validateScaleSel(const MCInst &Inst,
5596 const OperandVector &Operands) {
5597 unsigned Opc = Inst.getOpcode();
5598 int ScaleSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::scale_sel);
5599 if (ScaleSelIdx == -1)
5600 return true;
5601 int MaxSel = 0;
5602 switch (Opc) {
5603 case AMDGPU::V_CVT_SCALE_PK16_F16_FP6_e64_gfx1250:
5604 case AMDGPU::V_CVT_SCALE_PK16_BF16_FP6_e64_gfx1250:
5605 case AMDGPU::V_CVT_SCALE_PK16_F16_BF6_e64_gfx1250:
5606 case AMDGPU::V_CVT_SCALE_PK16_BF16_BF6_e64_gfx1250:
5607 case AMDGPU::V_CVT_SCALE_PK16_F32_FP6_e64_gfx1250:
5608 case AMDGPU::V_CVT_SCALE_PK16_F32_BF6_e64_gfx1250:
5609 case AMDGPU::V_CVT_SCALE_PK8_F16_FP4_e64_gfx1250:
5610 case AMDGPU::V_CVT_SCALE_PK8_BF16_FP4_e64_gfx1250:
5611 case AMDGPU::V_CVT_SCALE_PK8_F32_FP4_e64_gfx1250:
5612 MaxSel = 4;
5613 break;
5614 case AMDGPU::V_CVT_SCALE_PK8_F16_FP8_e64_gfx1250:
5615 case AMDGPU::V_CVT_SCALE_PK8_BF16_FP8_e64_gfx1250:
5616 case AMDGPU::V_CVT_SCALE_PK8_F16_BF8_e64_gfx1250:
5617 case AMDGPU::V_CVT_SCALE_PK8_BF16_BF8_e64_gfx1250:
5618 case AMDGPU::V_CVT_SCALE_PK8_F32_FP8_e64_gfx1250:
5619 case AMDGPU::V_CVT_SCALE_PK8_F32_BF8_e64_gfx1250:
5620 MaxSel = 8;
5621 break;
5622 default:
5623 return true;
5624 }
5625
5626 if (getSTI().hasFeature(Feature: AMDGPU::FeatureBlock16ConversionScaleInsts))
5627 MaxSel *= 2;
5628
5629 int ScaleSel = Inst.getOperand(i: ScaleSelIdx).getImm();
5630 if (ScaleSel < MaxSel)
5631 return true;
5632
5633 Error(L: getOperandLoc(Operands, MCOpIdx: ScaleSelIdx),
5634 Msg: "scale_sel maximum supported value is " + Twine(MaxSel - 1));
5635 return false;
5636}
5637
5638bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst, SMLoc IDLoc,
5639 const OperandVector &Operands) {
5640 if (!validateLdsDirect(Inst, Operands))
5641 return false;
5642 if (!validateTrue16OpSel(Inst)) {
5643 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyOpSel, Operands),
5644 Msg: "op_sel operand conflicts with 16-bit operand suffix");
5645 return false;
5646 }
5647 if (!validateSOPLiteral(Inst, Operands))
5648 return false;
5649 if (!validateVOPLiteral(Inst, Operands)) {
5650 return false;
5651 }
5652 if (!validateConstantBusLimitations(Inst, Operands)) {
5653 return false;
5654 }
5655 if (!validateVOPD(Inst, Operands)) {
5656 return false;
5657 }
5658 if (!validateIntClampSupported(Inst)) {
5659 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyClamp, Operands),
5660 Msg: "integer clamping is not supported on this GPU");
5661 return false;
5662 }
5663 if (!validateOpSel(Inst)) {
5664 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyOpSel, Operands),
5665 Msg: "invalid op_sel operand");
5666 return false;
5667 }
5668 if (!validateNeg(Inst, OpName: AMDGPU::OpName::neg_lo)) {
5669 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyNegLo, Operands),
5670 Msg: "invalid neg_lo operand");
5671 return false;
5672 }
5673 if (!validateNeg(Inst, OpName: AMDGPU::OpName::neg_hi)) {
5674 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyNegHi, Operands),
5675 Msg: "invalid neg_hi operand");
5676 return false;
5677 }
5678 if (!validateDPP(Inst, Operands)) {
5679 return false;
5680 }
5681 // For MUBUF/MTBUF d16 is a part of opcode, so there is nothing to validate.
5682 if (!validateMIMGD16(Inst)) {
5683 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyD16, Operands),
5684 Msg: "d16 modifier is not supported on this GPU");
5685 return false;
5686 }
5687 if (!validateMIMGDim(Inst, Operands)) {
5688 Error(L: IDLoc, Msg: "missing dim operand");
5689 return false;
5690 }
5691 if (!validateTensorR128(Inst)) {
5692 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyD16, Operands),
5693 Msg: "instruction must set modifier r128=0");
5694 return false;
5695 }
5696 if (!validateMIMGMSAA(Inst)) {
5697 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyDim, Operands),
5698 Msg: "invalid dim; must be MSAA type");
5699 return false;
5700 }
5701 if (!validateMIMGDataSize(Inst, IDLoc)) {
5702 return false;
5703 }
5704 if (!validateMIMGAddrSize(Inst, IDLoc))
5705 return false;
5706 if (!validateMIMGAtomicDMask(Inst)) {
5707 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyDMask, Operands),
5708 Msg: "invalid atomic image dmask");
5709 return false;
5710 }
5711 if (!validateMIMGGatherDMask(Inst)) {
5712 Error(L: getImmLoc(Type: AMDGPUOperand::ImmTyDMask, Operands),
5713 Msg: "invalid image_gather dmask: only one bit must be set");
5714 return false;
5715 }
5716 if (!validateMovrels(Inst, Operands)) {
5717 return false;
5718 }
5719 if (!validateOffset(Inst, Operands)) {
5720 return false;
5721 }
5722 if (!validateSMEMSData(Inst, Operands))
5723 return false;
5724 if (!validateBF16InlineConst(Inst, Operands)) {
5725 return false;
5726 }
5727 if (!validateMAIAccWrite(Inst, Operands)) {
5728 return false;
5729 }
5730 if (!validateMAISrc2(Inst, Operands)) {
5731 return false;
5732 }
5733 if (!validateMFMA(Inst, Operands)) {
5734 return false;
5735 }
5736 if (!validateCoherencyBits(Inst, Operands, IDLoc)) {
5737 return false;
5738 }
5739
5740 if (!validateAGPRLdSt(Inst)) {
5741 Error(
5742 L: IDLoc,
5743 Msg: getFeatureBits()[AMDGPU::FeatureGFX90AInsts]
5744 ? "invalid register class: data and dst should be all VGPR or AGPR"
5745 : "invalid register class: agpr loads and stores not supported on "
5746 "this GPU");
5747 return false;
5748 }
5749 if (!validateVGPRAlign(Inst)) {
5750 Error(L: IDLoc, Msg: "invalid register class: vgpr tuples must be 64 bit aligned");
5751 return false;
5752 }
5753 if (!validateDS(Inst, Operands)) {
5754 return false;
5755 }
5756
5757 if (!validateBLGP(Inst, Operands)) {
5758 return false;
5759 }
5760
5761 if (!validateDivScale(Inst)) {
5762 Error(L: IDLoc, Msg: "ABS not allowed in VOP3B instructions");
5763 return false;
5764 }
5765 if (!validateWaitCnt(Inst, Operands)) {
5766 return false;
5767 }
5768 if (!validateTFE(Inst, Operands)) {
5769 return false;
5770 }
5771 if (!validateWMMA(Inst, Operands)) {
5772 return false;
5773 }
5774 if (!validateMonitorSleep(Inst, Operands)) {
5775 return false;
5776 }
5777 if (!validateClusterBarrierIsFirst(Inst, Operands)) {
5778 return false;
5779 }
5780 if (!validateScaleSel(Inst, Operands)) {
5781 return false;
5782 }
5783
5784 return true;
5785}
5786
5787static std::string AMDGPUMnemonicSpellCheck(StringRef S,
5788 const FeatureBitset &FBS,
5789 unsigned VariantID = 0);
5790
5791static bool AMDGPUCheckMnemonic(StringRef Mnemonic,
5792 const FeatureBitset &AvailableFeatures,
5793 unsigned VariantID);
5794
5795bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
5796 const FeatureBitset &FBS) {
5797 return isSupportedMnemo(Mnemo, FBS, Variants: getAllVariants());
5798}
5799
5800bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
5801 const FeatureBitset &FBS,
5802 ArrayRef<unsigned> Variants) {
5803 for (auto Variant : Variants) {
5804 if (AMDGPUCheckMnemonic(Mnemonic: Mnemo, AvailableFeatures: FBS, VariantID: Variant))
5805 return true;
5806 }
5807
5808 return false;
5809}
5810
5811bool AMDGPUAsmParser::checkUnsupportedInstruction(StringRef Mnemo,
5812 SMLoc IDLoc) {
5813 FeatureBitset FBS = ComputeAvailableFeatures(FB: getFeatureBits());
5814
5815 // Check if requested instruction variant is supported.
5816 if (isSupportedMnemo(Mnemo, FBS, Variants: getMatchedVariants()))
5817 return false;
5818
5819 // This instruction is not supported.
5820 // Clear any other pending errors because they are no longer relevant.
5821 getParser().clearPendingErrors();
5822
5823 // Requested instruction variant is not supported.
5824 // Check if any other variants are supported.
5825 StringRef VariantName = getMatchedVariantName();
5826 if (!VariantName.empty() && isSupportedMnemo(Mnemo, FBS)) {
5827 return Error(L: IDLoc, Msg: Twine(VariantName,
5828 " variant of this instruction is not supported"));
5829 }
5830
5831 // Check if this instruction may be used with a different wavesize.
5832 if (isGFX10Plus() && getFeatureBits()[AMDGPU::FeatureWavefrontSize64] &&
5833 !getFeatureBits()[AMDGPU::FeatureWavefrontSize32]) {
5834 // FIXME: Use getAvailableFeatures, and do not manually recompute
5835 FeatureBitset FeaturesWS32 = getFeatureBits();
5836 FeaturesWS32.flip(I: AMDGPU::FeatureWavefrontSize64)
5837 .flip(I: AMDGPU::FeatureWavefrontSize32);
5838 FeatureBitset AvailableFeaturesWS32 =
5839 ComputeAvailableFeatures(FB: FeaturesWS32);
5840
5841 if (isSupportedMnemo(Mnemo, FBS: AvailableFeaturesWS32, Variants: getMatchedVariants()))
5842 return Error(L: IDLoc, Msg: "instruction requires wavesize=32");
5843 }
5844
5845 // Finally check if this instruction is supported on any other GPU.
5846 if (isSupportedMnemo(Mnemo, FBS: FeatureBitset().set())) {
5847 return Error(L: IDLoc, Msg: "instruction not supported on this GPU (" +
5848 getSTI().getCPU() + ")" + ": " + Mnemo);
5849 }
5850
5851 // Instruction not supported on any GPU. Probably a typo.
5852 std::string Suggestion = AMDGPUMnemonicSpellCheck(S: Mnemo, FBS);
5853 return Error(L: IDLoc, Msg: "invalid instruction" + Suggestion);
5854}
5855
5856static bool isInvalidVOPDY(const OperandVector &Operands,
5857 uint64_t InvalidOprIdx) {
5858 assert(InvalidOprIdx < Operands.size());
5859 const auto &Op = ((AMDGPUOperand &)*Operands[InvalidOprIdx]);
5860 if (Op.isToken() && InvalidOprIdx > 1) {
5861 const auto &PrevOp = ((AMDGPUOperand &)*Operands[InvalidOprIdx - 1]);
5862 return PrevOp.isToken() && PrevOp.getToken() == "::";
5863 }
5864 return false;
5865}
5866
5867bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
5868 OperandVector &Operands,
5869 MCStreamer &Out,
5870 uint64_t &ErrorInfo,
5871 bool MatchingInlineAsm) {
5872 MCInst Inst;
5873 Inst.setLoc(IDLoc);
5874 unsigned Result = Match_Success;
5875
5876 // Order match statuses from least to most specific and keep the most
5877 // specific one:
5878 // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature
5879 auto atLeastAsSpecific = [](unsigned New, unsigned Cur) {
5880 auto rank = [](unsigned M) {
5881 return M == Match_MnemonicFail ? 1
5882 : M == Match_InvalidOperand ? 2
5883 : M == Match_MissingFeature ? 3
5884 : 0; // Match_Success sentinel
5885 };
5886 return rank(New) >= rank(Cur);
5887 };
5888
5889 for (auto Variant : getMatchedVariants()) {
5890 uint64_t EI;
5891 auto R =
5892 MatchInstructionImpl(Operands, Inst, ErrorInfo&: EI, matchingInlineAsm: MatchingInlineAsm, VariantID: Variant);
5893 if (R == Match_Success || atLeastAsSpecific(R, Result)) {
5894 Result = R;
5895 ErrorInfo = EI;
5896 }
5897 if (R == Match_Success)
5898 break;
5899 }
5900
5901 if (Result == Match_Success) {
5902 if (!validateInstruction(Inst, IDLoc, Operands)) {
5903 return true;
5904 }
5905 emitTargetDirective();
5906 Out.emitInstruction(Inst, STI: getSTI());
5907 // Record for kernel prologue checking.
5908 OpcodeStream.push_back(Elt: Inst.getOpcode());
5909 return false;
5910 }
5911
5912 StringRef Mnemo = ((AMDGPUOperand &)*Operands[0]).getToken();
5913 if (checkUnsupportedInstruction(Mnemo, IDLoc)) {
5914 return true;
5915 }
5916
5917 switch (Result) {
5918 default:
5919 break;
5920 case Match_MissingFeature:
5921 // It has been verified that the specified instruction
5922 // mnemonic is valid. A match was found but it requires
5923 // features which are not supported on this GPU.
5924 return Error(L: IDLoc, Msg: "operands are not valid for this GPU or mode");
5925
5926 case Match_InvalidOperand: {
5927 SMLoc ErrorLoc = IDLoc;
5928 if (ErrorInfo != ~0ULL) {
5929 if (ErrorInfo >= Operands.size()) {
5930 return Error(L: IDLoc, Msg: "too few operands for instruction");
5931 }
5932 AMDGPUOperand &ErrorOp = (AMDGPUOperand &)*Operands[ErrorInfo];
5933 ErrorLoc = ErrorOp.getStartLoc();
5934 if (ErrorLoc == SMLoc())
5935 ErrorLoc = IDLoc;
5936
5937 if (isInvalidVOPDY(Operands, InvalidOprIdx: ErrorInfo))
5938 return Error(L: ErrorLoc, Msg: "invalid VOPDY instruction");
5939 }
5940 return Error(L: ErrorLoc, Msg: "invalid operand for instruction");
5941 }
5942
5943 case Match_MnemonicFail:
5944 llvm_unreachable("Invalid instructions should have been handled already");
5945 }
5946 llvm_unreachable("Implement any new match types added!");
5947}
5948
5949bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
5950 int64_t Tmp = -1;
5951 if (!isToken(Kind: AsmToken::Integer) && !isToken(Kind: AsmToken::Identifier)) {
5952 return true;
5953 }
5954 if (getParser().parseAbsoluteExpression(Res&: Tmp)) {
5955 return true;
5956 }
5957 Ret = static_cast<uint32_t>(Tmp);
5958 return false;
5959}
5960
5961bool AMDGPUAsmParser::ParseDirectiveAMDGCNTarget() {
5962 if (!getSTI().getTargetTriple().isAMDGCN())
5963 return TokError(Msg: "directive only supported for amdgcn architecture");
5964
5965 std::string TargetIDDirective;
5966 SMLoc TargetStart = getTok().getLoc();
5967 if (getParser().parseEscapedString(Data&: TargetIDDirective))
5968 return true;
5969
5970 std::optional<AMDGPU::TargetID> MaybeParsed =
5971 AMDGPU::TargetID::parseTargetIDString(TargetIDDirective);
5972 if (!MaybeParsed)
5973 return getParser().Error(L: TargetStart,
5974 Msg: "malformed target id '" + TargetIDDirective + "'");
5975
5976 const AMDGPU::TargetID &ParsedTargetID = *MaybeParsed;
5977 const Triple &TT = getSTI().getTargetTriple();
5978
5979 // The processor named in the target id must be covered by the triple's
5980 // subarch.
5981 if (!AMDGPU::isCPUValidForSubArch(SubArch: TT.getSubArch(),
5982 AK: ParsedTargetID.getGPUKind())) {
5983 return getParser().Error(
5984 L: TargetStart, Msg: "target id '" + TargetIDDirective +
5985 "' specifies a processor that is not valid for "
5986 "subarch '" +
5987 TT.getArchName() + "'");
5988 }
5989
5990 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
5991 getTargetStreamer().getTargetID();
5992
5993 Triple DirectiveTriple(ParsedTargetID.getTargetTripleString());
5994 const Triple &STITriple = getSTI().getTargetTriple();
5995 if (!DirectiveTriple.isCompatibleWith(Other: STITriple)) {
5996 return getParser().Error(
5997 L: TargetStart, Msg: ".amdgcn_target " + Twine(ParsedTargetID.toString()) +
5998 " is incompatible with " +
5999 Twine(CurrentTargetID->toString()));
6000 }
6001
6002 // Error if the ISA version doesn't match
6003 StringRef DirectiveProcessor =
6004 AMDGPU::getArchNameAMDGCN(AK: ParsedTargetID.getGPUKind());
6005 AMDGPU::IsaVersion DirectiveISA = AMDGPU::getIsaVersion(GPU: DirectiveProcessor);
6006 if (DirectiveISA != ISA) {
6007 return getParser().Error(L: TargetStart,
6008 Msg: ".amdgcn_target directive processor " +
6009 Twine(DirectiveProcessor) +
6010 " does not match the specified processor " +
6011 Twine(getSTI().getCPU()));
6012 }
6013
6014 // Warn if sramecc or xnack mismatch. These do not change the encoding.
6015 if (AMDGPU::IsaInfo::targetIDSettingsConflict(
6016 Lhs: ParsedTargetID.getXnackSetting(),
6017 Rhs: CurrentTargetID->getXnackSetting())) {
6018 Warning(L: TargetStart,
6019 Msg: ".amdgcn_target directive has conflicting xnack settings");
6020 }
6021 if (AMDGPU::IsaInfo::targetIDSettingsConflict(
6022 Lhs: ParsedTargetID.getSramEccSetting(),
6023 Rhs: CurrentTargetID->getSramEccSetting())) {
6024 Warning(L: TargetStart,
6025 Msg: ".amdgcn_target directive has conflicting sramecc settings");
6026 }
6027
6028 // Update the target streamer's TargetID with settings from the directive.
6029 // We don't update the MCSubtargetInfo because we've already validated
6030 // that the directive matches the command-line CPU.
6031 getTargetStreamer().getTargetID()->setXnackSetting(
6032 ParsedTargetID.getXnackSetting());
6033 getTargetStreamer().getTargetID()->setSramEccSetting(
6034 ParsedTargetID.getSramEccSetting());
6035
6036 return false;
6037}
6038
6039bool AMDGPUAsmParser::OutOfRangeError(SMRange Range) {
6040 return Error(L: Range.Start, Msg: "value out of range", Range);
6041}
6042
6043bool AMDGPUAsmParser::calculateGPRBlocks(
6044 const FeatureBitset &Features, const MCExpr *VCCUsed,
6045 const MCExpr *FlatScrUsed, bool XNACKUsed,
6046 std::optional<bool> EnableWavefrontSize32, const MCExpr *NextFreeVGPR,
6047 SMRange VGPRRange, const MCExpr *NextFreeSGPR, SMRange SGPRRange,
6048 const MCExpr *&VGPRBlocks, const MCExpr *&SGPRBlocks) {
6049 // TODO(scott.linder): These calculations are duplicated from
6050 // AMDGPUAsmPrinter::getSIProgramInfo and could be unified.
6051 MCContext &Ctx = getContext();
6052
6053 const MCExpr *NumSGPRs = NextFreeSGPR;
6054 int64_t EvaluatedSGPRs;
6055
6056 if (ISA.Major >= 10)
6057 NumSGPRs = MCConstantExpr::create(Value: 0, Ctx);
6058 else {
6059 unsigned MaxAddressableNumSGPRs = AMDGPU::getAddressableNumSGPRs(AK: Gfx);
6060
6061 if (NumSGPRs->evaluateAsAbsolute(Res&: EvaluatedSGPRs) && ISA.Major >= 8 &&
6062 !Features.test(I: FeatureSGPRInitBug) &&
6063 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6064 return OutOfRangeError(Range: SGPRRange);
6065
6066 const MCExpr *ExtraSGPRs =
6067 AMDGPUMCExpr::createExtraSGPRs(VCCUsed, FlatScrUsed, XNACKUsed, Ctx);
6068 NumSGPRs = MCBinaryExpr::createAdd(LHS: NumSGPRs, RHS: ExtraSGPRs, Ctx);
6069
6070 if (NumSGPRs->evaluateAsAbsolute(Res&: EvaluatedSGPRs) &&
6071 (ISA.Major <= 7 || Features.test(I: FeatureSGPRInitBug)) &&
6072 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6073 return OutOfRangeError(Range: SGPRRange);
6074
6075 if (Features.test(I: FeatureSGPRInitBug))
6076 NumSGPRs =
6077 MCConstantExpr::create(Value: IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG, Ctx);
6078 }
6079
6080 // The MCExpr equivalent of getNumSGPRBlocks/getNumVGPRBlocks:
6081 // (alignTo(max(1u, NumGPR), GPREncodingGranule) / GPREncodingGranule) - 1
6082 auto GetNumGPRBlocks = [&Ctx](const MCExpr *NumGPR,
6083 unsigned Granule) -> const MCExpr * {
6084 const MCExpr *OneConst = MCConstantExpr::create(Value: 1ul, Ctx);
6085 const MCExpr *GranuleConst = MCConstantExpr::create(Value: Granule, Ctx);
6086 const MCExpr *MaxNumGPR = AMDGPUMCExpr::createMax(Args: {NumGPR, OneConst}, Ctx);
6087 const MCExpr *AlignToGPR =
6088 AMDGPUMCExpr::createAlignTo(Value: MaxNumGPR, Align: GranuleConst, Ctx);
6089 const MCExpr *DivGPR =
6090 MCBinaryExpr::createDiv(LHS: AlignToGPR, RHS: GranuleConst, Ctx);
6091 const MCExpr *SubGPR = MCBinaryExpr::createSub(LHS: DivGPR, RHS: OneConst, Ctx);
6092 return SubGPR;
6093 };
6094
6095 VGPRBlocks = GetNumGPRBlocks(
6096 NextFreeVGPR,
6097 IsaInfo::getVGPREncodingGranule(STI: getSTI(), EnableWavefrontSize32));
6098 SGPRBlocks =
6099 GetNumGPRBlocks(NumSGPRs, IsaInfo::getSGPREncodingGranule(STI: getSTI()));
6100
6101 return false;
6102}
6103
6104bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
6105 if (!getSTI().getTargetTriple().isAMDGCN())
6106 return TokError(Msg: "directive only supported for amdgcn architecture");
6107
6108 if (!isHsaAbi(STI: getSTI()))
6109 return TokError(Msg: "directive only supported for amdhsa OS");
6110
6111 StringRef KernelName;
6112 if (getParser().parseIdentifier(Res&: KernelName))
6113 return true;
6114
6115 // Remember the kernel name so its prologue can be checked at end of file.
6116 // The matching label may have been parsed already or may follow later.
6117 AMDHSAKernelSymbols.insert(Ptr: getContext().getOrCreateSymbol(Name: KernelName));
6118
6119 AMDGPU::MCKernelDescriptor KD =
6120 AMDGPU::MCKernelDescriptor::getDefaultAmdhsaKernelDescriptor(
6121 STI: &getSTI(), Ctx&: getContext());
6122
6123 StringSet<> Seen;
6124
6125 const MCExpr *ZeroExpr = MCConstantExpr::create(Value: 0, Ctx&: getContext());
6126 const MCExpr *OneExpr = MCConstantExpr::create(Value: 1, Ctx&: getContext());
6127
6128 SMRange VGPRRange;
6129 const MCExpr *NextFreeVGPR = ZeroExpr;
6130 const MCExpr *AccumOffset = MCConstantExpr::create(Value: 0, Ctx&: getContext());
6131 const MCExpr *NamedBarCnt = ZeroExpr;
6132 uint64_t SharedVGPRCount = 0;
6133 uint64_t PreloadLength = 0;
6134 uint64_t PreloadOffset = 0;
6135 SMRange SGPRRange;
6136 const MCExpr *NextFreeSGPR = ZeroExpr;
6137
6138 // Count the number of user SGPRs implied from the enabled feature bits.
6139 unsigned ImpliedUserSGPRCount = 0;
6140
6141 // Track if the asm explicitly contains the directive for the user SGPR
6142 // count.
6143 std::optional<unsigned> ExplicitUserSGPRCount;
6144 const MCExpr *ReserveVCC = OneExpr;
6145 const MCExpr *ReserveFlatScr = OneExpr;
6146 std::optional<bool> EnableWavefrontSize32;
6147
6148 while (true) {
6149 while (trySkipToken(Kind: AsmToken::EndOfStatement))
6150 ;
6151
6152 StringRef ID;
6153 SMRange IDRange = getTok().getLocRange();
6154 if (!parseId(Val&: ID, ErrMsg: "expected .amdhsa_ directive or .end_amdhsa_kernel"))
6155 return true;
6156
6157 if (ID == ".end_amdhsa_kernel")
6158 break;
6159
6160 if (!Seen.insert(key: ID).second)
6161 return TokError(Msg: ".amdhsa_ directives cannot be repeated");
6162
6163 SMLoc ValStart = getLoc();
6164 const MCExpr *ExprVal;
6165 if (getParser().parseExpression(Res&: ExprVal))
6166 return true;
6167 SMLoc ValEnd = getLoc();
6168 SMRange ValRange = SMRange(ValStart, ValEnd);
6169
6170 int64_t IVal = 0;
6171 uint64_t Val = IVal;
6172 bool EvaluatableExpr;
6173 if ((EvaluatableExpr = ExprVal->evaluateAsAbsolute(Res&: IVal))) {
6174 if (IVal < 0)
6175 return OutOfRangeError(Range: ValRange);
6176 Val = IVal;
6177 }
6178
6179#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE) \
6180 if (!isUInt<ENTRY##_WIDTH>(Val)) \
6181 return OutOfRangeError(RANGE); \
6182 AMDGPU::MCKernelDescriptor::bits_set(FIELD, VALUE, ENTRY##_SHIFT, ENTRY, \
6183 getContext());
6184
6185// Some fields use the parsed value immediately which requires the expression to
6186// be solvable.
6187#define EXPR_RESOLVE_OR_ERROR(RESOLVED) \
6188 if (!(RESOLVED)) \
6189 return Error(IDRange.Start, "directive should have resolvable expression", \
6190 IDRange);
6191
6192 if (ID == ".amdhsa_group_segment_fixed_size") {
6193 if (!isUInt<sizeof(kernel_descriptor_t::group_segment_fixed_size) *
6194 CHAR_BIT>(x: Val))
6195 return OutOfRangeError(Range: ValRange);
6196 KD.group_segment_fixed_size = ExprVal;
6197 } else if (ID == ".amdhsa_private_segment_fixed_size") {
6198 if (!isUInt<sizeof(kernel_descriptor_t::private_segment_fixed_size) *
6199 CHAR_BIT>(x: Val))
6200 return OutOfRangeError(Range: ValRange);
6201 KD.private_segment_fixed_size = ExprVal;
6202 } else if (ID == ".amdhsa_kernarg_size") {
6203 if (!isUInt<sizeof(kernel_descriptor_t::kernarg_size) * CHAR_BIT>(x: Val))
6204 return OutOfRangeError(Range: ValRange);
6205 KD.kernarg_size = ExprVal;
6206 } else if (ID == ".amdhsa_user_sgpr_count") {
6207 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6208 ExplicitUserSGPRCount = Val;
6209 } else if (ID == ".amdhsa_user_sgpr_private_segment_buffer") {
6210 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6211 if (hasArchitectedFlatScratch())
6212 return Error(L: IDRange.Start,
6213 Msg: "directive is not supported with architected flat scratch",
6214 Range: IDRange);
6215 PARSE_BITS_ENTRY(KD.kernel_code_properties,
6216 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
6217 ExprVal, ValRange);
6218 if (Val)
6219 ImpliedUserSGPRCount += 4;
6220 } else if (ID == ".amdhsa_user_sgpr_kernarg_preload_length") {
6221 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6222 if (!hasKernargPreload())
6223 return Error(L: IDRange.Start, Msg: "directive requires gfx90a+", Range: IDRange);
6224
6225 if (Val > getMaxNumUserSGPRs())
6226 return OutOfRangeError(Range: ValRange);
6227 PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_LENGTH, ExprVal,
6228 ValRange);
6229 if (Val) {
6230 ImpliedUserSGPRCount += Val;
6231 PreloadLength = Val;
6232 }
6233 } else if (ID == ".amdhsa_user_sgpr_kernarg_preload_offset") {
6234 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6235 if (!hasKernargPreload())
6236 return Error(L: IDRange.Start, Msg: "directive requires gfx90a+", Range: IDRange);
6237
6238 if (Val >= 1024)
6239 return OutOfRangeError(Range: ValRange);
6240 PARSE_BITS_ENTRY(KD.kernarg_preload, KERNARG_PRELOAD_SPEC_OFFSET, ExprVal,
6241 ValRange);
6242 if (Val)
6243 PreloadOffset = Val;
6244 } else if (ID == ".amdhsa_user_sgpr_dispatch_ptr") {
6245 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6246 PARSE_BITS_ENTRY(KD.kernel_code_properties,
6247 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, ExprVal,
6248 ValRange);
6249 if (Val)
6250 ImpliedUserSGPRCount += 2;
6251 } else if (ID == ".amdhsa_user_sgpr_queue_ptr") {
6252 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6253 PARSE_BITS_ENTRY(KD.kernel_code_properties,
6254 KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, ExprVal,
6255 ValRange);
6256 if (Val)
6257 ImpliedUserSGPRCount += 2;
6258 } else if (ID == ".amdhsa_user_sgpr_kernarg_segment_ptr") {
6259 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6260 PARSE_BITS_ENTRY(KD.kernel_code_properties,
6261 KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
6262 ExprVal, ValRange);
6263 if (Val)
6264 ImpliedUserSGPRCount += 2;
6265 } else if (ID == ".amdhsa_user_sgpr_dispatch_id") {
6266 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6267 PARSE_BITS_ENTRY(KD.kernel_code_properties,
6268 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, ExprVal,
6269 ValRange);
6270 if (Val)
6271 ImpliedUserSGPRCount += 2;
6272 } else if (ID == ".amdhsa_user_sgpr_flat_scratch_init") {
6273 if (hasArchitectedFlatScratch())
6274 return Error(L: IDRange.Start,
6275 Msg: "directive is not supported with architected flat scratch",
6276 Range: IDRange);
6277 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6278 PARSE_BITS_ENTRY(KD.kernel_code_properties,
6279 KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT,
6280 ExprVal, ValRange);
6281 if (Val)
6282 ImpliedUserSGPRCount += 2;
6283 } else if (ID == ".amdhsa_user_sgpr_private_segment_size") {
6284 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6285 PARSE_BITS_ENTRY(KD.kernel_code_properties,
6286 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
6287 ExprVal, ValRange);
6288 if (Val)
6289 ImpliedUserSGPRCount += 1;
6290 } else if (ID == ".amdhsa_wavefront_size32") {
6291 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6292 if (ISA.Major < 10)
6293 return Error(L: IDRange.Start, Msg: "directive requires gfx10+", Range: IDRange);
6294 EnableWavefrontSize32 = Val;
6295 PARSE_BITS_ENTRY(KD.kernel_code_properties,
6296 KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, ExprVal,
6297 ValRange);
6298 } else if (ID == ".amdhsa_uses_dynamic_stack") {
6299 PARSE_BITS_ENTRY(KD.kernel_code_properties,
6300 KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, ExprVal,
6301 ValRange);
6302 } else if (ID == ".amdhsa_system_sgpr_private_segment_wavefront_offset") {
6303 if (hasArchitectedFlatScratch())
6304 return Error(L: IDRange.Start,
6305 Msg: "directive is not supported with architected flat scratch",
6306 Range: IDRange);
6307 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6308 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6309 ValRange);
6310 } else if (ID == ".amdhsa_enable_private_segment") {
6311 if (!hasArchitectedFlatScratch())
6312 return Error(
6313 L: IDRange.Start,
6314 Msg: "directive is not supported without architected flat scratch",
6315 Range: IDRange);
6316 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6317 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6318 ValRange);
6319 } else if (ID == ".amdhsa_system_sgpr_workgroup_id_x") {
6320 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6321 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, ExprVal,
6322 ValRange);
6323 } else if (ID == ".amdhsa_system_sgpr_workgroup_id_y") {
6324 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6325 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, ExprVal,
6326 ValRange);
6327 } else if (ID == ".amdhsa_system_sgpr_workgroup_id_z") {
6328 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6329 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, ExprVal,
6330 ValRange);
6331 } else if (ID == ".amdhsa_system_sgpr_workgroup_info") {
6332 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6333 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, ExprVal,
6334 ValRange);
6335 } else if (ID == ".amdhsa_system_vgpr_workitem_id") {
6336 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6337 COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, ExprVal,
6338 ValRange);
6339 } else if (ID == ".amdhsa_next_free_vgpr") {
6340 VGPRRange = ValRange;
6341 NextFreeVGPR = ExprVal;
6342 } else if (ID == ".amdhsa_next_free_sgpr") {
6343 SGPRRange = ValRange;
6344 NextFreeSGPR = ExprVal;
6345 } else if (ID == ".amdhsa_accum_offset") {
6346 if (!isGFX90A())
6347 return Error(L: IDRange.Start, Msg: "directive requires gfx90a+", Range: IDRange);
6348 AccumOffset = ExprVal;
6349 } else if (ID == ".amdhsa_named_barrier_count") {
6350 if (!isGFX1250Plus())
6351 return Error(L: IDRange.Start, Msg: "directive requires gfx1250+", Range: IDRange);
6352 NamedBarCnt = ExprVal;
6353 } else if (ID == ".amdhsa_reserve_vcc") {
6354 if (EvaluatableExpr && !isUInt<1>(x: Val))
6355 return OutOfRangeError(Range: ValRange);
6356 ReserveVCC = ExprVal;
6357 } else if (ID == ".amdhsa_reserve_flat_scratch") {
6358 if (ISA.Major < 7)
6359 return Error(L: IDRange.Start, Msg: "directive requires gfx7+", Range: IDRange);
6360 if (hasArchitectedFlatScratch())
6361 return Error(L: IDRange.Start,
6362 Msg: "directive is not supported with architected flat scratch",
6363 Range: IDRange);
6364 if (EvaluatableExpr && !isUInt<1>(x: Val))
6365 return OutOfRangeError(Range: ValRange);
6366 ReserveFlatScr = ExprVal;
6367 } else if (ID == ".amdhsa_reserve_xnack_mask") {
6368 if (ISA.Major < 8)
6369 return Error(L: IDRange.Start, Msg: "directive requires gfx8+", Range: IDRange);
6370 if (!isUInt<1>(x: Val))
6371 return OutOfRangeError(Range: ValRange);
6372 bool XnackOn = getTargetStreamer().getTargetID()->isXnackOnOrAny();
6373 if (Val != XnackOn) {
6374 return getParser().Error(
6375 L: IDRange.Start,
6376 Msg: ".amdhsa_reserve_xnack_mask does not match target id", Range: IDRange);
6377 }
6378 } else if (ID == ".amdhsa_float_round_mode_32") {
6379 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6380 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, ExprVal,
6381 ValRange);
6382 } else if (ID == ".amdhsa_float_round_mode_16_64") {
6383 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6384 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, ExprVal,
6385 ValRange);
6386 } else if (ID == ".amdhsa_float_denorm_mode_32") {
6387 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6388 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, ExprVal,
6389 ValRange);
6390 } else if (ID == ".amdhsa_float_denorm_mode_16_64") {
6391 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6392 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, ExprVal,
6393 ValRange);
6394 } else if (ID == ".amdhsa_dx10_clamp") {
6395 if (!getSTI().hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode))
6396 return Error(L: IDRange.Start, Msg: "directive unsupported on gfx1170+",
6397 Range: IDRange);
6398 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6399 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, ExprVal,
6400 ValRange);
6401 } else if (ID == ".amdhsa_ieee_mode") {
6402 if (!getSTI().hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode))
6403 return Error(L: IDRange.Start, Msg: "directive unsupported on gfx1170+",
6404 Range: IDRange);
6405 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6406 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, ExprVal,
6407 ValRange);
6408 } else if (ID == ".amdhsa_fp16_overflow") {
6409 if (ISA.Major < 9)
6410 return Error(L: IDRange.Start, Msg: "directive requires gfx9+", Range: IDRange);
6411 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6412 COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, ExprVal,
6413 ValRange);
6414 } else if (ID == ".amdhsa_tg_split") {
6415 if (!isGFX90A())
6416 return Error(L: IDRange.Start, Msg: "directive requires gfx90a+", Range: IDRange);
6417 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT,
6418 ExprVal, ValRange);
6419 } else if (ID == ".amdhsa_workgroup_processor_mode") {
6420 if (!supportsWGP(STI: getSTI()))
6421 return Error(L: IDRange.Start,
6422 Msg: "directive unsupported on " + getSTI().getCPU(), Range: IDRange);
6423 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6424 COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal,
6425 ValRange);
6426 } else if (ID == ".amdhsa_memory_ordered") {
6427 if (ISA.Major < 10)
6428 return Error(L: IDRange.Start, Msg: "directive requires gfx10+", Range: IDRange);
6429 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6430 COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, ExprVal,
6431 ValRange);
6432 } else if (ID == ".amdhsa_forward_progress") {
6433 if (ISA.Major < 10)
6434 return Error(L: IDRange.Start, Msg: "directive requires gfx10+", Range: IDRange);
6435 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6436 COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, ExprVal,
6437 ValRange);
6438 } else if (ID == ".amdhsa_shared_vgpr_count") {
6439 EXPR_RESOLVE_OR_ERROR(EvaluatableExpr);
6440 if (ISA.Major < 10 || ISA.Major >= 12)
6441 return Error(L: IDRange.Start, Msg: "directive requires gfx10 or gfx11",
6442 Range: IDRange);
6443 SharedVGPRCount = Val;
6444 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3,
6445 COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal,
6446 ValRange);
6447 } else if (ID == ".amdhsa_inst_pref_size") {
6448 if (ISA.Major < 11)
6449 return Error(L: IDRange.Start, Msg: "directive requires gfx11+", Range: IDRange);
6450 if (ISA.Major == 11) {
6451 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3,
6452 COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE, ExprVal,
6453 ValRange);
6454 } else {
6455 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3,
6456 COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE, ExprVal,
6457 ValRange);
6458 }
6459 } else if (ID == ".amdhsa_exception_fp_ieee_invalid_op") {
6460 PARSE_BITS_ENTRY(
6461 KD.compute_pgm_rsrc2,
6462 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION,
6463 ExprVal, ValRange);
6464 } else if (ID == ".amdhsa_exception_fp_denorm_src") {
6465 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6466 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
6467 ExprVal, ValRange);
6468 } else if (ID == ".amdhsa_exception_fp_ieee_div_zero") {
6469 PARSE_BITS_ENTRY(
6470 KD.compute_pgm_rsrc2,
6471 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO,
6472 ExprVal, ValRange);
6473 } else if (ID == ".amdhsa_exception_fp_ieee_overflow") {
6474 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6475 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
6476 ExprVal, ValRange);
6477 } else if (ID == ".amdhsa_exception_fp_ieee_underflow") {
6478 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6479 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
6480 ExprVal, ValRange);
6481 } else if (ID == ".amdhsa_exception_fp_ieee_inexact") {
6482 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6483 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
6484 ExprVal, ValRange);
6485 } else if (ID == ".amdhsa_exception_int_div_zero") {
6486 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
6487 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
6488 ExprVal, ValRange);
6489 } else if (ID == ".amdhsa_round_robin_scheduling") {
6490 if (ISA.Major < 12)
6491 return Error(L: IDRange.Start, Msg: "directive requires gfx12+", Range: IDRange);
6492 PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
6493 COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, ExprVal,
6494 ValRange);
6495 } else {
6496 return Error(L: IDRange.Start, Msg: "unknown .amdhsa_kernel directive", Range: IDRange);
6497 }
6498
6499#undef PARSE_BITS_ENTRY
6500 }
6501
6502 if (!Seen.contains(key: ".amdhsa_next_free_vgpr"))
6503 return TokError(Msg: ".amdhsa_next_free_vgpr directive is required");
6504
6505 if (!Seen.contains(key: ".amdhsa_next_free_sgpr"))
6506 return TokError(Msg: ".amdhsa_next_free_sgpr directive is required");
6507
6508 unsigned UserSGPRCount = ExplicitUserSGPRCount.value_or(u&: ImpliedUserSGPRCount);
6509 if (UserSGPRCount > getMaxNumUserSGPRs())
6510 return TokError(Msg: "too many user SGPRs enabled, found " +
6511 Twine(UserSGPRCount) + ", but only " +
6512 Twine(getMaxNumUserSGPRs()) + " are supported.");
6513
6514 // Consider the case where the total number of UserSGPRs with trailing
6515 // allocated preload SGPRs, is greater than the number of explicitly
6516 // referenced SGPRs.
6517 if (PreloadLength) {
6518 MCContext &Ctx = getContext();
6519 NextFreeSGPR = AMDGPUMCExpr::createMax(
6520 Args: {NextFreeSGPR, MCConstantExpr::create(Value: UserSGPRCount, Ctx)}, Ctx);
6521 }
6522
6523 const MCExpr *VGPRBlocks;
6524 const MCExpr *SGPRBlocks;
6525 if (calculateGPRBlocks(Features: getFeatureBits(), VCCUsed: ReserveVCC, FlatScrUsed: ReserveFlatScr,
6526 XNACKUsed: getTargetStreamer().getTargetID()->isXnackOnOrAny(),
6527 EnableWavefrontSize32, NextFreeVGPR, VGPRRange,
6528 NextFreeSGPR, SGPRRange, VGPRBlocks, SGPRBlocks))
6529 return true;
6530
6531 int64_t EvaluatedVGPRBlocks;
6532 bool VGPRBlocksEvaluatable =
6533 VGPRBlocks->evaluateAsAbsolute(Res&: EvaluatedVGPRBlocks);
6534 if (VGPRBlocksEvaluatable &&
6535 !isUInt<COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_WIDTH>(
6536 x: static_cast<uint64_t>(EvaluatedVGPRBlocks))) {
6537 return OutOfRangeError(Range: VGPRRange);
6538 }
6539 AMDGPU::MCKernelDescriptor::bits_set(
6540 Dst&: KD.compute_pgm_rsrc1, Value: VGPRBlocks,
6541 Shift: COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT,
6542 Mask: COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, Ctx&: getContext());
6543
6544 int64_t EvaluatedSGPRBlocks;
6545 if (SGPRBlocks->evaluateAsAbsolute(Res&: EvaluatedSGPRBlocks) &&
6546 !isUInt<COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_WIDTH>(
6547 x: static_cast<uint64_t>(EvaluatedSGPRBlocks)))
6548 return OutOfRangeError(Range: SGPRRange);
6549 AMDGPU::MCKernelDescriptor::bits_set(
6550 Dst&: KD.compute_pgm_rsrc1, Value: SGPRBlocks,
6551 Shift: COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT,
6552 Mask: COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, Ctx&: getContext());
6553
6554 if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount)
6555 return TokError(Msg: "amdgpu_user_sgpr_count smaller than implied by "
6556 "enabled user SGPRs");
6557
6558 if (isGFX1250Plus()) {
6559 AMDGPU::MCKernelDescriptor::bits_set(
6560 Dst&: KD.compute_pgm_rsrc2,
6561 Value: MCConstantExpr::create(Value: UserSGPRCount, Ctx&: getContext()),
6562 Shift: COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT_SHIFT,
6563 Mask: COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT, Ctx&: getContext());
6564 } else {
6565 AMDGPU::MCKernelDescriptor::bits_set(
6566 Dst&: KD.compute_pgm_rsrc2,
6567 Value: MCConstantExpr::create(Value: UserSGPRCount, Ctx&: getContext()),
6568 Shift: COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT_SHIFT,
6569 Mask: COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT, Ctx&: getContext());
6570 }
6571
6572 int64_t IVal = 0;
6573 if (!KD.kernarg_size->evaluateAsAbsolute(Res&: IVal))
6574 return TokError(Msg: "Kernarg size should be resolvable");
6575 uint64_t kernarg_size = IVal;
6576 if (PreloadLength && kernarg_size &&
6577 (PreloadLength * 4 + PreloadOffset * 4 > kernarg_size))
6578 return TokError(Msg: "Kernarg preload length + offset is larger than the "
6579 "kernarg segment size");
6580
6581 if (isGFX90A()) {
6582 if (!Seen.contains(key: ".amdhsa_accum_offset"))
6583 return TokError(Msg: ".amdhsa_accum_offset directive is required");
6584 int64_t EvaluatedAccum;
6585 bool AccumEvaluatable = AccumOffset->evaluateAsAbsolute(Res&: EvaluatedAccum);
6586 uint64_t UEvaluatedAccum = EvaluatedAccum;
6587 if (AccumEvaluatable &&
6588 (UEvaluatedAccum < 4 || UEvaluatedAccum > 256 || (UEvaluatedAccum & 3)))
6589 return TokError(Msg: "accum_offset should be in range [4..256] in "
6590 "increments of 4");
6591
6592 int64_t EvaluatedNumVGPR;
6593 if (NextFreeVGPR->evaluateAsAbsolute(Res&: EvaluatedNumVGPR) &&
6594 AccumEvaluatable &&
6595 UEvaluatedAccum >
6596 alignTo(Value: std::max(a: (uint64_t)1, b: (uint64_t)EvaluatedNumVGPR), Align: 4))
6597 return TokError(Msg: "accum_offset exceeds total VGPR allocation");
6598 const MCExpr *AdjustedAccum = MCBinaryExpr::createSub(
6599 LHS: MCBinaryExpr::createDiv(
6600 LHS: AccumOffset, RHS: MCConstantExpr::create(Value: 4, Ctx&: getContext()), Ctx&: getContext()),
6601 RHS: MCConstantExpr::create(Value: 1, Ctx&: getContext()), Ctx&: getContext());
6602 MCKernelDescriptor::bits_set(Dst&: KD.compute_pgm_rsrc3, Value: AdjustedAccum,
6603 Shift: COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
6604 Mask: COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
6605 Ctx&: getContext());
6606 }
6607
6608 if (isGFX1250Plus())
6609 MCKernelDescriptor::bits_set(Dst&: KD.compute_pgm_rsrc3, Value: NamedBarCnt,
6610 Shift: COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT,
6611 Mask: COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
6612 Ctx&: getContext());
6613
6614 if (ISA.Major >= 10 && ISA.Major < 12) {
6615 // SharedVGPRCount < 16 checked by PARSE_ENTRY_BITS
6616 if (SharedVGPRCount && EnableWavefrontSize32 && *EnableWavefrontSize32) {
6617 return TokError(Msg: "shared_vgpr_count directive not valid on "
6618 "wavefront size 32");
6619 }
6620
6621 if (VGPRBlocksEvaluatable &&
6622 (SharedVGPRCount * 2 + static_cast<uint64_t>(EvaluatedVGPRBlocks) >
6623 63)) {
6624 return TokError(Msg: "shared_vgpr_count*2 + "
6625 "compute_pgm_rsrc1.GRANULATED_WORKITEM_VGPR_COUNT cannot "
6626 "exceed 63\n");
6627 }
6628 }
6629
6630 emitTargetDirective();
6631 getTargetStreamer().EmitAmdhsaKernelDescriptor(STI: getSTI(), KernelName, KernelDescriptor: KD,
6632 NextVGPR: NextFreeVGPR, NextSGPR: NextFreeSGPR,
6633 ReserveVCC, ReserveFlatScr);
6634 return false;
6635}
6636
6637bool AMDGPUAsmParser::ParseDirectiveAMDHSACodeObjectVersion() {
6638 uint32_t Version;
6639 if (ParseAsAbsoluteExpression(Ret&: Version))
6640 return true;
6641
6642 getTargetStreamer().EmitDirectiveAMDHSACodeObjectVersion(COV: Version);
6643 emitTargetDirective();
6644 return false;
6645}
6646
6647bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
6648 AMDGPUMCKernelCodeT &C) {
6649 // max_scratch_backing_memory_byte_size is deprecated. Ignore it while parsing
6650 // assembly for backwards compatibility.
6651 if (ID == "max_scratch_backing_memory_byte_size") {
6652 Parser.eatToEndOfStatement();
6653 return false;
6654 }
6655
6656 SmallString<40> ErrStr;
6657 raw_svector_ostream Err(ErrStr);
6658 if (!C.ParseKernelCodeT(ID, MCParser&: getParser(), Err)) {
6659 return TokError(Msg: Err.str());
6660 }
6661 Lex();
6662
6663 if (ID == "enable_wavefront_size32") {
6664 if (C.code_properties & AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32) {
6665 if (!isGFX10Plus())
6666 return TokError(Msg: "enable_wavefront_size32=1 is only allowed on GFX10+");
6667 if (!isWave32())
6668 return TokError(Msg: "enable_wavefront_size32=1 requires +WavefrontSize32");
6669 } else {
6670 if (!isWave64())
6671 return TokError(Msg: "enable_wavefront_size32=0 requires +WavefrontSize64");
6672 }
6673 }
6674
6675 if (ID == "wavefront_size") {
6676 if (C.wavefront_size == 5) {
6677 if (!isGFX10Plus())
6678 return TokError(Msg: "wavefront_size=5 is only allowed on GFX10+");
6679 if (!isWave32())
6680 return TokError(Msg: "wavefront_size=5 requires +WavefrontSize32");
6681 } else if (C.wavefront_size == 6) {
6682 if (!isWave64())
6683 return TokError(Msg: "wavefront_size=6 requires +WavefrontSize64");
6684 }
6685 }
6686
6687 return false;
6688}
6689
6690bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
6691 AMDGPUMCKernelCodeT KernelCode;
6692 KernelCode.initDefault(STI: getSTI(), Ctx&: getContext());
6693
6694 while (true) {
6695 // Lex EndOfStatement. This is in a while loop, because lexing a comment
6696 // will set the current token to EndOfStatement.
6697 while (trySkipToken(Kind: AsmToken::EndOfStatement))
6698 ;
6699
6700 StringRef ID;
6701 if (!parseId(Val&: ID, ErrMsg: "expected value identifier or .end_amd_kernel_code_t"))
6702 return true;
6703
6704 if (ID == ".end_amd_kernel_code_t")
6705 break;
6706
6707 if (ParseAMDKernelCodeTValue(ID, C&: KernelCode))
6708 return true;
6709 }
6710
6711 KernelCode.validate(STI: &getSTI(), Ctx&: getContext());
6712 getTargetStreamer().EmitAMDKernelCodeT(Header&: KernelCode);
6713
6714 return false;
6715}
6716
6717bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
6718 StringRef KernelName;
6719 if (!parseId(Val&: KernelName, ErrMsg: "expected symbol name"))
6720 return true;
6721
6722 getTargetStreamer().EmitAMDGPUSymbolType(SymbolName: KernelName,
6723 Type: ELF::STT_AMDGPU_HSA_KERNEL);
6724
6725 KernelScope.initialize(Context&: getContext());
6726 return false;
6727}
6728
6729bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
6730 if (!getSTI().getTargetTriple().isAMDGCN()) {
6731 return Error(L: getLoc(),
6732 Msg: ".amd_amdgpu_isa directive is not available on non-amdgcn "
6733 "architectures");
6734 }
6735
6736 StringRef TargetIDDirective = getLexer().getTok().getStringContents();
6737
6738 std::optional<AMDGPU::TargetID> MaybeParsed =
6739 AMDGPU::TargetID::parseTargetIDString(TargetIDDirective);
6740 if (!MaybeParsed)
6741 return Error(L: getParser().getTok().getLoc(),
6742 Msg: "malformed target id '" + TargetIDDirective + "'");
6743
6744 const AMDGPU::TargetID &ParsedTargetID = *MaybeParsed;
6745 const Triple &TT = getSTI().getTargetTriple();
6746
6747 // The processor named in the target id must be covered by the triple's
6748 // subarch.
6749 if (!AMDGPU::isCPUValidForSubArch(SubArch: TT.getSubArch(),
6750 AK: ParsedTargetID.getGPUKind())) {
6751 return Error(L: getParser().getTok().getLoc(),
6752 Msg: "target id '" + TargetIDDirective +
6753 "' specifies a processor that is not valid for subarch '" +
6754 TT.getArchName() + "'");
6755 }
6756
6757 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
6758 getTargetStreamer().getTargetID();
6759
6760 Triple DirectiveTriple(ParsedTargetID.getTargetTripleString());
6761 const Triple &STITriple = getSTI().getTargetTriple();
6762 if (!DirectiveTriple.isCompatibleWith(Other: STITriple)) {
6763 return Error(L: getParser().getTok().getLoc(),
6764 Msg: ".amd_amdgpu_isa " + Twine(ParsedTargetID.toString()) +
6765 " is incompatible with " +
6766 Twine(CurrentTargetID->toString()));
6767 }
6768
6769 // Error if the ISA version doesn't match
6770 StringRef DirectiveProcessor =
6771 AMDGPU::getArchNameAMDGCN(AK: ParsedTargetID.getGPUKind());
6772 AMDGPU::IsaVersion DirectiveISA = AMDGPU::getIsaVersion(GPU: DirectiveProcessor);
6773 if (DirectiveISA != ISA) {
6774 return Error(L: getParser().getTok().getLoc(),
6775 Msg: ".amd_amdgpu_isa directive processor " +
6776 Twine(DirectiveProcessor) +
6777 " does not match the specified processor " +
6778 Twine(getSTI().getCPU()));
6779 }
6780
6781 getTargetStreamer().EmitISAVersion();
6782 Lex();
6783
6784 return false;
6785}
6786
6787bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
6788 assert(isHsaAbi(getSTI()));
6789
6790 std::string HSAMetadataString;
6791 if (ParseToEndDirective(AssemblerDirectiveBegin: HSAMD::V3::AssemblerDirectiveBegin,
6792 AssemblerDirectiveEnd: HSAMD::V3::AssemblerDirectiveEnd, CollectString&: HSAMetadataString))
6793 return true;
6794
6795 if (!getTargetStreamer().EmitHSAMetadataV3(HSAMetadataString))
6796 return Error(L: getLoc(), Msg: "invalid HSA metadata");
6797
6798 return false;
6799}
6800
6801/// Common code to parse out a block of text (typically YAML) between start and
6802/// end directives.
6803bool AMDGPUAsmParser::ParseToEndDirective(const char *AssemblerDirectiveBegin,
6804 const char *AssemblerDirectiveEnd,
6805 std::string &CollectString) {
6806
6807 raw_string_ostream CollectStream(CollectString);
6808
6809 getLexer().setSkipSpace(false);
6810
6811 bool FoundEnd = false;
6812 while (!isToken(Kind: AsmToken::Eof)) {
6813 while (isToken(Kind: AsmToken::Space)) {
6814 CollectStream << getTokenStr();
6815 Lex();
6816 }
6817
6818 if (trySkipId(Id: AssemblerDirectiveEnd)) {
6819 FoundEnd = true;
6820 break;
6821 }
6822
6823 CollectStream << Parser.parseStringToEndOfStatement()
6824 << getContext().getAsmInfo().getSeparatorString();
6825
6826 Parser.eatToEndOfStatement();
6827 }
6828
6829 getLexer().setSkipSpace(true);
6830
6831 if (isToken(Kind: AsmToken::Eof) && !FoundEnd) {
6832 return TokError(Msg: Twine("expected directive ") +
6833 Twine(AssemblerDirectiveEnd) + Twine(" not found"));
6834 }
6835
6836 return false;
6837}
6838
6839/// Parse the assembler directive for new MsgPack-format PAL metadata.
6840bool AMDGPUAsmParser::ParseDirectivePALMetadataBegin() {
6841 std::string String;
6842 if (ParseToEndDirective(AssemblerDirectiveBegin: AMDGPU::PALMD::AssemblerDirectiveBegin,
6843 AssemblerDirectiveEnd: AMDGPU::PALMD::AssemblerDirectiveEnd, CollectString&: String))
6844 return true;
6845
6846 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6847 if (!PALMetadata->setFromString(String))
6848 return Error(L: getLoc(), Msg: "invalid PAL metadata");
6849 return false;
6850}
6851
6852/// Parse the assembler directive for old linear-format PAL metadata.
6853bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
6854 if (getSTI().getTargetTriple().getOS() != Triple::AMDPAL) {
6855 return Error(L: getLoc(), Msg: (Twine(PALMD::AssemblerDirective) +
6856 Twine(" directive is "
6857 "not available on non-amdpal OSes"))
6858 .str());
6859 }
6860
6861 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6862 PALMetadata->setLegacy();
6863 for (;;) {
6864 uint32_t Key, Value;
6865 if (ParseAsAbsoluteExpression(Ret&: Key)) {
6866 return TokError(Msg: Twine("invalid value in ") +
6867 Twine(PALMD::AssemblerDirective));
6868 }
6869 if (!trySkipToken(Kind: AsmToken::Comma)) {
6870 return TokError(Msg: Twine("expected an even number of values in ") +
6871 Twine(PALMD::AssemblerDirective));
6872 }
6873 if (ParseAsAbsoluteExpression(Ret&: Value)) {
6874 return TokError(Msg: Twine("invalid value in ") +
6875 Twine(PALMD::AssemblerDirective));
6876 }
6877 PALMetadata->setRegister(Reg: Key, Val: Value);
6878 if (!trySkipToken(Kind: AsmToken::Comma))
6879 break;
6880 }
6881 return false;
6882}
6883
6884/// ParseDirectiveAMDGPULDS
6885/// ::= .amdgpu_lds identifier ',' size_expression [',' align_expression]
6886bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() {
6887 if (getParser().checkForValidSection())
6888 return true;
6889
6890 StringRef Name;
6891 SMLoc NameLoc = getLoc();
6892 if (getParser().parseIdentifier(Res&: Name))
6893 return TokError(Msg: "expected identifier in directive");
6894
6895 MCSymbol *Symbol = getContext().getOrCreateSymbol(Name);
6896 if (getParser().parseComma())
6897 return true;
6898
6899 unsigned LocalMemorySize = AMDGPU::IsaInfo::getLocalMemorySize(STI: getSTI());
6900
6901 int64_t Size;
6902 SMLoc SizeLoc = getLoc();
6903 if (getParser().parseAbsoluteExpression(Res&: Size))
6904 return true;
6905 if (Size < 0)
6906 return Error(L: SizeLoc, Msg: "size must be non-negative");
6907 if (Size > LocalMemorySize)
6908 return Error(L: SizeLoc, Msg: "size is too large");
6909
6910 int64_t Alignment = 4;
6911 if (trySkipToken(Kind: AsmToken::Comma)) {
6912 SMLoc AlignLoc = getLoc();
6913 if (getParser().parseAbsoluteExpression(Res&: Alignment))
6914 return true;
6915 if (Alignment < 0 || !isPowerOf2_64(Value: Alignment))
6916 return Error(L: AlignLoc, Msg: "alignment must be a power of two");
6917
6918 // Alignment larger than the size of LDS is possible in theory, as long
6919 // as the linker manages to place to symbol at address 0, but we do want
6920 // to make sure the alignment fits nicely into a 32-bit integer.
6921 if (Alignment >= 1u << 31)
6922 return Error(L: AlignLoc, Msg: "alignment is too large");
6923 }
6924
6925 if (parseEOL())
6926 return true;
6927
6928 Symbol->redefineIfPossible();
6929 if (!Symbol->isUndefined())
6930 return Error(L: NameLoc, Msg: "invalid symbol redefinition");
6931
6932 getTargetStreamer().emitAMDGPULDS(Symbol, Size, Alignment: Align(Alignment));
6933 return false;
6934}
6935
6936bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
6937 if (getParser().checkForValidSection())
6938 return true;
6939
6940 StringRef FuncName;
6941 if (getParser().parseIdentifier(Res&: FuncName))
6942 return TokError(Msg: "expected symbol name after .amdgpu_info");
6943
6944 MCSymbol *FuncSym = getContext().getOrCreateSymbol(Name: FuncName);
6945 AMDGPU::InfoSectionData ParsedInfoData;
6946 AMDGPU::FuncInfo FI;
6947 FI.Sym = FuncSym;
6948 bool HasScalarAttrs = false;
6949
6950 while (true) {
6951 while (trySkipToken(Kind: AsmToken::EndOfStatement))
6952 ;
6953
6954 StringRef ID;
6955 SMLoc IDLoc = getLoc();
6956 if (!parseId(Val&: ID, ErrMsg: "expected directive or .end_amdgpu_info"))
6957 return true;
6958
6959 if (ID == ".end_amdgpu_info")
6960 break;
6961
6962 // Every per-entry directive shares the `.amdgpu_` namespace prefix; strip
6963 // it once and dispatch on the distinguishing suffix below. The unstripped
6964 // ID is preserved for diagnostics.
6965 StringRef Dir = ID;
6966 if (!Dir.consume_front(Prefix: ".amdgpu_"))
6967 return Error(L: IDLoc, Msg: "unknown .amdgpu_info directive '" + ID + "'");
6968
6969 if (Dir == "flags") {
6970 int64_t Val;
6971 if (getParser().parseAbsoluteExpression(Res&: Val))
6972 return true;
6973 auto Flags = static_cast<AMDGPU::FuncInfoFlags>(Val);
6974 FI.UsesVCC = !!(Flags & AMDGPU::FuncInfoFlags::FUNC_USES_VCC);
6975 FI.UsesFlatScratch =
6976 !!(Flags & AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH);
6977 FI.HasDynStack = !!(Flags & AMDGPU::FuncInfoFlags::FUNC_HAS_DYN_STACK);
6978 HasScalarAttrs = true;
6979 } else if (Dir == "num_sgpr") {
6980 int64_t Val;
6981 if (getParser().parseAbsoluteExpression(Res&: Val))
6982 return true;
6983 FI.NumSGPR = static_cast<uint32_t>(Val);
6984 HasScalarAttrs = true;
6985 } else if (Dir == "num_vgpr") {
6986 int64_t Val;
6987 if (getParser().parseAbsoluteExpression(Res&: Val))
6988 return true;
6989 FI.NumArchVGPR = static_cast<uint32_t>(Val);
6990 HasScalarAttrs = true;
6991 } else if (Dir == "num_agpr") {
6992 int64_t Val;
6993 if (getParser().parseAbsoluteExpression(Res&: Val))
6994 return true;
6995 FI.NumAccVGPR = static_cast<uint32_t>(Val);
6996 HasScalarAttrs = true;
6997 } else if (Dir == "private_segment_size") {
6998 int64_t Val;
6999 if (getParser().parseAbsoluteExpression(Res&: Val))
7000 return true;
7001 FI.PrivateSegmentSize = static_cast<uint32_t>(Val);
7002 HasScalarAttrs = true;
7003 } else if (Dir == "use") {
7004 StringRef ResName;
7005 if (getParser().parseIdentifier(Res&: ResName))
7006 return TokError(Msg: "expected resource symbol for .amdgpu_use");
7007 ParsedInfoData.Uses.push_back(
7008 Elt: {FuncSym, getContext().getOrCreateSymbol(Name: ResName)});
7009 } else if (Dir == "call") {
7010 StringRef DstName;
7011 if (getParser().parseIdentifier(Res&: DstName))
7012 return TokError(Msg: "expected callee symbol for .amdgpu_call");
7013 ParsedInfoData.Calls.push_back(
7014 Elt: {FuncSym, getContext().getOrCreateSymbol(Name: DstName)});
7015 } else if (Dir == "indirect_call") {
7016 std::string TypeId;
7017 if (getParser().parseEscapedString(Data&: TypeId))
7018 return TokError(Msg: "expected type ID string for .amdgpu_indirect_call");
7019 ParsedInfoData.IndirectCalls.push_back(Elt: {FuncSym, std::move(TypeId)});
7020 } else if (Dir == "typeid") {
7021 std::string TypeId;
7022 if (getParser().parseEscapedString(Data&: TypeId))
7023 return TokError(Msg: "expected type ID string for .amdgpu_typeid");
7024 ParsedInfoData.TypeIds.push_back(Elt: {FuncSym, std::move(TypeId)});
7025 } else {
7026 return Error(L: IDLoc, Msg: "unknown .amdgpu_info directive '" + ID + "'");
7027 }
7028 }
7029
7030 if (HasScalarAttrs)
7031 ParsedInfoData.Funcs.push_back(Elt: std::move(FI));
7032
7033 AMDGPU::InfoSectionData &Data = InfoData ? *InfoData : InfoData.emplace();
7034 for (AMDGPU::FuncInfo &Func : ParsedInfoData.Funcs)
7035 Data.Funcs.push_back(Elt: std::move(Func));
7036 for (std::pair<MCSymbol *, MCSymbol *> &Use : ParsedInfoData.Uses)
7037 Data.Uses.push_back(Elt: Use);
7038 for (std::pair<MCSymbol *, MCSymbol *> &Call : ParsedInfoData.Calls)
7039 Data.Calls.push_back(Elt: Call);
7040 for (std::pair<MCSymbol *, std::string> &IndirectCall :
7041 ParsedInfoData.IndirectCalls)
7042 Data.IndirectCalls.push_back(Elt: std::move(IndirectCall));
7043 for (std::pair<MCSymbol *, std::string> &TypeId : ParsedInfoData.TypeIds)
7044 Data.TypeIds.push_back(Elt: std::move(TypeId));
7045
7046 return false;
7047}
7048
7049void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
7050 // Record every parsed label in the timeline so that, at end of file, the
7051 // instructions following a kernel's label can be located regardless of
7052 // whether the .amdhsa_kernel directive came before or after the label.
7053 OpcodeStreamSymbols.emplace_back(Args&: Symbol, Args&: IDLoc, Args: OpcodeStream.size());
7054}
7055
7056void AMDGPUAsmParser::checkKernelPrologues() {
7057 if (getFeatureBits()[AMDGPU::FeatureRequiresInitialUnclausedVmem]) {
7058 static const unsigned Required[] = {S_MOV_B64_gfx12, V_NOP_e32_gfx12,
7059 GLOBAL_PREFETCH_B8_SADDR_gfx1250};
7060 for (auto [Sym, Loc, Offset] : OpcodeStreamSymbols) {
7061 if (!AMDHSAKernelSymbols.contains(Ptr: Sym))
7062 continue;
7063 ArrayRef<unsigned> Prologue = ArrayRef(OpcodeStream).drop_front(N: Offset);
7064 if (!Prologue.empty() && Prologue.front() == S_SETREG_IMM32_B32_gfx12)
7065 Prologue = Prologue.drop_front();
7066 if (Prologue.take_front(N: std::size(Required)) != ArrayRef(Required)) {
7067 Warning(L: Loc, Msg: "kernel '" + Sym->getName() +
7068 "' does not begin with the required prologue "
7069 "sequence: s_mov_b64 followed by v_nop and "
7070 "global_prefetch_b8");
7071 }
7072 }
7073 }
7074 OpcodeStream.clear();
7075 OpcodeStreamSymbols.clear();
7076 AMDHSAKernelSymbols.clear();
7077}
7078
7079void AMDGPUAsmParser::onEndOfFile() {
7080 emitTargetDirective();
7081 checkKernelPrologues();
7082 if (InfoData)
7083 getTargetStreamer().emitAMDGPUInfo(Data: *InfoData);
7084}
7085
7086bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
7087 StringRef IDVal = DirectiveID.getString();
7088
7089 if (isHsaAbi(STI: getSTI())) {
7090 if (IDVal == ".amdhsa_kernel")
7091 return ParseDirectiveAMDHSAKernel();
7092
7093 if (IDVal == ".amdhsa_code_object_version")
7094 return ParseDirectiveAMDHSACodeObjectVersion();
7095
7096 // TODO: Restructure/combine with PAL metadata directive.
7097 if (IDVal == AMDGPU::HSAMD::V3::AssemblerDirectiveBegin)
7098 return ParseDirectiveHSAMetadata();
7099 } else {
7100 if (IDVal == ".amd_kernel_code_t")
7101 return ParseDirectiveAMDKernelCodeT();
7102
7103 if (IDVal == ".amdgpu_hsa_kernel")
7104 return ParseDirectiveAMDGPUHsaKernel();
7105
7106 if (IDVal == ".amd_amdgpu_isa")
7107 return ParseDirectiveISAVersion();
7108
7109 if (IDVal == AMDGPU::HSAMD::AssemblerDirectiveBegin) {
7110 return Error(L: getLoc(), Msg: (Twine(HSAMD::AssemblerDirectiveBegin) +
7111 Twine(" directive is "
7112 "not available on non-amdhsa OSes"))
7113 .str());
7114 }
7115 }
7116
7117 if (IDVal == ".amdgcn_target")
7118 return ParseDirectiveAMDGCNTarget();
7119
7120 if (IDVal == ".amdgpu_lds")
7121 return ParseDirectiveAMDGPULDS();
7122
7123 if (IDVal == ".amdgpu_info")
7124 return ParseDirectiveAMDGPUInfo();
7125
7126 if (IDVal == PALMD::AssemblerDirectiveBegin)
7127 return ParseDirectivePALMetadataBegin();
7128
7129 if (IDVal == PALMD::AssemblerDirective)
7130 return ParseDirectivePALMetadata();
7131
7132 return true;
7133}
7134
7135bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
7136 MCRegister Reg) {
7137 if (MRI.regsOverlap(RegA: TTMP12_TTMP13_TTMP14_TTMP15, RegB: Reg))
7138 return isGFX9Plus();
7139
7140 // GFX10+ has 2 more SGPRs 104 and 105.
7141 if (MRI.regsOverlap(RegA: SGPR104_SGPR105, RegB: Reg))
7142 return hasSGPR104_SGPR105();
7143
7144 switch (Reg.id()) {
7145 case SRC_SHARED_BASE_LO:
7146 case SRC_SHARED_BASE:
7147 case SRC_SHARED_LIMIT_LO:
7148 case SRC_SHARED_LIMIT:
7149 return isGFX9Plus();
7150 case SRC_PRIVATE_BASE_LO:
7151 case SRC_PRIVATE_BASE:
7152 case SRC_PRIVATE_LIMIT_LO:
7153 case SRC_PRIVATE_LIMIT:
7154 return AMDGPU::hasPrivateApertureRegs(STI: getSTI());
7155 case SRC_FLAT_SCRATCH_BASE_LO:
7156 case SRC_FLAT_SCRATCH_BASE_HI:
7157 return hasGloballyAddressableScratch();
7158 case SRC_POPS_EXITING_WAVE_ID:
7159 return hasPopsExitingWaveID(STI: getSTI());
7160 case TBA:
7161 case TBA_LO:
7162 case TBA_HI:
7163 case TMA:
7164 case TMA_LO:
7165 case TMA_HI:
7166 return !isGFX9Plus();
7167 case XNACK_MASK:
7168 case XNACK_MASK_LO:
7169 case XNACK_MASK_HI:
7170 return (isVI() || isGFX9()) &&
7171 getTargetStreamer().getTargetID()->isXnackSupported();
7172 case SGPR_NULL:
7173 return isGFX10Plus();
7174 case SRC_EXECZ:
7175 case SRC_VCCZ:
7176 return !isGFX11Plus();
7177 default:
7178 break;
7179 }
7180
7181 if (isCI())
7182 return true;
7183
7184 if (isSI() || isGFX10Plus()) {
7185 // No flat_scr on SI.
7186 // On GFX10Plus flat scratch is not a valid register operand and can only be
7187 // accessed with s_setreg/s_getreg.
7188 switch (Reg.id()) {
7189 case FLAT_SCR:
7190 case FLAT_SCR_LO:
7191 case FLAT_SCR_HI:
7192 return false;
7193 default:
7194 return true;
7195 }
7196 }
7197
7198 // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
7199 // SI/CI have.
7200 if (MRI.regsOverlap(RegA: SGPR102_SGPR103, RegB: Reg))
7201 return hasSGPR102_SGPR103();
7202
7203 return true;
7204}
7205
7206ParseStatus AMDGPUAsmParser::parseOperand(OperandVector &Operands,
7207 StringRef Mnemonic,
7208 OperandMode Mode) {
7209 ParseStatus Res = parseVOPD(Operands);
7210 if (Res.isSuccess() || Res.isFailure() || isToken(Kind: AsmToken::EndOfStatement))
7211 return Res;
7212
7213 // Try to parse with a custom parser
7214 Res = MatchOperandParserImpl(Operands, Mnemonic);
7215
7216 // If we successfully parsed the operand or if there as an error parsing,
7217 // we are done.
7218 //
7219 // If we are parsing after we reach EndOfStatement then this means we
7220 // are appending default values to the Operands list. This is only done
7221 // by custom parser, so we shouldn't continue on to the generic parsing.
7222 if (Res.isSuccess() || Res.isFailure() || isToken(Kind: AsmToken::EndOfStatement))
7223 return Res;
7224
7225 SMLoc RBraceLoc;
7226 SMLoc LBraceLoc = getLoc();
7227 if (Mode == OperandMode_NSA && trySkipToken(Kind: AsmToken::LBrac)) {
7228 unsigned Prefix = Operands.size();
7229
7230 for (;;) {
7231 auto Loc = getLoc();
7232 Res = parseReg(Operands);
7233 if (Res.isNoMatch())
7234 Error(L: Loc, Msg: "expected a register");
7235 if (!Res.isSuccess())
7236 return ParseStatus::Failure;
7237
7238 RBraceLoc = getLoc();
7239 if (trySkipToken(Kind: AsmToken::RBrac))
7240 break;
7241
7242 if (!skipToken(Kind: AsmToken::Comma,
7243 ErrMsg: "expected a comma or a closing square bracket"))
7244 return ParseStatus::Failure;
7245 }
7246
7247 if (Operands.size() - Prefix > 1) {
7248 Operands.insert(I: Operands.begin() + Prefix,
7249 Elt: AMDGPUOperand::CreateToken(AsmParser: this, Str: "[", Loc: LBraceLoc));
7250 Operands.push_back(Elt: AMDGPUOperand::CreateToken(AsmParser: this, Str: "]", Loc: RBraceLoc));
7251 }
7252
7253 return ParseStatus::Success;
7254 }
7255
7256 return parseRegOrImm(Operands);
7257}
7258
7259StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
7260 // Clear any forced encodings from the previous instruction.
7261 setForcedEncodingSize(0);
7262 setForcedDPP(false);
7263 setForcedSDWA(false);
7264
7265 if (Name.consume_back(Suffix: "_e64_dpp")) {
7266 setForcedDPP(true);
7267 setForcedEncodingSize(64);
7268 return Name;
7269 }
7270 if (Name.consume_back(Suffix: "_e64")) {
7271 setForcedEncodingSize(64);
7272 return Name;
7273 }
7274 if (Name.consume_back(Suffix: "_e32")) {
7275 setForcedEncodingSize(32);
7276 return Name;
7277 }
7278 if (Name.consume_back(Suffix: "_dpp")) {
7279 setForcedDPP(true);
7280 return Name;
7281 }
7282 if (Name.consume_back(Suffix: "_sdwa")) {
7283 setForcedSDWA(true);
7284 return Name;
7285 }
7286 return Name;
7287}
7288
7289static void applyMnemonicAliases(StringRef &Mnemonic,
7290 const FeatureBitset &Features,
7291 unsigned VariantID);
7292
7293bool AMDGPUAsmParser::parseInstruction(ParseInstructionInfo &Info,
7294 StringRef Name, SMLoc NameLoc,
7295 OperandVector &Operands) {
7296 // Add the instruction mnemonic
7297 Name = parseMnemonicSuffix(Name);
7298
7299 // If the target architecture uses MnemonicAlias, call it here to parse
7300 // operands correctly.
7301 applyMnemonicAliases(Mnemonic&: Name, Features: getAvailableFeatures(), VariantID: 0);
7302
7303 Operands.push_back(Elt: AMDGPUOperand::CreateToken(AsmParser: this, Str: Name, Loc: NameLoc));
7304
7305 bool IsMIMG = Name.starts_with(Prefix: "image_");
7306
7307 while (!trySkipToken(Kind: AsmToken::EndOfStatement)) {
7308 OperandMode Mode = OperandMode_Default;
7309 if (IsMIMG && isGFX10Plus() && Operands.size() == 2)
7310 Mode = OperandMode_NSA;
7311 ParseStatus Res = parseOperand(Operands, Mnemonic: Name, Mode);
7312
7313 if (!Res.isSuccess()) {
7314 checkUnsupportedInstruction(Mnemo: Name, IDLoc: NameLoc);
7315 if (!Parser.hasPendingError()) {
7316 // FIXME: use real operand location rather than the current location.
7317 StringRef Msg = Res.isFailure() ? "failed parsing operand."
7318 : "not a valid operand.";
7319 Error(L: getLoc(), Msg);
7320 }
7321 while (!trySkipToken(Kind: AsmToken::EndOfStatement)) {
7322 lex();
7323 }
7324 return true;
7325 }
7326
7327 // Eat the comma or space if there is one.
7328 trySkipToken(Kind: AsmToken::Comma);
7329 }
7330
7331 return false;
7332}
7333
7334//===----------------------------------------------------------------------===//
7335// Utility functions
7336//===----------------------------------------------------------------------===//
7337
7338ParseStatus AMDGPUAsmParser::parseTokenOp(StringRef Name,
7339 OperandVector &Operands) {
7340 SMLoc S = getLoc();
7341 if (!trySkipId(Id: Name))
7342 return ParseStatus::NoMatch;
7343
7344 Operands.push_back(Elt: AMDGPUOperand::CreateToken(AsmParser: this, Str: Name, Loc: S));
7345 return ParseStatus::Success;
7346}
7347
7348ParseStatus AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix,
7349 int64_t &IntVal) {
7350
7351 if (!trySkipId(Id: Prefix, Kind: AsmToken::Colon))
7352 return ParseStatus::NoMatch;
7353
7354 return parseExpr(Imm&: IntVal) ? ParseStatus::Success : ParseStatus::Failure;
7355}
7356
7357ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
7358 const char *Prefix, OperandVector &Operands, AMDGPUOperand::ImmTy ImmTy,
7359 std::function<bool(int64_t &)> ConvertResult) {
7360 SMLoc S = getLoc();
7361 int64_t Value = 0;
7362
7363 ParseStatus Res = parseIntWithPrefix(Prefix, IntVal&: Value);
7364 if (!Res.isSuccess())
7365 return Res;
7366
7367 if (ConvertResult && !ConvertResult(Value)) {
7368 Error(L: S, Msg: "invalid " + StringRef(Prefix) + " value.");
7369 }
7370
7371 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Value, Loc: S, Type: ImmTy));
7372 return ParseStatus::Success;
7373}
7374
7375ParseStatus AMDGPUAsmParser::parseOperandArrayWithPrefix(
7376 const char *Prefix, OperandVector &Operands, AMDGPUOperand::ImmTy ImmTy,
7377 bool (*ConvertResult)(int64_t &)) {
7378 SMLoc S = getLoc();
7379 if (!trySkipId(Id: Prefix, Kind: AsmToken::Colon))
7380 return ParseStatus::NoMatch;
7381
7382 if (!skipToken(Kind: AsmToken::LBrac, ErrMsg: "expected a left square bracket"))
7383 return ParseStatus::Failure;
7384
7385 unsigned Val = 0;
7386 const unsigned MaxSize = 4;
7387
7388 // FIXME: How to verify the number of elements matches the number of src
7389 // operands?
7390 for (int I = 0;; ++I) {
7391 int64_t Op;
7392 SMLoc Loc = getLoc();
7393 if (!parseExpr(Imm&: Op))
7394 return ParseStatus::Failure;
7395
7396 if (Op != 0 && Op != 1)
7397 return Error(L: Loc, Msg: "invalid " + StringRef(Prefix) + " value.");
7398
7399 Val |= (Op << I);
7400
7401 if (trySkipToken(Kind: AsmToken::RBrac))
7402 break;
7403
7404 if (I + 1 == MaxSize)
7405 return Error(L: getLoc(), Msg: "expected a closing square bracket");
7406
7407 if (!skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma"))
7408 return ParseStatus::Failure;
7409 }
7410
7411 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val, Loc: S, Type: ImmTy));
7412 return ParseStatus::Success;
7413}
7414
7415ParseStatus AMDGPUAsmParser::parseNamedBit(StringRef Name,
7416 OperandVector &Operands,
7417 AMDGPUOperand::ImmTy ImmTy,
7418 bool IgnoreNegative) {
7419 int64_t Bit;
7420 SMLoc S = getLoc();
7421
7422 if (trySkipId(Id: Name)) {
7423 Bit = 1;
7424 } else if (trySkipId(Pref: "no", Id: Name)) {
7425 if (IgnoreNegative)
7426 return ParseStatus::Success;
7427 Bit = 0;
7428 } else {
7429 return ParseStatus::NoMatch;
7430 }
7431
7432 if (Name == "r128" && !hasMIMG_R128())
7433 return Error(L: S, Msg: "r128 modifier is not supported on this GPU");
7434 if (Name == "a16" && !hasA16())
7435 return Error(L: S, Msg: "a16 modifier is not supported on this GPU");
7436
7437 if (Bit == 0 && Name == "gds") {
7438 StringRef Mnemo = ((AMDGPUOperand &)*Operands[0]).getToken();
7439 if (Mnemo.starts_with(Prefix: "ds_gws"))
7440 return Error(L: S, Msg: "nogds is not allowed");
7441 }
7442
7443 if (isGFX9() && ImmTy == AMDGPUOperand::ImmTyA16)
7444 ImmTy = AMDGPUOperand::ImmTyR128A16;
7445
7446 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Bit, Loc: S, Type: ImmTy));
7447 return ParseStatus::Success;
7448}
7449
7450unsigned AMDGPUAsmParser::getCPolKind(StringRef Id, StringRef Mnemo,
7451 bool &Disabling) const {
7452 Disabling = Id.consume_front(Prefix: "no");
7453
7454 if (isGFX940() && !Mnemo.starts_with(Prefix: "s_")) {
7455 return StringSwitch<unsigned>(Id)
7456 .Case(S: "nt", Value: AMDGPU::CPol::NT)
7457 .Case(S: "sc0", Value: AMDGPU::CPol::SC0)
7458 .Case(S: "sc1", Value: AMDGPU::CPol::SC1)
7459 .Default(Value: 0);
7460 }
7461
7462 return StringSwitch<unsigned>(Id)
7463 .Case(S: "dlc", Value: AMDGPU::CPol::DLC)
7464 .Case(S: "glc", Value: AMDGPU::CPol::GLC)
7465 .Case(S: "scc", Value: AMDGPU::CPol::SCC)
7466 .Case(S: "slc", Value: AMDGPU::CPol::SLC)
7467 .Default(Value: 0);
7468}
7469
7470ParseStatus AMDGPUAsmParser::parseCPol(OperandVector &Operands) {
7471 if (isGFX12Plus()) {
7472 SMLoc StringLoc = getLoc();
7473
7474 int64_t CPolVal = 0;
7475 ParseStatus ResTH = ParseStatus::NoMatch;
7476 ParseStatus ResScope = ParseStatus::NoMatch;
7477 ParseStatus ResNV = ParseStatus::NoMatch;
7478 ParseStatus ResScal = ParseStatus::NoMatch;
7479 ParseStatus ResCFS = ParseStatus::NoMatch;
7480
7481 for (;;) {
7482 if (ResTH.isNoMatch()) {
7483 int64_t TH;
7484 ResTH = parseTH(Operands, TH);
7485 if (ResTH.isFailure())
7486 return ResTH;
7487 if (ResTH.isSuccess()) {
7488 CPolVal |= TH;
7489 continue;
7490 }
7491 }
7492
7493 if (ResScope.isNoMatch()) {
7494 int64_t Scope;
7495 ResScope = parseScope(Operands, Scope);
7496 if (ResScope.isFailure())
7497 return ResScope;
7498 if (ResScope.isSuccess()) {
7499 CPolVal |= Scope;
7500 continue;
7501 }
7502 }
7503
7504 // NV bit exists on GFX12+, but does something starting from GFX1250.
7505 // Allow parsing on all GFX12 and fail on validation for better
7506 // diagnostics.
7507 if (ResNV.isNoMatch()) {
7508 if (trySkipId(Id: "nv")) {
7509 ResNV = ParseStatus::Success;
7510 CPolVal |= CPol::NV;
7511 continue;
7512 } else if (trySkipId(Pref: "no", Id: "nv")) {
7513 ResNV = ParseStatus::Success;
7514 continue;
7515 }
7516 }
7517
7518 if (ResScal.isNoMatch()) {
7519 if (trySkipId(Id: "scale_offset")) {
7520 ResScal = ParseStatus::Success;
7521 CPolVal |= CPol::SCAL;
7522 continue;
7523 } else if (trySkipId(Pref: "no", Id: "scale_offset")) {
7524 ResScal = ParseStatus::Success;
7525 continue;
7526 }
7527 }
7528
7529 if (ResCFS.isNoMatch()) {
7530 int64_t CFS;
7531 ResCFS = parseCFS(Operands, CFS);
7532 if (ResCFS.isFailure())
7533 return ResCFS;
7534 if (ResCFS.isSuccess()) {
7535 CPolVal |= CFS;
7536 continue;
7537 }
7538 }
7539
7540 break;
7541 }
7542
7543 if (ResTH.isNoMatch() && ResScope.isNoMatch() && ResNV.isNoMatch() &&
7544 ResScal.isNoMatch() && ResCFS.isNoMatch())
7545 return ParseStatus::NoMatch;
7546
7547 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: CPolVal, Loc: StringLoc,
7548 Type: AMDGPUOperand::ImmTyCPol));
7549 return ParseStatus::Success;
7550 }
7551
7552 StringRef Mnemo = ((AMDGPUOperand &)*Operands[0]).getToken();
7553 SMLoc OpLoc = getLoc();
7554 unsigned Enabled = 0, Seen = 0;
7555 for (;;) {
7556 SMLoc S = getLoc();
7557 bool Disabling;
7558 unsigned CPol = getCPolKind(Id: getId(), Mnemo, Disabling);
7559 if (!CPol)
7560 break;
7561
7562 lex();
7563
7564 if (!isGFX10Plus() && CPol == AMDGPU::CPol::DLC)
7565 return Error(L: S, Msg: "dlc modifier is not supported on this GPU");
7566
7567 if (!isGFX90A() && CPol == AMDGPU::CPol::SCC)
7568 return Error(L: S, Msg: "scc modifier is not supported on this GPU");
7569
7570 if (Seen & CPol)
7571 return Error(L: S, Msg: "duplicate cache policy modifier");
7572
7573 if (!Disabling)
7574 Enabled |= CPol;
7575
7576 Seen |= CPol;
7577 }
7578
7579 if (!Seen)
7580 return ParseStatus::NoMatch;
7581
7582 Operands.push_back(
7583 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Enabled, Loc: OpLoc, Type: AMDGPUOperand::ImmTyCPol));
7584 return ParseStatus::Success;
7585}
7586
7587ParseStatus AMDGPUAsmParser::parseScope(OperandVector &Operands,
7588 int64_t &Scope) {
7589 static const unsigned Scopes[] = {CPol::SCOPE_CU, CPol::SCOPE_SE,
7590 CPol::SCOPE_DEV, CPol::SCOPE_SYS};
7591
7592 ParseStatus Res = parseStringOrIntWithPrefix(
7593 Operands, Name: "scope", Ids: {"SCOPE_CU", "SCOPE_SE", "SCOPE_DEV", "SCOPE_SYS"},
7594 IntVal&: Scope);
7595
7596 if (Res.isSuccess())
7597 Scope = Scopes[Scope];
7598
7599 return Res;
7600}
7601
7602ParseStatus AMDGPUAsmParser::parseTH(OperandVector &Operands, int64_t &TH) {
7603 TH = AMDGPU::CPol::TH_RT; // default
7604
7605 StringRef Value;
7606 SMLoc StringLoc;
7607 ParseStatus Res = parseStringWithPrefix(Prefix: "th", Value, StringLoc);
7608 if (!Res.isSuccess())
7609 return Res;
7610
7611 if (Value == "TH_DEFAULT")
7612 TH = AMDGPU::CPol::TH_RT;
7613 else if (Value == "TH_STORE_LU" || Value == "TH_LOAD_WB" ||
7614 Value == "TH_LOAD_NT_WB") {
7615 return Error(L: StringLoc, Msg: "invalid th value");
7616 } else if (Value.consume_front(Prefix: "TH_ATOMIC_")) {
7617 TH = AMDGPU::CPol::TH_TYPE_ATOMIC;
7618 } else if (Value.consume_front(Prefix: "TH_LOAD_")) {
7619 TH = AMDGPU::CPol::TH_TYPE_LOAD;
7620 } else if (Value.consume_front(Prefix: "TH_STORE_")) {
7621 TH = AMDGPU::CPol::TH_TYPE_STORE;
7622 } else {
7623 return Error(L: StringLoc, Msg: "invalid th value");
7624 }
7625
7626 if (Value == "BYPASS")
7627 TH |= AMDGPU::CPol::TH_REAL_BYPASS;
7628
7629 if (TH != 0) {
7630 if (TH & AMDGPU::CPol::TH_TYPE_ATOMIC)
7631 TH |= StringSwitch<int64_t>(Value)
7632 .Case(S: "RETURN", Value: AMDGPU::CPol::TH_ATOMIC_RETURN)
7633 .Case(S: "RT", Value: AMDGPU::CPol::TH_RT)
7634 .Case(S: "RT_RETURN", Value: AMDGPU::CPol::TH_ATOMIC_RETURN)
7635 .Case(S: "NT", Value: AMDGPU::CPol::TH_ATOMIC_NT)
7636 .Case(S: "NT_RETURN", Value: AMDGPU::CPol::TH_ATOMIC_NT |
7637 AMDGPU::CPol::TH_ATOMIC_RETURN)
7638 .Case(S: "CASCADE_RT", Value: AMDGPU::CPol::TH_ATOMIC_CASCADE)
7639 .Case(S: "CASCADE_NT", Value: AMDGPU::CPol::TH_ATOMIC_CASCADE |
7640 AMDGPU::CPol::TH_ATOMIC_NT)
7641 .Default(Value: 0xffffffff);
7642 else
7643 TH |= StringSwitch<int64_t>(Value)
7644 .Case(S: "RT", Value: AMDGPU::CPol::TH_RT)
7645 .Case(S: "NT", Value: AMDGPU::CPol::TH_NT)
7646 .Case(S: "HT", Value: AMDGPU::CPol::TH_HT)
7647 .Case(S: "LU", Value: AMDGPU::CPol::TH_LU)
7648 .Case(S: "WB", Value: AMDGPU::CPol::TH_WB)
7649 .Case(S: "NT_RT", Value: AMDGPU::CPol::TH_NT_RT)
7650 .Case(S: "RT_NT", Value: AMDGPU::CPol::TH_RT_NT)
7651 .Case(S: "NT_HT", Value: AMDGPU::CPol::TH_NT_HT)
7652 .Case(S: "NT_WB", Value: AMDGPU::CPol::TH_NT_WB)
7653 .Case(S: "BYPASS", Value: AMDGPU::CPol::TH_BYPASS)
7654 .Default(Value: 0xffffffff);
7655 }
7656
7657 if (TH == 0xffffffff)
7658 return Error(L: StringLoc, Msg: "invalid th value");
7659
7660 return ParseStatus::Success;
7661}
7662
7663ParseStatus AMDGPUAsmParser::parseCFS(OperandVector &Operands, int64_t &CFS) {
7664 static const unsigned CFSVals[] = {CPol::CFS_256B, CPol::CFS_128B,
7665 CPol::CFS_64B, CPol::CFS_32B};
7666
7667 ParseStatus Res = parseStringOrIntWithPrefix(
7668 Operands, Name: "cfs", Ids: {"CFS_256B", "CFS_128B", "CFS_64B", "CFS_32B"}, IntVal&: CFS);
7669
7670 if (Res.isSuccess())
7671 CFS = CFSVals[CFS];
7672
7673 return Res;
7674}
7675
7676static void
7677addOptionalImmOperand(MCInst &Inst, const OperandVector &Operands,
7678 AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx,
7679 AMDGPUOperand::ImmTy ImmT, int64_t Default = 0,
7680 std::optional<unsigned> InsertAt = std::nullopt) {
7681 auto i = OptionalIdx.find(x: ImmT);
7682 if (i != OptionalIdx.end()) {
7683 unsigned Idx = i->second;
7684 const AMDGPUOperand &Op =
7685 static_cast<const AMDGPUOperand &>(*Operands[Idx]);
7686 if (InsertAt)
7687 Inst.insert(I: Inst.begin() + *InsertAt, Op: MCOperand::createImm(Val: Op.getImm()));
7688 else
7689 Op.addImmOperands(Inst, N: 1);
7690 } else {
7691 if (InsertAt.has_value())
7692 Inst.insert(I: Inst.begin() + *InsertAt, Op: MCOperand::createImm(Val: Default));
7693 else
7694 Inst.addOperand(Op: MCOperand::createImm(Val: Default));
7695 }
7696}
7697
7698ParseStatus AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix,
7699 StringRef &Value,
7700 SMLoc &StringLoc) {
7701 if (!trySkipId(Id: Prefix, Kind: AsmToken::Colon))
7702 return ParseStatus::NoMatch;
7703
7704 StringLoc = getLoc();
7705 return parseId(Val&: Value, ErrMsg: "expected an identifier") ? ParseStatus::Success
7706 : ParseStatus::Failure;
7707}
7708
7709ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7710 OperandVector &Operands, StringRef Name, ArrayRef<const char *> Ids,
7711 int64_t &IntVal) {
7712 if (!trySkipId(Id: Name, Kind: AsmToken::Colon))
7713 return ParseStatus::NoMatch;
7714
7715 SMLoc StringLoc = getLoc();
7716
7717 StringRef Value;
7718 if (isToken(Kind: AsmToken::Identifier)) {
7719 Value = getTokenStr();
7720 lex();
7721
7722 for (IntVal = 0; IntVal < (int64_t)Ids.size(); ++IntVal)
7723 if (Value == Ids[IntVal])
7724 break;
7725 } else if (!parseExpr(Imm&: IntVal))
7726 return ParseStatus::Failure;
7727
7728 if (IntVal < 0 || IntVal >= (int64_t)Ids.size())
7729 return Error(L: StringLoc, Msg: "invalid " + Twine(Name) + " value");
7730
7731 return ParseStatus::Success;
7732}
7733
7734ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7735 OperandVector &Operands, StringRef Name, ArrayRef<const char *> Ids,
7736 AMDGPUOperand::ImmTy Type) {
7737 SMLoc S = getLoc();
7738 int64_t IntVal;
7739
7740 ParseStatus Res = parseStringOrIntWithPrefix(Operands, Name, Ids, IntVal);
7741 if (Res.isSuccess())
7742 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: IntVal, Loc: S, Type));
7743
7744 return Res;
7745}
7746
7747//===----------------------------------------------------------------------===//
7748// MTBUF format
7749//===----------------------------------------------------------------------===//
7750
7751ParseStatus AMDGPUAsmParser::tryParseFmt(const char *Pref, int64_t MaxVal,
7752 int64_t &Fmt) {
7753 int64_t Val;
7754 SMLoc Loc = getLoc();
7755
7756 ParseStatus Res = parseIntWithPrefix(Prefix: Pref, IntVal&: Val);
7757 if (!Res.isSuccess())
7758 return Res;
7759
7760 if (Val < 0 || Val > MaxVal)
7761 return Error(L: Loc, Msg: Twine("out of range ", StringRef(Pref)));
7762
7763 Fmt = Val;
7764 return ParseStatus::Success;
7765}
7766
7767ParseStatus AMDGPUAsmParser::tryParseIndexKey(OperandVector &Operands,
7768 AMDGPUOperand::ImmTy ImmTy) {
7769 const char *Pref = "index_key";
7770 int64_t ImmVal = 0;
7771 SMLoc Loc = getLoc();
7772 auto Res = parseIntWithPrefix(Prefix: Pref, IntVal&: ImmVal);
7773 if (!Res.isSuccess())
7774 return Res;
7775
7776 if ((ImmTy == AMDGPUOperand::ImmTyIndexKey16bit ||
7777 ImmTy == AMDGPUOperand::ImmTyIndexKey32bit) &&
7778 (ImmVal < 0 || ImmVal > 1))
7779 return Error(L: Loc, Msg: Twine("out of range ", StringRef(Pref)));
7780
7781 if (ImmTy == AMDGPUOperand::ImmTyIndexKey8bit && (ImmVal < 0 || ImmVal > 3))
7782 return Error(L: Loc, Msg: Twine("out of range ", StringRef(Pref)));
7783
7784 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: ImmVal, Loc, Type: ImmTy));
7785 return ParseStatus::Success;
7786}
7787
7788ParseStatus AMDGPUAsmParser::parseIndexKey8bit(OperandVector &Operands) {
7789 return tryParseIndexKey(Operands, ImmTy: AMDGPUOperand::ImmTyIndexKey8bit);
7790}
7791
7792ParseStatus AMDGPUAsmParser::parseIndexKey16bit(OperandVector &Operands) {
7793 return tryParseIndexKey(Operands, ImmTy: AMDGPUOperand::ImmTyIndexKey16bit);
7794}
7795
7796ParseStatus AMDGPUAsmParser::parseIndexKey32bit(OperandVector &Operands) {
7797 return tryParseIndexKey(Operands, ImmTy: AMDGPUOperand::ImmTyIndexKey32bit);
7798}
7799
7800ParseStatus AMDGPUAsmParser::tryParseMatrixFMT(OperandVector &Operands,
7801 StringRef Name,
7802 AMDGPUOperand::ImmTy Type) {
7803 return parseStringOrIntWithPrefix(Operands, Name, Ids: WMMAMods::ModMatrixFmt,
7804 Type);
7805}
7806
7807ParseStatus AMDGPUAsmParser::parseMatrixAFMT(OperandVector &Operands) {
7808 return tryParseMatrixFMT(Operands, Name: "matrix_a_fmt",
7809 Type: AMDGPUOperand::ImmTyMatrixAFMT);
7810}
7811
7812ParseStatus AMDGPUAsmParser::parseMatrixBFMT(OperandVector &Operands) {
7813 return tryParseMatrixFMT(Operands, Name: "matrix_b_fmt",
7814 Type: AMDGPUOperand::ImmTyMatrixBFMT);
7815}
7816
7817ParseStatus AMDGPUAsmParser::tryParseMatrixScale(OperandVector &Operands,
7818 StringRef Name,
7819 AMDGPUOperand::ImmTy Type) {
7820 return parseStringOrIntWithPrefix(Operands, Name, Ids: WMMAMods::ModMatrixScale,
7821 Type);
7822}
7823
7824ParseStatus AMDGPUAsmParser::parseMatrixAScale(OperandVector &Operands) {
7825 return tryParseMatrixScale(Operands, Name: "matrix_a_scale",
7826 Type: AMDGPUOperand::ImmTyMatrixAScale);
7827}
7828
7829ParseStatus AMDGPUAsmParser::parseMatrixBScale(OperandVector &Operands) {
7830 return tryParseMatrixScale(Operands, Name: "matrix_b_scale",
7831 Type: AMDGPUOperand::ImmTyMatrixBScale);
7832}
7833
7834ParseStatus AMDGPUAsmParser::tryParseMatrixScaleFmt(OperandVector &Operands,
7835 StringRef Name,
7836 AMDGPUOperand::ImmTy Type) {
7837 return parseStringOrIntWithPrefix(Operands, Name, Ids: WMMAMods::ModMatrixScaleFmt,
7838 Type);
7839}
7840
7841ParseStatus AMDGPUAsmParser::parseMatrixAScaleFmt(OperandVector &Operands) {
7842 return tryParseMatrixScaleFmt(Operands, Name: "matrix_a_scale_fmt",
7843 Type: AMDGPUOperand::ImmTyMatrixAScaleFmt);
7844}
7845
7846ParseStatus AMDGPUAsmParser::parseMatrixBScaleFmt(OperandVector &Operands) {
7847 return tryParseMatrixScaleFmt(Operands, Name: "matrix_b_scale_fmt",
7848 Type: AMDGPUOperand::ImmTyMatrixBScaleFmt);
7849}
7850
7851// dfmt and nfmt (in a tbuffer instruction) are parsed as one to allow their
7852// values to live in a joint format operand in the MCInst encoding.
7853ParseStatus AMDGPUAsmParser::parseDfmtNfmt(int64_t &Format) {
7854 using namespace llvm::AMDGPU::MTBUFFormat;
7855
7856 int64_t Dfmt = DFMT_DEFAULT;
7857 int64_t Nfmt = NFMT_DEFAULT;
7858
7859 auto TryAfterComma = [&](const char *Pref, int64_t MaxVal,
7860 int64_t &Fmt) -> ParseStatus {
7861 if (isId(Id: Pref) && peekToken().is(K: AsmToken::Colon))
7862 return Error(L: getLoc(), Msg: "expected a comma");
7863 AsmToken NextTokens[2];
7864 peekTokens(Tokens: NextTokens);
7865 if (!isToken(Kind: AsmToken::Comma) || !isId(Token: NextTokens[0], Id: Pref) ||
7866 !NextTokens[1].is(K: AsmToken::Colon))
7867 return ParseStatus::Success;
7868 lex();
7869 return tryParseFmt(Pref, MaxVal, Fmt);
7870 };
7871
7872 // dfmt and nfmt can appear in either order, and each is optional. If both
7873 // are present, they are separated by a comma.
7874 ParseStatus Res = tryParseFmt(Pref: "dfmt", MaxVal: DFMT_MAX, Fmt&: Dfmt);
7875 if (Res.isSuccess()) {
7876 Res = TryAfterComma("nfmt", NFMT_MAX, Nfmt);
7877 } else if (Res.isNoMatch()) {
7878 Res = tryParseFmt(Pref: "nfmt", MaxVal: NFMT_MAX, Fmt&: Nfmt);
7879 if (Res.isSuccess())
7880 Res = TryAfterComma("dfmt", DFMT_MAX, Dfmt);
7881 }
7882 if (!Res.isSuccess())
7883 return Res;
7884
7885 Format = encodeDfmtNfmt(Dfmt, Nfmt);
7886 return ParseStatus::Success;
7887}
7888
7889ParseStatus AMDGPUAsmParser::parseUfmt(int64_t &Format) {
7890 return tryParseFmt(Pref: "format", MaxVal: MTBUFFormat::UFMT_MAX, Fmt&: Format);
7891}
7892
7893bool AMDGPUAsmParser::matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt,
7894 StringRef FormatStr, SMLoc Loc) {
7895 using namespace llvm::AMDGPU::MTBUFFormat;
7896 int64_t Format;
7897
7898 Format = getDfmt(Name: FormatStr);
7899 if (Format != DFMT_UNDEF) {
7900 Dfmt = Format;
7901 return true;
7902 }
7903
7904 Format = getNfmt(Name: FormatStr, STI: getSTI());
7905 if (Format != NFMT_UNDEF) {
7906 Nfmt = Format;
7907 return true;
7908 }
7909
7910 Error(L: Loc, Msg: "unsupported format");
7911 return false;
7912}
7913
7914ParseStatus AMDGPUAsmParser::parseSymbolicSplitFormat(StringRef FormatStr,
7915 SMLoc FormatLoc,
7916 int64_t &Format) {
7917 using namespace llvm::AMDGPU::MTBUFFormat;
7918
7919 int64_t Dfmt = DFMT_UNDEF;
7920 int64_t Nfmt = NFMT_UNDEF;
7921 if (!matchDfmtNfmt(Dfmt, Nfmt, FormatStr, Loc: FormatLoc))
7922 return ParseStatus::Failure;
7923
7924 if (trySkipToken(Kind: AsmToken::Comma)) {
7925 StringRef Str;
7926 SMLoc Loc = getLoc();
7927 if (!parseId(Val&: Str, ErrMsg: "expected a format string") ||
7928 !matchDfmtNfmt(Dfmt, Nfmt, FormatStr: Str, Loc))
7929 return ParseStatus::Failure;
7930 if (Dfmt == DFMT_UNDEF)
7931 return Error(L: Loc, Msg: "duplicate numeric format");
7932 if (Nfmt == NFMT_UNDEF)
7933 return Error(L: Loc, Msg: "duplicate data format");
7934 }
7935
7936 Dfmt = (Dfmt == DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7937 Nfmt = (Nfmt == NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7938
7939 if (isGFX10Plus()) {
7940 auto Ufmt = convertDfmtNfmt2Ufmt(Dfmt, Nfmt, STI: getSTI());
7941 if (Ufmt == UFMT_UNDEF)
7942 return Error(L: FormatLoc, Msg: "unsupported format");
7943 Format = Ufmt;
7944 } else {
7945 Format = encodeDfmtNfmt(Dfmt, Nfmt);
7946 }
7947
7948 return ParseStatus::Success;
7949}
7950
7951ParseStatus AMDGPUAsmParser::parseSymbolicUnifiedFormat(StringRef FormatStr,
7952 SMLoc Loc,
7953 int64_t &Format) {
7954 using namespace llvm::AMDGPU::MTBUFFormat;
7955
7956 auto Id = getUnifiedFormat(Name: FormatStr, STI: getSTI());
7957 if (Id == UFMT_UNDEF)
7958 return ParseStatus::NoMatch;
7959
7960 if (!isGFX10Plus())
7961 return Error(L: Loc, Msg: "unified format is not supported on this GPU");
7962
7963 Format = Id;
7964 return ParseStatus::Success;
7965}
7966
7967ParseStatus AMDGPUAsmParser::parseNumericFormat(int64_t &Format) {
7968 using namespace llvm::AMDGPU::MTBUFFormat;
7969 SMLoc Loc = getLoc();
7970
7971 if (!parseExpr(Imm&: Format))
7972 return ParseStatus::Failure;
7973 if (!isValidFormatEncoding(Val: Format, STI: getSTI()))
7974 return Error(L: Loc, Msg: "out of range format");
7975
7976 return ParseStatus::Success;
7977}
7978
7979ParseStatus AMDGPUAsmParser::parseSymbolicOrNumericFormat(int64_t &Format) {
7980 using namespace llvm::AMDGPU::MTBUFFormat;
7981
7982 if (!trySkipId(Id: "format", Kind: AsmToken::Colon))
7983 return ParseStatus::NoMatch;
7984
7985 if (trySkipToken(Kind: AsmToken::LBrac)) {
7986 StringRef FormatStr;
7987 SMLoc Loc = getLoc();
7988 if (!parseId(Val&: FormatStr, ErrMsg: "expected a format string"))
7989 return ParseStatus::Failure;
7990
7991 auto Res = parseSymbolicUnifiedFormat(FormatStr, Loc, Format);
7992 if (Res.isNoMatch())
7993 Res = parseSymbolicSplitFormat(FormatStr, FormatLoc: Loc, Format);
7994 if (!Res.isSuccess())
7995 return Res;
7996
7997 if (!skipToken(Kind: AsmToken::RBrac, ErrMsg: "expected a closing square bracket"))
7998 return ParseStatus::Failure;
7999
8000 return ParseStatus::Success;
8001 }
8002
8003 return parseNumericFormat(Format);
8004}
8005
8006ParseStatus AMDGPUAsmParser::parseFormat(OperandVector &Operands) {
8007 SMLoc Loc = getLoc();
8008 int64_t Format;
8009 ParseStatus Res = parseSymbolicOrNumericFormat(Format);
8010 if (!Res.isSuccess())
8011 return Res;
8012
8013 Operands.push_back(
8014 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Format, Loc, Type: AMDGPUOperand::ImmTyFormat));
8015 return ParseStatus::Success;
8016}
8017
8018ParseStatus AMDGPUAsmParser::parseFormatOp(OperandVector &Operands) {
8019 SMLoc Loc = getLoc();
8020 int64_t Format;
8021 ParseStatus Res = isGFX10Plus() ? parseUfmt(Format) : parseDfmtNfmt(Format);
8022 if (!Res.isSuccess())
8023 return Res;
8024
8025 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Format, Loc,
8026 Type: AMDGPUOperand::ImmTyFormatOp));
8027 return ParseStatus::Success;
8028}
8029
8030ParseStatus AMDGPUAsmParser::parseFlatOffset(OperandVector &Operands) {
8031 ParseStatus Res =
8032 parseIntWithPrefix(Prefix: "offset", Operands, ImmTy: AMDGPUOperand::ImmTyOffset);
8033 if (Res.isNoMatch()) {
8034 Res = parseIntWithPrefix(Prefix: "inst_offset", Operands,
8035 ImmTy: AMDGPUOperand::ImmTyInstOffset);
8036 }
8037 return Res;
8038}
8039
8040ParseStatus AMDGPUAsmParser::parseR128A16(OperandVector &Operands) {
8041 ParseStatus Res =
8042 parseNamedBit(Name: "r128", Operands, ImmTy: AMDGPUOperand::ImmTyR128A16);
8043 if (Res.isNoMatch())
8044 Res = parseNamedBit(Name: "a16", Operands, ImmTy: AMDGPUOperand::ImmTyA16);
8045 return Res;
8046}
8047
8048ParseStatus AMDGPUAsmParser::parseBLGP(OperandVector &Operands) {
8049 ParseStatus Res =
8050 parseIntWithPrefix(Prefix: "blgp", Operands, ImmTy: AMDGPUOperand::ImmTyBLGP);
8051 if (Res.isNoMatch()) {
8052 Res =
8053 parseOperandArrayWithPrefix(Prefix: "neg", Operands, ImmTy: AMDGPUOperand::ImmTyBLGP);
8054 }
8055 return Res;
8056}
8057
8058//===----------------------------------------------------------------------===//
8059// Exp
8060//===----------------------------------------------------------------------===//
8061
8062void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
8063 OptionalImmIndexMap OptionalIdx;
8064
8065 unsigned OperandIdx[4];
8066 unsigned EnMask = 0;
8067 int SrcIdx = 0;
8068
8069 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
8070 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
8071
8072 // Add the register arguments
8073 if (Op.isReg()) {
8074 assert(SrcIdx < 4);
8075 OperandIdx[SrcIdx] = Inst.size();
8076 Op.addRegOperands(Inst, N: 1);
8077 ++SrcIdx;
8078 continue;
8079 }
8080
8081 if (Op.isOff()) {
8082 assert(SrcIdx < 4);
8083 OperandIdx[SrcIdx] = Inst.size();
8084 Inst.addOperand(Op: MCOperand::createReg(Reg: MCRegister()));
8085 ++SrcIdx;
8086 continue;
8087 }
8088
8089 if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
8090 Op.addImmOperands(Inst, N: 1);
8091 continue;
8092 }
8093
8094 if (Op.isToken() && (Op.getToken() == "done" || Op.getToken() == "row_en"))
8095 continue;
8096
8097 // Handle optional arguments
8098 OptionalIdx[Op.getImmTy()] = i;
8099 }
8100
8101 assert(SrcIdx == 4);
8102
8103 bool Compr = false;
8104 if (OptionalIdx.find(x: AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
8105 Compr = true;
8106 Inst.getOperand(i: OperandIdx[1]) = Inst.getOperand(i: OperandIdx[2]);
8107 Inst.getOperand(i: OperandIdx[2]).setReg(MCRegister());
8108 Inst.getOperand(i: OperandIdx[3]).setReg(MCRegister());
8109 }
8110
8111 for (auto i = 0; i < SrcIdx; ++i) {
8112 if (Inst.getOperand(i: OperandIdx[i]).getReg()) {
8113 EnMask |= Compr ? (0x3 << i * 2) : (0x1 << i);
8114 }
8115 }
8116
8117 addOptionalImmOperand(Inst, Operands, OptionalIdx, ImmT: AMDGPUOperand::ImmTyExpVM);
8118 addOptionalImmOperand(Inst, Operands, OptionalIdx,
8119 ImmT: AMDGPUOperand::ImmTyExpCompr);
8120
8121 Inst.addOperand(Op: MCOperand::createImm(Val: EnMask));
8122}
8123
8124//===----------------------------------------------------------------------===//
8125// s_waitcnt
8126//===----------------------------------------------------------------------===//
8127
8128static bool encodeCnt(const AMDGPU::IsaVersion ISA, int64_t &IntVal,
8129 int64_t CntVal, bool Saturate,
8130 unsigned (*encode)(const IsaVersion &Version, unsigned,
8131 unsigned),
8132 unsigned (*decode)(const IsaVersion &Version, unsigned)) {
8133 bool Failed = false;
8134
8135 IntVal = encode(ISA, IntVal, CntVal);
8136 if (CntVal != decode(ISA, IntVal)) {
8137 if (Saturate) {
8138 IntVal = encode(ISA, IntVal, -1);
8139 } else {
8140 Failed = true;
8141 }
8142 }
8143 return Failed;
8144}
8145
8146bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
8147
8148 SMLoc CntLoc = getLoc();
8149 StringRef CntName = getTokenStr();
8150
8151 if (!skipToken(Kind: AsmToken::Identifier, ErrMsg: "expected a counter name") ||
8152 !skipToken(Kind: AsmToken::LParen, ErrMsg: "expected a left parenthesis"))
8153 return false;
8154
8155 int64_t CntVal;
8156 SMLoc ValLoc = getLoc();
8157 if (!parseExpr(Imm&: CntVal))
8158 return false;
8159
8160 bool Failed = true;
8161 bool Sat = CntName.ends_with(Suffix: "_sat");
8162
8163 if (CntName == "vmcnt" || CntName == "vmcnt_sat") {
8164 Failed = encodeCnt(ISA, IntVal, CntVal, Saturate: Sat, encode: encodeVmcnt, decode: decodeVmcnt);
8165 } else if (CntName == "expcnt" || CntName == "expcnt_sat") {
8166 Failed = encodeCnt(ISA, IntVal, CntVal, Saturate: Sat, encode: encodeExpcnt, decode: decodeExpcnt);
8167 } else if (CntName == "lgkmcnt" || CntName == "lgkmcnt_sat") {
8168 Failed = encodeCnt(ISA, IntVal, CntVal, Saturate: Sat, encode: encodeLgkmcnt, decode: decodeLgkmcnt);
8169 } else {
8170 Error(L: CntLoc, Msg: "invalid counter name " + CntName);
8171 return false;
8172 }
8173
8174 if (Failed) {
8175 Error(L: ValLoc, Msg: "too large value for " + CntName);
8176 return false;
8177 }
8178
8179 if (!skipToken(Kind: AsmToken::RParen, ErrMsg: "expected a closing parenthesis"))
8180 return false;
8181
8182 if (trySkipToken(Kind: AsmToken::Amp) || trySkipToken(Kind: AsmToken::Comma)) {
8183 if (isToken(Kind: AsmToken::EndOfStatement)) {
8184 Error(L: getLoc(), Msg: "expected a counter name");
8185 return false;
8186 }
8187 }
8188
8189 return true;
8190}
8191
8192ParseStatus AMDGPUAsmParser::parseSWaitCnt(OperandVector &Operands) {
8193 int64_t Waitcnt = getWaitcntBitMask(Version: ISA);
8194 SMLoc S = getLoc();
8195
8196 if (isToken(Kind: AsmToken::Identifier) && peekToken().is(K: AsmToken::LParen)) {
8197 while (!isToken(Kind: AsmToken::EndOfStatement)) {
8198 if (!parseCnt(IntVal&: Waitcnt))
8199 return ParseStatus::Failure;
8200 }
8201 } else {
8202 if (!parseExpr(Imm&: Waitcnt))
8203 return ParseStatus::Failure;
8204 }
8205
8206 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Waitcnt, Loc: S));
8207 return ParseStatus::Success;
8208}
8209
8210bool AMDGPUAsmParser::parseDelay(int64_t &Delay) {
8211 SMLoc FieldLoc = getLoc();
8212 StringRef FieldName = getTokenStr();
8213 if (!skipToken(Kind: AsmToken::Identifier, ErrMsg: "expected a field name") ||
8214 !skipToken(Kind: AsmToken::LParen, ErrMsg: "expected a left parenthesis"))
8215 return false;
8216
8217 SMLoc ValueLoc = getLoc();
8218 StringRef ValueName = getTokenStr();
8219 if (!skipToken(Kind: AsmToken::Identifier, ErrMsg: "expected a value name") ||
8220 !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected a right parenthesis"))
8221 return false;
8222
8223 unsigned Shift;
8224 if (FieldName == "instid0") {
8225 Shift = 0;
8226 } else if (FieldName == "instskip") {
8227 Shift = 4;
8228 } else if (FieldName == "instid1") {
8229 Shift = 7;
8230 } else {
8231 Error(L: FieldLoc, Msg: "invalid field name " + FieldName);
8232 return false;
8233 }
8234
8235 int Value;
8236 if (Shift == 4) {
8237 // Parse values for instskip.
8238 Value = StringSwitch<int>(ValueName)
8239 .Case(S: "SAME", Value: 0)
8240 .Case(S: "NEXT", Value: 1)
8241 .Case(S: "SKIP_1", Value: 2)
8242 .Case(S: "SKIP_2", Value: 3)
8243 .Case(S: "SKIP_3", Value: 4)
8244 .Case(S: "SKIP_4", Value: 5)
8245 .Default(Value: -1);
8246 } else {
8247 // Parse values for instid0 and instid1.
8248 Value = StringSwitch<int>(ValueName)
8249 .Case(S: "NO_DEP", Value: 0)
8250 .Case(S: "VALU_DEP_1", Value: 1)
8251 .Case(S: "VALU_DEP_2", Value: 2)
8252 .Case(S: "VALU_DEP_3", Value: 3)
8253 .Case(S: "VALU_DEP_4", Value: 4)
8254 .Case(S: "TRANS32_DEP_1", Value: 5)
8255 .Case(S: "TRANS32_DEP_2", Value: 6)
8256 .Case(S: "TRANS32_DEP_3", Value: 7)
8257 .Case(S: "FMA_ACCUM_CYCLE_1", Value: 8)
8258 .Case(S: "SALU_CYCLE_1", Value: 9)
8259 .Case(S: "SALU_CYCLE_2", Value: 10)
8260 .Case(S: "SALU_CYCLE_3", Value: 11)
8261 .Default(Value: -1);
8262 }
8263 if (Value < 0) {
8264 Error(L: ValueLoc, Msg: "invalid value name " + ValueName);
8265 return false;
8266 }
8267
8268 Delay |= Value << Shift;
8269 return true;
8270}
8271
8272ParseStatus AMDGPUAsmParser::parseSDelayALU(OperandVector &Operands) {
8273 int64_t Delay = 0;
8274 SMLoc S = getLoc();
8275
8276 if (isToken(Kind: AsmToken::Identifier) && peekToken().is(K: AsmToken::LParen)) {
8277 do {
8278 if (!parseDelay(Delay))
8279 return ParseStatus::Failure;
8280 } while (trySkipToken(Kind: AsmToken::Pipe));
8281 } else {
8282 if (!parseExpr(Imm&: Delay))
8283 return ParseStatus::Failure;
8284 }
8285
8286 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Delay, Loc: S));
8287 return ParseStatus::Success;
8288}
8289
8290bool AMDGPUOperand::isSWaitCnt() const { return isImm(); }
8291
8292bool AMDGPUOperand::isSDelayALU() const { return isImm(); }
8293
8294//===----------------------------------------------------------------------===//
8295// DepCtr
8296//===----------------------------------------------------------------------===//
8297
8298void AMDGPUAsmParser::depCtrError(SMLoc Loc, int ErrorId,
8299 StringRef DepCtrName) {
8300 switch (ErrorId) {
8301 case OPR_ID_UNKNOWN:
8302 Error(L: Loc, Msg: Twine("invalid counter name ", DepCtrName));
8303 return;
8304 case OPR_ID_UNSUPPORTED:
8305 Error(L: Loc, Msg: Twine(DepCtrName, " is not supported on this GPU"));
8306 return;
8307 case OPR_ID_DUPLICATE:
8308 Error(L: Loc, Msg: Twine("duplicate counter name ", DepCtrName));
8309 return;
8310 case OPR_VAL_INVALID:
8311 Error(L: Loc, Msg: Twine("invalid value for ", DepCtrName));
8312 return;
8313 default:
8314 assert(false);
8315 }
8316}
8317
8318bool AMDGPUAsmParser::parseDepCtr(int64_t &DepCtr, unsigned &UsedOprMask) {
8319
8320 using namespace llvm::AMDGPU::DepCtr;
8321
8322 SMLoc DepCtrLoc = getLoc();
8323 StringRef DepCtrName = getTokenStr();
8324
8325 if (!skipToken(Kind: AsmToken::Identifier, ErrMsg: "expected a counter name") ||
8326 !skipToken(Kind: AsmToken::LParen, ErrMsg: "expected a left parenthesis"))
8327 return false;
8328
8329 int64_t ExprVal;
8330 if (!parseExpr(Imm&: ExprVal))
8331 return false;
8332
8333 unsigned PrevOprMask = UsedOprMask;
8334 int CntVal = encodeDepCtr(Name: DepCtrName, Val: ExprVal, UsedOprMask, STI: getSTI());
8335
8336 if (CntVal < 0) {
8337 depCtrError(Loc: DepCtrLoc, ErrorId: CntVal, DepCtrName);
8338 return false;
8339 }
8340
8341 if (!skipToken(Kind: AsmToken::RParen, ErrMsg: "expected a closing parenthesis"))
8342 return false;
8343
8344 if (trySkipToken(Kind: AsmToken::Amp) || trySkipToken(Kind: AsmToken::Comma)) {
8345 if (isToken(Kind: AsmToken::EndOfStatement)) {
8346 Error(L: getLoc(), Msg: "expected a counter name");
8347 return false;
8348 }
8349 }
8350
8351 int64_t CntValMask = PrevOprMask ^ UsedOprMask;
8352 DepCtr = (DepCtr & ~CntValMask) | CntVal;
8353 return true;
8354}
8355
8356ParseStatus AMDGPUAsmParser::parseDepCtr(OperandVector &Operands) {
8357 using namespace llvm::AMDGPU::DepCtr;
8358
8359 int64_t DepCtr = getDefaultDepCtrEncoding(STI: getSTI());
8360 SMLoc Loc = getLoc();
8361
8362 if (isToken(Kind: AsmToken::Identifier) && peekToken().is(K: AsmToken::LParen)) {
8363 unsigned UsedOprMask = 0;
8364 while (!isToken(Kind: AsmToken::EndOfStatement)) {
8365 if (!parseDepCtr(DepCtr, UsedOprMask))
8366 return ParseStatus::Failure;
8367 }
8368 } else {
8369 if (!parseExpr(Imm&: DepCtr))
8370 return ParseStatus::Failure;
8371 }
8372
8373 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: DepCtr, Loc));
8374 return ParseStatus::Success;
8375}
8376
8377bool AMDGPUOperand::isDepCtr() const { return isS16Imm(); }
8378
8379//===----------------------------------------------------------------------===//
8380// hwreg
8381//===----------------------------------------------------------------------===//
8382
8383ParseStatus AMDGPUAsmParser::parseHwregFunc(OperandInfoTy &HwReg,
8384 OperandInfoTy &Offset,
8385 OperandInfoTy &Width) {
8386 using namespace llvm::AMDGPU::Hwreg;
8387
8388 if (!trySkipId(Id: "hwreg", Kind: AsmToken::LParen))
8389 return ParseStatus::NoMatch;
8390
8391 // The register may be specified by name or using a numeric code
8392 HwReg.Loc = getLoc();
8393 if (isToken(Kind: AsmToken::Identifier) &&
8394 (HwReg.Val = getHwregId(Name: getTokenStr(), STI: getSTI())) != OPR_ID_UNKNOWN) {
8395 HwReg.IsSymbolic = true;
8396 lex(); // skip register name
8397 } else if (!parseExpr(Imm&: HwReg.Val, Expected: "a register name")) {
8398 return ParseStatus::Failure;
8399 }
8400
8401 if (trySkipToken(Kind: AsmToken::RParen))
8402 return ParseStatus::Success;
8403
8404 // parse optional params
8405 if (!skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma or a closing parenthesis"))
8406 return ParseStatus::Failure;
8407
8408 Offset.Loc = getLoc();
8409 if (!parseExpr(Imm&: Offset.Val))
8410 return ParseStatus::Failure;
8411
8412 if (!skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma"))
8413 return ParseStatus::Failure;
8414
8415 Width.Loc = getLoc();
8416 if (!parseExpr(Imm&: Width.Val) ||
8417 !skipToken(Kind: AsmToken::RParen, ErrMsg: "expected a closing parenthesis"))
8418 return ParseStatus::Failure;
8419
8420 return ParseStatus::Success;
8421}
8422
8423ParseStatus AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
8424 using namespace llvm::AMDGPU::Hwreg;
8425
8426 int64_t ImmVal = 0;
8427 SMLoc Loc = getLoc();
8428
8429 StructuredOpField HwReg("id", "hardware register", HwregId::Width,
8430 HwregId::Default);
8431 StructuredOpField Offset("offset", "bit offset", HwregOffset::Width,
8432 HwregOffset::Default);
8433 struct : StructuredOpField {
8434 using StructuredOpField::StructuredOpField;
8435 bool validate(AMDGPUAsmParser &Parser) const override {
8436 if (!isUIntN(N: Width, x: Val - 1))
8437 return Error(Parser, Err: "only values from 1 to 32 are legal");
8438 return true;
8439 }
8440 } Width("size", "bitfield width", HwregSize::Width, HwregSize::Default);
8441 ParseStatus Res = parseStructuredOpFields(Fields: {&HwReg, &Offset, &Width});
8442
8443 if (Res.isNoMatch())
8444 Res = parseHwregFunc(HwReg, Offset, Width);
8445
8446 if (Res.isSuccess()) {
8447 if (!validateStructuredOpFields(Fields: {&HwReg, &Offset, &Width}))
8448 return ParseStatus::Failure;
8449 ImmVal = HwregEncoding::encode(Values: HwReg.Val, Values: Offset.Val, Values: Width.Val);
8450 }
8451
8452 if (Res.isNoMatch() &&
8453 parseExpr(Imm&: ImmVal, Expected: "a hwreg macro, structured immediate"))
8454 Res = ParseStatus::Success;
8455
8456 if (!Res.isSuccess())
8457 return ParseStatus::Failure;
8458
8459 if (!isUInt<16>(x: ImmVal))
8460 return Error(L: Loc, Msg: "invalid immediate: only 16-bit values are legal");
8461 Operands.push_back(
8462 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: ImmVal, Loc, Type: AMDGPUOperand::ImmTyHwreg));
8463 return ParseStatus::Success;
8464}
8465
8466bool AMDGPUOperand::isHwreg() const { return isImmTy(ImmT: ImmTyHwreg); }
8467
8468//===----------------------------------------------------------------------===//
8469// sendmsg
8470//===----------------------------------------------------------------------===//
8471
8472bool AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &Msg, OperandInfoTy &Op,
8473 OperandInfoTy &Stream) {
8474 using namespace llvm::AMDGPU::SendMsg;
8475
8476 Msg.Loc = getLoc();
8477 if (isToken(Kind: AsmToken::Identifier) &&
8478 (Msg.Val = getMsgId(Name: getTokenStr(), STI: getSTI())) != OPR_ID_UNKNOWN) {
8479 Msg.IsSymbolic = true;
8480 lex(); // skip message name
8481 } else if (!parseExpr(Imm&: Msg.Val, Expected: "a message name")) {
8482 return false;
8483 }
8484
8485 if (trySkipToken(Kind: AsmToken::Comma)) {
8486 Op.IsDefined = true;
8487 Op.Loc = getLoc();
8488 if (isToken(Kind: AsmToken::Identifier) &&
8489 (Op.Val = getMsgOpId(MsgId: Msg.Val, Name: getTokenStr(), STI: getSTI())) !=
8490 OPR_ID_UNKNOWN) {
8491 lex(); // skip operation name
8492 } else if (!parseExpr(Imm&: Op.Val, Expected: "an operation name")) {
8493 return false;
8494 }
8495
8496 if (trySkipToken(Kind: AsmToken::Comma)) {
8497 Stream.IsDefined = true;
8498 Stream.Loc = getLoc();
8499 if (!parseExpr(Imm&: Stream.Val))
8500 return false;
8501 }
8502 }
8503
8504 return skipToken(Kind: AsmToken::RParen, ErrMsg: "expected a closing parenthesis");
8505}
8506
8507bool AMDGPUAsmParser::validateSendMsg(const OperandInfoTy &Msg,
8508 const OperandInfoTy &Op,
8509 const OperandInfoTy &Stream) {
8510 using namespace llvm::AMDGPU::SendMsg;
8511
8512 // Validation strictness depends on whether message is specified
8513 // in a symbolic or in a numeric form. In the latter case
8514 // only encoding possibility is checked.
8515 bool Strict = Msg.IsSymbolic;
8516
8517 if (Strict) {
8518 if (Msg.Val == OPR_ID_UNSUPPORTED) {
8519 Error(L: Msg.Loc, Msg: "specified message id is not supported on this GPU");
8520 return false;
8521 }
8522 } else {
8523 if (!isValidMsgId(MsgId: Msg.Val, STI: getSTI())) {
8524 Error(L: Msg.Loc, Msg: "invalid message id");
8525 return false;
8526 }
8527 }
8528 if (Strict && (msgRequiresOp(MsgId: Msg.Val, STI: getSTI()) != Op.IsDefined)) {
8529 if (Op.IsDefined) {
8530 Error(L: Op.Loc, Msg: "message does not support operations");
8531 } else {
8532 Error(L: Msg.Loc, Msg: "missing message operation");
8533 }
8534 return false;
8535 }
8536 if (!isValidMsgOp(MsgId: Msg.Val, OpId: Op.Val, STI: getSTI(), Strict)) {
8537 if (Op.Val == OPR_ID_UNSUPPORTED)
8538 Error(L: Op.Loc, Msg: "specified operation id is not supported on this GPU");
8539 else
8540 Error(L: Op.Loc, Msg: "invalid operation id");
8541 return false;
8542 }
8543 if (Strict && !msgSupportsStream(MsgId: Msg.Val, OpId: Op.Val, STI: getSTI()) &&
8544 Stream.IsDefined) {
8545 Error(L: Stream.Loc, Msg: "message operation does not support streams");
8546 return false;
8547 }
8548 if (!isValidMsgStream(MsgId: Msg.Val, OpId: Op.Val, StreamId: Stream.Val, STI: getSTI(), Strict)) {
8549 Error(L: Stream.Loc, Msg: "invalid message stream id");
8550 return false;
8551 }
8552 return true;
8553}
8554
8555ParseStatus AMDGPUAsmParser::parseSendMsg(OperandVector &Operands) {
8556 using namespace llvm::AMDGPU::SendMsg;
8557
8558 int64_t ImmVal = 0;
8559 SMLoc Loc = getLoc();
8560
8561 if (trySkipId(Id: "sendmsg", Kind: AsmToken::LParen)) {
8562 OperandInfoTy Msg(OPR_ID_UNKNOWN);
8563 OperandInfoTy Op(OP_NONE_);
8564 OperandInfoTy Stream(STREAM_ID_NONE_);
8565 if (parseSendMsgBody(Msg, Op, Stream) && validateSendMsg(Msg, Op, Stream)) {
8566 ImmVal = encodeMsg(MsgId: Msg.Val, OpId: Op.Val, StreamId: Stream.Val);
8567 } else {
8568 return ParseStatus::Failure;
8569 }
8570 } else if (parseExpr(Imm&: ImmVal, Expected: "a sendmsg macro")) {
8571 if (ImmVal < 0 || !isUInt<16>(x: ImmVal))
8572 return Error(L: Loc, Msg: "invalid immediate: only 16-bit values are legal");
8573 } else {
8574 return ParseStatus::Failure;
8575 }
8576
8577 Operands.push_back(
8578 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: ImmVal, Loc, Type: AMDGPUOperand::ImmTySendMsg));
8579 return ParseStatus::Success;
8580}
8581
8582bool AMDGPUOperand::isSendMsg() const { return isImmTy(ImmT: ImmTySendMsg); }
8583
8584ParseStatus AMDGPUAsmParser::parseWaitEvent(OperandVector &Operands) {
8585 using namespace llvm::AMDGPU::WaitEvent;
8586
8587 SMLoc Loc = getLoc();
8588 int64_t ImmVal = 0;
8589
8590 StructuredOpField DontWaitExportReady("dont_wait_export_ready", "bit value",
8591 1, 0);
8592 StructuredOpField ExportReady("export_ready", "bit value", 1, 0);
8593
8594 StructuredOpField *TargetBitfield =
8595 isGFX11() ? &DontWaitExportReady : &ExportReady;
8596
8597 ParseStatus Res = parseStructuredOpFields(Fields: {TargetBitfield});
8598 if (Res.isNoMatch() && parseExpr(Imm&: ImmVal, Expected: "structured immediate"))
8599 Res = ParseStatus::Success;
8600 else if (Res.isSuccess()) {
8601 if (!validateStructuredOpFields(Fields: {TargetBitfield}))
8602 return ParseStatus::Failure;
8603 ImmVal = TargetBitfield->Val;
8604 }
8605
8606 if (!Res.isSuccess())
8607 return ParseStatus::Failure;
8608
8609 if (!isUInt<16>(x: ImmVal))
8610 return Error(L: Loc, Msg: "invalid immediate: only 16-bit values are legal");
8611
8612 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: ImmVal, Loc,
8613 Type: AMDGPUOperand::ImmTyWaitEvent));
8614 return ParseStatus::Success;
8615}
8616
8617bool AMDGPUOperand::isWaitEvent() const { return isImmTy(ImmT: ImmTyWaitEvent); }
8618
8619//===----------------------------------------------------------------------===//
8620// v_interp
8621//===----------------------------------------------------------------------===//
8622
8623ParseStatus AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
8624 StringRef Str;
8625 SMLoc S = getLoc();
8626
8627 if (!parseId(Val&: Str))
8628 return ParseStatus::NoMatch;
8629
8630 int Slot = StringSwitch<int>(Str)
8631 .Case(S: "p10", Value: 0)
8632 .Case(S: "p20", Value: 1)
8633 .Case(S: "p0", Value: 2)
8634 .Default(Value: -1);
8635
8636 if (Slot == -1)
8637 return Error(L: S, Msg: "invalid interpolation slot");
8638
8639 Operands.push_back(
8640 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Slot, Loc: S, Type: AMDGPUOperand::ImmTyInterpSlot));
8641 return ParseStatus::Success;
8642}
8643
8644ParseStatus AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
8645 StringRef Str;
8646 SMLoc S = getLoc();
8647
8648 if (!parseId(Val&: Str))
8649 return ParseStatus::NoMatch;
8650
8651 if (!Str.starts_with(Prefix: "attr"))
8652 return Error(L: S, Msg: "invalid interpolation attribute");
8653
8654 StringRef Chan = Str.take_back(N: 2);
8655 int AttrChan = StringSwitch<int>(Chan)
8656 .Case(S: ".x", Value: 0)
8657 .Case(S: ".y", Value: 1)
8658 .Case(S: ".z", Value: 2)
8659 .Case(S: ".w", Value: 3)
8660 .Default(Value: -1);
8661 if (AttrChan == -1)
8662 return Error(L: S, Msg: "invalid or missing interpolation attribute channel");
8663
8664 Str = Str.drop_back(N: 2).drop_front(N: 4);
8665
8666 uint8_t Attr;
8667 if (Str.getAsInteger(Radix: 10, Result&: Attr))
8668 return Error(L: S, Msg: "invalid or missing interpolation attribute number");
8669
8670 if (Attr > 32)
8671 return Error(L: S, Msg: "out of bounds interpolation attribute number");
8672
8673 SMLoc SChan = SMLoc::getFromPointer(Ptr: Chan.data());
8674
8675 Operands.push_back(
8676 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Attr, Loc: S, Type: AMDGPUOperand::ImmTyInterpAttr));
8677 Operands.push_back(Elt: AMDGPUOperand::CreateImm(
8678 AsmParser: this, Val: AttrChan, Loc: SChan, Type: AMDGPUOperand::ImmTyInterpAttrChan));
8679 return ParseStatus::Success;
8680}
8681
8682//===----------------------------------------------------------------------===//
8683// exp
8684//===----------------------------------------------------------------------===//
8685
8686ParseStatus AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
8687 using namespace llvm::AMDGPU::Exp;
8688
8689 StringRef Str;
8690 SMLoc S = getLoc();
8691
8692 if (!parseId(Val&: Str))
8693 return ParseStatus::NoMatch;
8694
8695 unsigned Id = getTgtId(Name: Str);
8696 if (Id == ET_INVALID || !isSupportedTgtId(Id, STI: getSTI()))
8697 return Error(L: S, Msg: (Id == ET_INVALID)
8698 ? "invalid exp target"
8699 : "exp target is not supported on this GPU");
8700
8701 Operands.push_back(
8702 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Id, Loc: S, Type: AMDGPUOperand::ImmTyExpTgt));
8703 return ParseStatus::Success;
8704}
8705
8706//===----------------------------------------------------------------------===//
8707// parser helpers
8708//===----------------------------------------------------------------------===//
8709
8710bool AMDGPUAsmParser::isId(const AsmToken &Token, const StringRef Id) const {
8711 return Token.is(K: AsmToken::Identifier) && Token.getString() == Id;
8712}
8713
8714bool AMDGPUAsmParser::isId(const StringRef Id) const {
8715 return isId(Token: getToken(), Id);
8716}
8717
8718bool AMDGPUAsmParser::isToken(const AsmToken::TokenKind Kind) const {
8719 return getTokenKind() == Kind;
8720}
8721
8722StringRef AMDGPUAsmParser::getId() const {
8723 return isToken(Kind: AsmToken::Identifier) ? getTokenStr() : StringRef();
8724}
8725
8726bool AMDGPUAsmParser::trySkipId(const StringRef Id) {
8727 if (isId(Id)) {
8728 lex();
8729 return true;
8730 }
8731 return false;
8732}
8733
8734bool AMDGPUAsmParser::trySkipId(const StringRef Pref, const StringRef Id) {
8735 if (isToken(Kind: AsmToken::Identifier)) {
8736 StringRef Tok = getTokenStr();
8737 if (Tok.starts_with(Prefix: Pref) && Tok.drop_front(N: Pref.size()) == Id) {
8738 lex();
8739 return true;
8740 }
8741 }
8742 return false;
8743}
8744
8745bool AMDGPUAsmParser::trySkipId(const StringRef Id,
8746 const AsmToken::TokenKind Kind) {
8747 if (isId(Id) && peekToken().is(K: Kind)) {
8748 lex();
8749 lex();
8750 return true;
8751 }
8752 return false;
8753}
8754
8755bool AMDGPUAsmParser::trySkipToken(const AsmToken::TokenKind Kind) {
8756 if (isToken(Kind)) {
8757 lex();
8758 return true;
8759 }
8760 return false;
8761}
8762
8763bool AMDGPUAsmParser::skipToken(const AsmToken::TokenKind Kind,
8764 const StringRef ErrMsg) {
8765 if (!trySkipToken(Kind)) {
8766 Error(L: getLoc(), Msg: ErrMsg);
8767 return false;
8768 }
8769 return true;
8770}
8771
8772bool AMDGPUAsmParser::parseExpr(int64_t &Imm, StringRef Expected) {
8773 SMLoc S = getLoc();
8774
8775 const MCExpr *Expr;
8776 if (Parser.parseExpression(Res&: Expr))
8777 return false;
8778
8779 if (Expr->evaluateAsAbsolute(Res&: Imm))
8780 return true;
8781
8782 if (Expected.empty()) {
8783 Error(L: S, Msg: "expected absolute expression");
8784 } else {
8785 Error(L: S,
8786 Msg: Twine("expected ", Expected) + Twine(" or an absolute expression"));
8787 }
8788 return false;
8789}
8790
8791bool AMDGPUAsmParser::parseExpr(OperandVector &Operands) {
8792 SMLoc S = getLoc();
8793
8794 const MCExpr *Expr;
8795 if (Parser.parseExpression(Res&: Expr))
8796 return false;
8797
8798 int64_t IntVal;
8799 if (Expr->evaluateAsAbsolute(Res&: IntVal)) {
8800 Operands.push_back(Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: IntVal, Loc: S));
8801 } else {
8802 Operands.push_back(Elt: AMDGPUOperand::CreateExpr(AsmParser: this, Expr, S));
8803 }
8804 return true;
8805}
8806
8807bool AMDGPUAsmParser::parseString(StringRef &Val, const StringRef ErrMsg) {
8808 if (isToken(Kind: AsmToken::String)) {
8809 Val = getToken().getStringContents();
8810 lex();
8811 return true;
8812 }
8813 Error(L: getLoc(), Msg: ErrMsg);
8814 return false;
8815}
8816
8817bool AMDGPUAsmParser::parseId(StringRef &Val, const StringRef ErrMsg) {
8818 if (isToken(Kind: AsmToken::Identifier)) {
8819 Val = getTokenStr();
8820 lex();
8821 return true;
8822 }
8823 if (!ErrMsg.empty())
8824 Error(L: getLoc(), Msg: ErrMsg);
8825 return false;
8826}
8827
8828AsmToken AMDGPUAsmParser::getToken() const { return Parser.getTok(); }
8829
8830AsmToken AMDGPUAsmParser::peekToken(bool ShouldSkipSpace) {
8831 return isToken(Kind: AsmToken::EndOfStatement)
8832 ? getToken()
8833 : getLexer().peekTok(ShouldSkipSpace);
8834}
8835
8836void AMDGPUAsmParser::peekTokens(MutableArrayRef<AsmToken> Tokens) {
8837 auto TokCount = getLexer().peekTokens(Buf: Tokens);
8838
8839 for (auto Idx = TokCount; Idx < Tokens.size(); ++Idx)
8840 Tokens[Idx] = AsmToken(AsmToken::Error, "");
8841}
8842
8843AsmToken::TokenKind AMDGPUAsmParser::getTokenKind() const {
8844 return getLexer().getKind();
8845}
8846
8847SMLoc AMDGPUAsmParser::getLoc() const { return getToken().getLoc(); }
8848
8849StringRef AMDGPUAsmParser::getTokenStr() const {
8850 return getToken().getString();
8851}
8852
8853void AMDGPUAsmParser::lex() { Parser.Lex(); }
8854
8855const AMDGPUOperand &
8856AMDGPUAsmParser::findMCOperand(const OperandVector &Operands,
8857 int MCOpIdx) const {
8858 for (const auto &Op : Operands) {
8859 const AMDGPUOperand &TargetOp = static_cast<AMDGPUOperand &>(*Op);
8860 if (TargetOp.getMCOpIdx() == MCOpIdx)
8861 return TargetOp;
8862 }
8863 llvm_unreachable("no such MC operand!");
8864}
8865
8866SMLoc AMDGPUAsmParser::getInstLoc(const OperandVector &Operands) const {
8867 return ((AMDGPUOperand &)*Operands[0]).getStartLoc();
8868}
8869
8870// Returns one of the given locations that comes later in the source.
8871SMLoc AMDGPUAsmParser::getLaterLoc(SMLoc a, SMLoc b) {
8872 return a.getPointer() < b.getPointer() ? b : a;
8873}
8874
8875SMLoc AMDGPUAsmParser::getOperandLoc(const OperandVector &Operands,
8876 int MCOpIdx) const {
8877 return findMCOperand(Operands, MCOpIdx).getStartLoc();
8878}
8879
8880SMLoc AMDGPUAsmParser::getOperandLoc(
8881 std::function<bool(const AMDGPUOperand &)> Test,
8882 const OperandVector &Operands) const {
8883 for (unsigned i = Operands.size() - 1; i > 0; --i) {
8884 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
8885 if (Test(Op))
8886 return Op.getStartLoc();
8887 }
8888 return getInstLoc(Operands);
8889}
8890
8891SMLoc AMDGPUAsmParser::getImmLoc(AMDGPUOperand::ImmTy Type,
8892 const OperandVector &Operands) const {
8893 auto Test = [=](const AMDGPUOperand &Op) { return Op.isImmTy(ImmT: Type); };
8894 return getOperandLoc(Test, Operands);
8895}
8896
8897ParseStatus
8898AMDGPUAsmParser::parseStructuredOpFields(ArrayRef<StructuredOpField *> Fields) {
8899 if (!trySkipToken(Kind: AsmToken::LCurly))
8900 return ParseStatus::NoMatch;
8901
8902 bool First = true;
8903 while (!trySkipToken(Kind: AsmToken::RCurly)) {
8904 if (!First &&
8905 !skipToken(Kind: AsmToken::Comma, ErrMsg: "comma or closing brace expected"))
8906 return ParseStatus::Failure;
8907
8908 StringRef Id = getTokenStr();
8909 SMLoc IdLoc = getLoc();
8910 if (!skipToken(Kind: AsmToken::Identifier, ErrMsg: "field name expected") ||
8911 !skipToken(Kind: AsmToken::Colon, ErrMsg: "colon expected"))
8912 return ParseStatus::Failure;
8913
8914 const auto *I =
8915 find_if(Range&: Fields, P: [Id](StructuredOpField *F) { return F->Id == Id; });
8916 if (I == Fields.end())
8917 return Error(L: IdLoc, Msg: "unknown field");
8918 if ((*I)->IsDefined)
8919 return Error(L: IdLoc, Msg: "duplicate field");
8920
8921 // TODO: Support symbolic values.
8922 (*I)->Loc = getLoc();
8923 if (!parseExpr(Imm&: (*I)->Val))
8924 return ParseStatus::Failure;
8925 (*I)->IsDefined = true;
8926
8927 First = false;
8928 }
8929 return ParseStatus::Success;
8930}
8931
8932bool AMDGPUAsmParser::validateStructuredOpFields(
8933 ArrayRef<const StructuredOpField *> Fields) {
8934 return all_of(Range&: Fields, P: [this](const StructuredOpField *F) {
8935 return F->validate(Parser&: *this);
8936 });
8937}
8938
8939//===----------------------------------------------------------------------===//
8940// swizzle
8941//===----------------------------------------------------------------------===//
8942
8943LLVM_READNONE
8944static unsigned encodeBitmaskPerm(const unsigned AndMask, const unsigned OrMask,
8945 const unsigned XorMask) {
8946 using namespace llvm::AMDGPU::Swizzle;
8947
8948 return BITMASK_PERM_ENC | (AndMask << BITMASK_AND_SHIFT) |
8949 (OrMask << BITMASK_OR_SHIFT) | (XorMask << BITMASK_XOR_SHIFT);
8950}
8951
8952bool AMDGPUAsmParser::parseSwizzleOperand(int64_t &Op, const unsigned MinVal,
8953 const unsigned MaxVal,
8954 const Twine &ErrMsg, SMLoc &Loc) {
8955 if (!skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma")) {
8956 return false;
8957 }
8958 Loc = getLoc();
8959 if (!parseExpr(Imm&: Op)) {
8960 return false;
8961 }
8962 if (Op < MinVal || Op > MaxVal) {
8963 Error(L: Loc, Msg: ErrMsg);
8964 return false;
8965 }
8966
8967 return true;
8968}
8969
8970bool AMDGPUAsmParser::parseSwizzleOperands(const unsigned OpNum, int64_t *Op,
8971 const unsigned MinVal,
8972 const unsigned MaxVal,
8973 const StringRef ErrMsg) {
8974 SMLoc Loc;
8975 for (unsigned i = 0; i < OpNum; ++i) {
8976 if (!parseSwizzleOperand(Op&: Op[i], MinVal, MaxVal, ErrMsg, Loc))
8977 return false;
8978 }
8979
8980 return true;
8981}
8982
8983bool AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &Imm) {
8984 using namespace llvm::AMDGPU::Swizzle;
8985
8986 int64_t Lane[LANE_NUM];
8987 if (parseSwizzleOperands(OpNum: LANE_NUM, Op: Lane, MinVal: 0, MaxVal: LANE_MAX,
8988 ErrMsg: "expected a 2-bit lane id")) {
8989 Imm = QUAD_PERM_ENC;
8990 for (unsigned I = 0; I < LANE_NUM; ++I) {
8991 Imm |= Lane[I] << (LANE_SHIFT * I);
8992 }
8993 return true;
8994 }
8995 return false;
8996}
8997
8998bool AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &Imm) {
8999 using namespace llvm::AMDGPU::Swizzle;
9000
9001 SMLoc Loc;
9002 int64_t GroupSize;
9003 int64_t LaneIdx;
9004
9005 if (!parseSwizzleOperand(Op&: GroupSize, MinVal: 2, MaxVal: 32,
9006 ErrMsg: "group size must be in the interval [2,32]", Loc)) {
9007 return false;
9008 }
9009 if (!isPowerOf2_64(Value: GroupSize)) {
9010 Error(L: Loc, Msg: "group size must be a power of two");
9011 return false;
9012 }
9013 if (parseSwizzleOperand(Op&: LaneIdx, MinVal: 0, MaxVal: GroupSize - 1,
9014 ErrMsg: "lane id must be in the interval [0,group size - 1]",
9015 Loc)) {
9016 Imm = encodeBitmaskPerm(AndMask: BITMASK_MAX - GroupSize + 1, OrMask: LaneIdx, XorMask: 0);
9017 return true;
9018 }
9019 return false;
9020}
9021
9022bool AMDGPUAsmParser::parseSwizzleReverse(int64_t &Imm) {
9023 using namespace llvm::AMDGPU::Swizzle;
9024
9025 SMLoc Loc;
9026 int64_t GroupSize;
9027
9028 if (!parseSwizzleOperand(Op&: GroupSize, MinVal: 2, MaxVal: 32,
9029 ErrMsg: "group size must be in the interval [2,32]", Loc)) {
9030 return false;
9031 }
9032 if (!isPowerOf2_64(Value: GroupSize)) {
9033 Error(L: Loc, Msg: "group size must be a power of two");
9034 return false;
9035 }
9036
9037 Imm = encodeBitmaskPerm(AndMask: BITMASK_MAX, OrMask: 0, XorMask: GroupSize - 1);
9038 return true;
9039}
9040
9041bool AMDGPUAsmParser::parseSwizzleSwap(int64_t &Imm) {
9042 using namespace llvm::AMDGPU::Swizzle;
9043
9044 SMLoc Loc;
9045 int64_t GroupSize;
9046
9047 if (!parseSwizzleOperand(Op&: GroupSize, MinVal: 1, MaxVal: 16,
9048 ErrMsg: "group size must be in the interval [1,16]", Loc)) {
9049 return false;
9050 }
9051 if (!isPowerOf2_64(Value: GroupSize)) {
9052 Error(L: Loc, Msg: "group size must be a power of two");
9053 return false;
9054 }
9055
9056 Imm = encodeBitmaskPerm(AndMask: BITMASK_MAX, OrMask: 0, XorMask: GroupSize);
9057 return true;
9058}
9059
9060bool AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &Imm) {
9061 using namespace llvm::AMDGPU::Swizzle;
9062
9063 if (!skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma")) {
9064 return false;
9065 }
9066
9067 StringRef Ctl;
9068 SMLoc StrLoc = getLoc();
9069 if (!parseString(Val&: Ctl)) {
9070 return false;
9071 }
9072 if (Ctl.size() != BITMASK_WIDTH) {
9073 Error(L: StrLoc, Msg: "expected a 5-character mask");
9074 return false;
9075 }
9076
9077 unsigned AndMask = 0;
9078 unsigned OrMask = 0;
9079 unsigned XorMask = 0;
9080
9081 for (size_t i = 0; i < Ctl.size(); ++i) {
9082 unsigned Mask = 1 << (BITMASK_WIDTH - 1 - i);
9083 switch (Ctl[i]) {
9084 default:
9085 Error(L: StrLoc, Msg: "invalid mask");
9086 return false;
9087 case '0':
9088 break;
9089 case '1':
9090 OrMask |= Mask;
9091 break;
9092 case 'p':
9093 AndMask |= Mask;
9094 break;
9095 case 'i':
9096 AndMask |= Mask;
9097 XorMask |= Mask;
9098 break;
9099 }
9100 }
9101
9102 Imm = encodeBitmaskPerm(AndMask, OrMask, XorMask);
9103 return true;
9104}
9105
9106bool AMDGPUAsmParser::parseSwizzleFFT(int64_t &Imm) {
9107 using namespace llvm::AMDGPU::Swizzle;
9108
9109 if (!AMDGPU::isGFX9Plus(STI: getSTI())) {
9110 Error(L: getLoc(), Msg: "FFT mode swizzle not supported on this GPU");
9111 return false;
9112 }
9113
9114 int64_t Swizzle;
9115 SMLoc Loc;
9116 if (!parseSwizzleOperand(Op&: Swizzle, MinVal: 0, MaxVal: FFT_SWIZZLE_MAX,
9117 ErrMsg: "FFT swizzle must be in the interval [0," +
9118 Twine(FFT_SWIZZLE_MAX) + Twine(']'),
9119 Loc))
9120 return false;
9121
9122 Imm = FFT_MODE_ENC | Swizzle;
9123 return true;
9124}
9125
9126bool AMDGPUAsmParser::parseSwizzleRotate(int64_t &Imm) {
9127 using namespace llvm::AMDGPU::Swizzle;
9128
9129 if (!AMDGPU::isGFX9Plus(STI: getSTI())) {
9130 Error(L: getLoc(), Msg: "Rotate mode swizzle not supported on this GPU");
9131 return false;
9132 }
9133
9134 SMLoc Loc;
9135 int64_t Direction;
9136
9137 if (!parseSwizzleOperand(Op&: Direction, MinVal: 0, MaxVal: 1,
9138 ErrMsg: "direction must be 0 (left) or 1 (right)", Loc))
9139 return false;
9140
9141 int64_t RotateSize;
9142 if (!parseSwizzleOperand(
9143 Op&: RotateSize, MinVal: 0, MaxVal: ROTATE_MAX_SIZE,
9144 ErrMsg: "number of threads to rotate must be in the interval [0," +
9145 Twine(ROTATE_MAX_SIZE) + Twine(']'),
9146 Loc))
9147 return false;
9148
9149 Imm = ROTATE_MODE_ENC | (Direction << ROTATE_DIR_SHIFT) |
9150 (RotateSize << ROTATE_SIZE_SHIFT);
9151 return true;
9152}
9153
9154bool AMDGPUAsmParser::parseSwizzleOffset(int64_t &Imm) {
9155
9156 SMLoc OffsetLoc = getLoc();
9157
9158 if (!parseExpr(Imm, Expected: "a swizzle macro")) {
9159 return false;
9160 }
9161 if (!isUInt<16>(x: Imm)) {
9162 Error(L: OffsetLoc, Msg: "expected a 16-bit offset");
9163 return false;
9164 }
9165 return true;
9166}
9167
9168bool AMDGPUAsmParser::parseSwizzleMacro(int64_t &Imm) {
9169 using namespace llvm::AMDGPU::Swizzle;
9170
9171 if (skipToken(Kind: AsmToken::LParen, ErrMsg: "expected a left parentheses")) {
9172
9173 SMLoc ModeLoc = getLoc();
9174 bool Ok = false;
9175
9176 if (trySkipId(Id: IdSymbolic[ID_QUAD_PERM])) {
9177 Ok = parseSwizzleQuadPerm(Imm);
9178 } else if (trySkipId(Id: IdSymbolic[ID_BITMASK_PERM])) {
9179 Ok = parseSwizzleBitmaskPerm(Imm);
9180 } else if (trySkipId(Id: IdSymbolic[ID_BROADCAST])) {
9181 Ok = parseSwizzleBroadcast(Imm);
9182 } else if (trySkipId(Id: IdSymbolic[ID_SWAP])) {
9183 Ok = parseSwizzleSwap(Imm);
9184 } else if (trySkipId(Id: IdSymbolic[ID_REVERSE])) {
9185 Ok = parseSwizzleReverse(Imm);
9186 } else if (trySkipId(Id: IdSymbolic[ID_FFT])) {
9187 Ok = parseSwizzleFFT(Imm);
9188 } else if (trySkipId(Id: IdSymbolic[ID_ROTATE])) {
9189 Ok = parseSwizzleRotate(Imm);
9190 } else {
9191 Error(L: ModeLoc, Msg: "expected a swizzle mode");
9192 }
9193
9194 return Ok && skipToken(Kind: AsmToken::RParen, ErrMsg: "expected a closing parentheses");
9195 }
9196
9197 return false;
9198}
9199
9200ParseStatus AMDGPUAsmParser::parseSwizzle(OperandVector &Operands) {
9201 SMLoc S = getLoc();
9202 int64_t Imm = 0;
9203
9204 if (trySkipId(Id: "offset")) {
9205
9206 bool Ok = false;
9207 if (skipToken(Kind: AsmToken::Colon, ErrMsg: "expected a colon")) {
9208 if (trySkipId(Id: "swizzle")) {
9209 Ok = parseSwizzleMacro(Imm);
9210 } else {
9211 Ok = parseSwizzleOffset(Imm);
9212 }
9213 }
9214
9215 Operands.push_back(
9216 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Imm, Loc: S, Type: AMDGPUOperand::ImmTySwizzle));
9217
9218 return Ok ? ParseStatus::Success : ParseStatus::Failure;
9219 }
9220 return ParseStatus::NoMatch;
9221}
9222
9223bool AMDGPUOperand::isSwizzle() const { return isImmTy(ImmT: ImmTySwizzle); }
9224
9225//===----------------------------------------------------------------------===//
9226// VGPR Index Mode
9227//===----------------------------------------------------------------------===//
9228
9229int64_t AMDGPUAsmParser::parseGPRIdxMacro() {
9230
9231 using namespace llvm::AMDGPU::VGPRIndexMode;
9232
9233 if (trySkipToken(Kind: AsmToken::RParen)) {
9234 return OFF;
9235 }
9236
9237 int64_t Imm = 0;
9238
9239 while (true) {
9240 unsigned Mode = 0;
9241 SMLoc S = getLoc();
9242
9243 for (unsigned ModeId = ID_MIN; ModeId <= ID_MAX; ++ModeId) {
9244 if (trySkipId(Id: IdSymbolic[ModeId])) {
9245 Mode = 1 << ModeId;
9246 break;
9247 }
9248 }
9249
9250 if (Mode == 0) {
9251 Error(L: S, Msg: (Imm == 0)
9252 ? "expected a VGPR index mode or a closing parenthesis"
9253 : "expected a VGPR index mode");
9254 return UNDEF;
9255 }
9256
9257 if (Imm & Mode) {
9258 Error(L: S, Msg: "duplicate VGPR index mode");
9259 return UNDEF;
9260 }
9261 Imm |= Mode;
9262
9263 if (trySkipToken(Kind: AsmToken::RParen))
9264 break;
9265 if (!skipToken(Kind: AsmToken::Comma,
9266 ErrMsg: "expected a comma or a closing parenthesis"))
9267 return UNDEF;
9268 }
9269
9270 return Imm;
9271}
9272
9273ParseStatus AMDGPUAsmParser::parseGPRIdxMode(OperandVector &Operands) {
9274
9275 using namespace llvm::AMDGPU::VGPRIndexMode;
9276
9277 int64_t Imm = 0;
9278 SMLoc S = getLoc();
9279
9280 if (trySkipId(Id: "gpr_idx", Kind: AsmToken::LParen)) {
9281 Imm = parseGPRIdxMacro();
9282 if (Imm == UNDEF)
9283 return ParseStatus::Failure;
9284 } else {
9285 if (getParser().parseAbsoluteExpression(Res&: Imm))
9286 return ParseStatus::Failure;
9287 if (Imm < 0 || !isUInt<4>(x: Imm))
9288 return Error(L: S, Msg: "invalid immediate: only 4-bit values are legal");
9289 }
9290
9291 Operands.push_back(
9292 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Imm, Loc: S, Type: AMDGPUOperand::ImmTyGprIdxMode));
9293 return ParseStatus::Success;
9294}
9295
9296bool AMDGPUOperand::isGPRIdxMode() const { return isImmTy(ImmT: ImmTyGprIdxMode); }
9297
9298//===----------------------------------------------------------------------===//
9299// sopp branch targets
9300//===----------------------------------------------------------------------===//
9301
9302ParseStatus AMDGPUAsmParser::parseSOPPBrTarget(OperandVector &Operands) {
9303
9304 // Make sure we are not parsing something
9305 // that looks like a label or an expression but is not.
9306 // This will improve error messages.
9307 if (isRegister() || isModifier())
9308 return ParseStatus::NoMatch;
9309
9310 if (!parseExpr(Operands))
9311 return ParseStatus::Failure;
9312
9313 AMDGPUOperand &Opr = ((AMDGPUOperand &)*Operands[Operands.size() - 1]);
9314 assert(Opr.isImm() || Opr.isExpr());
9315 SMLoc Loc = Opr.getStartLoc();
9316
9317 // Currently we do not support arbitrary expressions as branch targets.
9318 // Only labels and absolute expressions are accepted.
9319 if (Opr.isExpr() && !Opr.isSymbolRefExpr()) {
9320 Error(L: Loc, Msg: "expected an absolute expression or a label");
9321 } else if (Opr.isImm() && !Opr.isS16Imm()) {
9322 Error(L: Loc, Msg: "expected a 16-bit signed jump offset");
9323 }
9324
9325 return ParseStatus::Success;
9326}
9327
9328//===----------------------------------------------------------------------===//
9329// Boolean holding registers
9330//===----------------------------------------------------------------------===//
9331
9332ParseStatus AMDGPUAsmParser::parseBoolReg(OperandVector &Operands) {
9333 return parseReg(Operands);
9334}
9335
9336//===----------------------------------------------------------------------===//
9337// mubuf
9338//===----------------------------------------------------------------------===//
9339
9340void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
9341 bool IsAtomic) {
9342 OptionalImmIndexMap OptionalIdx;
9343 unsigned FirstOperandIdx = 1;
9344 bool IsAtomicReturn = false;
9345
9346 if (IsAtomic) {
9347 IsAtomicReturn = SIInstrFlags::isAtomicRet(O: MII, O: Inst);
9348 }
9349
9350 for (unsigned i = FirstOperandIdx, e = Operands.size(); i != e; ++i) {
9351 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
9352
9353 // Add the register arguments
9354 if (Op.isReg()) {
9355 Op.addRegOperands(Inst, N: 1);
9356 // Insert a tied src for atomic return dst.
9357 // This cannot be postponed as subsequent calls to
9358 // addImmOperands rely on correct number of MC operands.
9359 if (IsAtomicReturn && i == FirstOperandIdx)
9360 Op.addRegOperands(Inst, N: 1);
9361 continue;
9362 }
9363
9364 // Handle the case where soffset is an immediate
9365 if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
9366 Op.addImmOperands(Inst, N: 1);
9367 continue;
9368 }
9369
9370 // Handle tokens like 'offen' which are sometimes hard-coded into the
9371 // asm string. There are no MCInst operands for these.
9372 if (Op.isToken()) {
9373 continue;
9374 }
9375 assert(Op.isImm());
9376
9377 // Handle optional arguments
9378 OptionalIdx[Op.getImmTy()] = i;
9379 }
9380
9381 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9382 ImmT: AMDGPUOperand::ImmTyOffset);
9383 addOptionalImmOperand(Inst, Operands, OptionalIdx, ImmT: AMDGPUOperand::ImmTyCPol,
9384 Default: 0);
9385 // Parse a dummy operand as a placeholder for the SWZ operand. This enforces
9386 // agreement between MCInstrDesc.getNumOperands and MCInst.getNumOperands.
9387 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
9388 // The LDS variants carry a trailing IsAsync operand. Parse a dummy the same
9389 // way as the SWZ operand.
9390 if (AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::IsAsync))
9391 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
9392}
9393
9394//===----------------------------------------------------------------------===//
9395// smrd
9396//===----------------------------------------------------------------------===//
9397
9398bool AMDGPUOperand::isSMRDOffset8() const {
9399 return isImmLiteral() && isUInt<8>(x: getImm());
9400}
9401
9402bool AMDGPUOperand::isSMEMOffset() const {
9403 // Offset range is checked later by validator.
9404 return isImmLiteral();
9405}
9406
9407bool AMDGPUOperand::isSMRDLiteralOffset() const {
9408 // 32-bit literals are only supported on CI and we only want to use them
9409 // when the offset is > 8-bits.
9410 return isImmLiteral() && !isUInt<8>(x: getImm()) && isUInt<32>(x: getImm());
9411}
9412
9413//===----------------------------------------------------------------------===//
9414// vop3
9415//===----------------------------------------------------------------------===//
9416
9417static bool ConvertOmodMul(int64_t &Mul) {
9418 if (Mul != 1 && Mul != 2 && Mul != 4)
9419 return false;
9420
9421 Mul >>= 1;
9422 return true;
9423}
9424
9425static bool ConvertOmodDiv(int64_t &Div) {
9426 if (Div == 1) {
9427 Div = 0;
9428 return true;
9429 }
9430
9431 if (Div == 2) {
9432 Div = 3;
9433 return true;
9434 }
9435
9436 return false;
9437}
9438
9439// For pre-gfx11 targets, both bound_ctrl:0 and bound_ctrl:1 are encoded as 1.
9440// This is intentional and ensures compatibility with sp3.
9441// See bug 35397 for details.
9442bool AMDGPUAsmParser::convertDppBoundCtrl(int64_t &BoundCtrl) {
9443 if (BoundCtrl == 0 || BoundCtrl == 1) {
9444 if (!isGFX11Plus())
9445 BoundCtrl = 1;
9446 return true;
9447 }
9448 return false;
9449}
9450
9451void AMDGPUAsmParser::onBeginOfFile() {
9452 if (!getParser().getStreamer().getTargetStreamer())
9453 return;
9454
9455 if (!getTargetStreamer().getTargetID())
9456 getTargetStreamer().initializeTargetID(STI: getSTI(),
9457 /*ApplyFeatureString=*/true);
9458}
9459
9460void AMDGPUAsmParser::emitTargetDirective() {
9461 if (!getParser().getStreamer().getTargetStreamer() ||
9462 getSTI().getTargetTriple().getArch() == Triple::r600)
9463 return;
9464
9465 if (isHsaAbi(STI: getSTI()))
9466 getTargetStreamer().emitDirectiveAMDGCNTargetOnce();
9467}
9468
9469/// Parse AMDGPU specific expressions.
9470///
9471/// expr ::= or(expr, ...) |
9472/// max(expr, ...) |
9473/// min(expr, ...)
9474///
9475bool AMDGPUAsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) {
9476 using AGVK = AMDGPUMCExpr::VariantKind;
9477
9478 if (isToken(Kind: AsmToken::Identifier)) {
9479 StringRef TokenId = getTokenStr();
9480 AGVK VK = StringSwitch<AGVK>(TokenId)
9481 .Case(S: "max", Value: AGVK::AGVK_Max)
9482 .Case(S: "min", Value: AGVK::AGVK_Min)
9483 .Case(S: "or", Value: AGVK::AGVK_Or)
9484 .Case(S: "extrasgprs", Value: AGVK::AGVK_ExtraSGPRs)
9485 .Case(S: "totalnumvgprs", Value: AGVK::AGVK_TotalNumVGPRs)
9486 .Case(S: "alignto", Value: AGVK::AGVK_AlignTo)
9487 .Case(S: "occupancy", Value: AGVK::AGVK_Occupancy)
9488 .Case(S: "instprefsize", Value: AGVK::AGVK_InstPrefSize)
9489 .Default(Value: AGVK::AGVK_None);
9490
9491 if (VK != AGVK::AGVK_None && peekToken().is(K: AsmToken::LParen)) {
9492 SmallVector<const MCExpr *, 4> Exprs;
9493 uint64_t CommaCount = 0;
9494 lex(); // Eat Arg ('or', 'max', 'occupancy', etc.)
9495 lex(); // Eat '('
9496 while (true) {
9497 if (trySkipToken(Kind: AsmToken::RParen)) {
9498 if (Exprs.empty()) {
9499 Error(L: getToken().getLoc(),
9500 Msg: "empty " + Twine(TokenId) + " expression");
9501 return true;
9502 }
9503 if (CommaCount + 1 != Exprs.size()) {
9504 Error(L: getToken().getLoc(),
9505 Msg: "mismatch of commas in " + Twine(TokenId) + " expression");
9506 return true;
9507 }
9508 if (unsigned Expected = AMDGPUMCExpr::getNumExpectedArgs(Kind: VK);
9509 Expected && Exprs.size() != Expected) {
9510 Error(L: getToken().getLoc(), Msg: Twine(TokenId) + " expression expects " +
9511 Twine(Expected) + " operands");
9512 return true;
9513 }
9514 Res = AMDGPUMCExpr::create(Kind: VK, Args: Exprs, Ctx&: getContext());
9515 return false;
9516 }
9517 const MCExpr *Expr;
9518 if (getParser().parseExpression(Res&: Expr, EndLoc))
9519 return true;
9520 Exprs.push_back(Elt: Expr);
9521 bool LastTokenWasComma = trySkipToken(Kind: AsmToken::Comma);
9522 if (LastTokenWasComma)
9523 CommaCount++;
9524 if (!LastTokenWasComma && !isToken(Kind: AsmToken::RParen)) {
9525 Error(L: getToken().getLoc(),
9526 Msg: "unexpected token in " + Twine(TokenId) + " expression");
9527 return true;
9528 }
9529 }
9530 }
9531 }
9532 return getParser().parsePrimaryExpr(Res, EndLoc, TypeInfo: nullptr);
9533}
9534
9535ParseStatus AMDGPUAsmParser::parseOModSI(OperandVector &Operands) {
9536 StringRef Name = getTokenStr();
9537 if (Name == "mul") {
9538 return parseIntWithPrefix(Prefix: "mul", Operands, ImmTy: AMDGPUOperand::ImmTyOModSI,
9539 ConvertResult: ConvertOmodMul);
9540 }
9541
9542 if (Name == "div") {
9543 return parseIntWithPrefix(Prefix: "div", Operands, ImmTy: AMDGPUOperand::ImmTyOModSI,
9544 ConvertResult: ConvertOmodDiv);
9545 }
9546
9547 return ParseStatus::NoMatch;
9548}
9549
9550// Determines which bit DST_OP_SEL occupies in the op_sel operand according to
9551// the number of src operands present, then copies that bit into src0_modifiers.
9552static void cvtVOP3DstOpSelOnly(MCInst &Inst, const MCRegisterInfo &MRI) {
9553 int Opc = Inst.getOpcode();
9554 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel);
9555 if (OpSelIdx == -1)
9556 return;
9557
9558 int SrcNum;
9559 const AMDGPU::OpName Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9560 AMDGPU::OpName::src2};
9561 for (SrcNum = 0; SrcNum < 3 && AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: Ops[SrcNum]);
9562 ++SrcNum)
9563 ;
9564 assert(SrcNum > 0);
9565
9566 unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm();
9567
9568 int DstIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst);
9569 if (DstIdx == -1)
9570 return;
9571
9572 const MCOperand &DstOp = Inst.getOperand(i: DstIdx);
9573 int ModIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src0_modifiers);
9574 uint32_t ModVal = Inst.getOperand(i: ModIdx).getImm();
9575 if (DstOp.isReg() &&
9576 MRI.getRegClass(i: AMDGPU::VGPR_16RegClassID).contains(Reg: DstOp.getReg())) {
9577 if (AMDGPU::isHi16Reg(Reg: DstOp.getReg(), MRI))
9578 ModVal |= SISrcMods::DST_OP_SEL;
9579 } else {
9580 if ((OpSel & (1 << SrcNum)) != 0)
9581 ModVal |= SISrcMods::DST_OP_SEL;
9582 }
9583 Inst.getOperand(i: ModIdx).setImm(ModVal);
9584}
9585
9586void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst,
9587 const OperandVector &Operands) {
9588 cvtVOP3P(Inst, Operands);
9589 cvtVOP3DstOpSelOnly(Inst, MRI: *getMRI());
9590}
9591
9592void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands,
9593 OptionalImmIndexMap &OptionalIdx) {
9594 cvtVOP3P(Inst, Operands, OptionalIdx);
9595 cvtVOP3DstOpSelOnly(Inst, MRI: *getMRI());
9596}
9597
9598static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
9599 return
9600 // 1. This operand is input modifiers
9601 Desc.operands()[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
9602 // 2. This is not last operand
9603 && Desc.NumOperands > (OpNum + 1)
9604 // 3. Next operand is register class
9605 && Desc.operands()[OpNum + 1].RegClass != -1
9606 // 4. Next register is not tied to any other operand
9607 && Desc.getOperandConstraint(OpNum: OpNum + 1,
9608 Constraint: MCOI::OperandConstraint::TIED_TO) == -1;
9609}
9610
9611void AMDGPUAsmParser::cvtOpSelHelper(MCInst &Inst, unsigned OpSel) {
9612 unsigned Opc = Inst.getOpcode();
9613 constexpr AMDGPU::OpName Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9614 AMDGPU::OpName::src2};
9615 constexpr AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9616 AMDGPU::OpName::src1_modifiers,
9617 AMDGPU::OpName::src2_modifiers};
9618 for (int J = 0; J < 3; ++J) {
9619 int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: Ops[J]);
9620 if (OpIdx == -1)
9621 // Some instructions, e.g. v_interp_p2_f16 in GFX9, have src0, src2, but
9622 // no src1. So continue instead of break.
9623 continue;
9624
9625 int ModIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: ModOps[J]);
9626 uint32_t ModVal = Inst.getOperand(i: ModIdx).getImm();
9627
9628 if ((OpSel & (1 << J)) != 0)
9629 ModVal |= SISrcMods::OP_SEL_0;
9630 // op_sel[3] is encoded in src0_modifiers.
9631 if (ModOps[J] == AMDGPU::OpName::src0_modifiers && (OpSel & (1 << 3)) != 0)
9632 ModVal |= SISrcMods::DST_OP_SEL;
9633
9634 Inst.getOperand(i: ModIdx).setImm(ModVal);
9635 }
9636}
9637
9638void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst,
9639 const OperandVector &Operands) {
9640 OptionalImmIndexMap OptionalIdx;
9641 unsigned Opc = Inst.getOpcode();
9642
9643 unsigned I = 1;
9644 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
9645 for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
9646 ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, N: 1);
9647 }
9648
9649 for (unsigned E = Operands.size(); I != E; ++I) {
9650 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
9651 if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) {
9652 Op.addRegOrImmWithFPInputModsOperands(Inst, N: 2);
9653 } else if (Op.isInterpSlot() || Op.isInterpAttr() ||
9654 Op.isInterpAttrChan()) {
9655 Inst.addOperand(Op: MCOperand::createImm(Val: Op.getImm()));
9656 } else if (Op.isImmModifier()) {
9657 OptionalIdx[Op.getImmTy()] = I;
9658 } else {
9659 llvm_unreachable("unhandled operand type");
9660 }
9661 }
9662
9663 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::high))
9664 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9665 ImmT: AMDGPUOperand::ImmTyHigh);
9666
9667 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::clamp))
9668 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9669 ImmT: AMDGPUOperand::ImmTyClamp);
9670
9671 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::omod))
9672 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9673 ImmT: AMDGPUOperand::ImmTyOModSI);
9674
9675 // Some v_interp instructions use op_sel[3] for dst.
9676 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::op_sel)) {
9677 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9678 ImmT: AMDGPUOperand::ImmTyOpSel);
9679 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel);
9680 unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm();
9681
9682 cvtOpSelHelper(Inst, OpSel);
9683 }
9684}
9685
9686void AMDGPUAsmParser::cvtVINTERP(MCInst &Inst, const OperandVector &Operands) {
9687 OptionalImmIndexMap OptionalIdx;
9688 unsigned Opc = Inst.getOpcode();
9689
9690 unsigned I = 1;
9691 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
9692 for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
9693 ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, N: 1);
9694 }
9695
9696 for (unsigned E = Operands.size(); I != E; ++I) {
9697 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
9698 if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) {
9699 Op.addRegOrImmWithFPInputModsOperands(Inst, N: 2);
9700 } else if (Op.isImmModifier()) {
9701 OptionalIdx[Op.getImmTy()] = I;
9702 } else {
9703 llvm_unreachable("unhandled operand type");
9704 }
9705 }
9706
9707 addOptionalImmOperand(Inst, Operands, OptionalIdx, ImmT: AMDGPUOperand::ImmTyClamp);
9708
9709 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel);
9710 if (OpSelIdx != -1)
9711 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9712 ImmT: AMDGPUOperand::ImmTyOpSel);
9713
9714 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9715 ImmT: AMDGPUOperand::ImmTyWaitEXP);
9716
9717 if (OpSelIdx == -1)
9718 return;
9719
9720 unsigned OpSel = Inst.getOperand(i: OpSelIdx).getImm();
9721 cvtOpSelHelper(Inst, OpSel);
9722}
9723
9724void AMDGPUAsmParser::cvtScaledMFMA(MCInst &Inst,
9725 const OperandVector &Operands) {
9726 OptionalImmIndexMap OptionalIdx;
9727 unsigned Opc = Inst.getOpcode();
9728 unsigned I = 1;
9729 int CbszOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::cbsz);
9730
9731 const MCInstrDesc &Desc = MII.get(Opcode: Opc);
9732
9733 for (unsigned J = 0; J < Desc.getNumDefs(); ++J)
9734 static_cast<AMDGPUOperand &>(*Operands[I++]).addRegOperands(Inst, N: 1);
9735
9736 for (unsigned E = Operands.size(); I != E; ++I) {
9737 AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands[I]);
9738 int NumOperands = Inst.getNumOperands();
9739 // The order of operands in MCInst and parsed operands are different.
9740 // Adding dummy cbsz and blgp operands at corresponding MCInst operand
9741 // indices for parsing scale values correctly.
9742 if (NumOperands == CbszOpIdx) {
9743 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
9744 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
9745 }
9746 if (isRegOrImmWithInputMods(Desc, OpNum: NumOperands)) {
9747 Op.addRegOrImmWithFPInputModsOperands(Inst, N: 2);
9748 } else if (Op.isImmModifier()) {
9749 OptionalIdx[Op.getImmTy()] = I;
9750 } else {
9751 Op.addRegOrImmOperands(Inst, N: 1);
9752 }
9753 }
9754
9755 // Insert CBSZ and BLGP operands for F8F6F4 variants
9756 auto CbszIdx = OptionalIdx.find(x: AMDGPUOperand::ImmTyCBSZ);
9757 if (CbszIdx != OptionalIdx.end()) {
9758 int CbszVal = ((AMDGPUOperand &)*Operands[CbszIdx->second]).getImm();
9759 Inst.getOperand(i: CbszOpIdx).setImm(CbszVal);
9760 }
9761
9762 int BlgpOpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::blgp);
9763 auto BlgpIdx = OptionalIdx.find(x: AMDGPUOperand::ImmTyBLGP);
9764 if (BlgpIdx != OptionalIdx.end()) {
9765 int BlgpVal = ((AMDGPUOperand &)*Operands[BlgpIdx->second]).getImm();
9766 Inst.getOperand(i: BlgpOpIdx).setImm(BlgpVal);
9767 }
9768
9769 // Add dummy src_modifiers
9770 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
9771 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
9772
9773 // Handle op_sel fields
9774
9775 unsigned OpSel = 0;
9776 auto OpselIdx = OptionalIdx.find(x: AMDGPUOperand::ImmTyOpSel);
9777 if (OpselIdx != OptionalIdx.end()) {
9778 OpSel = static_cast<const AMDGPUOperand &>(*Operands[OpselIdx->second])
9779 .getImm();
9780 }
9781
9782 unsigned OpSelHi = 0;
9783 auto OpselHiIdx = OptionalIdx.find(x: AMDGPUOperand::ImmTyOpSelHi);
9784 if (OpselHiIdx != OptionalIdx.end()) {
9785 OpSelHi = static_cast<const AMDGPUOperand &>(*Operands[OpselHiIdx->second])
9786 .getImm();
9787 }
9788 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9789 AMDGPU::OpName::src1_modifiers};
9790
9791 for (unsigned J = 0; J < 2; ++J) {
9792 unsigned ModVal = 0;
9793 if (OpSel & (1 << J))
9794 ModVal |= SISrcMods::OP_SEL_0;
9795 if (OpSelHi & (1 << J))
9796 ModVal |= SISrcMods::OP_SEL_1;
9797
9798 const int ModIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: ModOps[J]);
9799 Inst.getOperand(i: ModIdx).setImm(ModVal);
9800 }
9801}
9802
9803void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands,
9804 OptionalImmIndexMap &OptionalIdx) {
9805 unsigned Opc = Inst.getOpcode();
9806
9807 unsigned I = 1;
9808 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
9809 for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
9810 ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, N: 1);
9811 }
9812
9813 for (unsigned E = Operands.size(); I != E; ++I) {
9814 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
9815 if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) {
9816 Op.addRegOrImmWithFPInputModsOperands(Inst, N: 2);
9817 } else if (Op.isImmModifier()) {
9818 OptionalIdx[Op.getImmTy()] = I;
9819 } else {
9820 Op.addRegOrImmOperands(Inst, N: 1);
9821 }
9822 }
9823
9824 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::scale_sel))
9825 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9826 ImmT: AMDGPUOperand::ImmTyScaleSel);
9827
9828 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::clamp))
9829 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9830 ImmT: AMDGPUOperand::ImmTyClamp);
9831
9832 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::byte_sel)) {
9833 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::vdst_in))
9834 Inst.addOperand(Op: Inst.getOperand(i: 0));
9835 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9836 ImmT: AMDGPUOperand::ImmTyByteSel);
9837 }
9838
9839 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::omod))
9840 addOptionalImmOperand(Inst, Operands, OptionalIdx,
9841 ImmT: AMDGPUOperand::ImmTyOModSI);
9842
9843 // Special case v_mac_{f16, f32} and v_fmac_{f16, f32} (gfx906/gfx10+):
9844 // it has src2 register operand that is tied to dst operand
9845 // we don't allow modifiers for this operand in assembler so src2_modifiers
9846 // should be 0.
9847 if (isMAC(Opc)) {
9848 auto *it = Inst.begin();
9849 std::advance(
9850 i&: it, n: AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2_modifiers));
9851 it = Inst.insert(I: it, Op: MCOperand::createImm(Val: 0)); // no modifiers for src2
9852 ++it;
9853 // Copy the operand to ensure it's not invalidated when Inst grows.
9854 Inst.insert(I: it, Op: MCOperand(Inst.getOperand(i: 0))); // src2 = dst
9855 }
9856}
9857
9858void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
9859 OptionalImmIndexMap OptionalIdx;
9860 cvtVOP3(Inst, Operands, OptionalIdx);
9861}
9862
9863void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands,
9864 OptionalImmIndexMap &OptIdx) {
9865 const int Opc = Inst.getOpcode();
9866
9867 const bool IsPacked = SIInstrFlags::isPacked(O: MII, O: Inst);
9868
9869 if (Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_F16_vi ||
9870 Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_BF16_vi ||
9871 Opc == AMDGPU::V_CVT_SR_BF8_F32_vi ||
9872 Opc == AMDGPU::V_CVT_SR_FP8_F32_vi ||
9873 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx11 ||
9874 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx11 ||
9875 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx12 ||
9876 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx12 ||
9877 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx13 ||
9878 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx13) {
9879 Inst.addOperand(Op: MCOperand::createImm(Val: 0)); // Placeholder for src2_mods
9880 Inst.addOperand(Op: Inst.getOperand(i: 0));
9881 }
9882
9883 // Append vdst_in only if a previous converter (cvtVOP3DPP for DPP variants,
9884 // cvtVOP3 for byte_sel variants) hasn't already placed it. Use the position
9885 // of the named operand to detect that, the same way cvtVOP3DPP does
9886 // internally.
9887 int VdstInIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst_in);
9888 if (VdstInIdx != -1 && VdstInIdx == static_cast<int>(Inst.getNumOperands()))
9889 Inst.addOperand(Op: Inst.getOperand(i: 0));
9890
9891 int BitOp3Idx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::bitop3);
9892 if (BitOp3Idx != -1) {
9893 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyBitOp3);
9894 }
9895
9896 // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3
9897 // instruction, and then figure out where to actually put the modifiers
9898
9899 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel);
9900 if (OpSelIdx != -1) {
9901 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyOpSel);
9902 }
9903
9904 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::op_sel_hi);
9905 if (OpSelHiIdx != -1) {
9906 int DefaultVal = IsPacked ? -1 : 0;
9907 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyOpSelHi,
9908 Default: DefaultVal);
9909 }
9910
9911 int MatrixAFMTIdx =
9912 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_a_fmt);
9913 if (MatrixAFMTIdx != -1) {
9914 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
9915 ImmT: AMDGPUOperand::ImmTyMatrixAFMT, Default: 0);
9916 }
9917
9918 int MatrixBFMTIdx =
9919 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_b_fmt);
9920 if (MatrixBFMTIdx != -1) {
9921 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
9922 ImmT: AMDGPUOperand::ImmTyMatrixBFMT, Default: 0);
9923 }
9924
9925 int MatrixAScaleIdx =
9926 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_a_scale);
9927 if (MatrixAScaleIdx != -1) {
9928 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
9929 ImmT: AMDGPUOperand::ImmTyMatrixAScale, Default: 0);
9930 }
9931
9932 int MatrixBScaleIdx =
9933 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_b_scale);
9934 if (MatrixBScaleIdx != -1) {
9935 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
9936 ImmT: AMDGPUOperand::ImmTyMatrixBScale, Default: 0);
9937 }
9938
9939 int MatrixAScaleFmtIdx =
9940 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_a_scale_fmt);
9941 if (MatrixAScaleFmtIdx != -1) {
9942 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
9943 ImmT: AMDGPUOperand::ImmTyMatrixAScaleFmt, Default: 0);
9944 }
9945
9946 int MatrixBScaleFmtIdx =
9947 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::matrix_b_scale_fmt);
9948 if (MatrixBScaleFmtIdx != -1) {
9949 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
9950 ImmT: AMDGPUOperand::ImmTyMatrixBScaleFmt, Default: 0);
9951 }
9952
9953 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::matrix_a_reuse))
9954 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
9955 ImmT: AMDGPUOperand::ImmTyMatrixAReuse, Default: 0);
9956
9957 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::matrix_b_reuse))
9958 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
9959 ImmT: AMDGPUOperand::ImmTyMatrixBReuse, Default: 0);
9960
9961 int NegLoIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::neg_lo);
9962 if (NegLoIdx != -1)
9963 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyNegLo);
9964
9965 int NegHiIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::neg_hi);
9966 if (NegHiIdx != -1)
9967 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyNegHi);
9968
9969 const AMDGPU::OpName Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9970 AMDGPU::OpName::src2};
9971 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9972 AMDGPU::OpName::src1_modifiers,
9973 AMDGPU::OpName::src2_modifiers};
9974
9975 unsigned OpSel = 0;
9976 unsigned OpSelHi = 0;
9977 unsigned NegLo = 0;
9978 unsigned NegHi = 0;
9979
9980 if (OpSelIdx != -1)
9981 OpSel = Inst.getOperand(i: OpSelIdx).getImm();
9982
9983 if (OpSelHiIdx != -1)
9984 OpSelHi = Inst.getOperand(i: OpSelHiIdx).getImm();
9985
9986 if (NegLoIdx != -1)
9987 NegLo = Inst.getOperand(i: NegLoIdx).getImm();
9988
9989 if (NegHiIdx != -1)
9990 NegHi = Inst.getOperand(i: NegHiIdx).getImm();
9991
9992 for (int J = 0; J < 3; ++J) {
9993 int OpIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: Ops[J]);
9994 if (OpIdx == -1)
9995 break;
9996
9997 int ModIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: ModOps[J]);
9998
9999 if (ModIdx == -1)
10000 continue;
10001
10002 // For MAC instructions, src2 is tied to vdst and its op_sel bit
10003 // is not encoded.
10004 if (AMDGPU::isMAC(Opc) && ModOps[J] == AMDGPU::OpName::src2_modifiers)
10005 continue;
10006
10007 uint32_t ModVal = 0;
10008
10009 const MCOperand &SrcOp = Inst.getOperand(i: OpIdx);
10010 if (SrcOp.isReg() && getMRI()
10011 ->getRegClass(i: AMDGPU::VGPR_16RegClassID)
10012 .contains(Reg: SrcOp.getReg())) {
10013 bool VGPRSuffixIsHi = AMDGPU::isHi16Reg(Reg: SrcOp.getReg(), MRI: *getMRI());
10014 if (VGPRSuffixIsHi)
10015 ModVal |= SISrcMods::OP_SEL_0;
10016 } else {
10017 if ((OpSel & (1 << J)) != 0)
10018 ModVal |= SISrcMods::OP_SEL_0;
10019 }
10020
10021 if ((OpSelHi & (1 << J)) != 0)
10022 ModVal |= SISrcMods::OP_SEL_1;
10023
10024 if ((NegLo & (1 << J)) != 0)
10025 ModVal |= SISrcMods::NEG;
10026
10027 if ((NegHi & (1 << J)) != 0)
10028 ModVal |= SISrcMods::NEG_HI;
10029
10030 Inst.getOperand(i: ModIdx).setImm(Inst.getOperand(i: ModIdx).getImm() | ModVal);
10031 }
10032}
10033
10034void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands) {
10035 OptionalImmIndexMap OptIdx;
10036 cvtVOP3(Inst, Operands, OptionalIdx&: OptIdx);
10037 cvtVOP3P(Inst, Operands, OptIdx);
10038}
10039
10040static void addSrcModifiersAndSrc(MCInst &Inst, const OperandVector &Operands,
10041 unsigned i, unsigned Opc,
10042 AMDGPU::OpName OpName) {
10043 if (AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: OpName) != -1)
10044 ((AMDGPUOperand &)*Operands[i]).addRegOrImmWithFPInputModsOperands(Inst, N: 2);
10045 else
10046 ((AMDGPUOperand &)*Operands[i]).addRegOperands(Inst, N: 1);
10047}
10048
10049void AMDGPUAsmParser::cvtSWMMAC(MCInst &Inst, const OperandVector &Operands) {
10050 unsigned Opc = Inst.getOpcode();
10051
10052 ((AMDGPUOperand &)*Operands[1]).addRegOperands(Inst, N: 1);
10053 addSrcModifiersAndSrc(Inst, Operands, i: 2, Opc, OpName: AMDGPU::OpName::src0_modifiers);
10054 addSrcModifiersAndSrc(Inst, Operands, i: 3, Opc, OpName: AMDGPU::OpName::src1_modifiers);
10055 ((AMDGPUOperand &)*Operands[1]).addRegOperands(Inst, N: 1); // srcTiedDef
10056 ((AMDGPUOperand &)*Operands[4]).addRegOperands(Inst, N: 1); // src2
10057
10058 OptionalImmIndexMap OptIdx;
10059 for (unsigned i = 5; i < Operands.size(); ++i) {
10060 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
10061 OptIdx[Op.getImmTy()] = i;
10062 }
10063
10064 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::index_key_8bit))
10065 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
10066 ImmT: AMDGPUOperand::ImmTyIndexKey8bit);
10067
10068 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::index_key_16bit))
10069 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
10070 ImmT: AMDGPUOperand::ImmTyIndexKey16bit);
10071
10072 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::index_key_32bit))
10073 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx,
10074 ImmT: AMDGPUOperand::ImmTyIndexKey32bit);
10075
10076 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::clamp))
10077 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyClamp);
10078
10079 cvtVOP3P(Inst, Operands, OptIdx);
10080}
10081
10082//===----------------------------------------------------------------------===//
10083// VOPD
10084//===----------------------------------------------------------------------===//
10085
10086ParseStatus AMDGPUAsmParser::parseVOPD(OperandVector &Operands) {
10087 if (!hasVOPD(STI: getSTI()))
10088 return ParseStatus::NoMatch;
10089
10090 if (isToken(Kind: AsmToken::Colon) && peekToken(ShouldSkipSpace: false).is(K: AsmToken::Colon)) {
10091 SMLoc S = getLoc();
10092 lex();
10093 lex();
10094 Operands.push_back(Elt: AMDGPUOperand::CreateToken(AsmParser: this, Str: "::", Loc: S));
10095 SMLoc OpYLoc = getLoc();
10096 StringRef OpYName;
10097 if (isToken(Kind: AsmToken::Identifier) && !Parser.parseIdentifier(Res&: OpYName)) {
10098 Operands.push_back(Elt: AMDGPUOperand::CreateToken(AsmParser: this, Str: OpYName, Loc: OpYLoc));
10099 return ParseStatus::Success;
10100 }
10101 return Error(L: OpYLoc, Msg: "expected a VOPDY instruction after ::");
10102 }
10103 return ParseStatus::NoMatch;
10104}
10105
10106// Create VOPD MCInst operands using parsed assembler operands.
10107void AMDGPUAsmParser::cvtVOPD(MCInst &Inst, const OperandVector &Operands) {
10108 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
10109
10110 auto addOp = [&](uint16_t ParsedOprIdx) { // NOLINT:function pointer
10111 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[ParsedOprIdx]);
10112 if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) {
10113 Op.addRegOrImmWithFPInputModsOperands(Inst, N: 2);
10114 return;
10115 }
10116 if (Op.isReg()) {
10117 Op.addRegOperands(Inst, N: 1);
10118 return;
10119 }
10120 if (Op.isImm() || Op.isExpr()) {
10121 Op.addImmOperands(Inst, N: 1);
10122 return;
10123 }
10124 llvm_unreachable("Unhandled operand type in cvtVOPD");
10125 };
10126
10127 const auto &InstInfo = getVOPDInstInfo(VOPDOpcode: Inst.getOpcode(), InstrInfo: &MII);
10128
10129 // MCInst operands are ordered as follows:
10130 // dstX, dstY, src0X [, other OpX operands], src0Y [, other OpY operands]
10131
10132 for (auto CompIdx : VOPD::COMPONENTS) {
10133 addOp(InstInfo[CompIdx].getIndexOfDstInParsedOperands());
10134 }
10135
10136 for (auto CompIdx : VOPD::COMPONENTS) {
10137 const auto &CInfo = InstInfo[CompIdx];
10138 auto CompSrcOperandsNum = InstInfo[CompIdx].getCompParsedSrcOperandsNum();
10139 for (unsigned CompSrcIdx = 0; CompSrcIdx < CompSrcOperandsNum; ++CompSrcIdx)
10140 addOp(CInfo.getIndexOfSrcInParsedOperands(CompSrcIdx));
10141 if (CInfo.hasSrc2Acc())
10142 addOp(CInfo.getIndexOfDstInParsedOperands());
10143 }
10144
10145 int BitOp3Idx =
10146 AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::bitop3);
10147 if (BitOp3Idx != -1) {
10148 OptionalImmIndexMap OptIdx;
10149 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands.back());
10150 if (Op.isImm())
10151 OptIdx[Op.getImmTy()] = Operands.size() - 1;
10152
10153 addOptionalImmOperand(Inst, Operands, OptionalIdx&: OptIdx, ImmT: AMDGPUOperand::ImmTyBitOp3);
10154 }
10155}
10156
10157//===----------------------------------------------------------------------===//
10158// dpp
10159//===----------------------------------------------------------------------===//
10160
10161bool AMDGPUOperand::isDPP8() const { return isImmTy(ImmT: ImmTyDPP8); }
10162
10163bool AMDGPUOperand::isDPPCtrl() const {
10164 using namespace AMDGPU::DPP;
10165
10166 bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(x: getImm());
10167 if (result) {
10168 int64_t Imm = getImm();
10169 return (Imm >= DppCtrl::QUAD_PERM_FIRST &&
10170 Imm <= DppCtrl::QUAD_PERM_LAST) ||
10171 (Imm >= DppCtrl::ROW_SHL_FIRST && Imm <= DppCtrl::ROW_SHL_LAST) ||
10172 (Imm >= DppCtrl::ROW_SHR_FIRST && Imm <= DppCtrl::ROW_SHR_LAST) ||
10173 (Imm >= DppCtrl::ROW_ROR_FIRST && Imm <= DppCtrl::ROW_ROR_LAST) ||
10174 (Imm == DppCtrl::WAVE_SHL1) || (Imm == DppCtrl::WAVE_ROL1) ||
10175 (Imm == DppCtrl::WAVE_SHR1) || (Imm == DppCtrl::WAVE_ROR1) ||
10176 (Imm == DppCtrl::ROW_MIRROR) || (Imm == DppCtrl::ROW_HALF_MIRROR) ||
10177 (Imm == DppCtrl::BCAST15) || (Imm == DppCtrl::BCAST31) ||
10178 (Imm >= DppCtrl::ROW_SHARE_FIRST &&
10179 Imm <= DppCtrl::ROW_SHARE_LAST) ||
10180 (Imm >= DppCtrl::ROW_XMASK_FIRST && Imm <= DppCtrl::ROW_XMASK_LAST);
10181 }
10182 return false;
10183}
10184
10185//===----------------------------------------------------------------------===//
10186// mAI
10187//===----------------------------------------------------------------------===//
10188
10189bool AMDGPUOperand::isBLGP() const {
10190 return isImm() && getImmTy() == ImmTyBLGP && isUInt<3>(x: getImm());
10191}
10192
10193bool AMDGPUOperand::isS16Imm() const {
10194 return isImmLiteral() && (isInt<16>(x: getImm()) || isUInt<16>(x: getImm()));
10195}
10196
10197bool AMDGPUOperand::isU16Imm() const {
10198 return isImmLiteral() && isUInt<16>(x: getImm());
10199}
10200
10201//===----------------------------------------------------------------------===//
10202// dim
10203//===----------------------------------------------------------------------===//
10204
10205bool AMDGPUAsmParser::parseDimId(unsigned &Encoding) {
10206 // We want to allow "dim:1D" etc.,
10207 // but the initial 1 is tokenized as an integer.
10208 std::string Token;
10209 if (isToken(Kind: AsmToken::Integer)) {
10210 SMLoc Loc = getToken().getEndLoc();
10211 Token = std::string(getTokenStr());
10212 lex();
10213 if (getLoc() != Loc)
10214 return false;
10215 }
10216
10217 StringRef Suffix;
10218 if (!parseId(Val&: Suffix))
10219 return false;
10220 Token += Suffix;
10221
10222 StringRef DimId = Token;
10223 DimId.consume_front(Prefix: "SQ_RSRC_IMG_");
10224
10225 const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfoByAsmSuffix(AsmSuffix: DimId);
10226 if (!DimInfo)
10227 return false;
10228
10229 Encoding = DimInfo->Encoding;
10230 return true;
10231}
10232
10233ParseStatus AMDGPUAsmParser::parseDim(OperandVector &Operands) {
10234 if (!isGFX10Plus())
10235 return ParseStatus::NoMatch;
10236
10237 SMLoc S = getLoc();
10238
10239 if (!trySkipId(Id: "dim", Kind: AsmToken::Colon))
10240 return ParseStatus::NoMatch;
10241
10242 unsigned Encoding;
10243 SMLoc Loc = getLoc();
10244 if (!parseDimId(Encoding))
10245 return Error(L: Loc, Msg: "invalid dim value");
10246
10247 Operands.push_back(
10248 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Encoding, Loc: S, Type: AMDGPUOperand::ImmTyDim));
10249 return ParseStatus::Success;
10250}
10251
10252//===----------------------------------------------------------------------===//
10253// dpp
10254//===----------------------------------------------------------------------===//
10255
10256ParseStatus AMDGPUAsmParser::parseDPP8(OperandVector &Operands) {
10257 SMLoc S = getLoc();
10258
10259 if (!isGFX10Plus() || !trySkipId(Id: "dpp8", Kind: AsmToken::Colon))
10260 return ParseStatus::NoMatch;
10261
10262 // dpp8:[%d,%d,%d,%d,%d,%d,%d,%d]
10263
10264 int64_t Sels[8];
10265
10266 if (!skipToken(Kind: AsmToken::LBrac, ErrMsg: "expected an opening square bracket"))
10267 return ParseStatus::Failure;
10268
10269 for (size_t i = 0; i < 8; ++i) {
10270 if (i > 0 && !skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma"))
10271 return ParseStatus::Failure;
10272
10273 SMLoc Loc = getLoc();
10274 if (getParser().parseAbsoluteExpression(Res&: Sels[i]))
10275 return ParseStatus::Failure;
10276 if (0 > Sels[i] || 7 < Sels[i])
10277 return Error(L: Loc, Msg: "expected a 3-bit value");
10278 }
10279
10280 if (!skipToken(Kind: AsmToken::RBrac, ErrMsg: "expected a closing square bracket"))
10281 return ParseStatus::Failure;
10282
10283 unsigned DPP8 = 0;
10284 for (size_t i = 0; i < 8; ++i)
10285 DPP8 |= (Sels[i] << (i * 3));
10286
10287 Operands.push_back(
10288 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: DPP8, Loc: S, Type: AMDGPUOperand::ImmTyDPP8));
10289 return ParseStatus::Success;
10290}
10291
10292bool AMDGPUAsmParser::isSupportedDPPCtrl(StringRef Ctrl,
10293 const OperandVector &Operands) {
10294 if (Ctrl == "row_newbcast")
10295 return isGFX90A();
10296
10297 if (Ctrl == "row_share" || Ctrl == "row_xmask")
10298 return isGFX10Plus();
10299
10300 if (Ctrl == "wave_shl" || Ctrl == "wave_shr" || Ctrl == "wave_rol" ||
10301 Ctrl == "wave_ror" || Ctrl == "row_bcast")
10302 return isVI() || isGFX9();
10303
10304 return Ctrl == "row_mirror" || Ctrl == "row_half_mirror" ||
10305 Ctrl == "quad_perm" || Ctrl == "row_shl" || Ctrl == "row_shr" ||
10306 Ctrl == "row_ror";
10307}
10308
10309int64_t AMDGPUAsmParser::parseDPPCtrlPerm() {
10310 // quad_perm:[%d,%d,%d,%d]
10311
10312 if (!skipToken(Kind: AsmToken::LBrac, ErrMsg: "expected an opening square bracket"))
10313 return -1;
10314
10315 int64_t Val = 0;
10316 for (int i = 0; i < 4; ++i) {
10317 if (i > 0 && !skipToken(Kind: AsmToken::Comma, ErrMsg: "expected a comma"))
10318 return -1;
10319
10320 int64_t Temp;
10321 SMLoc Loc = getLoc();
10322 if (getParser().parseAbsoluteExpression(Res&: Temp))
10323 return -1;
10324 if (Temp < 0 || Temp > 3) {
10325 Error(L: Loc, Msg: "expected a 2-bit value");
10326 return -1;
10327 }
10328
10329 Val += (Temp << i * 2);
10330 }
10331
10332 if (!skipToken(Kind: AsmToken::RBrac, ErrMsg: "expected a closing square bracket"))
10333 return -1;
10334
10335 return Val;
10336}
10337
10338int64_t AMDGPUAsmParser::parseDPPCtrlSel(StringRef Ctrl) {
10339 using namespace AMDGPU::DPP;
10340
10341 // sel:%d
10342
10343 int64_t Val;
10344 SMLoc Loc = getLoc();
10345
10346 if (getParser().parseAbsoluteExpression(Res&: Val))
10347 return -1;
10348
10349 struct DppCtrlCheck {
10350 int64_t Ctrl;
10351 int Lo;
10352 int Hi;
10353 };
10354
10355 DppCtrlCheck Check =
10356 StringSwitch<DppCtrlCheck>(Ctrl)
10357 .Case(S: "wave_shl", Value: {.Ctrl: DppCtrl::WAVE_SHL1, .Lo: 1, .Hi: 1})
10358 .Case(S: "wave_rol", Value: {.Ctrl: DppCtrl::WAVE_ROL1, .Lo: 1, .Hi: 1})
10359 .Case(S: "wave_shr", Value: {.Ctrl: DppCtrl::WAVE_SHR1, .Lo: 1, .Hi: 1})
10360 .Case(S: "wave_ror", Value: {.Ctrl: DppCtrl::WAVE_ROR1, .Lo: 1, .Hi: 1})
10361 .Case(S: "row_shl", Value: {.Ctrl: DppCtrl::ROW_SHL0, .Lo: 1, .Hi: 15})
10362 .Case(S: "row_shr", Value: {.Ctrl: DppCtrl::ROW_SHR0, .Lo: 1, .Hi: 15})
10363 .Case(S: "row_ror", Value: {.Ctrl: DppCtrl::ROW_ROR0, .Lo: 1, .Hi: 15})
10364 .Case(S: "row_share", Value: {.Ctrl: DppCtrl::ROW_SHARE_FIRST, .Lo: 0, .Hi: 15})
10365 .Case(S: "row_xmask", Value: {.Ctrl: DppCtrl::ROW_XMASK_FIRST, .Lo: 0, .Hi: 15})
10366 .Case(S: "row_newbcast", Value: {.Ctrl: DppCtrl::ROW_NEWBCAST_FIRST, .Lo: 0, .Hi: 15})
10367 .Default(Value: {.Ctrl: -1, .Lo: 0, .Hi: 0});
10368
10369 bool Valid;
10370 if (Check.Ctrl == -1) {
10371 Valid = (Ctrl == "row_bcast" && (Val == 15 || Val == 31));
10372 Val = (Val == 15) ? DppCtrl::BCAST15 : DppCtrl::BCAST31;
10373 } else {
10374 Valid = Check.Lo <= Val && Val <= Check.Hi;
10375 Val = (Check.Lo == Check.Hi) ? Check.Ctrl : (Check.Ctrl | Val);
10376 }
10377
10378 if (!Valid) {
10379 Error(L: Loc, Msg: Twine("invalid ", Ctrl) + Twine(" value"));
10380 return -1;
10381 }
10382
10383 return Val;
10384}
10385
10386ParseStatus AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
10387 using namespace AMDGPU::DPP;
10388
10389 if (!isToken(Kind: AsmToken::Identifier) ||
10390 !isSupportedDPPCtrl(Ctrl: getTokenStr(), Operands))
10391 return ParseStatus::NoMatch;
10392
10393 SMLoc S = getLoc();
10394 int64_t Val = -1;
10395 StringRef Ctrl;
10396
10397 parseId(Val&: Ctrl);
10398
10399 if (Ctrl == "row_mirror") {
10400 Val = DppCtrl::ROW_MIRROR;
10401 } else if (Ctrl == "row_half_mirror") {
10402 Val = DppCtrl::ROW_HALF_MIRROR;
10403 } else {
10404 if (skipToken(Kind: AsmToken::Colon, ErrMsg: "expected a colon")) {
10405 if (Ctrl == "quad_perm") {
10406 Val = parseDPPCtrlPerm();
10407 } else {
10408 Val = parseDPPCtrlSel(Ctrl);
10409 }
10410 }
10411 }
10412
10413 if (Val == -1)
10414 return ParseStatus::Failure;
10415
10416 Operands.push_back(
10417 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val, Loc: S, Type: AMDGPUOperand::ImmTyDppCtrl));
10418 return ParseStatus::Success;
10419}
10420
10421void AMDGPUAsmParser::cvtVOP3DPP(MCInst &Inst, const OperandVector &Operands,
10422 bool IsDPP8) {
10423 OptionalImmIndexMap OptionalIdx;
10424 unsigned Opc = Inst.getOpcode();
10425 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
10426
10427 // MAC instructions are special because they have 'old'
10428 // operand which is not tied to dst (but assumed to be).
10429 // They also have dummy unused src2_modifiers.
10430 int OldIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::old);
10431 int Src2ModIdx =
10432 AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::src2_modifiers);
10433 bool IsMAC = OldIdx != -1 && Src2ModIdx != -1 &&
10434 Desc.getOperandConstraint(OpNum: OldIdx, Constraint: MCOI::TIED_TO) == -1;
10435
10436 unsigned I = 1;
10437 for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
10438 ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, N: 1);
10439 }
10440
10441 int Fi = 0;
10442 int VdstInIdx = AMDGPU::getNamedOperandIdx(Opcode: Opc, Name: AMDGPU::OpName::vdst_in);
10443 bool IsVOP3CvtSrDpp = Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx12 ||
10444 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx13 ||
10445 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx12 ||
10446 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx13 ||
10447 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx12 ||
10448 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx13 ||
10449 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx12 ||
10450 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx13;
10451
10452 for (unsigned E = Operands.size(); I != E; ++I) {
10453
10454 if (IsMAC) {
10455 int NumOperands = Inst.getNumOperands();
10456 if (OldIdx == NumOperands) {
10457 // Handle old operand
10458 constexpr int DST_IDX = 0;
10459 Inst.addOperand(Op: Inst.getOperand(i: DST_IDX));
10460 } else if (Src2ModIdx == NumOperands) {
10461 // Add unused dummy src2_modifiers
10462 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
10463 }
10464 }
10465
10466 if (VdstInIdx == static_cast<int>(Inst.getNumOperands())) {
10467 Inst.addOperand(Op: Inst.getOperand(i: 0));
10468 }
10469
10470 if (IsVOP3CvtSrDpp) {
10471 if (Src2ModIdx == static_cast<int>(Inst.getNumOperands())) {
10472 Inst.addOperand(Op: MCOperand::createImm(Val: 0));
10473 Inst.addOperand(Op: MCOperand::createReg(Reg: MCRegister()));
10474 }
10475 }
10476
10477 auto TiedTo =
10478 Desc.getOperandConstraint(OpNum: Inst.getNumOperands(), Constraint: MCOI::TIED_TO);
10479 if (TiedTo != -1) {
10480 assert((unsigned)TiedTo < Inst.getNumOperands());
10481 // handle tied old or src2 for MAC instructions
10482 Inst.addOperand(Op: Inst.getOperand(i: TiedTo));
10483 }
10484 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
10485 // Add the register arguments
10486 if (IsDPP8 && Op.isDppFI()) {
10487 Fi = Op.getImm();
10488 } else if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) {
10489 Op.addRegOrImmWithFPInputModsOperands(Inst, N: 2);
10490 } else if (Op.isReg()) {
10491 Op.addRegOperands(Inst, N: 1);
10492 } else if (Op.isImm() &&
10493 Desc.operands()[Inst.getNumOperands()].RegClass != -1) {
10494 Op.addImmOperands(Inst, N: 1);
10495 } else if (Op.isImm()) {
10496 OptionalIdx[Op.getImmTy()] = I;
10497 } else {
10498 llvm_unreachable("unhandled operand type");
10499 }
10500 }
10501
10502 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::clamp) && !IsVOP3CvtSrDpp)
10503 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10504 ImmT: AMDGPUOperand::ImmTyClamp);
10505
10506 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::byte_sel)) {
10507 if (VdstInIdx == static_cast<int>(Inst.getNumOperands()))
10508 Inst.addOperand(Op: Inst.getOperand(i: 0));
10509 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10510 ImmT: AMDGPUOperand::ImmTyByteSel);
10511 }
10512
10513 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::omod))
10514 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10515 ImmT: AMDGPUOperand::ImmTyOModSI);
10516
10517 if (SIInstrFlags::isVOP3P(O: Desc))
10518 cvtVOP3P(Inst, Operands, OptIdx&: OptionalIdx);
10519 else if (SIInstrFlags::isVOP3(O: Desc))
10520 cvtVOP3OpSel(Inst, Operands, OptionalIdx);
10521 else if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::op_sel)) {
10522 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10523 ImmT: AMDGPUOperand::ImmTyOpSel);
10524 }
10525
10526 if (IsDPP8) {
10527 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10528 ImmT: AMDGPUOperand::ImmTyDPP8);
10529 using namespace llvm::AMDGPU::DPP;
10530 Inst.addOperand(Op: MCOperand::createImm(Val: Fi ? DPP8_FI_1 : DPP8_FI_0));
10531 } else {
10532 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10533 ImmT: AMDGPUOperand::ImmTyDppCtrl, Default: 0xe4);
10534 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10535 ImmT: AMDGPUOperand::ImmTyDppRowMask, Default: 0xf);
10536 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10537 ImmT: AMDGPUOperand::ImmTyDppBankMask, Default: 0xf);
10538 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10539 ImmT: AMDGPUOperand::ImmTyDppBoundCtrl);
10540
10541 if (AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::fi))
10542 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10543 ImmT: AMDGPUOperand::ImmTyDppFI);
10544 }
10545}
10546
10547void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands,
10548 bool IsDPP8) {
10549 OptionalImmIndexMap OptionalIdx;
10550
10551 unsigned I = 1;
10552 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
10553 for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
10554 ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, N: 1);
10555 }
10556
10557 int Fi = 0;
10558 for (unsigned E = Operands.size(); I != E; ++I) {
10559 auto TiedTo =
10560 Desc.getOperandConstraint(OpNum: Inst.getNumOperands(), Constraint: MCOI::TIED_TO);
10561 if (TiedTo != -1) {
10562 assert((unsigned)TiedTo < Inst.getNumOperands());
10563 // handle tied old or src2 for MAC instructions
10564 Inst.addOperand(Op: Inst.getOperand(i: TiedTo));
10565 }
10566 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
10567 // Add the register arguments
10568 if (Op.isReg() && validateVccOperand(Reg: Op.getReg())) {
10569 // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
10570 // Skip it.
10571 continue;
10572 }
10573
10574 if (IsDPP8) {
10575 if (Op.isDPP8()) {
10576 Op.addImmOperands(Inst, N: 1);
10577 } else if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) {
10578 Op.addRegWithFPInputModsOperands(Inst, N: 2);
10579 } else if (Op.isDppFI()) {
10580 Fi = Op.getImm();
10581 } else if (Op.isReg()) {
10582 Op.addRegOperands(Inst, N: 1);
10583 } else {
10584 llvm_unreachable("Invalid operand type");
10585 }
10586 } else {
10587 if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) {
10588 Op.addRegWithFPInputModsOperands(Inst, N: 2);
10589 } else if (Op.isReg()) {
10590 Op.addRegOperands(Inst, N: 1);
10591 } else if (Op.isDPPCtrl()) {
10592 Op.addImmOperands(Inst, N: 1);
10593 } else if (Op.isImm()) {
10594 // Handle optional arguments
10595 OptionalIdx[Op.getImmTy()] = I;
10596 } else {
10597 llvm_unreachable("Invalid operand type");
10598 }
10599 }
10600 }
10601
10602 if (IsDPP8) {
10603 using namespace llvm::AMDGPU::DPP;
10604 Inst.addOperand(Op: MCOperand::createImm(Val: Fi ? DPP8_FI_1 : DPP8_FI_0));
10605 } else {
10606 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10607 ImmT: AMDGPUOperand::ImmTyDppRowMask, Default: 0xf);
10608 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10609 ImmT: AMDGPUOperand::ImmTyDppBankMask, Default: 0xf);
10610 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10611 ImmT: AMDGPUOperand::ImmTyDppBoundCtrl);
10612 if (AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::fi)) {
10613 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10614 ImmT: AMDGPUOperand::ImmTyDppFI);
10615 }
10616 }
10617}
10618
10619//===----------------------------------------------------------------------===//
10620// sdwa
10621//===----------------------------------------------------------------------===//
10622
10623ParseStatus AMDGPUAsmParser::parseSDWASel(OperandVector &Operands,
10624 StringRef Prefix,
10625 AMDGPUOperand::ImmTy Type) {
10626 return parseStringOrIntWithPrefix(
10627 Operands, Name: Prefix,
10628 Ids: {"BYTE_0", "BYTE_1", "BYTE_2", "BYTE_3", "WORD_0", "WORD_1", "DWORD"},
10629 Type);
10630}
10631
10632ParseStatus AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
10633 return parseStringOrIntWithPrefix(
10634 Operands, Name: "dst_unused", Ids: {"UNUSED_PAD", "UNUSED_SEXT", "UNUSED_PRESERVE"},
10635 Type: AMDGPUOperand::ImmTySDWADstUnused);
10636}
10637
10638void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
10639 cvtSDWA(Inst, Operands, BasicInstType: SDWAInstType::VOP1);
10640}
10641
10642void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
10643 cvtSDWA(Inst, Operands, BasicInstType: SDWAInstType::VOP2);
10644}
10645
10646void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst,
10647 const OperandVector &Operands) {
10648 cvtSDWA(Inst, Operands, BasicInstType: SDWAInstType::VOP2, SkipDstVcc: true, SkipSrcVcc: true);
10649}
10650
10651void AMDGPUAsmParser::cvtSdwaVOP2e(MCInst &Inst,
10652 const OperandVector &Operands) {
10653 cvtSDWA(Inst, Operands, BasicInstType: SDWAInstType::VOP2, SkipDstVcc: false, SkipSrcVcc: true);
10654}
10655
10656void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
10657 cvtSDWA(Inst, Operands, BasicInstType: SDWAInstType::VOPC, SkipDstVcc: isVI());
10658}
10659
10660void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
10661 SDWAInstType BasicInstType, bool SkipDstVcc,
10662 bool SkipSrcVcc) {
10663 using namespace llvm::AMDGPU::SDWA;
10664
10665 OptionalImmIndexMap OptionalIdx;
10666 bool SkipVcc = SkipDstVcc || SkipSrcVcc;
10667 bool SkippedVcc = false;
10668
10669 unsigned I = 1;
10670 const MCInstrDesc &Desc = MII.get(Opcode: Inst.getOpcode());
10671 for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
10672 ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, N: 1);
10673 }
10674
10675 for (unsigned E = Operands.size(); I != E; ++I) {
10676 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
10677 if (SkipVcc && !SkippedVcc && Op.isReg() &&
10678 (Op.getReg() == AMDGPU::VCC || Op.getReg() == AMDGPU::VCC_LO)) {
10679 // VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
10680 // Skip it if it's 2nd (e.g. v_add_i32_sdwa v1, vcc, v2, v3)
10681 // or 4th (v_addc_u32_sdwa v1, vcc, v2, v3, vcc) operand.
10682 // Skip VCC only if we didn't skip it on previous iteration.
10683 // Note that src0 and src1 occupy 2 slots each because of modifiers.
10684 if (BasicInstType == SDWAInstType::VOP2 &&
10685 ((SkipDstVcc && Inst.getNumOperands() == 1) ||
10686 (SkipSrcVcc && Inst.getNumOperands() == 5))) {
10687 SkippedVcc = true;
10688 continue;
10689 }
10690 if (BasicInstType == SDWAInstType::VOPC && Inst.getNumOperands() == 0) {
10691 SkippedVcc = true;
10692 continue;
10693 }
10694 }
10695 if (isRegOrImmWithInputMods(Desc, OpNum: Inst.getNumOperands())) {
10696 Op.addRegOrImmWithInputModsOperands(Inst, N: 2);
10697 } else if (Op.isImm()) {
10698 // Handle optional arguments
10699 OptionalIdx[Op.getImmTy()] = I;
10700 } else {
10701 llvm_unreachable("Invalid operand type");
10702 }
10703 SkippedVcc = false;
10704 }
10705
10706 const unsigned Opc = Inst.getOpcode();
10707 if (Opc != AMDGPU::V_NOP_sdwa_gfx10 && Opc != AMDGPU::V_NOP_sdwa_gfx9 &&
10708 Opc != AMDGPU::V_NOP_sdwa_vi) {
10709 // v_nop_sdwa_sdwa_vi/gfx9 has no optional sdwa arguments
10710 switch (BasicInstType) {
10711 case SDWAInstType::VOP1:
10712 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::clamp))
10713 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10714 ImmT: AMDGPUOperand::ImmTyClamp, Default: 0);
10715
10716 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::omod))
10717 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10718 ImmT: AMDGPUOperand::ImmTyOModSI, Default: 0);
10719
10720 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::dst_sel))
10721 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10722 ImmT: AMDGPUOperand::ImmTySDWADstSel, Default: SdwaSel::DWORD);
10723
10724 if (AMDGPU::hasNamedOperand(Opcode: Opc, NamedIdx: AMDGPU::OpName::dst_unused))
10725 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10726 ImmT: AMDGPUOperand::ImmTySDWADstUnused,
10727 Default: DstUnused::UNUSED_PRESERVE);
10728
10729 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10730 ImmT: AMDGPUOperand::ImmTySDWASrc0Sel, Default: SdwaSel::DWORD);
10731 break;
10732
10733 case SDWAInstType::VOP2:
10734 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10735 ImmT: AMDGPUOperand::ImmTyClamp, Default: 0);
10736
10737 if (AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::omod))
10738 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10739 ImmT: AMDGPUOperand::ImmTyOModSI, Default: 0);
10740
10741 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10742 ImmT: AMDGPUOperand::ImmTySDWADstSel, Default: SdwaSel::DWORD);
10743 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10744 ImmT: AMDGPUOperand::ImmTySDWADstUnused,
10745 Default: DstUnused::UNUSED_PRESERVE);
10746 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10747 ImmT: AMDGPUOperand::ImmTySDWASrc0Sel, Default: SdwaSel::DWORD);
10748 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10749 ImmT: AMDGPUOperand::ImmTySDWASrc1Sel, Default: SdwaSel::DWORD);
10750 break;
10751
10752 case SDWAInstType::VOPC:
10753 if (AMDGPU::hasNamedOperand(Opcode: Inst.getOpcode(), NamedIdx: AMDGPU::OpName::clamp))
10754 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10755 ImmT: AMDGPUOperand::ImmTyClamp, Default: 0);
10756 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10757 ImmT: AMDGPUOperand::ImmTySDWASrc0Sel, Default: SdwaSel::DWORD);
10758 addOptionalImmOperand(Inst, Operands, OptionalIdx,
10759 ImmT: AMDGPUOperand::ImmTySDWASrc1Sel, Default: SdwaSel::DWORD);
10760 break;
10761 }
10762 }
10763
10764 // special case v_mac_{f16, f32}:
10765 // it has src2 register operand that is tied to dst operand
10766 if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
10767 Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
10768 auto *it = Inst.begin();
10769 std::advance(
10770 i&: it, n: AMDGPU::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: AMDGPU::OpName::src2));
10771 Inst.insert(I: it, Op: Inst.getOperand(i: 0)); // src2 = dst
10772 }
10773}
10774
10775/// Force static initialization.
10776extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
10777LLVMInitializeAMDGPUAsmParser() {
10778 RegisterMCAsmParser<AMDGPUAsmParser> A(getTheR600Target());
10779 RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
10780 RegisterMCAsmParser<AMDGPUAsmParser> C(getTheGCNLegacyTarget());
10781}
10782
10783#define GET_MATCHER_IMPLEMENTATION
10784#define GET_MNEMONIC_SPELL_CHECKER
10785#define GET_MNEMONIC_CHECKER
10786#include "AMDGPUGenAsmMatcher.inc"
10787
10788ParseStatus AMDGPUAsmParser::parseCustomOperand(OperandVector &Operands,
10789 unsigned MCK) {
10790 switch (MCK) {
10791 case MCK_addr64:
10792 return parseTokenOp(Name: "addr64", Operands);
10793 case MCK_done:
10794 return parseNamedBit(Name: "done", Operands, ImmTy: AMDGPUOperand::ImmTyDone, IgnoreNegative: true);
10795 case MCK_idxen:
10796 return parseTokenOp(Name: "idxen", Operands);
10797 case MCK_lds:
10798 return parseNamedBit(Name: "lds", Operands, ImmTy: AMDGPUOperand::ImmTyLDS,
10799 /*IgnoreNegative=*/true);
10800 case MCK_offen:
10801 return parseTokenOp(Name: "offen", Operands);
10802 case MCK_off:
10803 return parseTokenOp(Name: "off", Operands);
10804 case MCK_row_95_en:
10805 return parseNamedBit(Name: "row_en", Operands, ImmTy: AMDGPUOperand::ImmTyRowEn, IgnoreNegative: true);
10806 case MCK_gds:
10807 return parseNamedBit(Name: "gds", Operands, ImmTy: AMDGPUOperand::ImmTyGDS);
10808 case MCK_tfe:
10809 return parseNamedBit(Name: "tfe", Operands, ImmTy: AMDGPUOperand::ImmTyTFE);
10810 }
10811 return tryCustomParseOperand(Operands, MCK);
10812}
10813
10814// This function should be defined after auto-generated include so that we have
10815// MatchClassKind enum defined
10816unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
10817 unsigned Kind) {
10818 // Tokens like "glc" would be parsed as immediate operands in ParseOperand().
10819 // But MatchInstructionImpl() expects to meet token and fails to validate
10820 // operand. This method checks if we are given immediate operand but expect to
10821 // get corresponding token.
10822 AMDGPUOperand &Operand = (AMDGPUOperand &)Op;
10823 switch (Kind) {
10824 case MCK_addr64:
10825 return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
10826 case MCK_gds:
10827 return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
10828 case MCK_lds:
10829 return Operand.isLDS() ? Match_Success : Match_InvalidOperand;
10830 case MCK_idxen:
10831 return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
10832 case MCK_offen:
10833 return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
10834 case MCK_tfe:
10835 return Operand.isTFE() ? Match_Success : Match_InvalidOperand;
10836 case MCK_done:
10837 return Operand.isDone() ? Match_Success : Match_InvalidOperand;
10838 case MCK_row_95_en:
10839 return Operand.isRowEn() ? Match_Success : Match_InvalidOperand;
10840 case MCK_SSrc_b32:
10841 // When operands have expression values, they will return true for isToken,
10842 // because it is not possible to distinguish between a token and an
10843 // expression at parse time. MatchInstructionImpl() will always try to
10844 // match an operand as a token, when isToken returns true, and when the
10845 // name of the expression is not a valid token, the match will fail,
10846 // so we need to handle it here.
10847 return Operand.isSSrc_b32() ? Match_Success : Match_InvalidOperand;
10848 case MCK_SSrc_f32:
10849 return Operand.isSSrc_f32() ? Match_Success : Match_InvalidOperand;
10850 case MCK_SOPPBrTarget:
10851 return Operand.isSOPPBrTarget() ? Match_Success : Match_InvalidOperand;
10852 case MCK_VReg32OrOff:
10853 return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
10854 case MCK_InterpSlot:
10855 return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
10856 case MCK_InterpAttr:
10857 return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
10858 case MCK_InterpAttrChan:
10859 return Operand.isInterpAttrChan() ? Match_Success : Match_InvalidOperand;
10860 case MCK_SReg_64:
10861 case MCK_SReg_64_XEXEC:
10862 // Null is defined as a 32-bit register but
10863 // it should also be enabled with 64-bit operands or larger.
10864 // The following code enables it for SReg_64 and larger operands
10865 // used as source and destination. Remaining source
10866 // operands are handled in isInlinableImm.
10867 case MCK_SReg_96:
10868 case MCK_SReg_128:
10869 case MCK_SReg_256:
10870 case MCK_SReg_512:
10871 return Operand.isNull() ? Match_Success : Match_InvalidOperand;
10872 default:
10873 return Match_InvalidOperand;
10874 }
10875}
10876
10877//===----------------------------------------------------------------------===//
10878// endpgm
10879//===----------------------------------------------------------------------===//
10880
10881ParseStatus AMDGPUAsmParser::parseEndpgm(OperandVector &Operands) {
10882 SMLoc S = getLoc();
10883 int64_t Imm = 0;
10884
10885 if (!parseExpr(Imm)) {
10886 // The operand is optional, if not present default to 0
10887 Imm = 0;
10888 }
10889
10890 if (!isUInt<16>(x: Imm))
10891 return Error(L: S, Msg: "expected a 16-bit value");
10892
10893 Operands.push_back(
10894 Elt: AMDGPUOperand::CreateImm(AsmParser: this, Val: Imm, Loc: S, Type: AMDGPUOperand::ImmTyEndpgm));
10895 return ParseStatus::Success;
10896}
10897
10898bool AMDGPUOperand::isEndpgm() const { return isImmTy(ImmT: ImmTyEndpgm); }
10899
10900//===----------------------------------------------------------------------===//
10901// Split Barrier
10902//===----------------------------------------------------------------------===//
10903
10904bool AMDGPUOperand::isSplitBarrier() const {
10905 if (!isImm())
10906 return false;
10907
10908 int64_t Imm = getImm();
10909 return isUInt<5>(x: Imm) || (AMDGPU::Barrier::CLUSTER_TRAP <= Imm &&
10910 Imm <= AMDGPU::Barrier::WORKGROUP);
10911}
10912