1//===-- AMDGPUMCCodeEmitter.cpp - AMDGPU Code Emitter ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// The AMDGPU code emitter produces machine code that can be executed
11/// directly on the GPU device.
12//
13//===----------------------------------------------------------------------===//
14
15#include "MCTargetDesc/AMDGPUFixupKinds.h"
16#include "MCTargetDesc/AMDGPUMCExpr.h"
17#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
18#include "SIDefines.h"
19#include "Utils/AMDGPUBaseInfo.h"
20#include "llvm/ADT/APInt.h"
21#include "llvm/MC/MCCodeEmitter.h"
22#include "llvm/MC/MCContext.h"
23#include "llvm/MC/MCExpr.h"
24#include "llvm/MC/MCInstrInfo.h"
25#include "llvm/MC/MCRegisterInfo.h"
26#include "llvm/MC/MCSubtargetInfo.h"
27#include "llvm/Support/Casting.h"
28#include "llvm/Support/EndianStream.h"
29#include <optional>
30
31using namespace llvm;
32
33namespace {
34
35class AMDGPUMCCodeEmitter : public MCCodeEmitter {
36 const MCRegisterInfo &MRI;
37 const MCInstrInfo &MCII;
38
39public:
40 AMDGPUMCCodeEmitter(const MCInstrInfo &MCII, const MCRegisterInfo &MRI)
41 : MRI(MRI), MCII(MCII) {}
42
43 /// Encode the instruction and write it to the OS.
44 void encodeInstruction(const MCInst &MI, SmallVectorImpl<char> &CB,
45 SmallVectorImpl<MCFixup> &Fixups,
46 const MCSubtargetInfo &STI) const override;
47
48 void getMachineOpValue(const MCInst &MI, const MCOperand &MO, APInt &Op,
49 SmallVectorImpl<MCFixup> &Fixups,
50 const MCSubtargetInfo &STI) const;
51
52 void getMachineOpValueT16(const MCInst &MI, unsigned OpNo, APInt &Op,
53 SmallVectorImpl<MCFixup> &Fixups,
54 const MCSubtargetInfo &STI) const;
55
56 void getMachineOpValueT16Lo128(const MCInst &MI, unsigned OpNo, APInt &Op,
57 SmallVectorImpl<MCFixup> &Fixups,
58 const MCSubtargetInfo &STI) const;
59
60 /// Use a fixup to encode the simm16 field for SOPP branch
61 /// instructions.
62 void getSOPPBrEncoding(const MCInst &MI, unsigned OpNo, APInt &Op,
63 SmallVectorImpl<MCFixup> &Fixups,
64 const MCSubtargetInfo &STI) const;
65
66 void getSMEMOffsetEncoding(const MCInst &MI, unsigned OpNo, APInt &Op,
67 SmallVectorImpl<MCFixup> &Fixups,
68 const MCSubtargetInfo &STI) const;
69
70 void getSDWASrcEncoding(const MCInst &MI, unsigned OpNo, APInt &Op,
71 SmallVectorImpl<MCFixup> &Fixups,
72 const MCSubtargetInfo &STI) const;
73
74 void getSDWAVopcDstEncoding(const MCInst &MI, unsigned OpNo, APInt &Op,
75 SmallVectorImpl<MCFixup> &Fixups,
76 const MCSubtargetInfo &STI) const;
77
78 void getAVOperandEncoding(const MCInst &MI, unsigned OpNo, APInt &Op,
79 SmallVectorImpl<MCFixup> &Fixups,
80 const MCSubtargetInfo &STI) const;
81
82private:
83 uint64_t getImplicitOpSelHiEncoding(int Opcode) const;
84 void getMachineOpValueCommon(const MCInst &MI, const MCOperand &MO,
85 unsigned OpNo, APInt &Op,
86 SmallVectorImpl<MCFixup> &Fixups,
87 const MCSubtargetInfo &STI) const;
88
89 /// Encode an fp or int literal.
90 std::optional<uint64_t>
91 getLitEncoding(const MCInstrDesc &Desc, const MCOperand &MO, unsigned OpNo,
92 const MCSubtargetInfo &STI,
93 bool HasMandatoryLiteral = false) const;
94
95 void getBinaryCodeForInstr(const MCInst &MI, SmallVectorImpl<MCFixup> &Fixups,
96 APInt &Inst, APInt &Scratch,
97 const MCSubtargetInfo &STI) const;
98
99 template <bool HasSrc0, bool HasSrc1, bool HasSrc2>
100 APInt postEncodeVOP3(const MCInst &MI, APInt EncodedValue,
101 const MCSubtargetInfo &STI) const;
102
103 APInt postEncodeVOPCX(const MCInst &MI, APInt EncodedValue,
104 const MCSubtargetInfo &STI) const;
105};
106
107} // end anonymous namespace
108
109MCCodeEmitter *llvm::createAMDGPUMCCodeEmitter(const MCInstrInfo &MCII,
110 MCContext &Ctx) {
111 return new AMDGPUMCCodeEmitter(MCII, *Ctx.getRegisterInfo());
112}
113
114static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
115 const MCExpr *Value, uint16_t Kind, bool PCRel = false) {
116 Fixups.push_back(Elt: MCFixup::create(Offset, Value, Kind, PCRel));
117}
118
119// Returns the encoding value to use if the given integer is an integer inline
120// immediate value, or 0 if it is not.
121template <typename IntTy>
122static uint32_t getIntInlineImmEncoding(IntTy Imm) {
123 if (Imm >= 0 && Imm <= 64)
124 return 128 + Imm;
125
126 if (Imm >= -16 && Imm <= -1)
127 return 192 + std::abs(Imm);
128
129 return 0;
130}
131
132static uint32_t getLit16Encoding(uint16_t Val, const MCSubtargetInfo &STI) {
133 uint16_t IntImm = getIntInlineImmEncoding(Imm: static_cast<int16_t>(Val));
134 if (IntImm != 0)
135 return IntImm;
136
137 if (Val == 0x3800) // 0.5
138 return 240;
139
140 if (Val == 0xB800) // -0.5
141 return 241;
142
143 if (Val == 0x3C00) // 1.0
144 return 242;
145
146 if (Val == 0xBC00) // -1.0
147 return 243;
148
149 if (Val == 0x4000) // 2.0
150 return 244;
151
152 if (Val == 0xC000) // -2.0
153 return 245;
154
155 if (Val == 0x4400) // 4.0
156 return 246;
157
158 if (Val == 0xC400) // -4.0
159 return 247;
160
161 if (Val == 0x3118 && // 1.0 / (2.0 * pi)
162 STI.hasFeature(Feature: AMDGPU::FeatureInv2PiInlineImm))
163 return 248;
164
165 return 255;
166}
167
168static uint32_t getLitBF16Encoding(uint16_t Val) {
169 uint16_t IntImm = getIntInlineImmEncoding(Imm: static_cast<int16_t>(Val));
170 if (IntImm != 0)
171 return IntImm;
172
173 // clang-format off
174 switch (Val) {
175 case 0x3F00: return 240; // 0.5
176 case 0xBF00: return 241; // -0.5
177 case 0x3F80: return 242; // 1.0
178 case 0xBF80: return 243; // -1.0
179 case 0x4000: return 244; // 2.0
180 case 0xC000: return 245; // -2.0
181 case 0x4080: return 246; // 4.0
182 case 0xC080: return 247; // -4.0
183 case 0x3E22: return 248; // 1.0 / (2.0 * pi)
184 default: return 255;
185 }
186 // clang-format on
187}
188
189static uint32_t getLit32Encoding(uint32_t Val, const MCSubtargetInfo &STI) {
190 uint32_t IntImm = getIntInlineImmEncoding(Imm: static_cast<int32_t>(Val));
191 if (IntImm != 0)
192 return IntImm;
193
194 if (Val == llvm::bit_cast<uint32_t>(from: 0.5f))
195 return 240;
196
197 if (Val == llvm::bit_cast<uint32_t>(from: -0.5f))
198 return 241;
199
200 if (Val == llvm::bit_cast<uint32_t>(from: 1.0f))
201 return 242;
202
203 if (Val == llvm::bit_cast<uint32_t>(from: -1.0f))
204 return 243;
205
206 if (Val == llvm::bit_cast<uint32_t>(from: 2.0f))
207 return 244;
208
209 if (Val == llvm::bit_cast<uint32_t>(from: -2.0f))
210 return 245;
211
212 if (Val == llvm::bit_cast<uint32_t>(from: 4.0f))
213 return 246;
214
215 if (Val == llvm::bit_cast<uint32_t>(from: -4.0f))
216 return 247;
217
218 if (Val == 0x3e22f983 && // 1.0 / (2.0 * pi)
219 STI.hasFeature(Feature: AMDGPU::FeatureInv2PiInlineImm))
220 return 248;
221
222 return 255;
223}
224
225static uint32_t getLit16IntEncoding(uint32_t Val, const MCSubtargetInfo &STI) {
226 return getLit32Encoding(Val, STI);
227}
228
229static uint32_t getLit64Encoding(const MCInstrDesc &Desc, uint64_t Val,
230 const MCSubtargetInfo &STI, bool IsFP) {
231 uint32_t IntImm = getIntInlineImmEncoding(Imm: static_cast<int64_t>(Val));
232 if (IntImm != 0)
233 return IntImm;
234
235 if (Val == llvm::bit_cast<uint64_t>(from: 0.5))
236 return 240;
237
238 if (Val == llvm::bit_cast<uint64_t>(from: -0.5))
239 return 241;
240
241 if (Val == llvm::bit_cast<uint64_t>(from: 1.0))
242 return 242;
243
244 if (Val == llvm::bit_cast<uint64_t>(from: -1.0))
245 return 243;
246
247 if (Val == llvm::bit_cast<uint64_t>(from: 2.0))
248 return 244;
249
250 if (Val == llvm::bit_cast<uint64_t>(from: -2.0))
251 return 245;
252
253 if (Val == llvm::bit_cast<uint64_t>(from: 4.0))
254 return 246;
255
256 if (Val == llvm::bit_cast<uint64_t>(from: -4.0))
257 return 247;
258
259 if (Val == 0x3fc45f306dc9c882 && // 1.0 / (2.0 * pi)
260 STI.hasFeature(Feature: AMDGPU::FeatureInv2PiInlineImm))
261 return 248;
262
263 // The rest part needs to align with AMDGPUInstPrinter::printLiteral64.
264
265 bool CanUse64BitLiterals =
266 STI.hasFeature(Feature: AMDGPU::Feature64BitLiterals) &&
267 !(Desc.TSFlags & (SIInstrFlags::VOP3 | SIInstrFlags::VOP3P));
268 if (IsFP) {
269 return CanUse64BitLiterals && Lo_32(Value: Val) ? 254 : 255;
270 }
271
272 return CanUse64BitLiterals && (!isInt<32>(x: Val) || !isUInt<32>(x: Val)) ? 254
273 : 255;
274}
275
276std::optional<uint64_t> AMDGPUMCCodeEmitter::getLitEncoding(
277 const MCInstrDesc &Desc, const MCOperand &MO, unsigned OpNo,
278 const MCSubtargetInfo &STI, bool HasMandatoryLiteral) const {
279 const MCOperandInfo &OpInfo = Desc.operands()[OpNo];
280 int64_t Imm = 0;
281 if (MO.isExpr()) {
282 if (!MO.getExpr()->evaluateAsAbsolute(Res&: Imm) ||
283 AMDGPU::isLitExpr(Expr: MO.getExpr())) {
284 if (OpInfo.OperandType == AMDGPU::OPERAND_KIMM16 ||
285 OpInfo.OperandType == AMDGPU::OPERAND_KIMM32 ||
286 OpInfo.OperandType == AMDGPU::OPERAND_KIMM64)
287 return Imm;
288 if (STI.hasFeature(Feature: AMDGPU::Feature64BitLiterals) &&
289 AMDGPU::getOperandSize(OpInfo) == 8 &&
290 AMDGPU::getExprKind(Expr: MO.getExpr()) != AMDGPUMCExpr::AGVK_Lit)
291 return 254;
292 return 255;
293 }
294 } else {
295 assert(!MO.isDFPImm());
296
297 if (!MO.isImm())
298 return {};
299
300 Imm = MO.getImm();
301 }
302
303 switch (OpInfo.OperandType) {
304 case AMDGPU::OPERAND_REG_IMM_INT32:
305 case AMDGPU::OPERAND_REG_IMM_FP32:
306 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
307 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
308 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
309 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
310 case AMDGPU::OPERAND_REG_IMM_V2INT32:
311 case AMDGPU::OPERAND_REG_IMM_V2FP32:
312 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
313 return getLit32Encoding(Val: static_cast<uint32_t>(Imm), STI);
314
315 case AMDGPU::OPERAND_REG_IMM_INT64:
316 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
317 case AMDGPU::OPERAND_REG_IMM_V2INT64:
318 return getLit64Encoding(Desc, Val: static_cast<uint64_t>(Imm), STI, IsFP: false);
319
320 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
321 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
322 return getLit64Encoding(Desc, Val: static_cast<uint64_t>(Imm), STI, IsFP: true);
323
324 case AMDGPU::OPERAND_REG_IMM_FP64:
325 case AMDGPU::OPERAND_REG_IMM_V2FP64: {
326 auto Enc = getLit64Encoding(Desc, Val: static_cast<uint64_t>(Imm), STI, IsFP: true);
327 return (HasMandatoryLiteral && Enc == 255) ? 254 : Enc;
328 }
329
330 case AMDGPU::OPERAND_REG_IMM_INT16:
331 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
332 return getLit16IntEncoding(Val: static_cast<uint32_t>(Imm), STI);
333
334 case AMDGPU::OPERAND_REG_IMM_FP16:
335 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
336 // FIXME Is this correct? What do inline immediates do on SI for f16 src
337 // which does not have f16 support?
338 return getLit16Encoding(Val: static_cast<uint16_t>(Imm), STI);
339
340 case AMDGPU::OPERAND_REG_IMM_BF16:
341 case AMDGPU::OPERAND_REG_INLINE_C_BF16:
342 // We don't actually need to check Inv2Pi here because BF16 instructions can
343 // only be emitted for targets that already support the feature.
344 return getLitBF16Encoding(Val: static_cast<uint16_t>(Imm));
345
346 case AMDGPU::OPERAND_REG_IMM_V2INT16:
347 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
348 return AMDGPU::getInlineEncodingV2I16(Literal: static_cast<uint32_t>(Imm))
349 .value_or(u: 255);
350
351 case AMDGPU::OPERAND_REG_IMM_V2FP16:
352 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
353 return AMDGPU::getInlineEncodingV2F16(Literal: static_cast<uint32_t>(Imm))
354 .value_or(u: 255);
355
356 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
357 // V_PK_FMAC_F16 has different inline constant behavior on pre-GFX11 vs
358 // GFX11+: pre-GFX11 produces (f16, 0), GFX11+ duplicates f16 to both
359 // halves.
360 return AMDGPU::getPKFMACF16InlineEncoding(Literal: static_cast<uint32_t>(Imm),
361 IsGFX11Plus: AMDGPU::isGFX11Plus(STI))
362 .value_or(u: 255);
363
364 case AMDGPU::OPERAND_REG_IMM_V2BF16:
365 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
366 return AMDGPU::getInlineEncodingV2BF16(Literal: static_cast<uint32_t>(Imm))
367 .value_or(u: 255);
368
369 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
370 return 255;
371
372 case AMDGPU::OPERAND_KIMM32:
373 case AMDGPU::OPERAND_KIMM16:
374 case AMDGPU::OPERAND_KIMM64:
375 return Imm;
376 default:
377 llvm_unreachable("invalid operand size");
378 }
379}
380
381uint64_t AMDGPUMCCodeEmitter::getImplicitOpSelHiEncoding(int Opcode) const {
382 using namespace AMDGPU::VOP3PEncoding;
383
384 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::op_sel_hi)) {
385 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src2))
386 return 0;
387 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src1))
388 return OP_SEL_HI_2;
389 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src0))
390 return OP_SEL_HI_1 | OP_SEL_HI_2;
391 }
392 return OP_SEL_HI_0 | OP_SEL_HI_1 | OP_SEL_HI_2;
393}
394
395void AMDGPUMCCodeEmitter::encodeInstruction(const MCInst &MI,
396 SmallVectorImpl<char> &CB,
397 SmallVectorImpl<MCFixup> &Fixups,
398 const MCSubtargetInfo &STI) const {
399 int Opcode = MI.getOpcode();
400 APInt Encoding, Scratch;
401 getBinaryCodeForInstr(MI, Fixups, Inst&: Encoding, Scratch, STI);
402 const MCInstrDesc &Desc = MCII.get(Opcode: MI.getOpcode());
403 unsigned bytes = Desc.getSize();
404
405 // Set unused op_sel_hi bits to 1 for VOP3P and MAI instructions.
406 // Note that accvgpr_read/write are MAI, have src0, but do not use op_sel.
407 if (((Desc.TSFlags & SIInstrFlags::VOP3P) ||
408 Opcode == AMDGPU::V_ACCVGPR_READ_B32_vi ||
409 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_vi) &&
410 // Matrix B format operand reuses op_sel_hi.
411 !AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::matrix_b_fmt) &&
412 // Matrix B scale operand reuses op_sel_hi.
413 !AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::matrix_b_scale) &&
414 // Matrix B reuse operand reuses op_sel_hi.
415 !AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::matrix_b_reuse)) {
416 Encoding |= getImplicitOpSelHiEncoding(Opcode);
417 }
418
419 for (unsigned i = 0; i < bytes; i++) {
420 CB.push_back(Elt: (uint8_t)Encoding.extractBitsAsZExtValue(numBits: 8, bitPosition: 8 * i));
421 }
422
423 // NSA encoding.
424 if (AMDGPU::isGFX10Plus(STI) && Desc.TSFlags & SIInstrFlags::MIMG) {
425 int vaddr0 = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
426 Name: AMDGPU::OpName::vaddr0);
427 int srsrc = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
428 Name: AMDGPU::OpName::srsrc);
429 assert(vaddr0 >= 0 && srsrc > vaddr0);
430 unsigned NumExtraAddrs = srsrc - vaddr0 - 1;
431 unsigned NumPadding = (-NumExtraAddrs) & 3;
432
433 for (unsigned i = 0; i < NumExtraAddrs; ++i) {
434 getMachineOpValue(MI, MO: MI.getOperand(i: vaddr0 + 1 + i), Op&: Encoding, Fixups,
435 STI);
436 CB.push_back(Elt: (uint8_t)Encoding.getLimitedValue());
437 }
438 CB.append(NumInputs: NumPadding, Elt: 0);
439 }
440
441 if ((bytes > 8 && STI.hasFeature(Feature: AMDGPU::FeatureVOP3Literal)) ||
442 (bytes > 4 && !STI.hasFeature(Feature: AMDGPU::FeatureVOP3Literal)))
443 return;
444
445 // Do not print literals from SISrc Operands for insts with mandatory literals
446 if (AMDGPU::hasNamedOperand(Opcode: MI.getOpcode(), NamedIdx: AMDGPU::OpName::imm))
447 return;
448
449 // Check for additional literals
450 for (unsigned i = 0, e = Desc.getNumOperands(); i < e; ++i) {
451
452 // Check if this operand should be encoded as [SV]Src
453 if (!AMDGPU::isSISrcOperand(Desc, OpNo: i))
454 continue;
455
456 // Is this operand a literal immediate?
457 const MCOperand &Op = MI.getOperand(i);
458 auto Enc = getLitEncoding(Desc, MO: Op, OpNo: i, STI);
459 if (!Enc || (*Enc != 255 && *Enc != 254))
460 continue;
461
462 // Yes! Encode it
463 int64_t Imm = 0;
464
465 bool IsLit = false;
466 if (Op.isImm())
467 Imm = Op.getImm();
468 else if (Op.isExpr()) {
469 if (const auto *C = dyn_cast<MCConstantExpr>(Val: Op.getExpr())) {
470 Imm = C->getValue();
471 } else if (AMDGPU::isLitExpr(Expr: Op.getExpr())) {
472 IsLit = true;
473 Imm = AMDGPU::getLitValue(Expr: Op.getExpr());
474 }
475 } else // Exprs will be replaced with a fixup value.
476 llvm_unreachable("Must be immediate or expr");
477
478 if (*Enc == 254) {
479 assert(STI.hasFeature(AMDGPU::Feature64BitLiterals));
480 support::endian::write<uint64_t>(Out&: CB, V: Imm, E: llvm::endianness::little);
481 } else {
482 auto OpType =
483 static_cast<AMDGPU::OperandType>(Desc.operands()[i].OperandType);
484 Imm = AMDGPU::encode32BitLiteral(Imm, Type: OpType, IsLit);
485 support::endian::write<uint32_t>(Out&: CB, V: Imm, E: llvm::endianness::little);
486 }
487
488 // Only one literal value allowed
489 break;
490 }
491}
492
493void AMDGPUMCCodeEmitter::getSOPPBrEncoding(const MCInst &MI, unsigned OpNo,
494 APInt &Op,
495 SmallVectorImpl<MCFixup> &Fixups,
496 const MCSubtargetInfo &STI) const {
497 const MCOperand &MO = MI.getOperand(i: OpNo);
498
499 if (MO.isExpr()) {
500 const MCExpr *Expr = MO.getExpr();
501 addFixup(Fixups, Offset: 0, Value: Expr, Kind: AMDGPU::fixup_si_sopp_br, PCRel: true);
502 Op = APInt::getZero(numBits: 96);
503 } else {
504 getMachineOpValue(MI, MO, Op, Fixups, STI);
505 }
506}
507
508void AMDGPUMCCodeEmitter::getSMEMOffsetEncoding(
509 const MCInst &MI, unsigned OpNo, APInt &Op,
510 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {
511 auto Offset = MI.getOperand(i: OpNo).getImm();
512 // VI only supports 20-bit unsigned offsets.
513 assert(!AMDGPU::isVI(STI) || isUInt<20>(Offset));
514 Op = Offset;
515}
516
517void AMDGPUMCCodeEmitter::getSDWASrcEncoding(const MCInst &MI, unsigned OpNo,
518 APInt &Op,
519 SmallVectorImpl<MCFixup> &Fixups,
520 const MCSubtargetInfo &STI) const {
521 using namespace AMDGPU::SDWA;
522
523 uint64_t RegEnc = 0;
524
525 const MCOperand &MO = MI.getOperand(i: OpNo);
526
527 if (MO.isReg()) {
528 MCRegister Reg = MO.getReg();
529 RegEnc |= MRI.getEncodingValue(Reg);
530 RegEnc &= SDWA9EncValues::SRC_VGPR_MASK;
531 if (AMDGPU::isSGPR(Reg: AMDGPU::mc2PseudoReg(Reg), TRI: &MRI)) {
532 RegEnc |= SDWA9EncValues::SRC_SGPR_MASK;
533 }
534 Op = RegEnc;
535 return;
536 } else {
537 const MCInstrDesc &Desc = MCII.get(Opcode: MI.getOpcode());
538 auto Enc = getLitEncoding(Desc, MO, OpNo, STI);
539 if (Enc && *Enc != 255) {
540 Op = *Enc | SDWA9EncValues::SRC_SGPR_MASK;
541 return;
542 }
543 }
544
545 llvm_unreachable("Unsupported operand kind");
546}
547
548void AMDGPUMCCodeEmitter::getSDWAVopcDstEncoding(
549 const MCInst &MI, unsigned OpNo, APInt &Op,
550 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {
551 using namespace AMDGPU::SDWA;
552
553 uint64_t RegEnc = 0;
554
555 const MCOperand &MO = MI.getOperand(i: OpNo);
556
557 MCRegister Reg = MO.getReg();
558 if (Reg != AMDGPU::VCC && Reg != AMDGPU::VCC_LO) {
559 RegEnc |= MRI.getEncodingValue(Reg);
560 RegEnc &= SDWA9EncValues::VOPC_DST_SGPR_MASK;
561 RegEnc |= SDWA9EncValues::VOPC_DST_VCC_MASK;
562 }
563 Op = RegEnc;
564}
565
566void AMDGPUMCCodeEmitter::getAVOperandEncoding(
567 const MCInst &MI, unsigned OpNo, APInt &Op,
568 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {
569 MCRegister Reg = MI.getOperand(i: OpNo).getReg();
570 unsigned Enc = MRI.getEncodingValue(Reg);
571 unsigned Idx = Enc & AMDGPU::HWEncoding::LO256_REG_IDX_MASK;
572 bool IsVGPROrAGPR =
573 Enc & (AMDGPU::HWEncoding::IS_VGPR | AMDGPU::HWEncoding::IS_AGPR);
574
575 // VGPR and AGPR have the same encoding, but SrcA and SrcB operands of mfma
576 // instructions use acc[0:1] modifier bits to distinguish. These bits are
577 // encoded as a virtual 9th bit of the register for these operands.
578 bool IsAGPR = Enc & AMDGPU::HWEncoding::IS_AGPR;
579
580 Op = Idx | (IsVGPROrAGPR << 8) | (IsAGPR << 9);
581}
582
583static bool needsPCRel(const MCExpr *Expr) {
584 switch (Expr->getKind()) {
585 case MCExpr::SymbolRef: {
586 auto *SE = cast<MCSymbolRefExpr>(Val: Expr);
587 auto Spec = AMDGPU::getSpecifier(SRE: SE);
588 return Spec != AMDGPUMCExpr::S_ABS32_LO &&
589 Spec != AMDGPUMCExpr::S_ABS32_HI && Spec != AMDGPUMCExpr::S_ABS64;
590 }
591 case MCExpr::Binary: {
592 auto *BE = cast<MCBinaryExpr>(Val: Expr);
593 if (BE->getOpcode() == MCBinaryExpr::Sub)
594 return false;
595 return needsPCRel(Expr: BE->getLHS()) || needsPCRel(Expr: BE->getRHS());
596 }
597 case MCExpr::Unary:
598 return needsPCRel(Expr: cast<MCUnaryExpr>(Val: Expr)->getSubExpr());
599 case MCExpr::Specifier:
600 case MCExpr::Target:
601 case MCExpr::Constant:
602 return false;
603 }
604 llvm_unreachable("invalid kind");
605}
606
607void AMDGPUMCCodeEmitter::getMachineOpValue(const MCInst &MI,
608 const MCOperand &MO, APInt &Op,
609 SmallVectorImpl<MCFixup> &Fixups,
610 const MCSubtargetInfo &STI) const {
611 if (MO.isReg()){
612 unsigned Enc = MRI.getEncodingValue(Reg: MO.getReg());
613 unsigned Idx = Enc & AMDGPU::HWEncoding::LO256_REG_IDX_MASK;
614 bool IsVGPROrAGPR =
615 Enc & (AMDGPU::HWEncoding::IS_VGPR | AMDGPU::HWEncoding::IS_AGPR);
616 Op = Idx | (IsVGPROrAGPR << 8);
617 return;
618 }
619 unsigned OpNo = &MO - MI.begin();
620 getMachineOpValueCommon(MI, MO, OpNo, Op, Fixups, STI);
621}
622
623void AMDGPUMCCodeEmitter::getMachineOpValueT16(
624 const MCInst &MI, unsigned OpNo, APInt &Op,
625 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {
626 const MCOperand &MO = MI.getOperand(i: OpNo);
627 if (MO.isReg()) {
628 unsigned Enc = MRI.getEncodingValue(Reg: MO.getReg());
629 unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK;
630 bool IsVGPR = Enc & AMDGPU::HWEncoding::IS_VGPR;
631 Op = Idx | (IsVGPR << 8);
632 return;
633 }
634 getMachineOpValueCommon(MI, MO, OpNo, Op, Fixups, STI);
635 // VGPRs include the suffix/op_sel bit in the register encoding, but
636 // immediates and SGPRs include it in src_modifiers. Therefore, copy the
637 // op_sel bit from the src operands into src_modifier operands if Op is
638 // src_modifiers and the corresponding src is a VGPR
639 int SrcMOIdx = -1;
640 assert(OpNo < INT_MAX);
641 if ((int)OpNo == AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
642 Name: AMDGPU::OpName::src0_modifiers)) {
643 SrcMOIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src0);
644 int VDstMOIdx =
645 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::vdst);
646 if (VDstMOIdx != -1) {
647 auto DstReg = MI.getOperand(i: VDstMOIdx).getReg();
648 if (AMDGPU::isHi16Reg(Reg: DstReg, MRI))
649 Op |= SISrcMods::DST_OP_SEL;
650 }
651 } else if ((int)OpNo == AMDGPU::getNamedOperandIdx(
652 Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src1_modifiers))
653 SrcMOIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src1);
654 else if ((int)OpNo == AMDGPU::getNamedOperandIdx(
655 Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src2_modifiers))
656 SrcMOIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src2);
657 if (SrcMOIdx == -1)
658 return;
659
660 const MCOperand &SrcMO = MI.getOperand(i: SrcMOIdx);
661 if (!SrcMO.isReg())
662 return;
663 auto SrcReg = SrcMO.getReg();
664 if (AMDGPU::isSGPR(Reg: SrcReg, TRI: &MRI))
665 return;
666 if (AMDGPU::isHi16Reg(Reg: SrcReg, MRI))
667 Op |= SISrcMods::OP_SEL_0;
668}
669
670void AMDGPUMCCodeEmitter::getMachineOpValueT16Lo128(
671 const MCInst &MI, unsigned OpNo, APInt &Op,
672 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {
673 const MCOperand &MO = MI.getOperand(i: OpNo);
674 if (MO.isReg()) {
675 uint16_t Encoding = MRI.getEncodingValue(Reg: MO.getReg());
676 unsigned RegIdx = Encoding & AMDGPU::HWEncoding::LO256_REG_IDX_MASK;
677 bool IsHi = Encoding & AMDGPU::HWEncoding::IS_HI16;
678 bool IsVGPR = Encoding & AMDGPU::HWEncoding::IS_VGPR;
679 assert((!IsVGPR || isUInt<7>(RegIdx)) && "VGPR0-VGPR127 expected!");
680 Op = (IsVGPR ? 0x100 : 0) | (IsHi ? 0x80 : 0) | RegIdx;
681 return;
682 }
683 getMachineOpValueCommon(MI, MO, OpNo, Op, Fixups, STI);
684}
685
686void AMDGPUMCCodeEmitter::getMachineOpValueCommon(
687 const MCInst &MI, const MCOperand &MO, unsigned OpNo, APInt &Op,
688 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {
689 bool isLikeImm = false;
690 int64_t Val;
691
692 if (MO.isImm()) {
693 Val = MO.getImm();
694 isLikeImm = true;
695 } else if (MO.isExpr() && MO.getExpr()->evaluateAsAbsolute(Res&: Val)) {
696 isLikeImm = true;
697 } else if (MO.isExpr()) {
698 // FIXME: If this is expression is PCRel or not should not depend on what
699 // the expression looks like. Given that this is just a general expression,
700 // it should probably be FK_Data_4 and whatever is producing
701 //
702 // s_add_u32 s2, s2, (extern_const_addrspace+16
703 //
704 // And expecting a PCRel should instead produce
705 //
706 // .Ltmp1:
707 // s_add_u32 s2, s2, (extern_const_addrspace+16)-.Ltmp1
708 bool PCRel = needsPCRel(Expr: MO.getExpr());
709 const MCInstrDesc &Desc = MCII.get(Opcode: MI.getOpcode());
710 uint32_t Offset = Desc.getSize();
711 assert(Offset == 4 || Offset == 8);
712 unsigned Size = AMDGPU::getOperandSize(Desc, OpNo);
713 MCFixupKind Kind = MCFixup::getDataKindForSize(Size);
714 addFixup(Fixups, Offset, Value: MO.getExpr(), Kind, PCRel);
715 }
716
717 const MCInstrDesc &Desc = MCII.get(Opcode: MI.getOpcode());
718 if (AMDGPU::isSISrcOperand(Desc, OpNo)) {
719 bool HasMandatoryLiteral =
720 AMDGPU::hasNamedOperand(Opcode: MI.getOpcode(), NamedIdx: AMDGPU::OpName::imm);
721 if (auto Enc = getLitEncoding(Desc, MO, OpNo, STI, HasMandatoryLiteral)) {
722 Op = *Enc;
723 return;
724 }
725
726 llvm_unreachable("Operand not supported for SISrc");
727 }
728
729 if (isLikeImm) {
730 Op = Val;
731 return;
732 }
733
734 llvm_unreachable("Encoding of this operand type is not supported yet.");
735}
736
737template <bool HasSrc0, bool HasSrc1, bool HasSrc2>
738APInt AMDGPUMCCodeEmitter::postEncodeVOP3(const MCInst &MI, APInt EncodedValue,
739 const MCSubtargetInfo &STI) const {
740 if (!AMDGPU::isGFX10Plus(STI))
741 return EncodedValue;
742 // Set unused source fields in VOP3 encodings to inline immediate 0 to avoid
743 // hardware conservatively assuming the instruction reads SGPRs.
744 constexpr uint64_t InlineImmediate0 = 0x80;
745 if (!HasSrc0)
746 EncodedValue |= InlineImmediate0 << 32;
747 if (!HasSrc1)
748 EncodedValue |= InlineImmediate0 << 41;
749 if (!HasSrc2)
750 EncodedValue |= InlineImmediate0 << 50;
751 return EncodedValue;
752}
753
754APInt AMDGPUMCCodeEmitter::postEncodeVOPCX(const MCInst &MI, APInt EncodedValue,
755 const MCSubtargetInfo &STI) const {
756 // GFX10+ v_cmpx opcodes promoted to VOP3 have implied dst=EXEC.
757 // Documentation requires dst to be encoded as EXEC (0x7E),
758 // but it looks like the actual value encoded for dst operand
759 // is ignored by HW. It was decided to define dst as "do not care"
760 // in td files to allow disassembler accept any dst value.
761 // However, dst is encoded as EXEC for compatibility with SP3.
762 [[maybe_unused]] const MCInstrDesc &Desc = MCII.get(Opcode: MI.getOpcode());
763 assert((Desc.TSFlags & SIInstrFlags::VOP3) &&
764 Desc.hasImplicitDefOfPhysReg(AMDGPU::EXEC));
765 EncodedValue |= MRI.getEncodingValue(Reg: AMDGPU::EXEC_LO) &
766 AMDGPU::HWEncoding::LO256_REG_IDX_MASK;
767 return postEncodeVOP3<true, true, false>(MI, EncodedValue, STI);
768}
769
770#include "AMDGPUGenMCCodeEmitter.inc"
771