1//===-- AMDGPUMCCodeEmitter.cpp - AMDGPU Code Emitter ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// The AMDGPU code emitter produces machine code that can be executed
11/// directly on the GPU device.
12//
13//===----------------------------------------------------------------------===//
14
15#include "MCTargetDesc/AMDGPUFixupKinds.h"
16#include "MCTargetDesc/AMDGPUMCExpr.h"
17#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
18#include "SIDefines.h"
19#include "Utils/AMDGPUBaseInfo.h"
20#include "llvm/ADT/APInt.h"
21#include "llvm/MC/MCCodeEmitter.h"
22#include "llvm/MC/MCContext.h"
23#include "llvm/MC/MCExpr.h"
24#include "llvm/MC/MCInstrInfo.h"
25#include "llvm/MC/MCRegisterInfo.h"
26#include "llvm/MC/MCSubtargetInfo.h"
27#include "llvm/Support/Casting.h"
28#include "llvm/Support/EndianStream.h"
29#include <optional>
30
31using namespace llvm;
32
33namespace {
34
35class AMDGPUMCCodeEmitter : public MCCodeEmitter {
36 const MCRegisterInfo &MRI;
37 const MCInstrInfo &MCII;
38
39public:
40 AMDGPUMCCodeEmitter(const MCInstrInfo &MCII, const MCRegisterInfo &MRI)
41 : MRI(MRI), MCII(MCII) {}
42
43 /// Encode the instruction and write it to the OS.
44 void encodeInstruction(const MCInst &MI, SmallVectorImpl<char> &CB,
45 SmallVectorImpl<MCFixup> &Fixups,
46 const MCSubtargetInfo &STI) const override;
47
48 void getMachineOpValue(const MCInst &MI, const MCOperand &MO, APInt &Op,
49 SmallVectorImpl<MCFixup> &Fixups,
50 const MCSubtargetInfo &STI) const;
51
52 void getMachineOpValueT16(const MCInst &MI, unsigned OpNo, APInt &Op,
53 SmallVectorImpl<MCFixup> &Fixups,
54 const MCSubtargetInfo &STI) const;
55
56 void getMachineOpValueT16Lo128(const MCInst &MI, unsigned OpNo, APInt &Op,
57 SmallVectorImpl<MCFixup> &Fixups,
58 const MCSubtargetInfo &STI) const;
59
60 void getMachineOpValueRsrcRegOp(const MCInst &MI, unsigned OpNo, APInt &Op,
61 SmallVectorImpl<MCFixup> &Fixups,
62 const MCSubtargetInfo &STI) const;
63
64 /// Use a fixup to encode the simm16 field for SOPP branch
65 /// instructions.
66 void getSOPPBrEncoding(const MCInst &MI, unsigned OpNo, APInt &Op,
67 SmallVectorImpl<MCFixup> &Fixups,
68 const MCSubtargetInfo &STI) const;
69
70 void getSMEMOffsetEncoding(const MCInst &MI, unsigned OpNo, APInt &Op,
71 SmallVectorImpl<MCFixup> &Fixups,
72 const MCSubtargetInfo &STI) const;
73
74 void getSDWASrcEncoding(const MCInst &MI, unsigned OpNo, APInt &Op,
75 SmallVectorImpl<MCFixup> &Fixups,
76 const MCSubtargetInfo &STI) const;
77
78 void getSDWAVopcDstEncoding(const MCInst &MI, unsigned OpNo, APInt &Op,
79 SmallVectorImpl<MCFixup> &Fixups,
80 const MCSubtargetInfo &STI) const;
81
82 void getAVOperandEncoding(const MCInst &MI, unsigned OpNo, APInt &Op,
83 SmallVectorImpl<MCFixup> &Fixups,
84 const MCSubtargetInfo &STI) const;
85
86private:
87 uint64_t getImplicitOpSelHiEncoding(int Opcode) const;
88 void getMachineOpValueCommon(const MCInst &MI, const MCOperand &MO,
89 unsigned OpNo, APInt &Op,
90 SmallVectorImpl<MCFixup> &Fixups,
91 const MCSubtargetInfo &STI) const;
92
93 /// Encode an fp or int literal.
94 std::optional<uint64_t>
95 getLitEncoding(const MCInstrDesc &Desc, const MCOperand &MO, unsigned OpNo,
96 const MCSubtargetInfo &STI,
97 bool HasMandatoryLiteral = false) const;
98
99 void getBinaryCodeForInstr(const MCInst &MI, SmallVectorImpl<MCFixup> &Fixups,
100 APInt &Inst, APInt &Scratch,
101 const MCSubtargetInfo &STI) const;
102
103 template <bool HasSrc0, bool HasSrc1, bool HasSrc2>
104 APInt postEncodeVOP3(const MCInst &MI, APInt EncodedValue,
105 const MCSubtargetInfo &STI) const;
106
107 APInt postEncodeVOPCX(const MCInst &MI, APInt EncodedValue,
108 const MCSubtargetInfo &STI) const;
109};
110
111} // end anonymous namespace
112
113MCCodeEmitter *llvm::createAMDGPUMCCodeEmitter(const MCInstrInfo &MCII,
114 MCContext &Ctx) {
115 return new AMDGPUMCCodeEmitter(MCII, *Ctx.getRegisterInfo());
116}
117
118static void addFixup(SmallVectorImpl<MCFixup> &Fixups, uint32_t Offset,
119 const MCExpr *Value, uint16_t Kind, bool PCRel = false) {
120 Fixups.push_back(Elt: MCFixup::create(Offset, Value, Kind, PCRel));
121}
122
123// Returns the encoding value to use if the given integer is an integer inline
124// immediate value, or 0 if it is not.
125template <typename IntTy>
126static uint32_t getIntInlineImmEncoding(IntTy Imm) {
127 if (Imm >= 0 && Imm <= 64)
128 return 128 + Imm;
129
130 if (Imm >= -16 && Imm <= -1)
131 return 192 + std::abs(Imm);
132
133 return 0;
134}
135
136static uint32_t getLit16Encoding(uint16_t Val, const MCSubtargetInfo &STI) {
137 uint16_t IntImm = getIntInlineImmEncoding(Imm: static_cast<int16_t>(Val));
138 if (IntImm != 0)
139 return IntImm;
140
141 if (Val == 0x3800) // 0.5
142 return 240;
143
144 if (Val == 0xB800) // -0.5
145 return 241;
146
147 if (Val == 0x3C00) // 1.0
148 return 242;
149
150 if (Val == 0xBC00) // -1.0
151 return 243;
152
153 if (Val == 0x4000) // 2.0
154 return 244;
155
156 if (Val == 0xC000) // -2.0
157 return 245;
158
159 if (Val == 0x4400) // 4.0
160 return 246;
161
162 if (Val == 0xC400) // -4.0
163 return 247;
164
165 if (Val == 0x3118 && // 1.0 / (2.0 * pi)
166 STI.hasFeature(Feature: AMDGPU::FeatureInv2PiInlineImm))
167 return 248;
168
169 return 255;
170}
171
172static uint32_t getLitBF16Encoding(uint16_t Val) {
173 uint16_t IntImm = getIntInlineImmEncoding(Imm: static_cast<int16_t>(Val));
174 if (IntImm != 0)
175 return IntImm;
176
177 // clang-format off
178 switch (Val) {
179 case 0x3F00: return 240; // 0.5
180 case 0xBF00: return 241; // -0.5
181 case 0x3F80: return 242; // 1.0
182 case 0xBF80: return 243; // -1.0
183 case 0x4000: return 244; // 2.0
184 case 0xC000: return 245; // -2.0
185 case 0x4080: return 246; // 4.0
186 case 0xC080: return 247; // -4.0
187 case 0x3E22: return 248; // 1.0 / (2.0 * pi)
188 default: return 255;
189 }
190 // clang-format on
191}
192
193static uint32_t getLit32Encoding(uint32_t Val, const MCSubtargetInfo &STI) {
194 uint32_t IntImm = getIntInlineImmEncoding(Imm: static_cast<int32_t>(Val));
195 if (IntImm != 0)
196 return IntImm;
197
198 if (Val == llvm::bit_cast<uint32_t>(from: 0.5f))
199 return 240;
200
201 if (Val == llvm::bit_cast<uint32_t>(from: -0.5f))
202 return 241;
203
204 if (Val == llvm::bit_cast<uint32_t>(from: 1.0f))
205 return 242;
206
207 if (Val == llvm::bit_cast<uint32_t>(from: -1.0f))
208 return 243;
209
210 if (Val == llvm::bit_cast<uint32_t>(from: 2.0f))
211 return 244;
212
213 if (Val == llvm::bit_cast<uint32_t>(from: -2.0f))
214 return 245;
215
216 if (Val == llvm::bit_cast<uint32_t>(from: 4.0f))
217 return 246;
218
219 if (Val == llvm::bit_cast<uint32_t>(from: -4.0f))
220 return 247;
221
222 if (Val == 0x3e22f983 && // 1.0 / (2.0 * pi)
223 STI.hasFeature(Feature: AMDGPU::FeatureInv2PiInlineImm))
224 return 248;
225
226 return 255;
227}
228
229static uint32_t getLit16IntEncoding(uint32_t Val, const MCSubtargetInfo &STI) {
230 return getLit32Encoding(Val, STI);
231}
232
233static uint32_t getLit64Encoding(const MCInstrDesc &Desc, uint64_t Val,
234 const MCSubtargetInfo &STI, bool IsFP) {
235 uint32_t IntImm = getIntInlineImmEncoding(Imm: static_cast<int64_t>(Val));
236 if (IntImm != 0)
237 return IntImm;
238
239 if (Val == llvm::bit_cast<uint64_t>(from: 0.5))
240 return 240;
241
242 if (Val == llvm::bit_cast<uint64_t>(from: -0.5))
243 return 241;
244
245 if (Val == llvm::bit_cast<uint64_t>(from: 1.0))
246 return 242;
247
248 if (Val == llvm::bit_cast<uint64_t>(from: -1.0))
249 return 243;
250
251 if (Val == llvm::bit_cast<uint64_t>(from: 2.0))
252 return 244;
253
254 if (Val == llvm::bit_cast<uint64_t>(from: -2.0))
255 return 245;
256
257 if (Val == llvm::bit_cast<uint64_t>(from: 4.0))
258 return 246;
259
260 if (Val == llvm::bit_cast<uint64_t>(from: -4.0))
261 return 247;
262
263 if (Val == 0x3fc45f306dc9c882 && // 1.0 / (2.0 * pi)
264 STI.hasFeature(Feature: AMDGPU::FeatureInv2PiInlineImm))
265 return 248;
266
267 // The rest part needs to align with AMDGPUInstPrinter::printLiteral64.
268
269 bool CanUse64BitLiterals = STI.hasFeature(Feature: AMDGPU::Feature64BitLiterals) &&
270 !SIInstrFlags::isVOP3Like(O: Desc);
271 if (IsFP) {
272 return CanUse64BitLiterals && Lo_32(Value: Val) ? 254 : 255;
273 }
274
275 return CanUse64BitLiterals && (!isInt<32>(x: Val) || !isUInt<32>(x: Val)) ? 254
276 : 255;
277}
278
279std::optional<uint64_t> AMDGPUMCCodeEmitter::getLitEncoding(
280 const MCInstrDesc &Desc, const MCOperand &MO, unsigned OpNo,
281 const MCSubtargetInfo &STI, bool HasMandatoryLiteral) const {
282 const MCOperandInfo &OpInfo = Desc.operands()[OpNo];
283 int64_t Imm = 0;
284 if (MO.isExpr()) {
285 if (!MO.getExpr()->evaluateAsAbsolute(Res&: Imm) ||
286 AMDGPU::isLitExpr(Expr: MO.getExpr())) {
287 if (OpInfo.OperandType == AMDGPU::OPERAND_KIMM16 ||
288 OpInfo.OperandType == AMDGPU::OPERAND_KIMM32 ||
289 OpInfo.OperandType == AMDGPU::OPERAND_KIMM64)
290 return Imm;
291 if (STI.hasFeature(Feature: AMDGPU::Feature64BitLiterals) &&
292 AMDGPU::getOperandSize(OpInfo) == 8 &&
293 AMDGPU::getExprKind(Expr: MO.getExpr()) != AMDGPUMCExpr::AGVK_Lit)
294 return 254;
295 return 255;
296 }
297 } else {
298 assert(!MO.isDFPImm());
299
300 if (!MO.isImm())
301 return {};
302
303 Imm = MO.getImm();
304 }
305
306 switch (OpInfo.OperandType) {
307 case AMDGPU::OPERAND_REG_IMM_INT32:
308 case AMDGPU::OPERAND_REG_IMM_FP32:
309 case AMDGPU::OPERAND_REG_INLINE_C_INT32:
310 case AMDGPU::OPERAND_REG_INLINE_C_FP32:
311 case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
312 case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
313 case AMDGPU::OPERAND_REG_IMM_V2INT32:
314 case AMDGPU::OPERAND_REG_IMM_V2FP32:
315 case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
316 return getLit32Encoding(Val: static_cast<uint32_t>(Imm), STI);
317
318 case AMDGPU::OPERAND_REG_IMM_INT64:
319 case AMDGPU::OPERAND_REG_INLINE_C_INT64:
320 case AMDGPU::OPERAND_REG_IMM_V2INT64:
321 return getLit64Encoding(Desc, Val: static_cast<uint64_t>(Imm), STI, IsFP: false);
322
323 case AMDGPU::OPERAND_REG_INLINE_C_FP64:
324 case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
325 return getLit64Encoding(Desc, Val: static_cast<uint64_t>(Imm), STI, IsFP: true);
326
327 case AMDGPU::OPERAND_REG_IMM_FP64:
328 case AMDGPU::OPERAND_REG_IMM_V2FP64: {
329 auto Enc = getLit64Encoding(Desc, Val: static_cast<uint64_t>(Imm), STI, IsFP: true);
330 return (HasMandatoryLiteral && Enc == 255) ? 254 : Enc;
331 }
332
333 case AMDGPU::OPERAND_REG_IMM_INT16:
334 case AMDGPU::OPERAND_REG_INLINE_C_INT16:
335 return getLit16IntEncoding(Val: static_cast<uint32_t>(Imm), STI);
336
337 case AMDGPU::OPERAND_REG_IMM_FP16:
338 case AMDGPU::OPERAND_REG_INLINE_C_FP16:
339 // FIXME Is this correct? What do inline immediates do on SI for f16 src
340 // which does not have f16 support?
341 return getLit16Encoding(Val: static_cast<uint16_t>(Imm), STI);
342
343 case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
344 return 255;
345
346 case AMDGPU::OPERAND_REG_IMM_BF16:
347 case AMDGPU::OPERAND_REG_INLINE_C_BF16:
348 // We don't actually need to check Inv2Pi here because BF16 instructions can
349 // only be emitted for targets that already support the feature.
350 return getLitBF16Encoding(Val: static_cast<uint16_t>(Imm));
351
352 case AMDGPU::OPERAND_REG_IMM_V2INT16:
353 case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
354 return AMDGPU::getInlineEncodingV2I16(Literal: static_cast<uint32_t>(Imm))
355 .value_or(u: 255);
356
357 case AMDGPU::OPERAND_REG_IMM_V2FP16:
358 case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
359 return AMDGPU::getInlineEncodingV2F16(Literal: static_cast<uint32_t>(Imm))
360 .value_or(u: 255);
361
362 case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
363 // V_PK_FMAC_F16 has different inline constant behavior on pre-GFX11 vs
364 // GFX11+: pre-GFX11 produces (f16, 0), GFX11+ duplicates f16 to both
365 // halves.
366 return AMDGPU::getPKFMACF16InlineEncoding(Literal: static_cast<uint32_t>(Imm),
367 IsGFX11Plus: AMDGPU::isGFX11Plus(STI))
368 .value_or(u: 255);
369
370 case AMDGPU::OPERAND_REG_IMM_V2BF16:
371 case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
372 return AMDGPU::getInlineEncodingV2BF16(Literal: static_cast<uint32_t>(Imm))
373 .value_or(u: 255);
374
375 case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
376 return 255;
377
378 case AMDGPU::OPERAND_KIMM32:
379 case AMDGPU::OPERAND_KIMM16:
380 case AMDGPU::OPERAND_KIMM64:
381 return Imm;
382 default:
383 llvm_unreachable("invalid operand size");
384 }
385}
386
387uint64_t AMDGPUMCCodeEmitter::getImplicitOpSelHiEncoding(int Opcode) const {
388 using namespace AMDGPU::VOP3PEncoding;
389
390 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::op_sel_hi)) {
391 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src2))
392 return 0;
393 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src1))
394 return OP_SEL_HI_2;
395 if (AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::src0))
396 return OP_SEL_HI_1 | OP_SEL_HI_2;
397 }
398 return OP_SEL_HI_0 | OP_SEL_HI_1 | OP_SEL_HI_2;
399}
400
401void AMDGPUMCCodeEmitter::encodeInstruction(const MCInst &MI,
402 SmallVectorImpl<char> &CB,
403 SmallVectorImpl<MCFixup> &Fixups,
404 const MCSubtargetInfo &STI) const {
405 int Opcode = MI.getOpcode();
406 APInt Encoding, Scratch;
407 getBinaryCodeForInstr(MI, Fixups, Inst&: Encoding, Scratch, STI);
408 const MCInstrDesc &Desc = MCII.get(Opcode: MI.getOpcode());
409 unsigned bytes = Desc.getSize();
410
411 // Set unused op_sel_hi bits to 1 for VOP3P and MAI instructions.
412 // Note that accvgpr_read/write are MAI, have src0, but do not use op_sel.
413 if ((SIInstrFlags::isVOP3P(O: Desc) || Opcode == AMDGPU::V_ACCVGPR_READ_B32_vi ||
414 Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_vi) &&
415 // Matrix B format operand reuses op_sel_hi.
416 !AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::matrix_b_fmt) &&
417 // Matrix B scale operand reuses op_sel_hi.
418 !AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::matrix_b_scale) &&
419 // Matrix B reuse operand reuses op_sel_hi.
420 !AMDGPU::hasNamedOperand(Opcode, NamedIdx: AMDGPU::OpName::matrix_b_reuse) &&
421 // Some VOP3P instructions have no op_sel_hi field, and SP3 encodes
422 // those bits as zero.
423 !AMDGPU::hasNoOpSelHi(Opc: Opcode)) {
424 Encoding |= getImplicitOpSelHiEncoding(Opcode);
425 }
426
427 for (unsigned i = 0; i < bytes; i++) {
428 CB.push_back(Elt: (uint8_t)Encoding.extractBitsAsZExtValue(numBits: 8, bitPosition: 8 * i));
429 }
430
431 // NSA encoding.
432 if (AMDGPU::isGFX10Plus(STI) && SIInstrFlags::isMIMG(O: Desc)) {
433 int vaddr0 = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
434 Name: AMDGPU::OpName::vaddr0);
435 int srsrc = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
436 Name: AMDGPU::OpName::srsrc);
437 assert(vaddr0 >= 0 && srsrc > vaddr0);
438 unsigned NumExtraAddrs = srsrc - vaddr0 - 1;
439 unsigned NumPadding = (-NumExtraAddrs) & 3;
440
441 for (unsigned i = 0; i < NumExtraAddrs; ++i) {
442 getMachineOpValue(MI, MO: MI.getOperand(i: vaddr0 + 1 + i), Op&: Encoding, Fixups,
443 STI);
444 CB.push_back(Elt: (uint8_t)Encoding.getLimitedValue());
445 }
446 CB.append(NumInputs: NumPadding, Elt: 0);
447 }
448
449 if ((bytes > 8 && STI.hasFeature(Feature: AMDGPU::FeatureVOP3Literal)) ||
450 (bytes > 4 && !STI.hasFeature(Feature: AMDGPU::FeatureVOP3Literal)))
451 return;
452
453 // Do not print literals from SISrc Operands for insts with mandatory literals
454 if (AMDGPU::hasNamedOperand(Opcode: MI.getOpcode(), NamedIdx: AMDGPU::OpName::imm))
455 return;
456
457 // Check for additional literals
458 for (unsigned i = 0, e = Desc.getNumOperands(); i < e; ++i) {
459
460 // Check if this operand should be encoded as [SV]Src
461 if (!AMDGPU::isSISrcOperand(Desc, OpNo: i))
462 continue;
463
464 // Is this operand a literal immediate?
465 const MCOperand &Op = MI.getOperand(i);
466 auto Enc = getLitEncoding(Desc, MO: Op, OpNo: i, STI);
467 if (!Enc || (*Enc != 255 && *Enc != 254))
468 continue;
469
470 // Yes! Encode it
471 int64_t Imm = 0;
472
473 bool IsLit = false;
474 if (Op.isImm())
475 Imm = Op.getImm();
476 else if (Op.isExpr()) {
477 if (const auto *C = dyn_cast<MCConstantExpr>(Val: Op.getExpr())) {
478 Imm = C->getValue();
479 } else if (AMDGPU::isLitExpr(Expr: Op.getExpr())) {
480 IsLit = true;
481 Imm = AMDGPU::getLitValue(Expr: Op.getExpr());
482 }
483 } else // Exprs will be replaced with a fixup value.
484 llvm_unreachable("Must be immediate or expr");
485
486 if (*Enc == 254) {
487 assert(STI.hasFeature(AMDGPU::Feature64BitLiterals));
488 support::endian::write<uint64_t>(Out&: CB, V: Imm, E: llvm::endianness::little);
489 } else {
490 auto OpType =
491 static_cast<AMDGPU::OperandType>(Desc.operands()[i].OperandType);
492 Imm = AMDGPU::encode32BitLiteral(Imm, Type: OpType, IsLit);
493 support::endian::write<uint32_t>(Out&: CB, V: Imm, E: llvm::endianness::little);
494 }
495
496 // Only one literal value allowed
497 break;
498 }
499}
500
501void AMDGPUMCCodeEmitter::getSOPPBrEncoding(const MCInst &MI, unsigned OpNo,
502 APInt &Op,
503 SmallVectorImpl<MCFixup> &Fixups,
504 const MCSubtargetInfo &STI) const {
505 const MCOperand &MO = MI.getOperand(i: OpNo);
506
507 if (MO.isExpr()) {
508 const MCExpr *Expr = MO.getExpr();
509 addFixup(Fixups, Offset: 0, Value: Expr, Kind: AMDGPU::fixup_si_sopp_br, PCRel: true);
510 Op = APInt::getZero(numBits: 96);
511 } else {
512 getMachineOpValue(MI, MO, Op, Fixups, STI);
513 }
514}
515
516void AMDGPUMCCodeEmitter::getSMEMOffsetEncoding(
517 const MCInst &MI, unsigned OpNo, APInt &Op,
518 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {
519 auto Offset = MI.getOperand(i: OpNo).getImm();
520 // VI only supports 20-bit unsigned offsets.
521 assert(!AMDGPU::isVI(STI) || isUInt<20>(Offset));
522 Op = Offset;
523}
524
525void AMDGPUMCCodeEmitter::getSDWASrcEncoding(const MCInst &MI, unsigned OpNo,
526 APInt &Op,
527 SmallVectorImpl<MCFixup> &Fixups,
528 const MCSubtargetInfo &STI) const {
529 using namespace AMDGPU::SDWA;
530
531 uint64_t RegEnc = 0;
532
533 const MCOperand &MO = MI.getOperand(i: OpNo);
534
535 if (MO.isReg()) {
536 MCRegister Reg = MO.getReg();
537 RegEnc |= MRI.getEncodingValue(Reg);
538 RegEnc &= SDWA9EncValues::SRC_VGPR_MASK;
539 if (AMDGPU::isSGPR(Reg: AMDGPU::mc2PseudoReg(Reg), TRI: &MRI)) {
540 RegEnc |= SDWA9EncValues::SRC_SGPR_MASK;
541 }
542 Op = RegEnc;
543 return;
544 } else {
545 const MCInstrDesc &Desc = MCII.get(Opcode: MI.getOpcode());
546 auto Enc = getLitEncoding(Desc, MO, OpNo, STI);
547 if (Enc && *Enc != 255) {
548 Op = *Enc | SDWA9EncValues::SRC_SGPR_MASK;
549 return;
550 }
551 }
552
553 llvm_unreachable("Unsupported operand kind");
554}
555
556void AMDGPUMCCodeEmitter::getSDWAVopcDstEncoding(
557 const MCInst &MI, unsigned OpNo, APInt &Op,
558 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {
559 using namespace AMDGPU::SDWA;
560
561 uint64_t RegEnc = 0;
562
563 const MCOperand &MO = MI.getOperand(i: OpNo);
564
565 MCRegister Reg = MO.getReg();
566 if (Reg != AMDGPU::VCC && Reg != AMDGPU::VCC_LO) {
567 RegEnc |= MRI.getEncodingValue(Reg);
568 RegEnc &= SDWA9EncValues::VOPC_DST_SGPR_MASK;
569 RegEnc |= SDWA9EncValues::VOPC_DST_VCC_MASK;
570 }
571 Op = RegEnc;
572}
573
574void AMDGPUMCCodeEmitter::getAVOperandEncoding(
575 const MCInst &MI, unsigned OpNo, APInt &Op,
576 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {
577 MCRegister Reg = MI.getOperand(i: OpNo).getReg();
578 unsigned Enc = MRI.getEncodingValue(Reg);
579 unsigned Idx = Enc & AMDGPU::HWEncoding::LO256_REG_IDX_MASK;
580 bool IsVGPROrAGPR =
581 Enc & (AMDGPU::HWEncoding::IS_VGPR | AMDGPU::HWEncoding::IS_AGPR);
582
583 // VGPR and AGPR have the same encoding, but SrcA and SrcB operands of mfma
584 // instructions use acc[0:1] modifier bits to distinguish. These bits are
585 // encoded as a virtual 9th bit of the register for these operands.
586 bool IsAGPR = Enc & AMDGPU::HWEncoding::IS_AGPR;
587
588 Op = Idx | (IsVGPROrAGPR << 8) | (IsAGPR << 9);
589}
590
591static bool needsPCRel(const MCExpr *Expr) {
592 switch (Expr->getKind()) {
593 case MCExpr::SymbolRef: {
594 auto *SE = cast<MCSymbolRefExpr>(Val: Expr);
595 auto Spec = AMDGPU::getSpecifier(SRE: SE);
596 return Spec != AMDGPUMCExpr::S_ABS32_LO &&
597 Spec != AMDGPUMCExpr::S_ABS32_HI && Spec != AMDGPUMCExpr::S_ABS64;
598 }
599 case MCExpr::Binary: {
600 auto *BE = cast<MCBinaryExpr>(Val: Expr);
601 if (BE->getOpcode() == MCBinaryExpr::Sub)
602 return false;
603 return needsPCRel(Expr: BE->getLHS()) || needsPCRel(Expr: BE->getRHS());
604 }
605 case MCExpr::Unary:
606 return needsPCRel(Expr: cast<MCUnaryExpr>(Val: Expr)->getSubExpr());
607 case MCExpr::Specifier:
608 case MCExpr::Target:
609 case MCExpr::Constant:
610 return false;
611 }
612 llvm_unreachable("invalid kind");
613}
614
615void AMDGPUMCCodeEmitter::getMachineOpValue(const MCInst &MI,
616 const MCOperand &MO, APInt &Op,
617 SmallVectorImpl<MCFixup> &Fixups,
618 const MCSubtargetInfo &STI) const {
619 if (MO.isReg()){
620 unsigned Enc = MRI.getEncodingValue(Reg: MO.getReg());
621 unsigned Idx = Enc & AMDGPU::HWEncoding::LO256_REG_IDX_MASK;
622 bool IsVGPROrAGPR =
623 Enc & (AMDGPU::HWEncoding::IS_VGPR | AMDGPU::HWEncoding::IS_AGPR);
624 Op = Idx | (IsVGPROrAGPR << 8);
625 return;
626 }
627 unsigned OpNo = &MO - MI.begin();
628 getMachineOpValueCommon(MI, MO, OpNo, Op, Fixups, STI);
629}
630
631void AMDGPUMCCodeEmitter::getMachineOpValueT16(
632 const MCInst &MI, unsigned OpNo, APInt &Op,
633 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {
634 const MCOperand &MO = MI.getOperand(i: OpNo);
635 if (MO.isReg()) {
636 unsigned Enc = MRI.getEncodingValue(Reg: MO.getReg());
637 unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK;
638 bool IsVGPR = Enc & AMDGPU::HWEncoding::IS_VGPR;
639 Op = Idx | (IsVGPR << 8);
640 return;
641 }
642 getMachineOpValueCommon(MI, MO, OpNo, Op, Fixups, STI);
643 // VGPRs include the suffix/op_sel bit in the register encoding, but
644 // immediates and SGPRs include it in src_modifiers. Therefore, copy the
645 // op_sel bit from the src operands into src_modifier operands if Op is
646 // src_modifiers and the corresponding src is a VGPR
647 int SrcMOIdx = -1;
648 assert(OpNo < INT_MAX);
649 if ((int)OpNo == AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
650 Name: AMDGPU::OpName::src0_modifiers)) {
651 SrcMOIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src0);
652 int VDstMOIdx =
653 AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::vdst);
654 if (VDstMOIdx != -1) {
655 auto DstReg = MI.getOperand(i: VDstMOIdx).getReg();
656 if (AMDGPU::isHi16Reg(Reg: DstReg, MRI))
657 Op |= SISrcMods::DST_OP_SEL;
658 }
659 } else if ((int)OpNo == AMDGPU::getNamedOperandIdx(
660 Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src1_modifiers))
661 SrcMOIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src1);
662 else if ((int)OpNo == AMDGPU::getNamedOperandIdx(
663 Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src2_modifiers))
664 SrcMOIdx = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: AMDGPU::OpName::src2);
665 if (SrcMOIdx == -1)
666 return;
667
668 const MCOperand &SrcMO = MI.getOperand(i: SrcMOIdx);
669 if (!SrcMO.isReg())
670 return;
671 auto SrcReg = SrcMO.getReg();
672 if (AMDGPU::isSGPR(Reg: SrcReg, TRI: &MRI))
673 return;
674 if (AMDGPU::isHi16Reg(Reg: SrcReg, MRI))
675 Op |= SISrcMods::OP_SEL_0;
676}
677
678void AMDGPUMCCodeEmitter::getMachineOpValueT16Lo128(
679 const MCInst &MI, unsigned OpNo, APInt &Op,
680 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {
681 const MCOperand &MO = MI.getOperand(i: OpNo);
682 if (MO.isReg()) {
683 uint16_t Encoding = MRI.getEncodingValue(Reg: MO.getReg());
684 unsigned RegIdx = Encoding & AMDGPU::HWEncoding::LO256_REG_IDX_MASK;
685 bool IsHi = Encoding & AMDGPU::HWEncoding::IS_HI16;
686 bool IsVGPR = Encoding & AMDGPU::HWEncoding::IS_VGPR;
687 assert((!IsVGPR || isUInt<7>(RegIdx)) && "VGPR0-VGPR127 expected!");
688 Op = (IsVGPR ? 0x100 : 0) | (IsHi ? 0x80 : 0) | RegIdx;
689 return;
690 }
691 getMachineOpValueCommon(MI, MO, OpNo, Op, Fixups, STI);
692}
693
694// Encode an indexed-resource (rsrcidx) operand into the 9-bit srsrc field:
695// bit 8 selects a VGPR (or AGPR) index register, bit 7 selects a 32-bit SGPR
696// index register, and bits 6-0 hold the register index.
697void AMDGPUMCCodeEmitter::getMachineOpValueRsrcRegOp(
698 const MCInst &MI, unsigned OpNo, APInt &Op,
699 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {
700 const MCOperand &MO = MI.getOperand(i: OpNo);
701 bool IsSReg32 =
702 MRI.getRegClass(i: AMDGPU::SReg_32RegClassID).contains(Reg: MO.getReg());
703 unsigned Enc = MRI.getEncodingValue(Reg: MO.getReg());
704 unsigned Idx = Enc & AMDGPU::HWEncoding::LO256_REG_IDX_MASK;
705 bool IsVGPROrAGPR =
706 Enc & (AMDGPU::HWEncoding::IS_VGPR | AMDGPU::HWEncoding::IS_AGPR);
707 Op = Idx | IsVGPROrAGPR << 8 | IsSReg32 << 7;
708}
709
710void AMDGPUMCCodeEmitter::getMachineOpValueCommon(
711 const MCInst &MI, const MCOperand &MO, unsigned OpNo, APInt &Op,
712 SmallVectorImpl<MCFixup> &Fixups, const MCSubtargetInfo &STI) const {
713 bool isLikeImm = false;
714 int64_t Val;
715
716 if (MO.isImm()) {
717 Val = MO.getImm();
718 isLikeImm = true;
719 } else if (MO.isExpr() && MO.getExpr()->evaluateAsAbsolute(Res&: Val)) {
720 isLikeImm = true;
721 } else if (MO.isExpr()) {
722 // FIXME: If this is expression is PCRel or not should not depend on what
723 // the expression looks like. Given that this is just a general expression,
724 // it should probably be FK_Data_4 and whatever is producing
725 //
726 // s_add_u32 s2, s2, (extern_const_addrspace+16
727 //
728 // And expecting a PCRel should instead produce
729 //
730 // .Ltmp1:
731 // s_add_u32 s2, s2, (extern_const_addrspace+16)-.Ltmp1
732 bool PCRel = needsPCRel(Expr: MO.getExpr());
733 const MCInstrDesc &Desc = MCII.get(Opcode: MI.getOpcode());
734 uint32_t Offset = Desc.getSize();
735 assert(Offset == 4 || Offset == 8);
736 unsigned Size = AMDGPU::getOperandSize(Desc, OpNo);
737 MCFixupKind Kind = MCFixup::getDataKindForSize(Size);
738 addFixup(Fixups, Offset, Value: MO.getExpr(), Kind, PCRel);
739 }
740
741 const MCInstrDesc &Desc = MCII.get(Opcode: MI.getOpcode());
742 if (AMDGPU::isSISrcOperand(Desc, OpNo)) {
743 bool HasMandatoryLiteral =
744 AMDGPU::hasNamedOperand(Opcode: MI.getOpcode(), NamedIdx: AMDGPU::OpName::imm);
745 if (auto Enc = getLitEncoding(Desc, MO, OpNo, STI, HasMandatoryLiteral)) {
746 Op = *Enc;
747 return;
748 }
749
750 llvm_unreachable("Operand not supported for SISrc");
751 }
752
753 if (isLikeImm) {
754 Op = Val;
755 return;
756 }
757
758 llvm_unreachable("Encoding of this operand type is not supported yet.");
759}
760
761template <bool HasSrc0, bool HasSrc1, bool HasSrc2>
762APInt AMDGPUMCCodeEmitter::postEncodeVOP3(const MCInst &MI, APInt EncodedValue,
763 const MCSubtargetInfo &STI) const {
764 if (!AMDGPU::isGFX10Plus(STI))
765 return EncodedValue;
766 // Set unused source fields in VOP3 encodings to inline immediate 0 to avoid
767 // hardware conservatively assuming the instruction reads SGPRs.
768 constexpr uint64_t InlineImmediate0 = 0x80;
769 if (!HasSrc0)
770 EncodedValue |= InlineImmediate0 << 32;
771 if (!HasSrc1)
772 EncodedValue |= InlineImmediate0 << 41;
773 if (!HasSrc2)
774 EncodedValue |= InlineImmediate0 << 50;
775 return EncodedValue;
776}
777
778APInt AMDGPUMCCodeEmitter::postEncodeVOPCX(const MCInst &MI, APInt EncodedValue,
779 const MCSubtargetInfo &STI) const {
780 // GFX10+ v_cmpx opcodes promoted to VOP3 have implied dst=EXEC.
781 // Documentation requires dst to be encoded as EXEC (0x7E),
782 // but it looks like the actual value encoded for dst operand
783 // is ignored by HW. It was decided to define dst as "do not care"
784 // in td files to allow disassembler accept any dst value.
785 // However, dst is encoded as EXEC for compatibility with SP3.
786 assert(SIInstrFlags::isVOP3(MCII, MI) &&
787 MCII.get(MI.getOpcode()).hasImplicitDefOfPhysReg(AMDGPU::EXEC));
788 EncodedValue |= MRI.getEncodingValue(Reg: AMDGPU::EXEC_LO) &
789 AMDGPU::HWEncoding::LO256_REG_IDX_MASK;
790 return postEncodeVOP3<true, true, false>(MI, EncodedValue, STI);
791}
792
793#include "AMDGPUGenMCCodeEmitter.inc"
794