1//===- AMDGPUMCInstLower.cpp - Lower AMDGPU MachineInstr to an MCInst -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Code to lower AMDGPU MachineInstrs to their corresponding MCInst.
11//
12//===----------------------------------------------------------------------===//
13//
14
15#include "AMDGPUMCInstLower.h"
16#include "AMDGPUAsmPrinter.h"
17#include "MCTargetDesc/AMDGPUInstPrinter.h"
18#include "MCTargetDesc/AMDGPUMCExpr.h"
19#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
20#include "SIMachineFunctionInfo.h"
21#include "llvm/CodeGen/MachineBasicBlock.h"
22#include "llvm/CodeGen/MachineInstr.h"
23#include "llvm/IR/Constants.h"
24#include "llvm/IR/Function.h"
25#include "llvm/IR/GlobalVariable.h"
26#include "llvm/MC/MCCodeEmitter.h"
27#include "llvm/MC/MCContext.h"
28#include "llvm/MC/MCExpr.h"
29#include "llvm/MC/MCInst.h"
30#include "llvm/MC/MCObjectStreamer.h"
31#include "llvm/MC/MCStreamer.h"
32#include "llvm/Support/AMDGPUAddrSpace.h"
33#include "llvm/Support/Endian.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/Format.h"
36#include <algorithm>
37
38using namespace llvm;
39
40#include "AMDGPUGenMCPseudoLowering.inc"
41
42AMDGPUMCInstLower::AMDGPUMCInstLower(MCContext &ctx,
43 const TargetSubtargetInfo &st,
44 const AsmPrinter &ap):
45 Ctx(ctx), ST(st), AP(ap) { }
46
47static AMDGPUMCExpr::Specifier getSpecifier(unsigned MOFlags) {
48 switch (MOFlags) {
49 default:
50 return AMDGPUMCExpr::S_None;
51 case SIInstrInfo::MO_GOTPCREL:
52 case SIInstrInfo::MO_GOTPCREL64:
53 return AMDGPUMCExpr::S_GOTPCREL;
54 case SIInstrInfo::MO_GOTPCREL32_LO:
55 return AMDGPUMCExpr::S_GOTPCREL32_LO;
56 case SIInstrInfo::MO_GOTPCREL32_HI:
57 return AMDGPUMCExpr::S_GOTPCREL32_HI;
58 case SIInstrInfo::MO_REL32_LO:
59 return AMDGPUMCExpr::S_REL32_LO;
60 case SIInstrInfo::MO_REL32_HI:
61 return AMDGPUMCExpr::S_REL32_HI;
62 case SIInstrInfo::MO_REL64:
63 return AMDGPUMCExpr::S_REL64;
64 case SIInstrInfo::MO_ABS32_LO:
65 return AMDGPUMCExpr::S_ABS32_LO;
66 case SIInstrInfo::MO_ABS32_HI:
67 return AMDGPUMCExpr::S_ABS32_HI;
68 case SIInstrInfo::MO_ABS64:
69 return AMDGPUMCExpr::S_ABS64;
70 }
71}
72
73bool AMDGPUMCInstLower::lowerOperand(const MachineOperand &MO,
74 MCOperand &MCOp) const {
75 switch (MO.getType()) {
76 default:
77 break;
78 case MachineOperand::MO_Immediate:
79 MCOp = MCOperand::createImm(Val: MO.getImm());
80 return true;
81 case MachineOperand::MO_Register:
82 MCOp = MCOperand::createReg(Reg: AMDGPU::getMCReg(Reg: MO.getReg(), STI: ST));
83 return true;
84 case MachineOperand::MO_MachineBasicBlock:
85 MCOp = MCOperand::createExpr(
86 Val: MCSymbolRefExpr::create(Symbol: MO.getMBB()->getSymbol(), Ctx));
87 return true;
88 case MachineOperand::MO_GlobalAddress: {
89 const GlobalValue *GV = MO.getGlobal();
90 SmallString<128> SymbolName;
91 AP.getNameWithPrefix(Name&: SymbolName, GV);
92 MCSymbol *Sym = Ctx.getOrCreateSymbol(Name: SymbolName);
93 const MCExpr *Expr =
94 MCSymbolRefExpr::create(Symbol: Sym, specifier: getSpecifier(MOFlags: MO.getTargetFlags()), Ctx);
95 int64_t Offset = MO.getOffset();
96 if (Offset != 0) {
97 Expr = MCBinaryExpr::createAdd(LHS: Expr,
98 RHS: MCConstantExpr::create(Value: Offset, Ctx), Ctx);
99 }
100 MCOp = MCOperand::createExpr(Val: Expr);
101 return true;
102 }
103 case MachineOperand::MO_ExternalSymbol: {
104 MCSymbol *Sym = Ctx.getOrCreateSymbol(Name: StringRef(MO.getSymbolName()));
105 const MCExpr *Expr =
106 MCSymbolRefExpr::create(Symbol: Sym, specifier: getSpecifier(MOFlags: MO.getTargetFlags()), Ctx);
107 MCOp = MCOperand::createExpr(Val: Expr);
108 return true;
109 }
110 case MachineOperand::MO_BlockAddress: {
111 MCSymbol *Sym = AP.GetBlockAddressSymbol(BA: MO.getBlockAddress());
112 const MCSymbolRefExpr *Expr =
113 MCSymbolRefExpr::create(Symbol: Sym, specifier: getSpecifier(MOFlags: MO.getTargetFlags()), Ctx);
114 assert(MO.getOffset() == 0);
115 MCOp = MCOperand::createExpr(Val: Expr);
116 return true;
117 }
118 case MachineOperand::MO_RegisterMask:
119 // Regmasks are like implicit defs.
120 return false;
121 case MachineOperand::MO_MCSymbol:
122 if (MO.getTargetFlags() == SIInstrInfo::MO_FAR_BRANCH_OFFSET) {
123 MCSymbol *Sym = MO.getMCSymbol();
124 MCOp = MCOperand::createExpr(Val: Sym->getVariableValue());
125 return true;
126 }
127 break;
128 }
129 llvm_unreachable("unknown operand type");
130}
131
132// Lower true16 D16 Pseudo instruction to d16_lo/d16_hi MCInst based on
133// Dst/Data's .l/.h selection
134void AMDGPUMCInstLower::lowerT16D16Helper(const MachineInstr *MI,
135 MCInst &OutMI) const {
136 unsigned Opcode = MI->getOpcode();
137 const auto *TII = static_cast<const SIInstrInfo*>(ST.getInstrInfo());
138 const SIRegisterInfo &TRI = TII->getRegisterInfo();
139 const auto *Info = AMDGPU::getT16D16Helper(T16Op: Opcode);
140
141 llvm::AMDGPU::OpName OpName;
142 if (TII->isDS(Opcode)) {
143 if (MI->mayLoad())
144 OpName = llvm::AMDGPU::OpName::vdst;
145 else if (MI->mayStore())
146 OpName = llvm::AMDGPU::OpName::data0;
147 else
148 llvm_unreachable("LDS load or store expected");
149 } else {
150 OpName = AMDGPU::hasNamedOperand(Opcode, NamedIdx: llvm::AMDGPU::OpName::vdata)
151 ? llvm::AMDGPU::OpName::vdata
152 : llvm::AMDGPU::OpName::vdst;
153 }
154
155 // select Dst/Data
156 int VDstOrVDataIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: OpName);
157 const MachineOperand &MIVDstOrVData = MI->getOperand(i: VDstOrVDataIdx);
158
159 // select hi/lo MCInst
160 bool IsHi = AMDGPU::isHi16Reg(Reg: MIVDstOrVData.getReg(), MRI: TRI);
161 Opcode = IsHi ? Info->HiOp : Info->LoOp;
162
163 int MCOpcode = TII->pseudoToMCOpcode(Opcode);
164 assert(MCOpcode != -1 &&
165 "Pseudo instruction doesn't have a target-specific version");
166 OutMI.setOpcode(MCOpcode);
167
168 // lower operands
169 for (int I = 0, E = MI->getNumExplicitOperands(); I < E; I++) {
170 const MachineOperand &MO = MI->getOperand(i: I);
171 MCOperand MCOp;
172 if (I == VDstOrVDataIdx)
173 MCOp = MCOperand::createReg(Reg: TRI.get32BitRegister(Reg: MIVDstOrVData.getReg()));
174 else
175 lowerOperand(MO, MCOp);
176 OutMI.addOperand(Op: MCOp);
177 }
178
179 if (AMDGPU::hasNamedOperand(Opcode: MCOpcode, NamedIdx: AMDGPU::OpName::vdst_in)) {
180 MCOperand MCOp;
181 lowerOperand(MO: MIVDstOrVData, MCOp);
182 OutMI.addOperand(Op: MCOp);
183 }
184}
185
186void AMDGPUMCInstLower::lowerT16FmaMixFP16(const MachineInstr *MI,
187 MCInst &OutMI) const {
188 unsigned Opcode = MI->getOpcode();
189 const auto *TII = static_cast<const SIInstrInfo *>(ST.getInstrInfo());
190 const SIRegisterInfo &TRI = TII->getRegisterInfo();
191
192 int VDstIdx = AMDGPU::getNamedOperandIdx(Opcode, Name: llvm::AMDGPU::OpName::vdst);
193 const MachineOperand &VDst = MI->getOperand(i: VDstIdx);
194 bool IsHi = AMDGPU::isHi16Reg(Reg: VDst.getReg(), MRI: TRI);
195 switch (Opcode) {
196 case AMDGPU::V_FMA_MIX_F16_t16:
197 Opcode = IsHi ? AMDGPU::V_FMA_MIXHI_F16 : AMDGPU::V_FMA_MIXLO_F16;
198 break;
199 case AMDGPU::V_FMA_MIX_BF16_t16:
200 Opcode = IsHi ? AMDGPU::V_FMA_MIXHI_BF16 : AMDGPU::V_FMA_MIXLO_BF16;
201 break;
202 }
203 int MCOpcode = TII->pseudoToMCOpcode(Opcode);
204 assert(MCOpcode != -1 &&
205 "Pseudo instruction doesn't have a target-specific version");
206 OutMI.setOpcode(MCOpcode);
207
208 // lower operands
209 for (int I = 0, E = MI->getNumExplicitOperands(); I < E; I++) {
210 const MachineOperand &MO = MI->getOperand(i: I);
211 MCOperand MCOp;
212 if (I == VDstIdx)
213 MCOp = MCOperand::createReg(Reg: TRI.get32BitRegister(Reg: VDst.getReg()));
214 else
215 lowerOperand(MO, MCOp);
216 OutMI.addOperand(Op: MCOp);
217 }
218}
219
220void AMDGPUMCInstLower::lower(const MachineInstr *MI, MCInst &OutMI) const {
221 unsigned Opcode = MI->getOpcode();
222 const auto *TII = static_cast<const SIInstrInfo *>(ST.getInstrInfo());
223
224 // FIXME: Should be able to handle this with lowerPseudoInstExpansion. We
225 // need to select it to the subtarget specific version, and there's no way to
226 // do that with a single pseudo source operation.
227 if (Opcode == AMDGPU::S_SETPC_B64_return)
228 Opcode = AMDGPU::S_SETPC_B64;
229 else if (Opcode == AMDGPU::SI_CALL) {
230 // SI_CALL is just S_SWAPPC_B64 with an additional operand to track the
231 // called function (which we need to remove here).
232 OutMI.setOpcode(TII->pseudoToMCOpcode(Opcode: AMDGPU::S_SWAPPC_B64));
233 MCOperand Dest, Src;
234 lowerOperand(MO: MI->getOperand(i: 0), MCOp&: Dest);
235 lowerOperand(MO: MI->getOperand(i: 1), MCOp&: Src);
236 OutMI.addOperand(Op: Dest);
237 OutMI.addOperand(Op: Src);
238 return;
239 } else if (Opcode == AMDGPU::SI_TCRETURN ||
240 Opcode == AMDGPU::SI_TCRETURN_GFX ||
241 Opcode == AMDGPU::SI_TCRETURN_CHAIN) {
242 // TODO: How to use branch immediate and avoid register+add?
243 Opcode = AMDGPU::S_SETPC_B64;
244 } else if (AMDGPU::getT16D16Helper(T16Op: Opcode)) {
245 lowerT16D16Helper(MI, OutMI);
246 return;
247 } else if (Opcode == AMDGPU::V_FMA_MIX_F16_t16 ||
248 Opcode == AMDGPU::V_FMA_MIX_BF16_t16) {
249 lowerT16FmaMixFP16(MI, OutMI);
250 return;
251 }
252
253 int MCOpcode = TII->pseudoToMCOpcode(Opcode);
254 if (MCOpcode == -1) {
255 LLVMContext &C = MI->getMF()->getFunction().getContext();
256 C.emitError(ErrorStr: "AMDGPUMCInstLower::lower - Pseudo instruction doesn't have "
257 "a target-specific version: " + Twine(MI->getOpcode()));
258 return;
259 }
260
261 OutMI.setOpcode(MCOpcode);
262
263 for (const MachineOperand &MO : MI->explicit_operands()) {
264 MCOperand MCOp;
265 lowerOperand(MO, MCOp);
266 OutMI.addOperand(Op: MCOp);
267 }
268
269 int FIIdx = AMDGPU::getNamedOperandIdx(Opcode: MCOpcode, Name: AMDGPU::OpName::fi);
270 if (FIIdx >= (int)OutMI.getNumOperands())
271 OutMI.addOperand(Op: MCOperand::createImm(Val: 0));
272}
273
274bool AMDGPUAsmPrinter::lowerOperand(const MachineOperand &MO,
275 MCOperand &MCOp) const {
276 const GCNSubtarget &STI = MF->getSubtarget<GCNSubtarget>();
277 AMDGPUMCInstLower MCInstLowering(OutContext, STI, *this);
278 return MCInstLowering.lowerOperand(MO, MCOp);
279}
280
281const MCExpr *AMDGPUAsmPrinter::lowerConstant(const Constant *CV,
282 const Constant *BaseCV,
283 uint64_t Offset) {
284
285 // Intercept LDS variables with known addresses
286 if (const GlobalVariable *GV = dyn_cast<const GlobalVariable>(Val: CV)) {
287 if (std::optional<uint32_t> Address =
288 AMDGPUMachineFunctionInfo::get32BitAbsoluteAddress(
289 GV: *GV, AS: AMDGPUAS::LOCAL_ADDRESS)) {
290 auto *IntTy = Type::getInt32Ty(C&: CV->getContext());
291 return AsmPrinter::lowerConstant(CV: ConstantInt::get(Ty: IntTy, V: *Address),
292 BaseCV, Offset);
293 }
294 }
295
296 if (const MCExpr *E = lowerAddrSpaceCast(CV, OutContext))
297 return E;
298 return AsmPrinter::lowerConstant(CV, BaseCV, Offset);
299}
300
301static void emitVGPRBlockComment(const MachineInstr *MI, const SIInstrInfo *TII,
302 const TargetRegisterInfo *TRI,
303 const SIMachineFunctionInfo *MFI,
304 MCStreamer &OS) {
305 // The instruction will only transfer a subset of the registers in the block,
306 // based on the mask that is stored in m0. We could search for the instruction
307 // that sets m0, but most of the time we'll already have the mask stored in
308 // the machine function info. Try to use that. This assumes that we only use
309 // block loads/stores for CSR spills.
310 Register RegBlock =
311 TII->getNamedOperand(MI: *MI, OperandName: MI->mayLoad() ? AMDGPU::OpName::vdst
312 : AMDGPU::OpName::vdata)
313 ->getReg();
314 Register FirstRegInBlock = TRI->getSubReg(Reg: RegBlock, Idx: AMDGPU::sub0);
315 uint32_t Mask = MFI->getMaskForVGPRBlockOps(RegisterBlock: RegBlock);
316
317 if (!Mask)
318 return; // Nothing to report
319
320 SmallString<512> TransferredRegs;
321 for (unsigned I = 0; I < sizeof(Mask) * 8; ++I) {
322 if (Mask & (1 << I)) {
323 (llvm::Twine(" ") + TRI->getRegAsmName(Reg: FirstRegInBlock + I))
324 .toVector(Out&: TransferredRegs);
325 }
326 }
327
328 OS.emitRawComment(T: " transferring at most " + TransferredRegs);
329}
330
331void AMDGPUAsmPrinter::emitInstruction(const MachineInstr *MI) {
332 if (MI->isCall())
333 collectCallEdge(MI: *MI);
334
335 // FIXME: Enable feature predicate checks once all the test pass.
336 // AMDGPU_MC::verifyInstructionPredicates(MI->getOpcode(),
337 // getSubtargetInfo().getFeatureBits());
338
339 if (MCInst OutInst; lowerPseudoInstExpansion(MI, Inst&: OutInst)) {
340 EmitToStreamer(S&: *OutStreamer, Inst: OutInst);
341 return;
342 }
343
344 const GCNSubtarget &STI = MF->getSubtarget<GCNSubtarget>();
345 AMDGPUMCInstLower MCInstLowering(OutContext, STI, *this);
346
347 StringRef Err;
348 if (!STI.getInstrInfo()->verifyInstruction(MI: *MI, ErrInfo&: Err)) {
349 LLVMContext &C = MI->getMF()->getFunction().getContext();
350 C.emitError(ErrorStr: "Illegal instruction detected: " + Err);
351 MI->print(OS&: errs());
352 }
353
354 if (MI->isBundle()) {
355 const MachineBasicBlock *MBB = MI->getParent();
356 MachineBasicBlock::const_instr_iterator I = ++MI->getIterator();
357 while (I != MBB->instr_end() && I->isInsideBundle()) {
358 emitInstruction(MI: &*I);
359 ++I;
360 }
361 } else {
362 // We don't want these pseudo instructions encoded. They are
363 // placeholder instructions and should only be printed as
364 // comments.
365 if (MI->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) {
366 if (isVerbose())
367 OutStreamer->emitRawComment(T: " return to shader part epilog");
368 return;
369 }
370
371 if (MI->getOpcode() == AMDGPU::WAVE_BARRIER) {
372 if (isVerbose())
373 OutStreamer->emitRawComment(T: " wave barrier");
374 return;
375 }
376
377 if (MI->getOpcode() == AMDGPU::ASYNCMARK) {
378 if (isVerbose())
379 OutStreamer->emitRawComment(T: " asyncmark");
380 return;
381 }
382
383 if (MI->getOpcode() == AMDGPU::WAIT_ASYNCMARK) {
384 if (isVerbose()) {
385 OutStreamer->emitRawComment(T: " wait_asyncmark(" +
386 Twine(MI->getOperand(i: 0).getImm()) + ")");
387 }
388 return;
389 }
390
391 if (MI->getOpcode() == AMDGPU::SCHED_BARRIER) {
392 if (isVerbose()) {
393 std::string HexString;
394 raw_string_ostream HexStream(HexString);
395 HexStream << format_hex(N: MI->getOperand(i: 0).getImm(), Width: 10, Upper: true);
396 OutStreamer->emitRawComment(T: " sched_barrier mask(" + HexString + ")");
397 }
398 return;
399 }
400
401 if (MI->getOpcode() == AMDGPU::SCHED_GROUP_BARRIER) {
402 if (isVerbose()) {
403 std::string HexString;
404 raw_string_ostream HexStream(HexString);
405 HexStream << format_hex(N: MI->getOperand(i: 0).getImm(), Width: 10, Upper: true);
406 OutStreamer->emitRawComment(
407 T: " sched_group_barrier mask(" + HexString + ") size(" +
408 Twine(MI->getOperand(i: 1).getImm()) + ") SyncID(" +
409 Twine(MI->getOperand(i: 2).getImm()) + ")");
410 }
411 return;
412 }
413
414 if (MI->getOpcode() == AMDGPU::IGLP_OPT) {
415 if (isVerbose()) {
416 std::string HexString;
417 raw_string_ostream HexStream(HexString);
418 HexStream << format_hex(N: MI->getOperand(i: 0).getImm(), Width: 10, Upper: true);
419 OutStreamer->emitRawComment(T: " iglp_opt mask(" + HexString + ")");
420 }
421 return;
422 }
423
424 if (MI->getOpcode() == AMDGPU::SI_MASKED_UNREACHABLE) {
425 if (isVerbose())
426 OutStreamer->emitRawComment(T: " divergent unreachable");
427 return;
428 }
429
430 if (MI->isMetaInstruction()) {
431 if (isVerbose())
432 OutStreamer->emitRawComment(T: " meta instruction");
433 return;
434 }
435
436 unsigned Opc = MI->getOpcode();
437 if (LLVM_UNLIKELY(Opc == TargetOpcode::STATEPOINT ||
438 Opc == TargetOpcode::STACKMAP ||
439 Opc == TargetOpcode::PATCHPOINT)) {
440 LLVMContext &Ctx = MI->getMF()->getFunction().getContext();
441 Ctx.emitError(ErrorStr: "unhandled statepoint-like instruction");
442 OutStreamer->emitRawComment(T: "unsupported statepoint/stackmap/patchpoint");
443 return;
444 }
445
446 if (isVerbose())
447 if (STI.getInstrInfo()->isBlockLoadStore(Opcode: MI->getOpcode()))
448 emitVGPRBlockComment(MI, TII: STI.getInstrInfo(), TRI: STI.getRegisterInfo(),
449 MFI: MF->getInfo<SIMachineFunctionInfo>(),
450 OS&: *OutStreamer);
451
452 if (isVerbose() && (MI->getOpcode() == AMDGPU::S_SET_VGPR_MSB ||
453 (MI->getOpcode() == AMDGPU::S_SETREG_IMM32_B32 &&
454 STI.has1024AddressableVGPRs()))) {
455 std::optional<unsigned> V;
456 if (MI->getOpcode() == AMDGPU::S_SETREG_IMM32_B32)
457 V = AMDGPU::convertSetRegImmToVgprMSBs(MI: *MI,
458 HasSetregVGPRMSBFixup: STI.hasSetregVGPRMSBFixup());
459 else
460 V = MI->getOperand(i: 0).getImm() & 0xff;
461 if (V.has_value())
462 OutStreamer->AddComment(
463 T: " msbs: dst=" + Twine(*V >> 6) + " src0=" + Twine(*V & 3) +
464 " src1=" + Twine((*V >> 2) & 3) + " src2=" + Twine((*V >> 4) & 3));
465 }
466
467 MCInst TmpInst;
468 MCInstLowering.lower(MI, OutMI&: TmpInst);
469 EmitToStreamer(S&: *OutStreamer, Inst: TmpInst);
470
471 if (DumpCodeInstEmitter) {
472 // Disassemble instruction/operands to text
473 DisasmLines.resize(new_size: DisasmLines.size() + 1);
474 std::string &DisasmLine = DisasmLines.back();
475 raw_string_ostream DisasmStream(DisasmLine);
476
477 AMDGPUInstPrinter InstPrinter(TM.getMCAsmInfo(), *STI.getInstrInfo(),
478 *STI.getRegisterInfo());
479 InstPrinter.printInst(MI: &TmpInst, Address: 0, Annot: StringRef(), STI, O&: DisasmStream);
480
481 // Disassemble instruction/operands to hex representation.
482 SmallVector<MCFixup, 4> Fixups;
483 SmallVector<char, 16> CodeBytes;
484
485 DumpCodeInstEmitter->encodeInstruction(
486 Inst: TmpInst, CB&: CodeBytes, Fixups, STI: MF->getSubtarget<MCSubtargetInfo>());
487 HexLines.resize(new_size: HexLines.size() + 1);
488 std::string &HexLine = HexLines.back();
489 raw_string_ostream HexStream(HexLine);
490
491 for (size_t i = 0; i < CodeBytes.size(); i += 4) {
492 unsigned int CodeDWord =
493 support::endian::read32le(P: CodeBytes.data() + i);
494 HexStream << format(Fmt: "%s%08X", Vals: (i > 0 ? " " : ""), Vals: CodeDWord);
495 }
496
497 DisasmLineMaxLen = std::max(a: DisasmLineMaxLen, b: DisasmLine.size());
498 }
499 }
500}
501