1//===-- AMDGPUAsmPrinter.cpp - AMDGPU assembly printer --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10///
11/// The AMDGPUAsmPrinter is used to print both assembly string and also binary
12/// code. When passed an MCAsmStreamer it prints assembly and when passed
13/// an MCObjectStreamer it outputs binary code.
14//
15//===----------------------------------------------------------------------===//
16//
17
18#include "AMDGPUAsmPrinter.h"
19#include "AMDGPU.h"
20#include "AMDGPUHSAMetadataStreamer.h"
21#include "AMDGPUMCResourceInfo.h"
22#include "AMDGPUResourceUsageAnalysis.h"
23#include "AMDGPUTargetMachine.h"
24#include "GCNSubtarget.h"
25#include "MCTargetDesc/AMDGPUInstPrinter.h"
26#include "MCTargetDesc/AMDGPUMCExpr.h"
27#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h"
28#include "MCTargetDesc/AMDGPUTargetStreamer.h"
29#include "R600AsmPrinter.h"
30#include "SIMachineFunctionInfo.h"
31#include "TargetInfo/AMDGPUTargetInfo.h"
32#include "Utils/AMDGPUBaseInfo.h"
33#include "Utils/AMDKernelCodeTUtils.h"
34#include "Utils/SIDefinesUtils.h"
35#include "llvm/ADT/StringSet.h"
36#include "llvm/Analysis/OptimizationRemarkEmitter.h"
37#include "llvm/BinaryFormat/ELF.h"
38#include "llvm/CodeGen/AsmPrinterAnalysis.h"
39#include "llvm/CodeGen/AsmPrinterHandler.h"
40#include "llvm/CodeGen/MachineFrameInfo.h"
41#include "llvm/CodeGen/MachineModuleInfo.h"
42#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
43#include "llvm/IR/DiagnosticInfo.h"
44#include "llvm/MC/MCAssembler.h"
45#include "llvm/MC/MCContext.h"
46#include "llvm/MC/MCSectionELF.h"
47#include "llvm/MC/MCStreamer.h"
48#include "llvm/MC/MCValue.h"
49#include "llvm/MC/TargetRegistry.h"
50#include "llvm/Support/AMDHSAKernelDescriptor.h"
51#include "llvm/Support/Compiler.h"
52#include "llvm/Target/TargetLoweringObjectFile.h"
53#include "llvm/Target/TargetMachine.h"
54#include "llvm/TargetParser/AMDGPUTargetParser.h"
55
56using namespace llvm;
57using namespace llvm::AMDGPU;
58
59// This should get the default rounding mode from the kernel. We just set the
60// default here, but this could change if the OpenCL rounding mode pragmas are
61// used.
62//
63// The denormal mode here should match what is reported by the OpenCL runtime
64// for the CL_FP_DENORM bit from CL_DEVICE_{HALF|SINGLE|DOUBLE}_FP_CONFIG, but
65// can also be override to flush with the -cl-denorms-are-zero compiler flag.
66//
67// AMD OpenCL only sets flush none and reports CL_FP_DENORM for double
68// precision, and leaves single precision to flush all and does not report
69// CL_FP_DENORM for CL_DEVICE_SINGLE_FP_CONFIG. Mesa's OpenCL currently reports
70// CL_FP_DENORM for both.
71//
72// FIXME: It seems some instructions do not support single precision denormals
73// regardless of the mode (exp_*_f32, rcp_*_f32, rsq_*_f32, rsq_*f32, sqrt_f32,
74// and sin_f32, cos_f32 on most parts).
75
76// We want to use these instructions, and using fp32 denormals also causes
77// instructions to run at the double precision rate for the device so it's
78// probably best to just report no single precision denormals.
79static uint32_t getFPMode(SIModeRegisterDefaults Mode) {
80 return FP_ROUND_MODE_SP(FP_ROUND_ROUND_TO_NEAREST) |
81 FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_NEAREST) |
82 FP_DENORM_MODE_SP(Mode.fpDenormModeSPValue()) |
83 FP_DENORM_MODE_DP(Mode.fpDenormModeDPValue());
84}
85
86static AsmPrinter *
87createAMDGPUAsmPrinterPass(TargetMachine &tm,
88 std::unique_ptr<MCStreamer> &&Streamer) {
89 return new AMDGPUAsmPrinter(tm, std::move(Streamer));
90}
91
92extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
93LLVMInitializeAMDGPUAsmPrinter() {
94 TargetRegistry::RegisterAsmPrinter(T&: getTheR600Target(),
95 Fn: llvm::createR600AsmPrinterPass);
96 TargetRegistry::RegisterAsmPrinter(T&: getTheGCNTarget(),
97 Fn: createAMDGPUAsmPrinterPass);
98 TargetRegistry::RegisterAsmPrinter(T&: getTheGCNLegacyTarget(),
99 Fn: createAMDGPUAsmPrinterPass);
100}
101
102namespace {
103class AMDGPUAsmPrinterHandler : public AsmPrinterHandler {
104protected:
105 AMDGPUAsmPrinter *Asm;
106
107public:
108 AMDGPUAsmPrinterHandler(AMDGPUAsmPrinter *A) : Asm(A) {}
109
110 void beginFunction(const MachineFunction *MF) override {}
111
112 void endFunction(const MachineFunction *MF) override { Asm->endFunction(MF); }
113
114 void endModule() override {}
115};
116} // End anonymous namespace
117
118AMDGPUAsmPrinter::AMDGPUAsmPrinter(TargetMachine &TM,
119 std::unique_ptr<MCStreamer> Streamer)
120 : AsmPrinter(TM, std::move(Streamer)) {
121 assert(OutStreamer && "AsmPrinter constructed without streamer");
122 GetResourceUsage = [this](MachineFunction &MF)
123 -> const AMDGPUResourceUsageAnalysisImpl::SIFunctionResourceInfo * {
124 if (auto *ResourceUsageW =
125 getAnalysisIfAvailable<AMDGPUResourceUsageAnalysisWrapperPass>())
126 return &ResourceUsageW->getResourceInfo();
127 return nullptr;
128 };
129}
130
131StringRef AMDGPUAsmPrinter::getPassName() const {
132 return "AMDGPU Assembly Printer";
133}
134
135const MCSubtargetInfo *AMDGPUAsmPrinter::getGlobalSTI() const {
136 return &TM.getMCSubtargetInfo();
137}
138
139AMDGPUTargetStreamer *AMDGPUAsmPrinter::getTargetStreamer() const {
140 if (!OutStreamer)
141 return nullptr;
142 return static_cast<AMDGPUTargetStreamer *>(OutStreamer->getTargetStreamer());
143}
144
145void AMDGPUAsmPrinter::emitStartOfAsmFile(Module &M) {
146 IsTargetStreamerInitialized = false;
147}
148
149void AMDGPUAsmPrinter::initTargetStreamer(Module &M) {
150 IsTargetStreamerInitialized = true;
151
152 // TODO: Which one is called first, emitStartOfAsmFile or
153 // emitFunctionBodyStart?
154 if (getTargetStreamer() && !getTargetStreamer()->getTargetID())
155 initializeTargetID(M);
156
157 const Triple &TT = M.getTargetTriple();
158 if (TT.getOS() != Triple::AMDHSA && TT.getOS() != Triple::AMDPAL)
159 return;
160
161 getTargetStreamer()->EmitDirectiveAMDGCNTarget();
162
163 if (TT.getOS() == Triple::AMDHSA) {
164 getTargetStreamer()->EmitDirectiveAMDHSACodeObjectVersion(
165 COV: CodeObjectVersion);
166 HSAMetadataStream->begin(Mod: M, TargetID: *getTargetStreamer()->getTargetID());
167 }
168
169 if (TT.getOS() == Triple::AMDPAL)
170 getTargetStreamer()->getPALMetadata()->readFromIR(M);
171}
172
173void AMDGPUAsmPrinter::emitEndOfAsmFile(Module &M) {
174 // Init target streamer if it has not yet happened
175 if (!IsTargetStreamerInitialized)
176 initTargetStreamer(M);
177
178 const Triple &TT = M.getTargetTriple();
179 if (TT.getOS() != Triple::AMDHSA)
180 getTargetStreamer()->EmitISAVersion();
181
182 // Emit HSA Metadata (NT_AMD_AMDGPU_HSA_METADATA).
183 // Emit HSA Metadata (NT_AMD_HSA_METADATA).
184 if (TT.getOS() == Triple::AMDHSA) {
185 HSAMetadataStream->end();
186 bool Success = HSAMetadataStream->emitTo(TargetStreamer&: *getTargetStreamer());
187 (void)Success;
188 assert(Success && "Malformed HSA Metadata");
189 }
190}
191
192void AMDGPUAsmPrinter::emitFunctionBodyStart() {
193 const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
194 const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
195 const Function &F = MF->getFunction();
196
197 // TODO: We're checking this late, would be nice to check it earlier.
198 if (STM.requiresCodeObjectV6() && CodeObjectVersion < AMDGPU::AMDHSA_COV6) {
199 reportFatalUsageError(
200 reason: STM.getCPU() + " is only available on code object version 6 or better");
201 }
202
203 // TODO: Which one is called first, emitStartOfAsmFile or
204 // emitFunctionBodyStart?
205 if (!getTargetStreamer()->getTargetID())
206 initializeTargetID(M: *F.getParent());
207
208 if (!MFI.isEntryFunction())
209 return;
210
211 if (STM.isMesaKernel(F) &&
212 (F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
213 F.getCallingConv() == CallingConv::SPIR_KERNEL)) {
214 AMDGPUMCKernelCodeT KernelCode;
215 getAmdKernelCode(Out&: KernelCode, KernelInfo: CurrentProgramInfo, MF: *MF);
216 KernelCode.validate(STI: &STM, Ctx&: MF->getContext());
217 getTargetStreamer()->EmitAMDKernelCodeT(Header&: KernelCode);
218 }
219
220 if (STM.isAmdHsaOS())
221 HSAMetadataStream->emitKernel(MF: *MF, ProgramInfo: CurrentProgramInfo);
222}
223
224/// Set bits in a kernel descriptor MCExpr field:
225/// return ((Dst & ~Mask) | (Value << Shift))
226static const MCExpr *setBits(const MCExpr *Dst, const MCExpr *Value,
227 uint32_t Mask, uint32_t Shift, MCContext &Ctx) {
228 const auto *Shft = MCConstantExpr::create(Value: Shift, Ctx);
229 const auto *Msk = MCConstantExpr::create(Value: Mask, Ctx);
230 Dst = MCBinaryExpr::createAnd(LHS: Dst, RHS: MCUnaryExpr::createNot(Expr: Msk, Ctx), Ctx);
231 Dst = MCBinaryExpr::createOr(LHS: Dst, RHS: MCBinaryExpr::createShl(LHS: Value, RHS: Shft, Ctx),
232 Ctx);
233 return Dst;
234}
235
236void AMDGPUAsmPrinter::endFunction(const MachineFunction *MF) {
237 const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
238 if (!MFI.isEntryFunction())
239 return;
240
241 assert(TM.getTargetTriple().getOS() == Triple::AMDHSA);
242
243 const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
244 MCContext &Ctx = MF->getContext();
245
246 AMDGPU::MCKernelDescriptor KD =
247 getAmdhsaKernelDescriptor(MF: *MF, PI: CurrentProgramInfo);
248
249 // Compute inst_pref_size using MCExpr label subtraction for exact code
250 // size. At this point .Lfunc_end has been emitted (by the base AsmPrinter)
251 // right after the function code, so (Lfunc_end - func_sym) gives the
252 // exact function code size in bytes.
253 if (STM.hasInstPrefSize()) {
254 const MCExpr *CodeSizeExpr = MCBinaryExpr::createSub(
255 LHS: MCSymbolRefExpr::create(Symbol: getFunctionEnd(), Ctx&: OutContext),
256 RHS: MCSymbolRefExpr::create(Symbol: CurrentFnSym, Ctx&: OutContext), Ctx&: OutContext);
257
258 uint32_t Mask, Shift, Width, CacheLineSize;
259 STM.getInstPrefSizeArgs(Mask, Shift, Width, CacheLineSize);
260 const MCExpr *InstPrefSize =
261 AMDGPUMCExpr::createInstPrefSize(CodeSizeBytes: CodeSizeExpr, Ctx);
262 KD.compute_pgm_rsrc3 =
263 setBits(Dst: KD.compute_pgm_rsrc3, Value: InstPrefSize, Mask, Shift, Ctx);
264 }
265
266 auto &Streamer = getTargetStreamer()->getStreamer();
267 auto &Context = Streamer.getContext();
268 auto &ObjectFileInfo = *Context.getObjectFileInfo();
269 auto &ReadOnlySection = *ObjectFileInfo.getReadOnlySection();
270
271 Streamer.pushSection();
272 Streamer.switchSection(Section: &ReadOnlySection);
273
274 // CP microcode requires the kernel descriptor to be allocated on 64 byte
275 // alignment.
276 Streamer.emitValueToAlignment(Alignment: Align(64), Fill: 0, FillLen: 1, MaxBytesToEmit: 0);
277 ReadOnlySection.ensureMinAlignment(MinAlignment: Align(64));
278
279 SmallString<128> KernelName;
280 getNameWithPrefix(Name&: KernelName, GV: &MF->getFunction());
281 getTargetStreamer()->EmitAmdhsaKernelDescriptor(
282 STI: STM, KernelName, KernelDescriptor: KD, NextVGPR: CurrentProgramInfo.NumVGPRsForWavesPerEU,
283 NextSGPR: MCBinaryExpr::createSub(
284 LHS: CurrentProgramInfo.NumSGPRsForWavesPerEU,
285 RHS: AMDGPUMCExpr::createExtraSGPRs(
286 VCCUsed: CurrentProgramInfo.VCCUsed, FlatScrUsed: CurrentProgramInfo.FlatUsed,
287 XNACKUsed: getTargetStreamer()->getTargetID()->isXnackOnOrAny(), Ctx&: Context),
288 Ctx&: Context),
289 ReserveVCC: CurrentProgramInfo.VCCUsed, ReserveFlatScr: CurrentProgramInfo.FlatUsed);
290
291 Streamer.popSection();
292}
293
294void AMDGPUAsmPrinter::emitImplicitDef(const MachineInstr *MI) const {
295 Register RegNo = MI->getOperand(i: 0).getReg();
296
297 SmallString<128> Str;
298 raw_svector_ostream OS(Str);
299 OS << "implicit-def: "
300 << printReg(Reg: RegNo, TRI: MF->getSubtarget().getRegisterInfo());
301
302 if (MI->getAsmPrinterFlags() & AMDGPU::SGPR_SPILL)
303 OS << " : SGPR spill to VGPR lane";
304
305 OutStreamer->AddComment(T: OS.str());
306 OutStreamer->addBlankLine();
307}
308
309void AMDGPUAsmPrinter::emitFunctionEntryLabel() {
310 if (TM.getTargetTriple().getOS() == Triple::AMDHSA) {
311 AsmPrinter::emitFunctionEntryLabel();
312 return;
313 }
314
315 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
316 const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
317 if (MFI->isEntryFunction() && STM.isAmdHsaOrMesa(F: MF->getFunction())) {
318 SmallString<128> SymbolName;
319 getNameWithPrefix(Name&: SymbolName, GV: &MF->getFunction()),
320 getTargetStreamer()->EmitAMDGPUSymbolType(SymbolName,
321 Type: ELF::STT_AMDGPU_HSA_KERNEL);
322 }
323 if (DumpCodeInstEmitter) {
324 // Disassemble function name label to text.
325 DisasmLines.push_back(x: MF->getName().str() + ":");
326 DisasmLineMaxLen = std::max(a: DisasmLineMaxLen, b: DisasmLines.back().size());
327 HexLines.emplace_back(args: "");
328 }
329
330 AsmPrinter::emitFunctionEntryLabel();
331}
332
333void AMDGPUAsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) {
334 if (DumpCodeInstEmitter && !isBlockOnlyReachableByFallthrough(MBB: &MBB)) {
335 // Write a line for the basic block label if it is not only fallthrough.
336 DisasmLines.push_back(x: (Twine("BB") + Twine(getFunctionNumber()) + "_" +
337 Twine(MBB.getNumber()) + ":")
338 .str());
339 DisasmLineMaxLen = std::max(a: DisasmLineMaxLen, b: DisasmLines.back().size());
340 HexLines.emplace_back(args: "");
341 }
342 AsmPrinter::emitBasicBlockStart(MBB);
343}
344
345void AMDGPUAsmPrinter::emitGlobalVariable(const GlobalVariable *GV) {
346 if (GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
347 if (GV->hasInitializer() && !isa<UndefValue>(Val: GV->getInitializer())) {
348 OutContext.reportError(L: {},
349 Msg: Twine(GV->getName()) +
350 ": unsupported initializer for address space");
351 return;
352 }
353
354 const Triple::OSType OS = TM.getTargetTriple().getOS();
355 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL) {
356 if (!AMDGPUTargetMachine::EnableObjectLinking)
357 return;
358 // With object linking, LDS definitions should have been externalized
359 // by earlier passes (e.g. LDS lowering, named barrier lowering).
360 // Only declarations reach here, emitted as SHN_AMDGPU_LDS symbols
361 // so the linker can assign their offsets.
362 assert(GV->isDeclaration() &&
363 "LDS definitions should have been externalized when object "
364 "linking is enabled");
365 }
366
367 MCSymbol *GVSym = getSymbol(GV);
368
369 GVSym->redefineIfPossible();
370 if (GVSym->isDefined() || GVSym->isVariable())
371 report_fatal_error(reason: "symbol '" + Twine(GVSym->getName()) +
372 "' is already defined");
373
374 const DataLayout &DL = GV->getDataLayout();
375 uint64_t Size = GV->getGlobalSize(DL);
376 Align Alignment = GV->getAlign().value_or(u: Align(4));
377
378 emitVisibility(Sym: GVSym, Visibility: GV->getVisibility(), IsDefinition: !GV->isDeclaration());
379 emitLinkage(GV, GVSym);
380 auto *TS = getTargetStreamer();
381 TS->emitAMDGPULDS(Symbol: GVSym, Size, Alignment);
382 return;
383 }
384
385 AsmPrinter::emitGlobalVariable(GV);
386}
387
388bool AMDGPUAsmPrinter::doInitialization(Module &M) {
389 const llvm::Triple &TT = M.getTargetTriple();
390 CodeObjectVersion = AMDGPU::getAMDHSACodeObjectVersion(M);
391
392 if (TT.getOS() == Triple::AMDHSA) {
393 switch (CodeObjectVersion) {
394 case AMDGPU::AMDHSA_COV4:
395 HSAMetadataStream = std::make_unique<HSAMD::MetadataStreamerMsgPackV4>();
396 break;
397 case AMDGPU::AMDHSA_COV5:
398 HSAMetadataStream = std::make_unique<HSAMD::MetadataStreamerMsgPackV5>();
399 break;
400 case AMDGPU::AMDHSA_COV6:
401 HSAMetadataStream = std::make_unique<HSAMD::MetadataStreamerMsgPackV6>();
402 break;
403 default:
404 reportFatalUsageError(reason: "unsupported code object version");
405 }
406
407 addAsmPrinterHandler(Handler: std::make_unique<AMDGPUAsmPrinterHandler>(args: this));
408 }
409
410 return AsmPrinter::doInitialization(M);
411}
412
413/// Mimics GCNSubtarget::computeOccupancy for MCExpr.
414///
415/// Remove dependency on GCNSubtarget and depend only only the necessary values
416/// for said occupancy computation. Should match computeOccupancy implementation
417/// without passing \p STM on.
418const AMDGPUMCExpr *createOccupancy(unsigned InitOcc, const MCExpr *NumSGPRs,
419 const MCExpr *NumVGPRs,
420 unsigned DynamicVGPRBlockSize,
421 const GCNSubtarget &STM, MCContext &Ctx) {
422 unsigned MaxWaves = STM.getMaxWavesPerEU();
423 unsigned Granule = IsaInfo::getVGPRAllocGranule(STI: STM, DynamicVGPRBlockSize);
424 unsigned TargetTotalNumVGPRs = IsaInfo::getTotalNumVGPRs(STI: STM);
425
426 // Bake the per-function SGPR budget into the operands so the late-evaluated
427 // MCExpr stays arithmetic. The trap reservation in particular is implicit on
428 // amdhsa and lives on STM, not on the assembler's MCSubtargetInfo.
429 AMDGPU::GPUKind Kind = STM.getTargetID().getGPUKind();
430 unsigned SGPRTotal = AMDGPU::getTotalNumSGPRs(AK: Kind);
431 unsigned SGPRGranule = AMDGPU::getSGPRAllocGranule(AK: Kind);
432 unsigned SGPRTrapReserve = STM.hasTrapHandler() ? IsaInfo::TRAP_NUM_SGPRS : 0;
433
434 auto CreateExpr = [&Ctx](unsigned Value) {
435 return MCConstantExpr::create(Value, Ctx);
436 };
437
438 // Zero SGPR count when SGPRs don't limit occupancy, so the MCExpr skips the
439 // SGPR term without having to test the generation itself.
440 const MCExpr *SGPRArg =
441 IsaInfo::isSGPROccupancyLimited(STI: STM) ? NumSGPRs : CreateExpr(0);
442
443 return AMDGPUMCExpr::create(Kind: AMDGPUMCExpr::AGVK_Occupancy,
444 Args: {CreateExpr(MaxWaves), CreateExpr(Granule),
445 CreateExpr(TargetTotalNumVGPRs),
446 CreateExpr(InitOcc), CreateExpr(SGPRTotal),
447 CreateExpr(SGPRGranule),
448 CreateExpr(SGPRTrapReserve), SGPRArg, NumVGPRs},
449 Ctx);
450}
451
452void AMDGPUAsmPrinter::validateMCResourceInfo(Function &F) {
453 if (F.isDeclaration() || !AMDGPU::isModuleEntryFunctionCC(CC: F.getCallingConv()))
454 return;
455
456 using RIK = MCResourceInfo::ResourceInfoKind;
457 const GCNSubtarget &STM = TM.getSubtarget<GCNSubtarget>(F);
458 MCSymbol *FnSym = TM.getSymbol(GV: &F);
459
460 auto TryGetMCExprValue = [](const MCExpr *Value, uint64_t &Res) -> bool {
461 int64_t Val;
462 if (Value->evaluateAsAbsolute(Res&: Val)) {
463 Res = Val;
464 return true;
465 }
466 return false;
467 };
468
469 const uint64_t MaxScratchPerWorkitem =
470 STM.getMaxWaveScratchSize() / STM.getWavefrontSize();
471 MCSymbol *ScratchSizeSymbol =
472 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_PrivateSegSize, OutContext);
473 uint64_t ScratchSize;
474 if (ScratchSizeSymbol->isVariable() &&
475 TryGetMCExprValue(ScratchSizeSymbol->getVariableValue(), ScratchSize) &&
476 ScratchSize > MaxScratchPerWorkitem) {
477 DiagnosticInfoStackSize DiagStackSize(F, ScratchSize, MaxScratchPerWorkitem,
478 DS_Error);
479 F.getContext().diagnose(DI: DiagStackSize);
480 }
481
482 // Validate addressable scalar registers (i.e., prior to added implicit
483 // SGPRs).
484 MCSymbol *NumSGPRSymbol =
485 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_NumSGPR, OutContext);
486 if (STM.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
487 !STM.hasSGPRInitBug()) {
488 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
489 uint64_t NumSgpr;
490 if (NumSGPRSymbol->isVariable() &&
491 TryGetMCExprValue(NumSGPRSymbol->getVariableValue(), NumSgpr) &&
492 NumSgpr > MaxAddressableNumSGPRs) {
493 F.getContext().diagnose(DI: DiagnosticInfoResourceLimit(
494 F, "addressable scalar registers", NumSgpr, MaxAddressableNumSGPRs,
495 DS_Error, DK_ResourceLimit));
496 return;
497 }
498 }
499
500 MCSymbol *VCCUsedSymbol =
501 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_UsesVCC, OutContext);
502 MCSymbol *FlatUsedSymbol =
503 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_UsesFlatScratch, OutContext);
504 uint64_t VCCUsed, FlatUsed, NumSgpr;
505
506 if (NumSGPRSymbol->isVariable() && VCCUsedSymbol->isVariable() &&
507 FlatUsedSymbol->isVariable() &&
508 TryGetMCExprValue(NumSGPRSymbol->getVariableValue(), NumSgpr) &&
509 TryGetMCExprValue(VCCUsedSymbol->getVariableValue(), VCCUsed) &&
510 TryGetMCExprValue(FlatUsedSymbol->getVariableValue(), FlatUsed)) {
511
512 // Recomputes NumSgprs + implicit SGPRs but all symbols should now be
513 // resolvable.
514 NumSgpr += IsaInfo::getNumExtraSGPRs(
515 STI: STM, VCCUsed, FlatScrUsed: FlatUsed,
516 XNACKUsed: getTargetStreamer()->getTargetID()->isXnackOnOrAny());
517 if (STM.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS ||
518 STM.hasSGPRInitBug()) {
519 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
520 if (NumSgpr > MaxAddressableNumSGPRs) {
521 F.getContext().diagnose(DI: DiagnosticInfoResourceLimit(
522 F, "scalar registers", NumSgpr, MaxAddressableNumSGPRs, DS_Error,
523 DK_ResourceLimit));
524 return;
525 }
526 }
527
528 MCSymbol *NumVgprSymbol =
529 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_NumVGPR, OutContext);
530 MCSymbol *NumAgprSymbol =
531 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_NumAGPR, OutContext);
532 uint64_t NumVgpr, NumAgpr;
533
534 MachineModuleInfo &MMI = *GetMMI();
535 MachineFunction *MF = MMI.getMachineFunction(F);
536 if (MF && NumVgprSymbol->isVariable() && NumAgprSymbol->isVariable() &&
537 TryGetMCExprValue(NumVgprSymbol->getVariableValue(), NumVgpr) &&
538 TryGetMCExprValue(NumAgprSymbol->getVariableValue(), NumAgpr)) {
539 const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
540 unsigned MaxWaves = MFI.getMaxWavesPerEU();
541 uint64_t TotalNumVgpr =
542 getTotalNumVGPRs(has90AInsts: STM.hasGFX90AInsts(), ArgNumAGPR: NumAgpr, ArgNumVGPR: NumVgpr);
543 uint64_t NumVGPRsForWavesPerEU =
544 std::max(l: {TotalNumVgpr, (uint64_t)1,
545 (uint64_t)STM.getMinNumVGPRs(
546 WavesPerEU: MaxWaves, DynamicVGPRBlockSize: MFI.getDynamicVGPRBlockSize())});
547 uint64_t NumSGPRsForWavesPerEU = std::max(
548 l: {NumSgpr, (uint64_t)1, (uint64_t)STM.getMinNumSGPRs(WavesPerEU: MaxWaves)});
549 const MCExpr *OccupancyExpr = createOccupancy(
550 InitOcc: STM.getOccupancyWithWorkGroupSizes(MF: *MF).second,
551 NumSGPRs: MCConstantExpr::create(Value: NumSGPRsForWavesPerEU, Ctx&: OutContext),
552 NumVGPRs: MCConstantExpr::create(Value: NumVGPRsForWavesPerEU, Ctx&: OutContext),
553 DynamicVGPRBlockSize: MFI.getDynamicVGPRBlockSize(), STM, Ctx&: OutContext);
554 uint64_t Occupancy;
555
556 const auto [MinWEU, MaxWEU] = AMDGPU::getIntegerPairAttribute(
557 F, Name: "amdgpu-waves-per-eu", Default: {0, 0}, OnlyFirstRequired: true);
558
559 if (TryGetMCExprValue(OccupancyExpr, Occupancy) && Occupancy < MinWEU) {
560 DiagnosticInfoOptimizationFailure Diag(
561 F, F.getSubprogram(),
562 "failed to meet occupancy target given by 'amdgpu-waves-per-eu' in "
563 "'" +
564 F.getName() + "': desired occupancy was " + Twine(MinWEU) +
565 ", final occupancy is " + Twine(Occupancy));
566 F.getContext().diagnose(DI: Diag);
567 return;
568 }
569 }
570 }
571}
572
573static void appendTypeEncoding(std::string &Enc, Type *Ty, const DataLayout &DL,
574 bool IsReturnType) {
575 if (Ty->isVoidTy()) {
576 Enc += 'v';
577 return;
578 }
579 unsigned Bits = DL.getTypeSizeInBits(Ty);
580 // Zero-sized non-void types (e.g. `{}` or `[0 x i8]`) consume no ABI
581 // registers. For returns, emit the same no-result marker as void so the
582 // parameter encoding still has an explicit return-type prefix.
583 if (Bits == 0) {
584 if (IsReturnType)
585 Enc += 'v';
586 return;
587 }
588 if (Bits <= 32)
589 Enc += 'i';
590 else if (Bits <= 64)
591 Enc += 'l';
592 else
593 Enc.append(n: divideCeil(Numerator: Bits, Denominator: 32), c: 'i');
594}
595
596static std::string computeTypeId(const FunctionType *FTy,
597 const DataLayout &DL) {
598 std::string Enc;
599 appendTypeEncoding(Enc, Ty: FTy->getReturnType(), DL, /*IsReturnType=*/true);
600 for (Type *ParamTy : FTy->params())
601 appendTypeEncoding(Enc, Ty: ParamTy, DL, /*IsReturnType=*/false);
602 return Enc;
603}
604
605void AMDGPUAsmPrinter::collectCallEdge(const MachineInstr &MI) {
606 if (!AMDGPUTargetMachine::EnableObjectLinking)
607 return;
608 const SIInstrInfo *TII = MF->getSubtarget<GCNSubtarget>().getInstrInfo();
609 const MachineOperand *Callee =
610 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::callee);
611 if (!Callee || !Callee->isGlobal())
612 return;
613 DirectCallEdges.insert(
614 X: {getSymbol(GV: &MF->getFunction()), getSymbol(GV: Callee->getGlobal())});
615}
616
617void AMDGPUAsmPrinter::emitAMDGPUInfo(Module &M) {
618 if (!AMDGPUTargetMachine::EnableObjectLinking)
619 return;
620
621 const NamedMDNode *LDSMD = M.getNamedMetadata(Name: "amdgpu.lds.uses");
622 bool HasLDSUses = LDSMD && LDSMD->getNumOperands() > 0;
623
624 const NamedMDNode *BarMD = M.getNamedMetadata(Name: "amdgpu.named_barrier.uses");
625 bool HasNamedBarriers = BarMD && BarMD->getNumOperands() > 0;
626
627 // Collect address-taken functions (with type IDs) and indirect call sites.
628 DenseMap<const Function *, std::string> AddrTakenTypeIds;
629 using IndirectCallInfo = std::pair<const Function *, std::string>;
630 SmallVector<IndirectCallInfo, 8> IndirectCalls;
631
632 for (const Function &F : M) {
633 bool IsKernel = AMDGPU::isKernel(CC: F.getCallingConv());
634
635 if (!IsKernel && F.hasAddressTaken(/*PutOffender=*/nullptr,
636 /*IgnoreCallbackUses=*/false,
637 /*IgnoreAssumeLikeCalls=*/true,
638 /*IgnoreLLVMUsed=*/IngoreLLVMUsed: true)) {
639 AddrTakenTypeIds[&F] =
640 computeTypeId(FTy: F.getFunctionType(), DL: M.getDataLayout());
641 }
642
643 if (F.isDeclaration())
644 continue;
645
646 StringSet<> SeenTypeIds;
647 for (const BasicBlock &BB : F) {
648 for (const Instruction &I : BB) {
649 const auto *CB = dyn_cast<CallBase>(Val: &I);
650 if (!CB || !CB->isIndirectCall())
651 continue;
652 std::string TId =
653 computeTypeId(FTy: CB->getFunctionType(), DL: M.getDataLayout());
654 if (SeenTypeIds.insert(key: TId).second)
655 IndirectCalls.push_back(Elt: {&F, std::move(TId)});
656 }
657 }
658 }
659
660 if (FunctionInfos.empty() && DirectCallEdges.empty() && !HasLDSUses &&
661 !HasNamedBarriers && AddrTakenTypeIds.empty() && IndirectCalls.empty())
662 return;
663
664 AMDGPU::InfoSectionData Data;
665 Data.Funcs = std::move(FunctionInfos);
666
667 for (auto &[F, TypeId] : AddrTakenTypeIds) {
668 MCSymbol *Sym = getSymbol(GV: F);
669 Data.TypeIds.push_back(Elt: {Sym, TypeId});
670 }
671
672 for (auto &[CallerSym, CalleeSym] : DirectCallEdges)
673 Data.Calls.push_back(Elt: {CallerSym, CalleeSym});
674 DirectCallEdges.clear();
675
676 if (HasLDSUses) {
677 for (const MDNode *N : LDSMD->operands()) {
678 auto *Func = mdconst::extract<Function>(MD: N->getOperand(I: 0));
679 auto *LdsVar = mdconst::extract<GlobalVariable>(MD: N->getOperand(I: 1));
680 Data.Uses.push_back(Elt: {getSymbol(GV: Func), getSymbol(GV: LdsVar)});
681 }
682 }
683
684 if (HasNamedBarriers) {
685 for (const MDNode *N : BarMD->operands()) {
686 auto *BarVar = mdconst::extract<GlobalVariable>(MD: N->getOperand(I: 0));
687 MCSymbol *BarSym = getSymbol(GV: BarVar);
688 for (unsigned I = 1, E = N->getNumOperands(); I < E; ++I) {
689 auto *Func = mdconst::extract<Function>(MD: N->getOperand(I));
690 Data.Uses.push_back(Elt: {getSymbol(GV: Func), BarSym});
691 }
692 }
693 }
694
695 for (auto &[Caller, Enc] : IndirectCalls) {
696 MCSymbol *CallerSym = getSymbol(GV: Caller);
697 Data.IndirectCalls.push_back(Elt: {CallerSym, Enc});
698 }
699
700 getTargetStreamer()->emitAMDGPUInfo(Data);
701}
702
703bool AMDGPUAsmPrinter::doFinalization(Module &M) {
704 const Triple &TT = M.getTargetTriple();
705
706 // Pad with s_code_end to help tools and guard against instruction prefetch
707 // causing stale data in caches. Arguably this should be done by the linker,
708 // which is why this isn't done for Mesa.
709 // Don't do it if there is no code.
710 const MCSubtargetInfo &STI = *getGlobalSTI();
711 if ((AMDGPU::isGFX10Plus(STI) || AMDGPU::isGFX90A(STI)) &&
712 (TT.getOS() == Triple::AMDHSA || TT.getOS() == Triple::AMDPAL)) {
713 MCSection *TextSect = getObjFileLowering().getTextSection();
714 if (TextSect->hasInstructions()) {
715 OutStreamer->switchSection(Section: TextSect);
716 getTargetStreamer()->EmitCodeEnd(STI);
717 }
718 }
719
720 // Emit the unified .amdgpu.info section (per-function resources, call graph,
721 // LDS/named-barrier use edges, indirect calls, and address-taken type IDs).
722 emitAMDGPUInfo(M);
723
724 // Assign expressions which can only be resolved when all other functions are
725 // known.
726 RI.finalize(OutContext);
727
728 // Switch section and emit all GPR maximums within the processed module.
729 OutStreamer->pushSection();
730 MCSectionELF *MaxGPRSection =
731 OutContext.getELFSection(Section: ".AMDGPU.gpr_maximums", Type: ELF::SHT_PROGBITS, Flags: 0);
732 OutStreamer->switchSection(Section: MaxGPRSection);
733 getTargetStreamer()->EmitMCResourceMaximums(
734 MaxVGPR: RI.getMaxVGPRSymbol(OutContext), MaxAGPR: RI.getMaxAGPRSymbol(OutContext),
735 MaxSGPR: RI.getMaxSGPRSymbol(OutContext), MaxNamedBarrier: RI.getMaxNamedBarrierSymbol(OutContext));
736 OutStreamer->popSection();
737
738 // In the object-linking pipeline per-function resource MCExprs reference
739 // external callee symbols that cannot be evaluated here, so cross-TU limit
740 // checks would silently no-op for every non-leaf function. Defer resource
741 // sanity checking to the linker, which re-validates against the aggregated
742 // call graph in the combined .amdgpu.info metadata.
743 if (!AMDGPUTargetMachine::EnableObjectLinking) {
744 for (Function &F : M.functions())
745 validateMCResourceInfo(F);
746 }
747
748 RI.reset();
749
750 return AsmPrinter::doFinalization(M);
751}
752
753SmallString<128> AMDGPUAsmPrinter::getMCExprStr(const MCExpr *Value) {
754 SmallString<128> Str;
755 raw_svector_ostream OSS(Str);
756 auto &Streamer = getTargetStreamer()->getStreamer();
757 auto &Context = Streamer.getContext();
758 const MCExpr *New = foldAMDGPUMCExpr(Expr: Value, Ctx&: Context);
759 printAMDGPUMCExpr(Expr: New, OS&: OSS, MAI: &MAI);
760 return Str;
761}
762
763// Print comments that apply to both callable functions and entry points.
764void AMDGPUAsmPrinter::emitCommonFunctionComments(
765 const MCExpr *NumVGPR, const MCExpr *NumAGPR, const MCExpr *TotalNumVGPR,
766 const MCExpr *NumSGPR, const MCExpr *ScratchSize, uint64_t CodeSize,
767 const AMDGPUMachineFunctionInfo *MFI) {
768 OutStreamer->emitRawComment(T: " codeLenInByte = " + Twine(CodeSize), TabPrefix: false);
769 OutStreamer->emitRawComment(T: " TotalNumSgprs: " + getMCExprStr(Value: NumSGPR),
770 TabPrefix: false);
771 OutStreamer->emitRawComment(T: " NumVgprs: " + getMCExprStr(Value: NumVGPR), TabPrefix: false);
772 if (NumAGPR && TotalNumVGPR) {
773 OutStreamer->emitRawComment(T: " NumAgprs: " + getMCExprStr(Value: NumAGPR), TabPrefix: false);
774 OutStreamer->emitRawComment(T: " TotalNumVgprs: " + getMCExprStr(Value: TotalNumVGPR),
775 TabPrefix: false);
776 }
777 OutStreamer->emitRawComment(T: " ScratchSize: " + getMCExprStr(Value: ScratchSize),
778 TabPrefix: false);
779 OutStreamer->emitRawComment(T: " MemoryBound: " + Twine(MFI->isMemoryBound()),
780 TabPrefix: false);
781}
782
783const MCExpr *AMDGPUAsmPrinter::getAmdhsaKernelCodeProperties(
784 const MachineFunction &MF) const {
785 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
786 MCContext &Ctx = MF.getContext();
787 uint16_t KernelCodeProperties = 0;
788 const GCNUserSGPRUsageInfo &UserSGPRInfo = MFI.getUserSGPRInfo();
789
790 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
791 KernelCodeProperties |=
792 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER;
793 }
794 if (UserSGPRInfo.hasDispatchPtr()) {
795 KernelCodeProperties |=
796 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR;
797 }
798 if (UserSGPRInfo.hasQueuePtr()) {
799 KernelCodeProperties |= amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR;
800 }
801 if (UserSGPRInfo.hasKernargSegmentPtr()) {
802 KernelCodeProperties |=
803 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR;
804 }
805 if (UserSGPRInfo.hasDispatchID()) {
806 KernelCodeProperties |=
807 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID;
808 }
809 if (UserSGPRInfo.hasFlatScratchInit()) {
810 KernelCodeProperties |=
811 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT;
812 }
813 if (UserSGPRInfo.hasPrivateSegmentSize()) {
814 KernelCodeProperties |=
815 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE;
816 }
817 if (MF.getSubtarget<GCNSubtarget>().isWave32()) {
818 KernelCodeProperties |=
819 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;
820 }
821
822 // CurrentProgramInfo.DynamicCallStack is a MCExpr and could be
823 // un-evaluatable at this point so it cannot be conditionally checked here.
824 // Instead, we'll directly shift the possibly unknown MCExpr into its place
825 // and bitwise-or it into KernelCodeProperties.
826 const MCExpr *KernelCodePropExpr =
827 MCConstantExpr::create(Value: KernelCodeProperties, Ctx);
828 const MCExpr *OrValue = MCConstantExpr::create(
829 Value: amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK_SHIFT, Ctx);
830 OrValue = MCBinaryExpr::createShl(LHS: CurrentProgramInfo.DynamicCallStack,
831 RHS: OrValue, Ctx);
832 KernelCodePropExpr = MCBinaryExpr::createOr(LHS: KernelCodePropExpr, RHS: OrValue, Ctx);
833
834 return KernelCodePropExpr;
835}
836
837MCKernelDescriptor
838AMDGPUAsmPrinter::getAmdhsaKernelDescriptor(const MachineFunction &MF,
839 const SIProgramInfo &PI) const {
840 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
841 const Function &F = MF.getFunction();
842 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
843 MCContext &Ctx = MF.getContext();
844
845 MCKernelDescriptor KernelDescriptor;
846
847 KernelDescriptor.group_segment_fixed_size =
848 MCConstantExpr::create(Value: PI.LDSSize, Ctx);
849 KernelDescriptor.private_segment_fixed_size = PI.ScratchSize;
850
851 Align MaxKernArgAlign;
852 KernelDescriptor.kernarg_size = MCConstantExpr::create(
853 Value: STM.getKernArgSegmentSize(F, MaxAlign&: MaxKernArgAlign), Ctx);
854
855 KernelDescriptor.compute_pgm_rsrc1 = PI.getComputePGMRSrc1(ST: STM, Ctx);
856 KernelDescriptor.compute_pgm_rsrc2 = PI.getComputePGMRSrc2(ST: STM, Ctx);
857 KernelDescriptor.kernel_code_properties = getAmdhsaKernelCodeProperties(MF);
858
859 int64_t PGM_Rsrc3 = 1;
860 bool EvaluatableRsrc3 =
861 CurrentProgramInfo.ComputePGMRSrc3->evaluateAsAbsolute(Res&: PGM_Rsrc3);
862 (void)PGM_Rsrc3;
863 (void)EvaluatableRsrc3;
864 assert(STM.getGeneration() >= AMDGPUSubtarget::GFX10 ||
865 STM.hasGFX90AInsts() || STM.hasGFX1250Insts() || !EvaluatableRsrc3 ||
866 static_cast<uint64_t>(PGM_Rsrc3) == 0);
867 KernelDescriptor.compute_pgm_rsrc3 = CurrentProgramInfo.ComputePGMRSrc3;
868
869 KernelDescriptor.kernarg_preload = MCConstantExpr::create(
870 Value: AMDGPU::hasKernargPreload(STI: STM) ? Info->getNumKernargPreloadedSGPRs() : 0,
871 Ctx);
872
873 return KernelDescriptor;
874}
875
876bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
877 // Init target streamer lazily on the first function so that previous passes
878 // can set metadata.
879 if (!IsTargetStreamerInitialized)
880 initTargetStreamer(M&: *MF.getFunction().getParent());
881
882 ResourceUsage = GetResourceUsage(MF);
883 CurrentProgramInfo.reset(MF);
884
885 const AMDGPUMachineFunctionInfo *MFI =
886 MF.getInfo<AMDGPUMachineFunctionInfo>();
887 MCContext &Ctx = MF.getContext();
888
889 // The starting address of all shader programs must be 256 bytes aligned.
890 // Regular functions just need the basic required instruction alignment.
891 MF.ensureAlignment(A: MFI->isEntryFunction() ? Align(256) : Align(4));
892
893 SetupMachineFunction(MF);
894
895 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
896 MCContext &Context = getObjFileLowering().getContext();
897 // FIXME: This should be an explicit check for Mesa.
898 if (!STM.isAmdHsaOS() && !STM.isAmdPalOS()) {
899 MCSectionELF *ConfigSection =
900 Context.getELFSection(Section: ".AMDGPU.config", Type: ELF::SHT_PROGBITS, Flags: 0);
901 OutStreamer->switchSection(Section: ConfigSection);
902 }
903
904 RI.gatherResourceInfo(MF, FRI: *ResourceUsage, OutContext);
905
906 if (AMDGPUTargetMachine::EnableObjectLinking) {
907 const AMDGPUResourceUsageAnalysisWrapperPass::FunctionResourceInfo &RU =
908 *ResourceUsage;
909 FunctionInfos.push_back(
910 Elt: {/*NumSGPR=*/static_cast<uint32_t>(RU.NumExplicitSGPR),
911 /*NumArchVGPR=*/static_cast<uint32_t>(RU.NumVGPR),
912 /*NumAccVGPR=*/static_cast<uint32_t>(RU.NumAGPR),
913 /*PrivateSegmentSize=*/static_cast<uint32_t>(RU.PrivateSegmentSize),
914 /*UsesVCC=*/RU.UsesVCC,
915 /*UsesFlatScratch=*/RU.UsesFlatScratch,
916 /*HasDynStack=*/RU.HasDynamicallySizedStack,
917 /*Sym=*/getSymbol(GV: &MF.getFunction())});
918 }
919
920 if (MFI->isModuleEntryFunction()) {
921 getSIProgramInfo(Out&: CurrentProgramInfo, MF);
922 }
923
924 if (STM.isAmdPalOS()) {
925 if (MFI->isEntryFunction())
926 EmitPALMetadata(MF, KernelInfo: CurrentProgramInfo);
927 else if (MFI->isModuleEntryFunction())
928 emitPALFunctionMetadata(MF);
929 } else if (!STM.isAmdHsaOS()) {
930 EmitProgramInfoSI(MF, KernelInfo: CurrentProgramInfo);
931 }
932
933 DumpCodeInstEmitter = nullptr;
934 if (STM.dumpCode()) {
935 // For -dumpcode, get the assembler out of the streamer. This only works
936 // with -filetype=obj.
937 MCAssembler *Assembler = OutStreamer->getAssemblerPtr();
938 if (Assembler)
939 DumpCodeInstEmitter = Assembler->getEmitterPtr();
940 }
941
942 DisasmLines.clear();
943 HexLines.clear();
944 DisasmLineMaxLen = 0;
945
946 emitFunctionBody();
947
948 emitResourceUsageRemarks(MF, CurrentProgramInfo, isModuleEntryFunction: MFI->isModuleEntryFunction(),
949 hasMAIInsts: STM.hasMAIInsts());
950
951 {
952 using RIK = MCResourceInfo::ResourceInfoKind;
953 getTargetStreamer()->EmitMCResourceInfo(
954 NumVGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumVGPR, OutContext),
955 NumAGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumAGPR, OutContext),
956 NumExplicitSGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumSGPR, OutContext),
957 NumNamedBarrier: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumNamedBarrier,
958 OutContext),
959 PrivateSegmentSize: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_PrivateSegSize,
960 OutContext),
961 UsesVCC: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_UsesVCC, OutContext),
962 UsesFlatScratch: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_UsesFlatScratch,
963 OutContext),
964 HasDynamicallySizedStack: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_HasDynSizedStack,
965 OutContext),
966 HasRecursion: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_HasRecursion,
967 OutContext),
968 HasIndirectCall: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_HasIndirectCall,
969 OutContext));
970 }
971
972 // Emit _dvgpr$ symbol when appropriate.
973 emitDVgprSymbol(MF);
974
975 if (isVerbose()) {
976 MCSectionELF *CommentSection =
977 Context.getELFSection(Section: ".AMDGPU.csdata", Type: ELF::SHT_PROGBITS, Flags: 0);
978 OutStreamer->switchSection(Section: CommentSection);
979
980 if (!MFI->isEntryFunction()) {
981 using RIK = MCResourceInfo::ResourceInfoKind;
982 OutStreamer->emitRawComment(T: " Function info:", TabPrefix: false);
983
984 emitCommonFunctionComments(
985 NumVGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumVGPR, OutContext)
986 ->getVariableValue(),
987 NumAGPR: STM.hasMAIInsts() ? RI.getSymbol(FuncName: CurrentFnSym->getName(),
988 RIK: RIK::RIK_NumAGPR, OutContext)
989 ->getVariableValue()
990 : nullptr,
991 TotalNumVGPR: RI.createTotalNumVGPRs(MF, Ctx),
992 NumSGPR: RI.createTotalNumSGPRs(
993 MF,
994 hasXnack: MF.getSubtarget<GCNSubtarget>().getTargetID().isXnackOnOrAny(),
995 Ctx),
996 ScratchSize: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_PrivateSegSize,
997 OutContext)
998 ->getVariableValue(),
999 CodeSize: CurrentProgramInfo.getFunctionCodeSize(MF), MFI);
1000 return false;
1001 }
1002
1003 OutStreamer->emitRawComment(T: " Kernel info:", TabPrefix: false);
1004 emitCommonFunctionComments(
1005 NumVGPR: CurrentProgramInfo.NumArchVGPR,
1006 NumAGPR: STM.hasMAIInsts() ? CurrentProgramInfo.NumAccVGPR : nullptr,
1007 TotalNumVGPR: CurrentProgramInfo.NumVGPR, NumSGPR: CurrentProgramInfo.NumSGPR,
1008 ScratchSize: CurrentProgramInfo.ScratchSize,
1009 CodeSize: CurrentProgramInfo.getFunctionCodeSize(MF), MFI);
1010
1011 OutStreamer->emitRawComment(
1012 T: " FloatMode: " + Twine(CurrentProgramInfo.FloatMode), TabPrefix: false);
1013 OutStreamer->emitRawComment(
1014 T: " IeeeMode: " + Twine(CurrentProgramInfo.IEEEMode), TabPrefix: false);
1015 OutStreamer->emitRawComment(
1016 T: " LDSByteSize: " + Twine(CurrentProgramInfo.LDSSize) +
1017 " bytes/workgroup (compile time only)",
1018 TabPrefix: false);
1019
1020 OutStreamer->emitRawComment(
1021 T: " SGPRBlocks: " + getMCExprStr(Value: CurrentProgramInfo.SGPRBlocks), TabPrefix: false);
1022
1023 OutStreamer->emitRawComment(
1024 T: " VGPRBlocks: " + getMCExprStr(Value: CurrentProgramInfo.VGPRBlocks), TabPrefix: false);
1025
1026 OutStreamer->emitRawComment(
1027 T: " NumSGPRsForWavesPerEU: " +
1028 getMCExprStr(Value: CurrentProgramInfo.NumSGPRsForWavesPerEU),
1029 TabPrefix: false);
1030 OutStreamer->emitRawComment(
1031 T: " NumVGPRsForWavesPerEU: " +
1032 getMCExprStr(Value: CurrentProgramInfo.NumVGPRsForWavesPerEU),
1033 TabPrefix: false);
1034
1035 if (STM.hasGFX90AInsts()) {
1036 const MCExpr *AdjustedAccum = MCBinaryExpr::createAdd(
1037 LHS: CurrentProgramInfo.AccumOffset, RHS: MCConstantExpr::create(Value: 1, Ctx), Ctx);
1038 AdjustedAccum = MCBinaryExpr::createMul(
1039 LHS: AdjustedAccum, RHS: MCConstantExpr::create(Value: 4, Ctx), Ctx);
1040 OutStreamer->emitRawComment(
1041 T: " AccumOffset: " + getMCExprStr(Value: AdjustedAccum), TabPrefix: false);
1042 }
1043
1044 if (STM.hasGFX1250Insts())
1045 OutStreamer->emitRawComment(
1046 T: " NamedBarCnt: " + getMCExprStr(Value: CurrentProgramInfo.NamedBarCnt),
1047 TabPrefix: false);
1048
1049 OutStreamer->emitRawComment(
1050 T: " Occupancy: " + getMCExprStr(Value: CurrentProgramInfo.Occupancy), TabPrefix: false);
1051
1052 OutStreamer->emitRawComment(
1053 T: " WaveLimiterHint : " + Twine(MFI->needsWaveLimiter()), TabPrefix: false);
1054
1055 OutStreamer->emitRawComment(
1056 T: " COMPUTE_PGM_RSRC2:SCRATCH_EN: " +
1057 getMCExprStr(Value: CurrentProgramInfo.ScratchEnable),
1058 TabPrefix: false);
1059 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:USER_SGPR: " +
1060 Twine(CurrentProgramInfo.UserSGPR),
1061 TabPrefix: false);
1062 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TRAP_HANDLER: " +
1063 Twine(CurrentProgramInfo.TrapHandlerEnable),
1064 TabPrefix: false);
1065 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TGID_X_EN: " +
1066 Twine(CurrentProgramInfo.TGIdXEnable),
1067 TabPrefix: false);
1068 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TGID_Y_EN: " +
1069 Twine(CurrentProgramInfo.TGIdYEnable),
1070 TabPrefix: false);
1071 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TGID_Z_EN: " +
1072 Twine(CurrentProgramInfo.TGIdZEnable),
1073 TabPrefix: false);
1074 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TIDIG_COMP_CNT: " +
1075 Twine(CurrentProgramInfo.TIdIGCompCount),
1076 TabPrefix: false);
1077
1078 [[maybe_unused]] int64_t PGMRSrc3;
1079 assert(STM.getGeneration() >= AMDGPUSubtarget::GFX10 ||
1080 STM.hasGFX90AInsts() || STM.hasGFX1250Insts() ||
1081 (CurrentProgramInfo.ComputePGMRSrc3->evaluateAsAbsolute(PGMRSrc3) &&
1082 static_cast<uint64_t>(PGMRSrc3) == 0));
1083 if (STM.hasGFX90AInsts()) {
1084 OutStreamer->emitRawComment(
1085 T: " COMPUTE_PGM_RSRC3_GFX90A:ACCUM_OFFSET: " +
1086 getMCExprStr(Value: MCKernelDescriptor::bits_get(
1087 Src: CurrentProgramInfo.ComputePGMRSrc3,
1088 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
1089 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, Ctx)),
1090 TabPrefix: false);
1091 OutStreamer->emitRawComment(
1092 T: " COMPUTE_PGM_RSRC3_GFX90A:TG_SPLIT: " +
1093 getMCExprStr(Value: MCKernelDescriptor::bits_get(
1094 Src: CurrentProgramInfo.ComputePGMRSrc3,
1095 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT,
1096 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, Ctx)),
1097 TabPrefix: false);
1098 }
1099 }
1100
1101 if (DumpCodeInstEmitter) {
1102
1103 OutStreamer->switchSection(
1104 Section: Context.getELFSection(Section: ".AMDGPU.disasm", Type: ELF::SHT_PROGBITS, Flags: 0));
1105
1106 for (size_t i = 0; i < DisasmLines.size(); ++i) {
1107 std::string Comment = "\n";
1108 if (!HexLines[i].empty()) {
1109 Comment = std::string(DisasmLineMaxLen - DisasmLines[i].size(), ' ');
1110 Comment += " ; " + HexLines[i] + "\n";
1111 }
1112
1113 OutStreamer->emitBytes(Data: StringRef(DisasmLines[i]));
1114 OutStreamer->emitBytes(Data: StringRef(Comment));
1115 }
1116 }
1117
1118 return false;
1119}
1120
1121// When appropriate, add a _dvgpr$ symbol, with the value of the function
1122// symbol, plus an offset encoding one less than the number of VGPR blocks used
1123// by the function in bits 5..3 of the symbol value. A "VGPR block" can be
1124// either 16 VGPRs (for a max of 128), or 32 VGPRs (for a max of 256). This is
1125// used by a front-end to have functions that are chained rather than called,
1126// and a dispatcher that dynamically resizes the VGPR count before dispatching
1127// to a function.
1128void AMDGPUAsmPrinter::emitDVgprSymbol(MachineFunction &MF) {
1129 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
1130 if (MFI.isDynamicVGPREnabled() &&
1131 MF.getFunction().getCallingConv() == CallingConv::AMDGPU_CS_Chain) {
1132 MCContext &Ctx = MF.getContext();
1133 unsigned BlockSize = MFI.getDynamicVGPRBlockSize();
1134
1135 const MCExpr *EncodedBlocks;
1136 MCValue NumVGPRs;
1137 if (CurrentProgramInfo.NumVGPRsForWavesPerEU->evaluateAsRelocatable(
1138 Res&: NumVGPRs, Asm: nullptr) &&
1139 NumVGPRs.isAbsolute()) {
1140
1141 // Calculate number of VGPR blocks.
1142 // Treat 0 VGPRs as 1 VGPR to avoid underflowing.
1143 unsigned NumBlocks =
1144 divideCeil(Numerator: std::max(a: unsigned(NumVGPRs.getConstant()), b: 1U), Denominator: BlockSize);
1145
1146 if (NumBlocks > AMDGPU::IsaInfo::MaxDynamicVGPRBlocks) {
1147 OutContext.reportError(
1148 L: {}, Msg: "DVGPR block count " + Twine(NumBlocks) +
1149 " exceeds maximum of " +
1150 Twine(AMDGPU::IsaInfo::MaxDynamicVGPRBlocks) +
1151 " for __dvgpr$ symbol for '" +
1152 Twine(CurrentFnSym->getName()) + "'");
1153 return;
1154 }
1155 unsigned EncodedNumBlocks = (NumBlocks - 1) << 3;
1156 EncodedBlocks = MCConstantExpr::create(Value: EncodedNumBlocks, Ctx);
1157 } else {
1158 // Value not yet available so build a symbolic MCExpr:
1159 // ((alignTo(max(NumVGPRs, 1), BlockSize) / BlockSize - 1) << 3
1160 const MCExpr *One = MCConstantExpr::create(Value: 1, Ctx);
1161 const MCExpr *BlockSizeConst = MCConstantExpr::create(Value: BlockSize, Ctx);
1162 const MCExpr *MaxVGPRs = AMDGPUMCExpr::createMax(
1163 Args: {CurrentProgramInfo.NumVGPRsForWavesPerEU, One}, Ctx);
1164 const MCExpr *NumBlocks = MCBinaryExpr::createDiv(
1165 LHS: AMDGPUMCExpr::createAlignTo(Value: MaxVGPRs, Align: BlockSizeConst, Ctx),
1166 RHS: BlockSizeConst, Ctx);
1167 EncodedBlocks =
1168 MCBinaryExpr::createShl(LHS: MCBinaryExpr::createSub(LHS: NumBlocks, RHS: One, Ctx),
1169 RHS: MCConstantExpr::create(Value: 3, Ctx), Ctx);
1170 }
1171
1172 // Add to function symbol to create _dvgpr$ symbol.
1173 const MCExpr *DVgprFuncVal = MCBinaryExpr::createAdd(
1174 LHS: MCSymbolRefExpr::create(Symbol: CurrentFnSym, Ctx), RHS: EncodedBlocks, Ctx);
1175 MCSymbol *DVgprFuncSym =
1176 Ctx.getOrCreateSymbol(Name: Twine("_dvgpr$") + CurrentFnSym->getName());
1177 OutStreamer->emitAssignment(Symbol: DVgprFuncSym, Value: DVgprFuncVal);
1178 emitVisibility(Sym: DVgprFuncSym, Visibility: MF.getFunction().getVisibility());
1179 emitLinkage(GV: &MF.getFunction(), GVSym: DVgprFuncSym);
1180 }
1181}
1182
1183// TODO: Fold this into emitFunctionBodyStart.
1184void AMDGPUAsmPrinter::initializeTargetID(const Module &M) {
1185 getTargetStreamer()->initializeTargetID(STI: *getGlobalSTI());
1186
1187 auto &TSTargetID = getTargetStreamer()->getTargetID();
1188
1189 // Error if -mattr specified xnack or sramecc.
1190 // TODO: Remove this when subtarget features removed.
1191 StringRef FeatureString = getGlobalSTI()->getFeatureString();
1192 if (FeatureString.contains(Other: "xnack")) {
1193 M.getContext().diagnose(DI: DiagnosticInfoGeneric(
1194 "xnack/sramecc should be specified via module flags. "
1195 "Use module flag 'amdgpu.xnack' instead of subtarget feature",
1196 DS_Error));
1197 }
1198 if (FeatureString.contains(Other: "sramecc")) {
1199 M.getContext().diagnose(DI: DiagnosticInfoGeneric(
1200 "xnack/sramecc should be specified via module flags. "
1201 "Use module flag 'amdgpu.sramecc' instead of subtarget feature",
1202 DS_Error));
1203 }
1204
1205 // Apply xnack/sramecc settings from module flags.
1206 if (getGlobalSTI()->getFeatureBits().test(I: AMDGPU::FeatureXNACKOnOffModes)) {
1207 AMDGPU::TargetIDSetting Setting =
1208 GCNTargetMachine::getTargetIDSettingFromModuleFlag(M, FlagName: "amdgpu.xnack");
1209 if (Setting != AMDGPU::TargetIDSetting::Any)
1210 TSTargetID->setXnackSetting(Setting);
1211 }
1212
1213 if (getGlobalSTI()->getFeatureBits().test(I: AMDGPU::FeatureSupportsSRAMECC)) {
1214 AMDGPU::TargetIDSetting Setting =
1215 GCNTargetMachine::getTargetIDSettingFromModuleFlag(M, FlagName: "amdgpu.sramecc");
1216 if (Setting != AMDGPU::TargetIDSetting::Any)
1217 TSTargetID->setSramEccSetting(Setting);
1218 }
1219}
1220
1221// AccumOffset computed for the MCExpr equivalent of:
1222// alignTo(std::max(1, NumVGPR), 4) / 4 - 1;
1223static const MCExpr *computeAccumOffset(const MCExpr *NumVGPR, MCContext &Ctx) {
1224 const MCExpr *ConstFour = MCConstantExpr::create(Value: 4, Ctx);
1225 const MCExpr *ConstOne = MCConstantExpr::create(Value: 1, Ctx);
1226
1227 // Can't be lower than 1 for subsequent alignTo.
1228 const MCExpr *MaximumTaken =
1229 AMDGPUMCExpr::createMax(Args: {ConstOne, NumVGPR}, Ctx);
1230
1231 // Practically, it's computing divideCeil(MaximumTaken, 4).
1232 const MCExpr *DivCeil = MCBinaryExpr::createDiv(
1233 LHS: AMDGPUMCExpr::createAlignTo(Value: MaximumTaken, Align: ConstFour, Ctx), RHS: ConstFour,
1234 Ctx);
1235
1236 return MCBinaryExpr::createSub(LHS: DivCeil, RHS: ConstOne, Ctx);
1237}
1238
1239void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
1240 const MachineFunction &MF) {
1241 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1242 MCContext &Ctx = MF.getContext();
1243
1244 auto CreateExpr = [&Ctx](int64_t Value) {
1245 return MCConstantExpr::create(Value, Ctx);
1246 };
1247
1248 auto TryGetMCExprValue = [](const MCExpr *Value, uint64_t &Res) -> bool {
1249 int64_t Val;
1250 if (Value->evaluateAsAbsolute(Res&: Val)) {
1251 Res = Val;
1252 return true;
1253 }
1254 return false;
1255 };
1256
1257 auto GetSymRefExpr =
1258 [&](MCResourceInfo::ResourceInfoKind RIK) -> const MCExpr * {
1259 MCSymbol *Sym = RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK, OutContext);
1260 return MCSymbolRefExpr::create(Symbol: Sym, Ctx);
1261 };
1262
1263 using RIK = MCResourceInfo::ResourceInfoKind;
1264 ProgInfo.NumArchVGPR = GetSymRefExpr(RIK::RIK_NumVGPR);
1265 ProgInfo.NumAccVGPR = GetSymRefExpr(RIK::RIK_NumAGPR);
1266 ProgInfo.NumVGPR = AMDGPUMCExpr::createTotalNumVGPR(
1267 NumAGPR: ProgInfo.NumAccVGPR, NumVGPR: ProgInfo.NumArchVGPR, Ctx);
1268
1269 ProgInfo.AccumOffset = computeAccumOffset(NumVGPR: ProgInfo.NumArchVGPR, Ctx);
1270 ProgInfo.TgSplit =
1271 STM.hasTgSplitSupport() && AMDGPU::isTgSplitEnabled(F: MF.getFunction());
1272 ProgInfo.NumSGPR = GetSymRefExpr(RIK::RIK_NumSGPR);
1273 ProgInfo.ScratchSize = GetSymRefExpr(RIK::RIK_PrivateSegSize);
1274 ProgInfo.VCCUsed = GetSymRefExpr(RIK::RIK_UsesVCC);
1275 ProgInfo.FlatUsed = GetSymRefExpr(RIK::RIK_UsesFlatScratch);
1276 ProgInfo.DynamicCallStack =
1277 MCBinaryExpr::createOr(LHS: GetSymRefExpr(RIK::RIK_HasDynSizedStack),
1278 RHS: GetSymRefExpr(RIK::RIK_HasRecursion), Ctx);
1279
1280 const MCExpr *BarBlkConst = MCConstantExpr::create(Value: 4, Ctx);
1281 const MCExpr *AlignToBlk = AMDGPUMCExpr::createAlignTo(
1282 Value: GetSymRefExpr(RIK::RIK_NumNamedBarrier), Align: BarBlkConst, Ctx);
1283 ProgInfo.NamedBarCnt = MCBinaryExpr::createDiv(LHS: AlignToBlk, RHS: BarBlkConst, Ctx);
1284
1285 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1286
1287 // The calculations related to SGPR/VGPR blocks are
1288 // duplicated in part in AMDGPUAsmParser::calculateGPRBlocks, and could be
1289 // unified.
1290 const MCExpr *ExtraSGPRs = AMDGPUMCExpr::createExtraSGPRs(
1291 VCCUsed: ProgInfo.VCCUsed, FlatScrUsed: ProgInfo.FlatUsed,
1292 XNACKUsed: getTargetStreamer()->getTargetID()->isXnackOnOrAny(), Ctx);
1293
1294 // Check the addressable register limit before we add ExtraSGPRs.
1295 if (STM.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
1296 !STM.hasSGPRInitBug()) {
1297 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
1298 uint64_t NumSgpr;
1299 if (TryGetMCExprValue(ProgInfo.NumSGPR, NumSgpr) &&
1300 NumSgpr > MaxAddressableNumSGPRs) {
1301 // This can happen due to a compiler bug or when using inline asm.
1302 LLVMContext &Ctx = MF.getFunction().getContext();
1303 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1304 MF.getFunction(), "addressable scalar registers", NumSgpr,
1305 MaxAddressableNumSGPRs, DS_Error, DK_ResourceLimit));
1306 ProgInfo.NumSGPR = CreateExpr(MaxAddressableNumSGPRs - 1);
1307 }
1308 }
1309
1310 // Account for extra SGPRs and VGPRs reserved for debugger use.
1311 ProgInfo.NumSGPR = MCBinaryExpr::createAdd(LHS: ProgInfo.NumSGPR, RHS: ExtraSGPRs, Ctx);
1312
1313 const Function &F = MF.getFunction();
1314
1315 // Ensure there are enough SGPRs and VGPRs for wave dispatch, where wave
1316 // dispatch registers as function args.
1317 unsigned WaveDispatchNumSGPR = MFI->getNumWaveDispatchSGPRs(),
1318 WaveDispatchNumVGPR = MFI->getNumWaveDispatchVGPRs();
1319
1320 if (WaveDispatchNumSGPR) {
1321 ProgInfo.NumSGPR = AMDGPUMCExpr::createMax(
1322 Args: {ProgInfo.NumSGPR,
1323 MCBinaryExpr::createAdd(LHS: CreateExpr(WaveDispatchNumSGPR), RHS: ExtraSGPRs,
1324 Ctx)},
1325 Ctx);
1326 }
1327
1328 if (WaveDispatchNumVGPR) {
1329 ProgInfo.NumArchVGPR = AMDGPUMCExpr::createMax(
1330 Args: {ProgInfo.NumVGPR, CreateExpr(WaveDispatchNumVGPR)}, Ctx);
1331
1332 ProgInfo.NumVGPR = AMDGPUMCExpr::createTotalNumVGPR(
1333 NumAGPR: ProgInfo.NumAccVGPR, NumVGPR: ProgInfo.NumArchVGPR, Ctx);
1334 }
1335
1336 // Adjust number of registers used to meet default/requested minimum/maximum
1337 // number of waves per execution unit request.
1338 unsigned MaxWaves = MFI->getMaxWavesPerEU();
1339 ProgInfo.NumSGPRsForWavesPerEU =
1340 AMDGPUMCExpr::createMax(Args: {ProgInfo.NumSGPR, CreateExpr(1ul),
1341 CreateExpr(STM.getMinNumSGPRs(WavesPerEU: MaxWaves))},
1342 Ctx);
1343 ProgInfo.NumVGPRsForWavesPerEU =
1344 AMDGPUMCExpr::createMax(Args: {ProgInfo.NumVGPR, CreateExpr(1ul),
1345 CreateExpr(STM.getMinNumVGPRs(
1346 WavesPerEU: MaxWaves, DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize()))},
1347 Ctx);
1348
1349 if (STM.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS ||
1350 STM.hasSGPRInitBug()) {
1351 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
1352 uint64_t NumSgpr;
1353 if (TryGetMCExprValue(ProgInfo.NumSGPR, NumSgpr) &&
1354 NumSgpr > MaxAddressableNumSGPRs) {
1355 // This can happen due to a compiler bug or when using inline asm to use
1356 // the registers which are usually reserved for vcc etc.
1357 LLVMContext &Ctx = MF.getFunction().getContext();
1358 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1359 MF.getFunction(), "scalar registers", NumSgpr, MaxAddressableNumSGPRs,
1360 DS_Error, DK_ResourceLimit));
1361 ProgInfo.NumSGPR = CreateExpr(MaxAddressableNumSGPRs);
1362 ProgInfo.NumSGPRsForWavesPerEU = CreateExpr(MaxAddressableNumSGPRs);
1363 }
1364 }
1365
1366 if (STM.hasSGPRInitBug()) {
1367 ProgInfo.NumSGPR =
1368 CreateExpr(AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG);
1369 ProgInfo.NumSGPRsForWavesPerEU =
1370 CreateExpr(AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG);
1371 }
1372
1373 if (MFI->getNumUserSGPRs() > STM.getMaxNumUserSGPRs()) {
1374 LLVMContext &Ctx = MF.getFunction().getContext();
1375 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1376 MF.getFunction(), "user SGPRs", MFI->getNumUserSGPRs(),
1377 STM.getMaxNumUserSGPRs(), DS_Error));
1378 }
1379
1380 if (MFI->getLDSSize() > STM.getAddressableLocalMemorySize()) {
1381 LLVMContext &Ctx = MF.getFunction().getContext();
1382 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1383 MF.getFunction(), "local memory", MFI->getLDSSize(),
1384 STM.getAddressableLocalMemorySize(), DS_Error));
1385 }
1386 // The MCExpr equivalent of getNumSGPRBlocks/getNumVGPRBlocks:
1387 // (alignTo(max(1u, NumGPR), GPREncodingGranule) / GPREncodingGranule) - 1
1388 auto GetNumGPRBlocks = [&CreateExpr, &Ctx](const MCExpr *NumGPR,
1389 unsigned Granule) {
1390 const MCExpr *OneConst = CreateExpr(1ul);
1391 const MCExpr *GranuleConst = CreateExpr(Granule);
1392 const MCExpr *MaxNumGPR = AMDGPUMCExpr::createMax(Args: {NumGPR, OneConst}, Ctx);
1393 const MCExpr *AlignToGPR =
1394 AMDGPUMCExpr::createAlignTo(Value: MaxNumGPR, Align: GranuleConst, Ctx);
1395 const MCExpr *DivGPR =
1396 MCBinaryExpr::createDiv(LHS: AlignToGPR, RHS: GranuleConst, Ctx);
1397 const MCExpr *SubGPR = MCBinaryExpr::createSub(LHS: DivGPR, RHS: OneConst, Ctx);
1398 return SubGPR;
1399 };
1400 // GFX10+ will always allocate 128 SGPRs and this field must be 0
1401 if (STM.getGeneration() >= AMDGPUSubtarget::GFX10) {
1402 ProgInfo.SGPRBlocks = CreateExpr(0ul);
1403 } else {
1404 ProgInfo.SGPRBlocks = GetNumGPRBlocks(ProgInfo.NumSGPRsForWavesPerEU,
1405 IsaInfo::getSGPREncodingGranule(STI: STM));
1406 }
1407 ProgInfo.VGPRBlocks = GetNumGPRBlocks(ProgInfo.NumVGPRsForWavesPerEU,
1408 IsaInfo::getVGPREncodingGranule(STI: STM));
1409
1410 const SIModeRegisterDefaults Mode = MFI->getMode();
1411
1412 // Set the value to initialize FP_ROUND and FP_DENORM parts of the mode
1413 // register.
1414 ProgInfo.FloatMode = getFPMode(Mode);
1415
1416 ProgInfo.IEEEMode = Mode.IEEE;
1417
1418 // Make clamp modifier on NaN input returns 0.
1419 ProgInfo.DX10Clamp = Mode.DX10Clamp;
1420 ProgInfo.SGPRSpill = MFI->getNumSpilledSGPRs();
1421 ProgInfo.VGPRSpill = MFI->getNumSpilledVGPRs();
1422
1423 ProgInfo.LDSSize = MFI->getLDSSize();
1424
1425 unsigned LDSGranularityBytes = getLdsDwGranularity(ST: STM) * 4;
1426 ProgInfo.LDSBlocks =
1427 alignTo(Value: ProgInfo.LDSSize, Align: LDSGranularityBytes) / LDSGranularityBytes;
1428
1429 // The MCExpr equivalent of divideCeil.
1430 auto DivideCeil = [&Ctx](const MCExpr *Numerator, const MCExpr *Denominator) {
1431 const MCExpr *Ceil =
1432 AMDGPUMCExpr::createAlignTo(Value: Numerator, Align: Denominator, Ctx);
1433 return MCBinaryExpr::createDiv(LHS: Ceil, RHS: Denominator, Ctx);
1434 };
1435
1436 // Scratch is allocated in 64-dword or 256-dword blocks.
1437 unsigned ScratchAlignShift =
1438 STM.getGeneration() >= AMDGPUSubtarget::GFX11 ? 8 : 10;
1439 // We need to program the hardware with the amount of scratch memory that
1440 // is used by the entire wave. ProgInfo.ScratchSize is the amount of
1441 // scratch memory used per thread.
1442 ProgInfo.ScratchBlocks = DivideCeil(
1443 MCBinaryExpr::createMul(LHS: ProgInfo.ScratchSize,
1444 RHS: CreateExpr(STM.getWavefrontSize()), Ctx),
1445 CreateExpr(1ULL << ScratchAlignShift));
1446
1447 if (STM.hasSupportsWGP()) {
1448 ProgInfo.WgpMode = STM.isCuModeEnabled() ? 0 : 1;
1449 }
1450
1451 if (getIsaVersion(GPU: getGlobalSTI()->getCPU()).Major >= 10) {
1452 ProgInfo.MemOrdered = 1;
1453 ProgInfo.FwdProgress = !F.hasFnAttribute(Kind: "amdgpu-no-fwd-progress");
1454 }
1455
1456 // 0 = X, 1 = XY, 2 = XYZ
1457 unsigned TIDIGCompCnt = 0;
1458 if (MFI->hasWorkItemIDZ())
1459 TIDIGCompCnt = 2;
1460 else if (MFI->hasWorkItemIDY())
1461 TIDIGCompCnt = 1;
1462
1463 // The private segment wave byte offset is the last of the system SGPRs. We
1464 // initially assumed it was allocated, and may have used it. It shouldn't harm
1465 // anything to disable it if we know the stack isn't used here. We may still
1466 // have emitted code reading it to initialize scratch, but if that's unused
1467 // reading garbage should be OK.
1468 ProgInfo.ScratchEnable = MCBinaryExpr::createLOr(
1469 LHS: MCBinaryExpr::createGT(LHS: ProgInfo.ScratchBlocks,
1470 RHS: MCConstantExpr::create(Value: 0, Ctx), Ctx),
1471 RHS: ProgInfo.DynamicCallStack, Ctx);
1472
1473 ProgInfo.UserSGPR = MFI->getNumUserSGPRs();
1474 // For AMDHSA, TRAP_HANDLER must be zero, as it is populated by the CP.
1475 ProgInfo.TrapHandlerEnable = STM.isAmdHsaOS() ? 0 : STM.hasTrapHandler();
1476 ProgInfo.TGIdXEnable = MFI->hasWorkGroupIDX();
1477 ProgInfo.TGIdYEnable = MFI->hasWorkGroupIDY();
1478 ProgInfo.TGIdZEnable = MFI->hasWorkGroupIDZ();
1479 ProgInfo.TGSizeEnable = MFI->hasWorkGroupInfo();
1480 ProgInfo.TIdIGCompCount = TIDIGCompCnt;
1481 ProgInfo.EXCPEnMSB = 0;
1482 // For AMDHSA, LDS_SIZE must be zero, as it is populated by the CP.
1483 ProgInfo.LdsSize = STM.isAmdHsaOS() ? 0 : ProgInfo.LDSBlocks;
1484 ProgInfo.EXCPEnable = 0;
1485
1486 if (STM.hasGFX90AInsts()) {
1487 ProgInfo.ComputePGMRSrc3 =
1488 setBits(Dst: ProgInfo.ComputePGMRSrc3, Value: ProgInfo.AccumOffset,
1489 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
1490 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT, Ctx);
1491 ProgInfo.ComputePGMRSrc3 =
1492 setBits(Dst: ProgInfo.ComputePGMRSrc3, Value: CreateExpr(ProgInfo.TgSplit),
1493 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT,
1494 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT, Ctx);
1495 }
1496
1497 if (STM.hasGFX1250Insts())
1498 ProgInfo.ComputePGMRSrc3 =
1499 setBits(Dst: ProgInfo.ComputePGMRSrc3, Value: ProgInfo.NamedBarCnt,
1500 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
1501 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT, Ctx);
1502
1503 ProgInfo.Occupancy = createOccupancy(
1504 InitOcc: STM.computeOccupancy(F, LDSSize: ProgInfo.LDSSize).second,
1505 NumSGPRs: ProgInfo.NumSGPRsForWavesPerEU, NumVGPRs: ProgInfo.NumVGPRsForWavesPerEU,
1506 DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize(), STM, Ctx);
1507
1508 const auto [MinWEU, MaxWEU] =
1509 AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-waves-per-eu", Default: {0, 0}, OnlyFirstRequired: true);
1510 uint64_t Occupancy;
1511 if (TryGetMCExprValue(ProgInfo.Occupancy, Occupancy) && Occupancy < MinWEU) {
1512 DiagnosticInfoOptimizationFailure Diag(
1513 F, F.getSubprogram(),
1514 "failed to meet occupancy target given by 'amdgpu-waves-per-eu' in "
1515 "'" +
1516 F.getName() + "': desired occupancy was " + Twine(MinWEU) +
1517 ", final occupancy is " + Twine(Occupancy));
1518 F.getContext().diagnose(DI: Diag);
1519 }
1520}
1521
1522static unsigned getRsrcReg(CallingConv::ID CallConv) {
1523 switch (CallConv) {
1524 default:
1525 [[fallthrough]];
1526 case CallingConv::AMDGPU_CS:
1527 return R_00B848_COMPUTE_PGM_RSRC1;
1528 case CallingConv::AMDGPU_LS:
1529 return R_00B528_SPI_SHADER_PGM_RSRC1_LS;
1530 case CallingConv::AMDGPU_HS:
1531 return R_00B428_SPI_SHADER_PGM_RSRC1_HS;
1532 case CallingConv::AMDGPU_ES:
1533 return R_00B328_SPI_SHADER_PGM_RSRC1_ES;
1534 case CallingConv::AMDGPU_GS:
1535 return R_00B228_SPI_SHADER_PGM_RSRC1_GS;
1536 case CallingConv::AMDGPU_VS:
1537 return R_00B128_SPI_SHADER_PGM_RSRC1_VS;
1538 case CallingConv::AMDGPU_PS:
1539 return R_00B028_SPI_SHADER_PGM_RSRC1_PS;
1540 }
1541}
1542
1543void AMDGPUAsmPrinter::EmitProgramInfoSI(
1544 const MachineFunction &MF, const SIProgramInfo &CurrentProgramInfo) {
1545 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1546 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1547 unsigned RsrcReg = getRsrcReg(CallConv: MF.getFunction().getCallingConv());
1548 MCContext &Ctx = MF.getContext();
1549
1550 // (((Value) & Mask) << Shift)
1551 auto SetBits = [&Ctx](const MCExpr *Value, uint32_t Mask, uint32_t Shift) {
1552 const MCExpr *msk = MCConstantExpr::create(Value: Mask, Ctx);
1553 const MCExpr *shft = MCConstantExpr::create(Value: Shift, Ctx);
1554 return MCBinaryExpr::createShl(LHS: MCBinaryExpr::createAnd(LHS: Value, RHS: msk, Ctx),
1555 RHS: shft, Ctx);
1556 };
1557
1558 auto EmitResolvedOrExpr = [this](const MCExpr *Value, unsigned Size) {
1559 int64_t Val;
1560 if (Value->evaluateAsAbsolute(Res&: Val))
1561 OutStreamer->emitIntValue(Value: static_cast<uint64_t>(Val), Size);
1562 else
1563 OutStreamer->emitValue(Value, Size);
1564 };
1565
1566 if (AMDGPU::isCompute(CC: MF.getFunction().getCallingConv())) {
1567 OutStreamer->emitInt32(R_00B848_COMPUTE_PGM_RSRC1);
1568
1569 EmitResolvedOrExpr(CurrentProgramInfo.getComputePGMRSrc1(ST: STM, Ctx),
1570 /*Size=*/4);
1571
1572 OutStreamer->emitInt32(R_00B84C_COMPUTE_PGM_RSRC2);
1573 EmitResolvedOrExpr(CurrentProgramInfo.getComputePGMRSrc2(ST: STM, Ctx),
1574 /*Size=*/4);
1575
1576 OutStreamer->emitInt32(R_00B860_COMPUTE_TMPRING_SIZE);
1577
1578 // Sets bits according to S_0286E8_WAVESIZE_* mask and shift values for the
1579 // appropriate generation.
1580 if (STM.getGeneration() >= AMDGPUSubtarget::GFX12) {
1581 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1582 /*Mask=*/0x3FFFF, /*Shift=*/12),
1583 /*Size=*/4);
1584 } else if (STM.getGeneration() == AMDGPUSubtarget::GFX11) {
1585 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1586 /*Mask=*/0x7FFF, /*Shift=*/12),
1587 /*Size=*/4);
1588 } else {
1589 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1590 /*Mask=*/0x1FFF, /*Shift=*/12),
1591 /*Size=*/4);
1592 }
1593
1594 // TODO: Should probably note flat usage somewhere. SC emits a "FlatPtr32 =
1595 // 0" comment but I don't see a corresponding field in the register spec.
1596 } else {
1597 OutStreamer->emitInt32(Value: RsrcReg);
1598
1599 const MCExpr *GPRBlocks = MCBinaryExpr::createOr(
1600 LHS: SetBits(CurrentProgramInfo.VGPRBlocks, /*Mask=*/0x3F, /*Shift=*/0),
1601 RHS: SetBits(CurrentProgramInfo.SGPRBlocks, /*Mask=*/0x0F, /*Shift=*/6),
1602 Ctx&: MF.getContext());
1603 EmitResolvedOrExpr(GPRBlocks, /*Size=*/4);
1604 OutStreamer->emitInt32(R_0286E8_SPI_TMPRING_SIZE);
1605
1606 // Sets bits according to S_0286E8_WAVESIZE_* mask and shift values for the
1607 // appropriate generation.
1608 if (STM.getGeneration() >= AMDGPUSubtarget::GFX12) {
1609 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1610 /*Mask=*/0x3FFFF, /*Shift=*/12),
1611 /*Size=*/4);
1612 } else if (STM.getGeneration() == AMDGPUSubtarget::GFX11) {
1613 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1614 /*Mask=*/0x7FFF, /*Shift=*/12),
1615 /*Size=*/4);
1616 } else {
1617 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1618 /*Mask=*/0x1FFF, /*Shift=*/12),
1619 /*Size=*/4);
1620 }
1621 }
1622
1623 if (MF.getFunction().getCallingConv() == CallingConv::AMDGPU_PS) {
1624 OutStreamer->emitInt32(R_00B02C_SPI_SHADER_PGM_RSRC2_PS);
1625 unsigned ExtraLDSSize = STM.getGeneration() >= AMDGPUSubtarget::GFX11
1626 ? divideCeil(Numerator: CurrentProgramInfo.LDSBlocks, Denominator: 2)
1627 : CurrentProgramInfo.LDSBlocks;
1628 OutStreamer->emitInt32(S_00B02C_EXTRA_LDS_SIZE(ExtraLDSSize));
1629 OutStreamer->emitInt32(R_0286CC_SPI_PS_INPUT_ENA);
1630 OutStreamer->emitInt32(Value: MFI->getPSInputEnable());
1631 OutStreamer->emitInt32(R_0286D0_SPI_PS_INPUT_ADDR);
1632 OutStreamer->emitInt32(Value: MFI->getPSInputAddr());
1633 }
1634
1635 OutStreamer->emitInt32(R_SPILLED_SGPRS);
1636 OutStreamer->emitInt32(Value: MFI->getNumSpilledSGPRs());
1637 OutStreamer->emitInt32(R_SPILLED_VGPRS);
1638 OutStreamer->emitInt32(Value: MFI->getNumSpilledVGPRs());
1639}
1640
1641// Helper function to add common PAL Metadata 3.0+
1642static void EmitPALMetadataCommon(AMDGPUPALMetadata *MD,
1643 const SIProgramInfo &CurrentProgramInfo,
1644 CallingConv::ID CC, const GCNSubtarget &ST,
1645 unsigned DynamicVGPRBlockSize) {
1646 if (ST.hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode))
1647 MD->setHwStage(CC, field: ".ieee_mode", Val: (bool)CurrentProgramInfo.IEEEMode);
1648
1649 MD->setHwStage(CC, field: ".wgp_mode", Val: (bool)CurrentProgramInfo.WgpMode);
1650 MD->setHwStage(CC, field: ".mem_ordered", Val: (bool)CurrentProgramInfo.MemOrdered);
1651 MD->setHwStage(CC, field: ".forward_progress", Val: (bool)CurrentProgramInfo.FwdProgress);
1652
1653 if (AMDGPU::isCompute(CC)) {
1654 MD->setHwStage(CC, field: ".trap_present",
1655 Val: (bool)CurrentProgramInfo.TrapHandlerEnable);
1656 MD->setHwStage(CC, field: ".excp_en", Val: CurrentProgramInfo.EXCPEnable);
1657
1658 if (DynamicVGPRBlockSize != 0)
1659 MD->setComputeRegisters(field: ".dynamic_vgpr_en", Val: true);
1660 }
1661
1662 MD->updateHwStageMaximum(
1663 CC, field: ".lds_size",
1664 Val: (unsigned)(CurrentProgramInfo.LdsSize * getLdsDwGranularity(ST) *
1665 sizeof(uint32_t)));
1666}
1667
1668// This is the equivalent of EmitProgramInfoSI above, but for when the OS type
1669// is AMDPAL. It stores each compute/SPI register setting and other PAL
1670// metadata items into the PALMD::Metadata, combining with any provided by the
1671// frontend as LLVM metadata. Once all functions are written, the PAL metadata
1672// is then written as a single block in the .note section.
1673void AMDGPUAsmPrinter::EmitPALMetadata(
1674 const MachineFunction &MF, const SIProgramInfo &CurrentProgramInfo) {
1675 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1676 auto CC = MF.getFunction().getCallingConv();
1677 auto *MD = getTargetStreamer()->getPALMetadata();
1678 auto &Ctx = MF.getContext();
1679
1680 MD->setEntryPoint(CC, Name: MF.getFunction().getName());
1681 MD->setNumUsedVgprs(CC, Val: CurrentProgramInfo.NumVGPRsForWavesPerEU, Ctx);
1682
1683 // For targets that support dynamic VGPRs, set the number of saved dynamic
1684 // VGPRs (if any) in the PAL metadata.
1685 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1686 if (MFI->isDynamicVGPREnabled() &&
1687 MFI->getScratchReservedForDynamicVGPRs() > 0)
1688 MD->setHwStage(CC, field: ".dynamic_vgpr_saved_count",
1689 Val: MFI->getScratchReservedForDynamicVGPRs() / 4);
1690
1691 // Only set AGPRs for supported devices
1692 if (STM.hasMAIInsts()) {
1693 MD->setNumUsedAgprs(CC, Val: CurrentProgramInfo.NumAccVGPR);
1694 }
1695
1696 MD->setNumUsedSgprs(CC, Val: CurrentProgramInfo.NumSGPRsForWavesPerEU, Ctx);
1697 if (MD->getPALMajorVersion() < 3) {
1698 MD->setRsrc1(CC, Val: CurrentProgramInfo.getPGMRSrc1(CC, ST: STM, Ctx), Ctx);
1699 if (AMDGPU::isCompute(CC)) {
1700 MD->setRsrc2(CC, Val: CurrentProgramInfo.getComputePGMRSrc2(ST: STM, Ctx), Ctx);
1701 } else {
1702 const MCExpr *HasScratchBlocks =
1703 MCBinaryExpr::createGT(LHS: CurrentProgramInfo.ScratchBlocks,
1704 RHS: MCConstantExpr::create(Value: 0, Ctx), Ctx);
1705 auto [Shift, Mask] = getShiftMask(C_00B84C_SCRATCH_EN);
1706 MD->setRsrc2(CC, Val: maskShiftSet(Val: HasScratchBlocks, Mask, Shift, Ctx), Ctx);
1707 }
1708 } else {
1709 MD->setHwStage(CC, field: ".debug_mode", Val: (bool)CurrentProgramInfo.DebugMode);
1710 MD->setHwStage(CC, field: ".scratch_en", Type: msgpack::Type::Boolean,
1711 Val: CurrentProgramInfo.ScratchEnable);
1712 EmitPALMetadataCommon(MD, CurrentProgramInfo, CC, ST: STM,
1713 DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize());
1714 }
1715
1716 // ScratchSize is in bytes, 16 aligned.
1717 MD->setScratchSize(
1718 CC,
1719 Val: AMDGPUMCExpr::createAlignTo(Value: CurrentProgramInfo.ScratchSize,
1720 Align: MCConstantExpr::create(Value: 16, Ctx), Ctx),
1721 Ctx);
1722
1723 if (MF.getFunction().getCallingConv() == CallingConv::AMDGPU_PS) {
1724 unsigned ExtraLDSSize = STM.getGeneration() >= AMDGPUSubtarget::GFX11
1725 ? divideCeil(Numerator: CurrentProgramInfo.LDSBlocks, Denominator: 2)
1726 : CurrentProgramInfo.LDSBlocks;
1727 if (MD->getPALMajorVersion() < 3) {
1728 MD->setRsrc2(
1729 CC,
1730 Val: MCConstantExpr::create(S_00B02C_EXTRA_LDS_SIZE(ExtraLDSSize), Ctx),
1731 Ctx);
1732 MD->setSpiPsInputEna(MFI->getPSInputEnable());
1733 MD->setSpiPsInputAddr(MFI->getPSInputAddr());
1734 } else {
1735 // Graphics registers
1736 const unsigned ExtraLdsDwGranularity =
1737 STM.getGeneration() >= AMDGPUSubtarget::GFX11 ? 256 : 128;
1738 MD->setGraphicsRegisters(
1739 field: ".ps_extra_lds_size",
1740 Val: (unsigned)(ExtraLDSSize * ExtraLdsDwGranularity * sizeof(uint32_t)));
1741
1742 // Set PsInputEna and PsInputAddr .spi_ps_input_ena and .spi_ps_input_addr
1743 static StringLiteral const PsInputFields[] = {
1744 ".persp_sample_ena", ".persp_center_ena",
1745 ".persp_centroid_ena", ".persp_pull_model_ena",
1746 ".linear_sample_ena", ".linear_center_ena",
1747 ".linear_centroid_ena", ".line_stipple_tex_ena",
1748 ".pos_x_float_ena", ".pos_y_float_ena",
1749 ".pos_z_float_ena", ".pos_w_float_ena",
1750 ".front_face_ena", ".ancillary_ena",
1751 ".sample_coverage_ena", ".pos_fixed_pt_ena"};
1752 unsigned PSInputEna = MFI->getPSInputEnable();
1753 unsigned PSInputAddr = MFI->getPSInputAddr();
1754 for (auto [Idx, Field] : enumerate(First: PsInputFields)) {
1755 MD->setGraphicsRegisters(field1: ".spi_ps_input_ena", field2: Field,
1756 Val: (bool)((PSInputEna >> Idx) & 1));
1757 MD->setGraphicsRegisters(field1: ".spi_ps_input_addr", field2: Field,
1758 Val: (bool)((PSInputAddr >> Idx) & 1));
1759 }
1760 }
1761 }
1762
1763 // For version 3 and above the wave front size is already set in the metadata
1764 if (MD->getPALMajorVersion() < 3 && STM.isWave32())
1765 MD->setWave32(MF.getFunction().getCallingConv());
1766}
1767
1768void AMDGPUAsmPrinter::emitPALFunctionMetadata(const MachineFunction &MF) {
1769 auto *MD = getTargetStreamer()->getPALMetadata();
1770 const MachineFrameInfo &MFI = MF.getFrameInfo();
1771 StringRef FnName = MF.getFunction().getName();
1772 MD->setFunctionScratchSize(FnName, Val: MFI.getStackSize());
1773 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1774 MCContext &Ctx = MF.getContext();
1775
1776 if (MD->getPALMajorVersion() < 3) {
1777 // Set compute registers
1778 MD->setRsrc1(
1779 CC: CallingConv::AMDGPU_CS,
1780 Val: CurrentProgramInfo.getPGMRSrc1(CC: CallingConv::AMDGPU_CS, ST, Ctx), Ctx);
1781 MD->setRsrc2(CC: CallingConv::AMDGPU_CS,
1782 Val: CurrentProgramInfo.getComputePGMRSrc2(ST, Ctx), Ctx);
1783 } else {
1784 EmitPALMetadataCommon(
1785 MD, CurrentProgramInfo, CC: CallingConv::AMDGPU_CS, ST,
1786 DynamicVGPRBlockSize: MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize());
1787 }
1788
1789 // Set optional info
1790 MD->setFunctionLdsSize(FnName, Val: CurrentProgramInfo.LDSSize);
1791 MD->setFunctionNumUsedVgprs(FnName, Val: CurrentProgramInfo.NumVGPRsForWavesPerEU);
1792 MD->setFunctionNumUsedSgprs(FnName, Val: CurrentProgramInfo.NumSGPRsForWavesPerEU);
1793}
1794
1795// This is supposed to be log2(Size)
1796static amd_element_byte_size_t getElementByteSizeValue(unsigned Size) {
1797 switch (Size) {
1798 case 4:
1799 return AMD_ELEMENT_4_BYTES;
1800 case 8:
1801 return AMD_ELEMENT_8_BYTES;
1802 case 16:
1803 return AMD_ELEMENT_16_BYTES;
1804 default:
1805 llvm_unreachable("invalid private_element_size");
1806 }
1807}
1808
1809void AMDGPUAsmPrinter::getAmdKernelCode(AMDGPUMCKernelCodeT &Out,
1810 const SIProgramInfo &CurrentProgramInfo,
1811 const MachineFunction &MF) const {
1812 const Function &F = MF.getFunction();
1813 assert(F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
1814 F.getCallingConv() == CallingConv::SPIR_KERNEL);
1815
1816 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1817 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1818 MCContext &Ctx = MF.getContext();
1819
1820 Out.initDefault(STI: STM, Ctx, /*InitMCExpr=*/false);
1821
1822 Out.compute_pgm_resource1_registers =
1823 CurrentProgramInfo.getComputePGMRSrc1(ST: STM, Ctx);
1824 Out.compute_pgm_resource2_registers =
1825 CurrentProgramInfo.getComputePGMRSrc2(ST: STM, Ctx);
1826 Out.code_properties |= AMD_CODE_PROPERTY_IS_PTR64;
1827
1828 Out.is_dynamic_callstack = CurrentProgramInfo.DynamicCallStack;
1829
1830 AMD_HSA_BITS_SET(Out.code_properties, AMD_CODE_PROPERTY_PRIVATE_ELEMENT_SIZE,
1831 getElementByteSizeValue(STM.getMaxPrivateElementSize(true)));
1832
1833 const GCNUserSGPRUsageInfo &UserSGPRInfo = MFI->getUserSGPRInfo();
1834 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
1835 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER;
1836 }
1837
1838 if (UserSGPRInfo.hasDispatchPtr())
1839 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR;
1840
1841 if (UserSGPRInfo.hasQueuePtr())
1842 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR;
1843
1844 if (UserSGPRInfo.hasKernargSegmentPtr())
1845 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR;
1846
1847 if (UserSGPRInfo.hasDispatchID())
1848 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID;
1849
1850 if (UserSGPRInfo.hasFlatScratchInit())
1851 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT;
1852
1853 if (UserSGPRInfo.hasPrivateSegmentSize())
1854 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE;
1855
1856 if (STM.isXNACKEnabled())
1857 Out.code_properties |= AMD_CODE_PROPERTY_IS_XNACK_SUPPORTED;
1858
1859 Align MaxKernArgAlign;
1860 Out.kernarg_segment_byte_size = STM.getKernArgSegmentSize(F, MaxAlign&: MaxKernArgAlign);
1861 Out.wavefront_sgpr_count = CurrentProgramInfo.NumSGPR;
1862 Out.workitem_vgpr_count = CurrentProgramInfo.NumVGPR;
1863 Out.workitem_private_segment_byte_size = CurrentProgramInfo.ScratchSize;
1864 Out.workgroup_group_segment_byte_size = CurrentProgramInfo.LDSSize;
1865
1866 // kernarg_segment_alignment is specified as log of the alignment.
1867 // The minimum alignment is 16.
1868 // FIXME: The metadata treats the minimum as 4?
1869 Out.kernarg_segment_alignment = Log2(A: std::max(a: Align(16), b: MaxKernArgAlign));
1870}
1871
1872bool AMDGPUAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
1873 const char *ExtraCode, raw_ostream &O) {
1874 // First try the generic code, which knows about modifiers like 'c' and 'n'.
1875 if (!AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, OS&: O))
1876 return false;
1877
1878 if (ExtraCode && ExtraCode[0]) {
1879 if (ExtraCode[1] != 0)
1880 return true; // Unknown modifier.
1881
1882 switch (ExtraCode[0]) {
1883 case 'r':
1884 break;
1885 default:
1886 return true;
1887 }
1888 }
1889
1890 // TODO: Should be able to support other operand types like globals.
1891 const MachineOperand &MO = MI->getOperand(i: OpNo);
1892 if (MO.isReg()) {
1893 AMDGPUInstPrinter::printRegOperand(Reg: MO.getReg(), O,
1894 MRI: *MF->getSubtarget().getRegisterInfo());
1895 return false;
1896 }
1897 if (MO.isImm()) {
1898 int64_t Val = MO.getImm();
1899 if (AMDGPU::isInlinableIntLiteral(Literal: Val)) {
1900 O << Val;
1901 } else if (isUInt<16>(x: Val)) {
1902 O << format(Fmt: "0x%" PRIx16, Vals: static_cast<uint16_t>(Val));
1903 } else if (isUInt<32>(x: Val)) {
1904 O << format(Fmt: "0x%" PRIx32, Vals: static_cast<uint32_t>(Val));
1905 } else {
1906 O << format(Fmt: "0x%" PRIx64, Vals: static_cast<uint64_t>(Val));
1907 }
1908 return false;
1909 }
1910 return true;
1911}
1912
1913void AMDGPUAsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const {
1914 AU.addRequired<AMDGPUResourceUsageAnalysisWrapperPass>();
1915 AU.addPreserved<AMDGPUResourceUsageAnalysisWrapperPass>();
1916 AU.addRequired<MachineModuleInfoWrapperPass>();
1917 AU.addPreserved<MachineModuleInfoWrapperPass>();
1918 AsmPrinter::getAnalysisUsage(AU);
1919}
1920
1921void AMDGPUAsmPrinter::emitResourceUsageRemarks(
1922 const MachineFunction &MF, const SIProgramInfo &CurrentProgramInfo,
1923 bool isModuleEntryFunction, bool hasMAIInsts) {
1924 if (!ORE)
1925 return;
1926
1927 const char *Name = "kernel-resource-usage";
1928 const char *Indent = " ";
1929
1930 // If the remark is not specifically enabled, do not output to yaml
1931 LLVMContext &Ctx = MF.getFunction().getContext();
1932 if (!Ctx.getDiagHandlerPtr()->isAnalysisRemarkEnabled(PassName: Name))
1933 return;
1934
1935 // Currently non-kernel functions have no resources to emit.
1936 if (!isEntryFunctionCC(CC: MF.getFunction().getCallingConv()))
1937 return;
1938
1939 auto EmitResourceUsageRemark = [&](StringRef RemarkName,
1940 StringRef RemarkLabel, auto Argument) {
1941 // Add an indent for every line besides the line with the kernel name. This
1942 // makes it easier to tell which resource usage go with which kernel since
1943 // the kernel name will always be displayed first.
1944 std::string LabelStr = RemarkLabel.str() + ": ";
1945 if (RemarkName != "FunctionName")
1946 LabelStr = Indent + LabelStr;
1947
1948 ORE->emit([&]() {
1949 return MachineOptimizationRemarkAnalysis(Name, RemarkName,
1950 MF.getFunction().getSubprogram(),
1951 &MF.front())
1952 << LabelStr << ore::NV(RemarkName, Argument);
1953 });
1954 };
1955
1956 // FIXME: Formatting here is pretty nasty because clang does not accept
1957 // newlines from diagnostics. This forces us to emit multiple diagnostic
1958 // remarks to simulate newlines. If and when clang does accept newlines, this
1959 // formatting should be aggregated into one remark with newlines to avoid
1960 // printing multiple diagnostic location and diag opts.
1961 EmitResourceUsageRemark("FunctionName", "Function Name",
1962 MF.getFunction().getName());
1963 EmitResourceUsageRemark("NumSGPR", "TotalSGPRs",
1964 getMCExprStr(Value: CurrentProgramInfo.NumSGPR));
1965 EmitResourceUsageRemark("NumVGPR", "VGPRs",
1966 getMCExprStr(Value: CurrentProgramInfo.NumArchVGPR));
1967 if (hasMAIInsts) {
1968 EmitResourceUsageRemark("NumAGPR", "AGPRs",
1969 getMCExprStr(Value: CurrentProgramInfo.NumAccVGPR));
1970 }
1971 EmitResourceUsageRemark("ScratchSize", "ScratchSize [bytes/lane]",
1972 getMCExprStr(Value: CurrentProgramInfo.ScratchSize));
1973 int64_t DynStack;
1974 bool DynStackEvaluatable =
1975 CurrentProgramInfo.DynamicCallStack->evaluateAsAbsolute(Res&: DynStack);
1976 StringRef DynamicStackStr =
1977 DynStackEvaluatable && DynStack ? "True" : "False";
1978 EmitResourceUsageRemark("DynamicStack", "Dynamic Stack", DynamicStackStr);
1979 EmitResourceUsageRemark("Occupancy", "Occupancy [waves/SIMD]",
1980 getMCExprStr(Value: CurrentProgramInfo.Occupancy));
1981 EmitResourceUsageRemark("SGPRSpill", "SGPRs Spill",
1982 CurrentProgramInfo.SGPRSpill);
1983 EmitResourceUsageRemark("VGPRSpill", "VGPRs Spill",
1984 CurrentProgramInfo.VGPRSpill);
1985 if (isModuleEntryFunction)
1986 EmitResourceUsageRemark("BytesLDS", "LDS Size [bytes/block]",
1987 CurrentProgramInfo.LDSSize);
1988}
1989
1990PreservedAnalyses AMDGPUAsmPrinterBeginPass::run(Module &M,
1991 ModuleAnalysisManager &MAM) {
1992
1993 AMDGPUAsmPrinter &AsmPrinter = static_cast<AMDGPUAsmPrinter &>(
1994 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
1995 setupModuleAsmPrinter(M, MAM, AsmPrinter);
1996 AsmPrinter.doInitialization(M);
1997 return PreservedAnalyses::all();
1998}
1999
2000PreservedAnalyses
2001AMDGPUAsmPrinterPass::run(MachineFunction &MF,
2002 MachineFunctionAnalysisManager &MFAM) {
2003 AMDGPUAsmPrinter &AsmPrinter = static_cast<AMDGPUAsmPrinter &>(
2004 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF)
2005 .getCachedResult<AsmPrinterAnalysis>(IR&: *MF.getFunction().getParent())
2006 ->getPrinter());
2007 setupMachineFunctionAsmPrinter(MFAM, MF, AsmPrinter);
2008 AsmPrinter.GetResourceUsage = [&MFAM](MachineFunction &MF)
2009 -> const AMDGPUResourceUsageAnalysisImpl::SIFunctionResourceInfo * {
2010 return &MFAM.getResult<AMDGPUResourceUsageAnalysis>(IR&: MF);
2011 };
2012 AsmPrinter.runOnMachineFunction(MF);
2013 return PreservedAnalyses::all();
2014}
2015
2016PreservedAnalyses AMDGPUAsmPrinterEndPass::run(Module &M,
2017 ModuleAnalysisManager &MAM) {
2018 AMDGPUAsmPrinter &AsmPrinter = static_cast<AMDGPUAsmPrinter &>(
2019 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
2020 setupModuleAsmPrinter(M, MAM, AsmPrinter);
2021 AsmPrinter.doFinalization(M);
2022 return PreservedAnalyses::all();
2023}
2024
2025char AMDGPUAsmPrinter::ID = 0;
2026
2027INITIALIZE_PASS(AMDGPUAsmPrinter, "amdgpu-asm-printer",
2028 "AMDGPU Assembly Printer", false, false)
2029