1//===-- AMDGPUAsmPrinter.cpp - AMDGPU assembly printer --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10///
11/// The AMDGPUAsmPrinter is used to print both assembly string and also binary
12/// code. When passed an MCAsmStreamer it prints assembly and when passed
13/// an MCObjectStreamer it outputs binary code.
14//
15//===----------------------------------------------------------------------===//
16//
17
18#include "AMDGPUAsmPrinter.h"
19#include "AMDGPU.h"
20#include "AMDGPUHSAMetadataStreamer.h"
21#include "AMDGPUMCResourceInfo.h"
22#include "AMDGPUResourceUsageAnalysis.h"
23#include "AMDGPUTargetMachine.h"
24#include "GCNSubtarget.h"
25#include "MCTargetDesc/AMDGPUInstPrinter.h"
26#include "MCTargetDesc/AMDGPUMCExpr.h"
27#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h"
28#include "MCTargetDesc/AMDGPUTargetStreamer.h"
29#include "R600AsmPrinter.h"
30#include "SIMachineFunctionInfo.h"
31#include "TargetInfo/AMDGPUTargetInfo.h"
32#include "Utils/AMDGPUBaseInfo.h"
33#include "Utils/AMDKernelCodeTUtils.h"
34#include "Utils/SIDefinesUtils.h"
35#include "llvm/Analysis/OptimizationRemarkEmitter.h"
36#include "llvm/BinaryFormat/ELF.h"
37#include "llvm/CodeGen/AsmPrinterAnalysis.h"
38#include "llvm/CodeGen/AsmPrinterHandler.h"
39#include "llvm/CodeGen/MachineFrameInfo.h"
40#include "llvm/CodeGen/MachineModuleInfo.h"
41#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
42#include "llvm/IR/DiagnosticInfo.h"
43#include "llvm/MC/MCAssembler.h"
44#include "llvm/MC/MCContext.h"
45#include "llvm/MC/MCSectionELF.h"
46#include "llvm/MC/MCStreamer.h"
47#include "llvm/MC/MCValue.h"
48#include "llvm/MC/TargetRegistry.h"
49#include "llvm/Support/AMDHSAKernelDescriptor.h"
50#include "llvm/Support/Compiler.h"
51#include "llvm/Target/TargetLoweringObjectFile.h"
52#include "llvm/Target/TargetMachine.h"
53#include "llvm/TargetParser/AMDGPUTargetParser.h"
54
55using namespace llvm;
56using namespace llvm::AMDGPU;
57
58// This should get the default rounding mode from the kernel. We just set the
59// default here, but this could change if the OpenCL rounding mode pragmas are
60// used.
61//
62// The denormal mode here should match what is reported by the OpenCL runtime
63// for the CL_FP_DENORM bit from CL_DEVICE_{HALF|SINGLE|DOUBLE}_FP_CONFIG, but
64// can also be override to flush with the -cl-denorms-are-zero compiler flag.
65//
66// AMD OpenCL only sets flush none and reports CL_FP_DENORM for double
67// precision, and leaves single precision to flush all and does not report
68// CL_FP_DENORM for CL_DEVICE_SINGLE_FP_CONFIG. Mesa's OpenCL currently reports
69// CL_FP_DENORM for both.
70//
71// FIXME: It seems some instructions do not support single precision denormals
72// regardless of the mode (exp_*_f32, rcp_*_f32, rsq_*_f32, rsq_*f32, sqrt_f32,
73// and sin_f32, cos_f32 on most parts).
74
75// We want to use these instructions, and using fp32 denormals also causes
76// instructions to run at the double precision rate for the device so it's
77// probably best to just report no single precision denormals.
78static uint32_t getFPMode(SIModeRegisterDefaults Mode) {
79 return FP_ROUND_MODE_SP(FP_ROUND_ROUND_TO_NEAREST) |
80 FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_NEAREST) |
81 FP_DENORM_MODE_SP(Mode.fpDenormModeSPValue()) |
82 FP_DENORM_MODE_DP(Mode.fpDenormModeDPValue());
83}
84
85static AsmPrinter *
86createAMDGPUAsmPrinterPass(TargetMachine &tm,
87 std::unique_ptr<MCStreamer> &&Streamer) {
88 return new AMDGPUAsmPrinter(tm, std::move(Streamer));
89}
90
91extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
92LLVMInitializeAMDGPUAsmPrinter() {
93 TargetRegistry::RegisterAsmPrinter(T&: getTheR600Target(),
94 Fn: llvm::createR600AsmPrinterPass);
95 TargetRegistry::RegisterAsmPrinter(T&: getTheGCNTarget(),
96 Fn: createAMDGPUAsmPrinterPass);
97 TargetRegistry::RegisterAsmPrinter(T&: getTheGCNLegacyTarget(),
98 Fn: createAMDGPUAsmPrinterPass);
99}
100
101namespace {
102class AMDGPUAsmPrinterHandler : public AsmPrinterHandler {
103protected:
104 AMDGPUAsmPrinter *Asm;
105
106public:
107 AMDGPUAsmPrinterHandler(AMDGPUAsmPrinter *A) : Asm(A) {}
108
109 void beginFunction(const MachineFunction *MF) override {}
110
111 void endFunction(const MachineFunction *MF) override { Asm->endFunction(MF); }
112
113 void endModule() override {}
114};
115} // End anonymous namespace
116
117AMDGPUAsmPrinter::AMDGPUAsmPrinter(TargetMachine &TM,
118 std::unique_ptr<MCStreamer> Streamer)
119 : AsmPrinter(TM, std::move(Streamer)) {
120 assert(OutStreamer && "AsmPrinter constructed without streamer");
121 // AMDGPU describes a pointer's address space with its own attribute rather
122 // than DW_AT_address_class.
123 TypeAddressSpaceAttribute = dwarf::DW_AT_LLVM_address_space;
124 GetResourceUsage = [this](MachineFunction &MF)
125 -> const AMDGPUResourceUsageAnalysisImpl::SIFunctionResourceInfo * {
126 if (auto *ResourceUsageW =
127 getAnalysisIfAvailable<AMDGPUResourceUsageAnalysisWrapperPass>())
128 return &ResourceUsageW->getResourceInfo();
129 return nullptr;
130 };
131}
132
133StringRef AMDGPUAsmPrinter::getPassName() const {
134 return "AMDGPU Assembly Printer";
135}
136
137const MCSubtargetInfo *AMDGPUAsmPrinter::getGlobalSTI() const {
138 return &TM.getMCSubtargetInfo();
139}
140
141AMDGPUTargetStreamer *AMDGPUAsmPrinter::getTargetStreamer() const {
142 if (!OutStreamer)
143 return nullptr;
144 return static_cast<AMDGPUTargetStreamer *>(OutStreamer->getTargetStreamer());
145}
146
147void AMDGPUAsmPrinter::emitStartOfAsmFile(Module &M) {
148 IsTargetStreamerInitialized = false;
149}
150
151void AMDGPUAsmPrinter::initTargetStreamer(Module &M) {
152 IsTargetStreamerInitialized = true;
153
154 // TODO: Which one is called first, emitStartOfAsmFile or
155 // emitFunctionBodyStart?
156 if (getTargetStreamer() && !getTargetStreamer()->getTargetID())
157 initializeTargetID(M);
158
159 const Triple &TT = M.getTargetTriple();
160 if (TT.getOS() != Triple::AMDHSA && TT.getOS() != Triple::AMDPAL)
161 return;
162
163 getTargetStreamer()->EmitDirectiveAMDGCNTarget();
164
165 if (TT.getOS() == Triple::AMDHSA) {
166 getTargetStreamer()->EmitDirectiveAMDHSACodeObjectVersion(
167 COV: CodeObjectVersion);
168 HSAMetadataStream->begin(Mod: M, TargetID: *getTargetStreamer()->getTargetID());
169 }
170
171 if (TT.getOS() == Triple::AMDPAL)
172 getTargetStreamer()->getPALMetadata()->readFromIR(M);
173}
174
175void AMDGPUAsmPrinter::emitEndOfAsmFile(Module &M) {
176 // Init target streamer if it has not yet happened
177 if (!IsTargetStreamerInitialized)
178 initTargetStreamer(M);
179
180 const Triple &TT = M.getTargetTriple();
181 if (TT.getOS() != Triple::AMDHSA)
182 getTargetStreamer()->EmitISAVersion();
183
184 // Emit HSA Metadata (NT_AMD_AMDGPU_HSA_METADATA).
185 // Emit HSA Metadata (NT_AMD_HSA_METADATA).
186 if (TT.getOS() == Triple::AMDHSA) {
187 HSAMetadataStream->end();
188 bool Success = HSAMetadataStream->emitTo(TargetStreamer&: *getTargetStreamer());
189 (void)Success;
190 assert(Success && "Malformed HSA Metadata");
191 }
192}
193
194void AMDGPUAsmPrinter::emitFunctionBodyStart() {
195 const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
196 const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
197 const Function &F = MF->getFunction();
198
199 // TODO: We're checking this late, would be nice to check it earlier.
200 if (STM.requiresCodeObjectV6() && CodeObjectVersion < AMDGPU::AMDHSA_COV6) {
201 reportFatalUsageError(
202 reason: STM.getCPU() + " is only available on code object version 6 or better");
203 }
204
205 // TODO: Which one is called first, emitStartOfAsmFile or
206 // emitFunctionBodyStart?
207 if (!getTargetStreamer()->getTargetID())
208 initializeTargetID(M: *F.getParent());
209
210 if (!MFI.isEntryFunction())
211 return;
212
213 if (STM.isMesaKernel(F) &&
214 (F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
215 F.getCallingConv() == CallingConv::SPIR_KERNEL)) {
216 AMDGPUMCKernelCodeT KernelCode;
217 getAmdKernelCode(Out&: KernelCode, KernelInfo: CurrentProgramInfo, MF: *MF);
218 KernelCode.validate(STI: &STM, Ctx&: MF->getContext());
219 getTargetStreamer()->EmitAMDKernelCodeT(Header&: KernelCode);
220 }
221
222 if (STM.isAmdHsaOS())
223 HSAMetadataStream->emitKernel(MF: *MF, ProgramInfo: CurrentProgramInfo);
224}
225
226/// Set bits in a kernel descriptor MCExpr field:
227/// return ((Dst & ~Mask) | (Value << Shift))
228static const MCExpr *setBits(const MCExpr *Dst, const MCExpr *Value,
229 uint32_t Mask, uint32_t Shift, MCContext &Ctx) {
230 const auto *Shft = MCConstantExpr::create(Value: Shift, Ctx);
231 const auto *Msk = MCConstantExpr::create(Value: Mask, Ctx);
232 Dst = MCBinaryExpr::createAnd(LHS: Dst, RHS: MCUnaryExpr::createNot(Expr: Msk, Ctx), Ctx);
233 Dst = MCBinaryExpr::createOr(LHS: Dst, RHS: MCBinaryExpr::createShl(LHS: Value, RHS: Shft, Ctx),
234 Ctx);
235 return Dst;
236}
237
238void AMDGPUAsmPrinter::endFunction(const MachineFunction *MF) {
239 const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
240 if (!MFI.isEntryFunction())
241 return;
242
243 assert(TM.getTargetTriple().getOS() == Triple::AMDHSA);
244
245 const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
246 MCContext &Ctx = MF->getContext();
247
248 AMDGPU::MCKernelDescriptor KD =
249 getAmdhsaKernelDescriptor(MF: *MF, PI: CurrentProgramInfo);
250
251 // Compute inst_pref_size using MCExpr label subtraction for exact code
252 // size. At this point .Lfunc_end has been emitted (by the base AsmPrinter)
253 // right after the function code, so (Lfunc_end - func_sym) gives the
254 // exact function code size in bytes.
255 if (STM.hasInstPrefSize()) {
256 const MCExpr *CodeSizeExpr = MCBinaryExpr::createSub(
257 LHS: MCSymbolRefExpr::create(Symbol: getFunctionEnd(), Ctx&: OutContext),
258 RHS: MCSymbolRefExpr::create(Symbol: CurrentFnSym, Ctx&: OutContext), Ctx&: OutContext);
259
260 uint32_t Mask, Shift, Width, CacheLineSize;
261 STM.getInstPrefSizeArgs(Mask, Shift, Width, CacheLineSize);
262 const MCExpr *InstPrefSize =
263 AMDGPUMCExpr::createInstPrefSize(CodeSizeBytes: CodeSizeExpr, Ctx);
264 KD.compute_pgm_rsrc3 =
265 setBits(Dst: KD.compute_pgm_rsrc3, Value: InstPrefSize, Mask, Shift, Ctx);
266 }
267
268 auto &Streamer = getTargetStreamer()->getStreamer();
269 auto &Context = Streamer.getContext();
270 auto &ObjectFileInfo = *Context.getObjectFileInfo();
271 auto &ReadOnlySection = *ObjectFileInfo.getReadOnlySection();
272
273 Streamer.pushSection();
274 Streamer.switchSection(Section: &ReadOnlySection);
275
276 // CP microcode requires the kernel descriptor to be allocated on 64 byte
277 // alignment.
278 Streamer.emitValueToAlignment(Alignment: Align(64), Fill: 0, FillLen: 1, MaxBytesToEmit: 0);
279 ReadOnlySection.ensureMinAlignment(MinAlignment: Align(64));
280
281 SmallString<128> KernelName;
282 getNameWithPrefix(Name&: KernelName, GV: &MF->getFunction());
283 getTargetStreamer()->EmitAmdhsaKernelDescriptor(
284 STI: STM, KernelName, KernelDescriptor: KD, NextVGPR: CurrentProgramInfo.NumVGPRsForWavesPerEU,
285 NextSGPR: MCBinaryExpr::createSub(
286 LHS: CurrentProgramInfo.NumSGPRsForWavesPerEU,
287 RHS: AMDGPUMCExpr::createExtraSGPRs(
288 VCCUsed: CurrentProgramInfo.VCCUsed, FlatScrUsed: CurrentProgramInfo.FlatUsed,
289 XNACKUsed: getTargetStreamer()->getTargetID()->isXnackOnOrAny(), Ctx&: Context),
290 Ctx&: Context),
291 ReserveVCC: CurrentProgramInfo.VCCUsed, ReserveFlatScr: CurrentProgramInfo.FlatUsed);
292
293 Streamer.popSection();
294}
295
296void AMDGPUAsmPrinter::emitImplicitDef(const MachineInstr *MI) const {
297 Register RegNo = MI->getOperand(i: 0).getReg();
298
299 SmallString<128> Str;
300 raw_svector_ostream OS(Str);
301 OS << "implicit-def: "
302 << printReg(Reg: RegNo, TRI: MF->getSubtarget().getRegisterInfo());
303
304 if (MI->getAsmPrinterFlags() & AMDGPU::SGPR_SPILL)
305 OS << " : SGPR spill to VGPR lane";
306
307 OutStreamer->AddComment(T: OS.str());
308 OutStreamer->addBlankLine();
309}
310
311void AMDGPUAsmPrinter::emitFunctionEntryLabel() {
312 if (TM.getTargetTriple().getOS() == Triple::AMDHSA) {
313 AsmPrinter::emitFunctionEntryLabel();
314 return;
315 }
316
317 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
318 const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
319 if (MFI->isEntryFunction() && STM.isAmdHsaOrMesa(F: MF->getFunction())) {
320 SmallString<128> SymbolName;
321 getNameWithPrefix(Name&: SymbolName, GV: &MF->getFunction()),
322 getTargetStreamer()->EmitAMDGPUSymbolType(SymbolName,
323 Type: ELF::STT_AMDGPU_HSA_KERNEL);
324 }
325 if (DumpCodeInstEmitter) {
326 // Disassemble function name label to text.
327 DisasmLines.push_back(x: MF->getName().str() + ":");
328 DisasmLineMaxLen = std::max(a: DisasmLineMaxLen, b: DisasmLines.back().size());
329 HexLines.emplace_back(args: "");
330 }
331
332 AsmPrinter::emitFunctionEntryLabel();
333}
334
335void AMDGPUAsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) {
336 if (DumpCodeInstEmitter && !isBlockOnlyReachableByFallthrough(MBB: &MBB)) {
337 // Write a line for the basic block label if it is not only fallthrough.
338 DisasmLines.push_back(x: (Twine("BB") + Twine(getFunctionNumber()) + "_" +
339 Twine(MBB.getNumber()) + ":")
340 .str());
341 DisasmLineMaxLen = std::max(a: DisasmLineMaxLen, b: DisasmLines.back().size());
342 HexLines.emplace_back(args: "");
343 }
344 AsmPrinter::emitBasicBlockStart(MBB);
345}
346
347void AMDGPUAsmPrinter::emitGlobalVariable(const GlobalVariable *GV) {
348 if (GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
349 if (GV->hasInitializer() && !isa<UndefValue>(Val: GV->getInitializer())) {
350 OutContext.reportError(L: {},
351 Msg: Twine(GV->getName()) +
352 ": unsupported initializer for address space");
353 return;
354 }
355
356 const Triple::OSType OS = TM.getTargetTriple().getOS();
357 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL) {
358 if (!AMDGPUTargetMachine::EnableObjectLinking)
359 return;
360 // With object linking, LDS definitions should have been externalized
361 // by earlier passes (e.g. LDS lowering, named barrier lowering).
362 // Only declarations reach here, emitted as SHN_AMDGPU_LDS symbols
363 // so the linker can assign their offsets.
364 assert(GV->isDeclaration() &&
365 "LDS definitions should have been externalized when object "
366 "linking is enabled");
367 }
368
369 MCSymbol *GVSym = getSymbol(GV);
370
371 GVSym->redefineIfPossible();
372 if (GVSym->isDefined() || GVSym->isVariable())
373 report_fatal_error(reason: "symbol '" + Twine(GVSym->getName()) +
374 "' is already defined");
375
376 const DataLayout &DL = GV->getDataLayout();
377 uint64_t Size = GV->getGlobalSize(DL);
378 Align Alignment = GV->getAlign().value_or(u: Align(4));
379
380 emitVisibility(Sym: GVSym, Visibility: GV->getVisibility(), IsDefinition: !GV->isDeclaration());
381 emitLinkage(GV, GVSym);
382 auto *TS = getTargetStreamer();
383 TS->emitAMDGPULDS(Symbol: GVSym, Size, Alignment);
384 return;
385 }
386
387 AsmPrinter::emitGlobalVariable(GV);
388}
389
390bool AMDGPUAsmPrinter::doInitialization(Module &M) {
391 const Triple &TT = M.getTargetTriple();
392 if (TT.getSubArch() == Triple::NoSubArch) {
393 Triple::SubArchType SubArch =
394 AMDGPU::getSubArch(AK: AMDGPU::parseArchAMDGCN(CPU: getGlobalSTI()->getCPU()));
395 if (SubArch != Triple::NoSubArch) {
396 Triple Fixed(TT);
397 Fixed.setArch(Kind: Triple::amdgpu, SubArch);
398 M.getContext().diagnose(DI: DiagnosticInfoGeneric(
399 "codegen with no subarch in the target triple is deprecated and will "
400 "become an error; use the target triple '" +
401 Fixed.str() + "' instead",
402 DS_Warning));
403 } else {
404 M.getContext().diagnose(DI: DiagnosticInfoGeneric(
405 "codegen with no subarch in the target triple is deprecated and will "
406 "become an error",
407 DS_Warning));
408 }
409 }
410
411 CodeObjectVersion = AMDGPU::getAMDHSACodeObjectVersion(M);
412
413 if (TT.getOS() == Triple::AMDHSA) {
414 switch (CodeObjectVersion) {
415 case AMDGPU::AMDHSA_COV4:
416 HSAMetadataStream = std::make_unique<HSAMD::MetadataStreamerMsgPackV4>();
417 break;
418 case AMDGPU::AMDHSA_COV5:
419 HSAMetadataStream = std::make_unique<HSAMD::MetadataStreamerMsgPackV5>();
420 break;
421 case AMDGPU::AMDHSA_COV6:
422 HSAMetadataStream = std::make_unique<HSAMD::MetadataStreamerMsgPackV6>();
423 break;
424 default:
425 reportFatalUsageError(reason: "unsupported code object version");
426 }
427
428 addAsmPrinterHandler(Handler: std::make_unique<AMDGPUAsmPrinterHandler>(args: this));
429 }
430
431 return AsmPrinter::doInitialization(M);
432}
433
434/// Mimics GCNSubtarget::computeOccupancy for MCExpr.
435///
436/// Remove dependency on GCNSubtarget and depend only only the necessary values
437/// for said occupancy computation. Should match computeOccupancy implementation
438/// without passing \p STM on.
439const AMDGPUMCExpr *createOccupancy(unsigned InitOcc, const MCExpr *NumSGPRs,
440 const MCExpr *NumVGPRs,
441 unsigned DynamicVGPRBlockSize,
442 const GCNSubtarget &STM, MCContext &Ctx) {
443 unsigned MaxWaves = STM.getMaxWavesPerEU();
444 unsigned Granule = IsaInfo::getVGPRAllocGranule(STI: STM, DynamicVGPRBlockSize);
445 unsigned TargetTotalNumVGPRs = STM.getTotalNumVGPRs();
446
447 // Bake the per-function SGPR budget into the operands so the late-evaluated
448 // MCExpr stays arithmetic. The trap reservation in particular is implicit on
449 // amdhsa and lives on STM, not on the assembler's MCSubtargetInfo.
450 AMDGPU::GPUKind Kind = STM.getTargetID().getGPUKind();
451 unsigned SGPRTotal = AMDGPU::getTotalNumSGPRs(AK: Kind);
452 unsigned SGPRGranule = AMDGPU::getSGPRAllocGranule(AK: Kind);
453 unsigned SGPRTrapReserve = STM.hasTrapHandler() ? IsaInfo::TRAP_NUM_SGPRS : 0;
454
455 auto CreateExpr = [&Ctx](unsigned Value) {
456 return MCConstantExpr::create(Value, Ctx);
457 };
458
459 // Zero SGPR count when SGPRs don't limit occupancy, so the MCExpr skips the
460 // SGPR term without having to test the generation itself.
461 const MCExpr *SGPRArg =
462 IsaInfo::isSGPROccupancyLimited(STI: STM) ? NumSGPRs : CreateExpr(0);
463
464 return AMDGPUMCExpr::create(Kind: AMDGPUMCExpr::AGVK_Occupancy,
465 Args: {CreateExpr(MaxWaves), CreateExpr(Granule),
466 CreateExpr(TargetTotalNumVGPRs),
467 CreateExpr(InitOcc), CreateExpr(SGPRTotal),
468 CreateExpr(SGPRGranule),
469 CreateExpr(SGPRTrapReserve), SGPRArg, NumVGPRs},
470 Ctx);
471}
472
473void AMDGPUAsmPrinter::validateMCResourceInfo(Function &F) {
474 if (F.isDeclaration() || !AMDGPU::isModuleEntryFunctionCC(CC: F.getCallingConv()))
475 return;
476
477 using RIK = MCResourceInfo::ResourceInfoKind;
478 const GCNSubtarget &STM = TM.getSubtarget<GCNSubtarget>(F);
479 MCSymbol *FnSym = TM.getSymbol(GV: &F);
480
481 auto TryGetMCExprValue = [](const MCExpr *Value, uint64_t &Res) -> bool {
482 int64_t Val;
483 if (Value->evaluateAsAbsolute(Res&: Val)) {
484 Res = Val;
485 return true;
486 }
487 return false;
488 };
489
490 const uint64_t MaxScratchPerWorkitem =
491 STM.getMaxWaveScratchSize() / STM.getWavefrontSize();
492 MCSymbol *ScratchSizeSymbol =
493 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_PrivateSegSize, OutContext);
494 uint64_t ScratchSize;
495 if (ScratchSizeSymbol->isVariable() &&
496 TryGetMCExprValue(ScratchSizeSymbol->getVariableValue(), ScratchSize) &&
497 ScratchSize > MaxScratchPerWorkitem) {
498 DiagnosticInfoStackSize DiagStackSize(F, ScratchSize, MaxScratchPerWorkitem,
499 DS_Error);
500 F.getContext().diagnose(DI: DiagStackSize);
501 }
502
503 // Validate addressable scalar registers (i.e., prior to added implicit
504 // SGPRs).
505 MCSymbol *NumSGPRSymbol =
506 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_NumSGPR, OutContext);
507 if (STM.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
508 !STM.hasSGPRInitBug()) {
509 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
510 uint64_t NumSgpr;
511 if (NumSGPRSymbol->isVariable() &&
512 TryGetMCExprValue(NumSGPRSymbol->getVariableValue(), NumSgpr) &&
513 NumSgpr > MaxAddressableNumSGPRs) {
514 F.getContext().diagnose(DI: DiagnosticInfoResourceLimit(
515 F, "addressable scalar registers", NumSgpr, MaxAddressableNumSGPRs,
516 DS_Error, DK_ResourceLimit));
517 return;
518 }
519 }
520
521 MCSymbol *VCCUsedSymbol =
522 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_UsesVCC, OutContext);
523 MCSymbol *FlatUsedSymbol =
524 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_UsesFlatScratch, OutContext);
525 uint64_t VCCUsed, FlatUsed, NumSgpr;
526
527 if (NumSGPRSymbol->isVariable() && VCCUsedSymbol->isVariable() &&
528 FlatUsedSymbol->isVariable() &&
529 TryGetMCExprValue(NumSGPRSymbol->getVariableValue(), NumSgpr) &&
530 TryGetMCExprValue(VCCUsedSymbol->getVariableValue(), VCCUsed) &&
531 TryGetMCExprValue(FlatUsedSymbol->getVariableValue(), FlatUsed)) {
532
533 // Recomputes NumSgprs + implicit SGPRs but all symbols should now be
534 // resolvable.
535 NumSgpr += IsaInfo::getNumExtraSGPRs(
536 STI: STM, VCCUsed, FlatScrUsed: FlatUsed,
537 XNACKUsed: getTargetStreamer()->getTargetID()->isXnackOnOrAny());
538 if (STM.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS ||
539 STM.hasSGPRInitBug()) {
540 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
541 if (NumSgpr > MaxAddressableNumSGPRs) {
542 F.getContext().diagnose(DI: DiagnosticInfoResourceLimit(
543 F, "scalar registers", NumSgpr, MaxAddressableNumSGPRs, DS_Error,
544 DK_ResourceLimit));
545 return;
546 }
547 }
548
549 MCSymbol *NumVgprSymbol =
550 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_NumVGPR, OutContext);
551 MCSymbol *NumAgprSymbol =
552 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_NumAGPR, OutContext);
553 uint64_t NumVgpr, NumAgpr;
554
555 MachineModuleInfo &MMI = *GetMMI();
556 MachineFunction *MF = MMI.getMachineFunction(F);
557 if (MF && NumVgprSymbol->isVariable() && NumAgprSymbol->isVariable() &&
558 TryGetMCExprValue(NumVgprSymbol->getVariableValue(), NumVgpr) &&
559 TryGetMCExprValue(NumAgprSymbol->getVariableValue(), NumAgpr)) {
560 const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
561 unsigned MaxWaves = MFI.getMaxWavesPerEU();
562 uint64_t TotalNumVgpr =
563 getTotalNumVGPRs(has90AInsts: STM.hasGFX90AInsts(), ArgNumAGPR: NumAgpr, ArgNumVGPR: NumVgpr);
564 uint64_t NumVGPRsForWavesPerEU =
565 std::max(l: {TotalNumVgpr, (uint64_t)1,
566 (uint64_t)STM.getMinNumVGPRs(
567 WavesPerEU: MaxWaves, DynamicVGPRBlockSize: MFI.getDynamicVGPRBlockSize())});
568 uint64_t NumSGPRsForWavesPerEU = std::max(
569 l: {NumSgpr, (uint64_t)1, (uint64_t)STM.getMinNumSGPRs(WavesPerEU: MaxWaves)});
570 const MCExpr *OccupancyExpr = createOccupancy(
571 InitOcc: STM.getOccupancyWithWorkGroupSizes(MF: *MF).second,
572 NumSGPRs: MCConstantExpr::create(Value: NumSGPRsForWavesPerEU, Ctx&: OutContext),
573 NumVGPRs: MCConstantExpr::create(Value: NumVGPRsForWavesPerEU, Ctx&: OutContext),
574 DynamicVGPRBlockSize: MFI.getDynamicVGPRBlockSize(), STM, Ctx&: OutContext);
575 uint64_t Occupancy;
576
577 const auto [MinWEU, MaxWEU] = AMDGPU::getIntegerPairAttribute(
578 F, Name: "amdgpu-waves-per-eu", Default: {0, 0}, OnlyFirstRequired: true);
579
580 if (TryGetMCExprValue(OccupancyExpr, Occupancy) && Occupancy < MinWEU) {
581 DiagnosticInfoOptimizationFailure Diag(
582 F, F.getSubprogram(),
583 "failed to meet occupancy target given by 'amdgpu-waves-per-eu' in "
584 "'" +
585 F.getName() + "': desired occupancy was " + Twine(MinWEU) +
586 ", final occupancy is " + Twine(Occupancy));
587 F.getContext().diagnose(DI: Diag);
588 return;
589 }
590 }
591 }
592}
593
594static void appendTypeEncoding(std::string &Enc, Type *Ty, const DataLayout &DL,
595 bool IsReturnType) {
596 if (Ty->isVoidTy()) {
597 Enc += 'v';
598 return;
599 }
600 unsigned Bits = DL.getTypeSizeInBits(Ty);
601 // Zero-sized non-void types (e.g. `{}` or `[0 x i8]`) consume no ABI
602 // registers. For returns, emit the same no-result marker as void so the
603 // parameter encoding still has an explicit return-type prefix.
604 if (Bits == 0) {
605 if (IsReturnType)
606 Enc += 'v';
607 return;
608 }
609 if (Bits <= 32)
610 Enc += 'i';
611 else if (Bits <= 64)
612 Enc += 'l';
613 else
614 Enc.append(n: divideCeil(Numerator: Bits, Denominator: 32), c: 'i');
615}
616
617static std::string computeTypeId(const FunctionType *FTy,
618 const DataLayout &DL) {
619 std::string Enc;
620 appendTypeEncoding(Enc, Ty: FTy->getReturnType(), DL, /*IsReturnType=*/true);
621 for (Type *ParamTy : FTy->params())
622 appendTypeEncoding(Enc, Ty: ParamTy, DL, /*IsReturnType=*/false);
623 return Enc;
624}
625
626void AMDGPUAsmPrinter::collectCallEdge(const MachineInstr &MI) {
627 if (!AMDGPUTargetMachine::EnableObjectLinking)
628 return;
629 const SIInstrInfo *TII = MF->getSubtarget<GCNSubtarget>().getInstrInfo();
630 const MachineOperand *Callee =
631 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::callee);
632 if (!Callee || !Callee->isGlobal())
633 return;
634 DirectCallEdges.insert(
635 X: {getSymbol(GV: &MF->getFunction()), getSymbol(GV: Callee->getGlobal())});
636}
637
638void AMDGPUAsmPrinter::emitAMDGPUInfo(Module &M) {
639 if (!AMDGPUTargetMachine::EnableObjectLinking)
640 return;
641
642 const NamedMDNode *LDSMD = M.getNamedMetadata(Name: "amdgpu.lds.uses");
643 bool HasLDSUses = LDSMD && LDSMD->getNumOperands() > 0;
644
645 const NamedMDNode *BarMD = M.getNamedMetadata(Name: "amdgpu.named_barrier.uses");
646 bool HasNamedBarriers = BarMD && BarMD->getNumOperands() > 0;
647
648 // Collect address-taken functions (with type IDs) and indirect call sites.
649 DenseMap<const Function *, std::string> AddrTakenTypeIds;
650 using IndirectCallInfo = std::pair<const Function *, std::string>;
651 SmallVector<IndirectCallInfo, 8> IndirectCalls;
652
653 for (const Function &F : M) {
654 bool IsKernel = AMDGPU::isKernel(CC: F.getCallingConv());
655
656 if (!IsKernel && F.hasAddressTaken(/*PutOffender=*/nullptr,
657 /*IgnoreCallbackUses=*/false,
658 /*IgnoreAssumeLikeCalls=*/true,
659 /*IgnoreLLVMUsed=*/IngoreLLVMUsed: true)) {
660 AddrTakenTypeIds[&F] =
661 computeTypeId(FTy: F.getFunctionType(), DL: M.getDataLayout());
662 }
663
664 if (F.isDeclaration())
665 continue;
666
667 StringSet<> SeenTypeIds;
668 for (const BasicBlock &BB : F) {
669 for (const Instruction &I : BB) {
670 const auto *CB = dyn_cast<CallBase>(Val: &I);
671 if (!CB || !CB->isIndirectCall())
672 continue;
673 std::string TId =
674 computeTypeId(FTy: CB->getFunctionType(), DL: M.getDataLayout());
675 if (SeenTypeIds.insert(key: TId).second)
676 IndirectCalls.push_back(Elt: {&F, std::move(TId)});
677 }
678 }
679 }
680
681 if (FunctionInfos.empty() && DirectCallEdges.empty() && !HasLDSUses &&
682 !HasNamedBarriers && AddrTakenTypeIds.empty() && IndirectCalls.empty())
683 return;
684
685 AMDGPU::InfoSectionData Data;
686 Data.Funcs = std::move(FunctionInfos);
687
688 for (auto &[F, TypeId] : AddrTakenTypeIds) {
689 MCSymbol *Sym = getSymbol(GV: F);
690 Data.TypeIds.push_back(Elt: {Sym, TypeId});
691 }
692
693 for (auto &[CallerSym, CalleeSym] : DirectCallEdges)
694 Data.Calls.push_back(Elt: {CallerSym, CalleeSym});
695 DirectCallEdges.clear();
696
697 if (HasLDSUses) {
698 for (const MDNode *N : LDSMD->operands()) {
699 auto *Func = mdconst::extract<Function>(MD: N->getOperand(I: 0));
700 auto *LdsVar = mdconst::extract<GlobalVariable>(MD: N->getOperand(I: 1));
701 Data.Uses.push_back(Elt: {getSymbol(GV: Func), getSymbol(GV: LdsVar)});
702 }
703 }
704
705 if (HasNamedBarriers) {
706 for (const MDNode *N : BarMD->operands()) {
707 auto *BarVar = mdconst::extract<GlobalVariable>(MD: N->getOperand(I: 0));
708 MCSymbol *BarSym = getSymbol(GV: BarVar);
709 for (unsigned I = 1, E = N->getNumOperands(); I < E; ++I) {
710 auto *Func = mdconst::extract<Function>(MD: N->getOperand(I));
711 Data.Uses.push_back(Elt: {getSymbol(GV: Func), BarSym});
712 }
713 }
714 }
715
716 for (auto &[Caller, Enc] : IndirectCalls) {
717 MCSymbol *CallerSym = getSymbol(GV: Caller);
718 Data.IndirectCalls.push_back(Elt: {CallerSym, Enc});
719 }
720
721 getTargetStreamer()->emitAMDGPUInfo(Data);
722}
723
724bool AMDGPUAsmPrinter::doFinalization(Module &M) {
725 const Triple &TT = M.getTargetTriple();
726
727 // Pad with s_code_end to help tools and guard against instruction prefetch
728 // causing stale data in caches. Arguably this should be done by the linker,
729 // which is why this isn't done for Mesa.
730 // Don't do it if there is no code.
731 const MCSubtargetInfo &STI = *getGlobalSTI();
732 if ((AMDGPU::isGFX10Plus(STI) || AMDGPU::isGFX90A(STI)) &&
733 (TT.getOS() == Triple::AMDHSA || TT.getOS() == Triple::AMDPAL)) {
734 MCSection *TextSect = getObjFileLowering().getTextSection();
735 if (TextSect->hasInstructions()) {
736 OutStreamer->switchSection(Section: TextSect);
737 getTargetStreamer()->EmitCodeEnd(STI);
738 }
739 }
740
741 // Emit the unified .amdgpu.info section (per-function resources, call graph,
742 // LDS/named-barrier use edges, indirect calls, and address-taken type IDs).
743 emitAMDGPUInfo(M);
744
745 // Assign expressions which can only be resolved when all other functions are
746 // known.
747 RI.finalize(OutContext);
748
749 // Switch section and emit all GPR maximums within the processed module.
750 OutStreamer->pushSection();
751 MCSectionELF *MaxGPRSection =
752 OutContext.getELFSection(Section: ".AMDGPU.gpr_maximums", Type: ELF::SHT_PROGBITS, Flags: 0);
753 OutStreamer->switchSection(Section: MaxGPRSection);
754 getTargetStreamer()->EmitMCResourceMaximums(
755 MaxVGPR: RI.getMaxVGPRSymbol(OutContext), MaxAGPR: RI.getMaxAGPRSymbol(OutContext),
756 MaxSGPR: RI.getMaxSGPRSymbol(OutContext), MaxNamedBarrier: RI.getMaxNamedBarrierSymbol(OutContext));
757 OutStreamer->popSection();
758
759 // In the object-linking pipeline per-function resource MCExprs reference
760 // external callee symbols that cannot be evaluated here, so cross-TU limit
761 // checks would silently no-op for every non-leaf function. Defer resource
762 // sanity checking to the linker, which re-validates against the aggregated
763 // call graph in the combined .amdgpu.info metadata.
764 if (!AMDGPUTargetMachine::EnableObjectLinking) {
765 for (Function &F : M.functions())
766 validateMCResourceInfo(F);
767 }
768
769 RI.reset();
770
771 return AsmPrinter::doFinalization(M);
772}
773
774SmallString<128> AMDGPUAsmPrinter::getMCExprStr(const MCExpr *Value) {
775 SmallString<128> Str;
776 raw_svector_ostream OSS(Str);
777 auto &Streamer = getTargetStreamer()->getStreamer();
778 auto &Context = Streamer.getContext();
779 const MCExpr *New = foldAMDGPUMCExpr(Expr: Value, Ctx&: Context);
780 printAMDGPUMCExpr(Expr: New, OS&: OSS, MAI: &MAI);
781 return Str;
782}
783
784// Print comments that apply to both callable functions and entry points.
785void AMDGPUAsmPrinter::emitCommonFunctionComments(
786 const MCExpr *NumVGPR, const MCExpr *NumAGPR, const MCExpr *TotalNumVGPR,
787 const MCExpr *NumSGPR, const MCExpr *ScratchSize, uint64_t CodeSize,
788 const AMDGPUMachineFunctionInfo *MFI) {
789 OutStreamer->emitRawComment(T: " codeLenInByte = " + Twine(CodeSize), TabPrefix: false);
790 OutStreamer->emitRawComment(T: " TotalNumSgprs: " + getMCExprStr(Value: NumSGPR),
791 TabPrefix: false);
792 OutStreamer->emitRawComment(T: " NumVgprs: " + getMCExprStr(Value: NumVGPR), TabPrefix: false);
793 if (NumAGPR && TotalNumVGPR) {
794 OutStreamer->emitRawComment(T: " NumAgprs: " + getMCExprStr(Value: NumAGPR), TabPrefix: false);
795 OutStreamer->emitRawComment(T: " TotalNumVgprs: " + getMCExprStr(Value: TotalNumVGPR),
796 TabPrefix: false);
797 }
798 OutStreamer->emitRawComment(T: " ScratchSize: " + getMCExprStr(Value: ScratchSize),
799 TabPrefix: false);
800 OutStreamer->emitRawComment(T: " MemoryBound: " + Twine(MFI->isMemoryBound()),
801 TabPrefix: false);
802}
803
804const MCExpr *AMDGPUAsmPrinter::getAmdhsaKernelCodeProperties(
805 const MachineFunction &MF) const {
806 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
807 MCContext &Ctx = MF.getContext();
808 uint16_t KernelCodeProperties = 0;
809 const GCNUserSGPRUsageInfo &UserSGPRInfo = MFI.getUserSGPRInfo();
810
811 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
812 KernelCodeProperties |=
813 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER;
814 }
815 if (UserSGPRInfo.hasDispatchPtr()) {
816 KernelCodeProperties |=
817 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR;
818 }
819 if (UserSGPRInfo.hasQueuePtr()) {
820 KernelCodeProperties |= amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR;
821 }
822 if (UserSGPRInfo.hasKernargSegmentPtr()) {
823 KernelCodeProperties |=
824 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR;
825 }
826 if (UserSGPRInfo.hasDispatchID()) {
827 KernelCodeProperties |=
828 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID;
829 }
830 if (UserSGPRInfo.hasFlatScratchInit()) {
831 KernelCodeProperties |=
832 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT;
833 }
834 if (UserSGPRInfo.hasPrivateSegmentSize()) {
835 KernelCodeProperties |=
836 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE;
837 }
838 if (MF.getSubtarget<GCNSubtarget>().isWave32()) {
839 KernelCodeProperties |=
840 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;
841 }
842
843 // CurrentProgramInfo.DynamicCallStack is a MCExpr and could be
844 // un-evaluatable at this point so it cannot be conditionally checked here.
845 // Instead, we'll directly shift the possibly unknown MCExpr into its place
846 // and bitwise-or it into KernelCodeProperties.
847 const MCExpr *KernelCodePropExpr =
848 MCConstantExpr::create(Value: KernelCodeProperties, Ctx);
849 const MCExpr *OrValue = MCConstantExpr::create(
850 Value: amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK_SHIFT, Ctx);
851 OrValue = MCBinaryExpr::createShl(LHS: CurrentProgramInfo.DynamicCallStack,
852 RHS: OrValue, Ctx);
853 KernelCodePropExpr = MCBinaryExpr::createOr(LHS: KernelCodePropExpr, RHS: OrValue, Ctx);
854
855 return KernelCodePropExpr;
856}
857
858MCKernelDescriptor
859AMDGPUAsmPrinter::getAmdhsaKernelDescriptor(const MachineFunction &MF,
860 const SIProgramInfo &PI) const {
861 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
862 const Function &F = MF.getFunction();
863 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
864 MCContext &Ctx = MF.getContext();
865
866 MCKernelDescriptor KernelDescriptor;
867
868 KernelDescriptor.group_segment_fixed_size =
869 MCConstantExpr::create(Value: PI.LDSSize, Ctx);
870 KernelDescriptor.private_segment_fixed_size = PI.ScratchSize;
871
872 Align MaxKernArgAlign;
873 KernelDescriptor.kernarg_size = MCConstantExpr::create(
874 Value: STM.getKernArgSegmentSize(F, MaxAlign&: MaxKernArgAlign), Ctx);
875
876 KernelDescriptor.compute_pgm_rsrc1 = PI.getComputePGMRSrc1(ST: STM, Ctx);
877 KernelDescriptor.compute_pgm_rsrc2 = PI.getComputePGMRSrc2(ST: STM, Ctx);
878 KernelDescriptor.kernel_code_properties = getAmdhsaKernelCodeProperties(MF);
879
880 int64_t PGM_Rsrc3 = 1;
881 bool EvaluatableRsrc3 =
882 CurrentProgramInfo.ComputePGMRSrc3->evaluateAsAbsolute(Res&: PGM_Rsrc3);
883 (void)PGM_Rsrc3;
884 (void)EvaluatableRsrc3;
885 assert(STM.getGeneration() >= AMDGPUSubtarget::GFX10 ||
886 STM.hasGFX90AInsts() || STM.hasGFX1250Insts() || !EvaluatableRsrc3 ||
887 static_cast<uint64_t>(PGM_Rsrc3) == 0);
888 KernelDescriptor.compute_pgm_rsrc3 = CurrentProgramInfo.ComputePGMRSrc3;
889
890 KernelDescriptor.kernarg_preload = MCConstantExpr::create(
891 Value: AMDGPU::hasKernargPreload(STI: STM) ? Info->getNumKernargPreloadedSGPRs() : 0,
892 Ctx);
893
894 return KernelDescriptor;
895}
896
897bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
898 // Init target streamer lazily on the first function so that previous passes
899 // can set metadata.
900 if (!IsTargetStreamerInitialized)
901 initTargetStreamer(M&: *MF.getFunction().getParent());
902
903 ResourceUsage = GetResourceUsage(MF);
904 CurrentProgramInfo.reset(MF);
905
906 const AMDGPUMachineFunctionInfo *MFI =
907 MF.getInfo<AMDGPUMachineFunctionInfo>();
908 MCContext &Ctx = MF.getContext();
909
910 // The starting address of all shader programs must be 256 bytes aligned.
911 // Regular functions just need the basic required instruction alignment.
912 MF.ensureAlignment(A: MFI->isEntryFunction() ? Align(256) : Align(4));
913
914 SetupMachineFunction(MF);
915
916 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
917 MCContext &Context = getObjFileLowering().getContext();
918 // FIXME: This should be an explicit check for Mesa.
919 if (!STM.isAmdHsaOS() && !STM.isAmdPalOS()) {
920 MCSectionELF *ConfigSection =
921 Context.getELFSection(Section: ".AMDGPU.config", Type: ELF::SHT_PROGBITS, Flags: 0);
922 OutStreamer->switchSection(Section: ConfigSection);
923 }
924
925 RI.gatherResourceInfo(MF, FRI: *ResourceUsage, OutContext);
926
927 if (AMDGPUTargetMachine::EnableObjectLinking) {
928 const AMDGPUResourceUsageAnalysisWrapperPass::FunctionResourceInfo &RU =
929 *ResourceUsage;
930 FunctionInfos.push_back(
931 Elt: {/*NumSGPR=*/static_cast<uint32_t>(RU.NumExplicitSGPR),
932 /*NumArchVGPR=*/static_cast<uint32_t>(RU.NumVGPR),
933 /*NumAccVGPR=*/static_cast<uint32_t>(RU.NumAGPR),
934 /*PrivateSegmentSize=*/static_cast<uint32_t>(RU.PrivateSegmentSize),
935 /*UsesVCC=*/RU.UsesVCC,
936 /*UsesFlatScratch=*/RU.UsesFlatScratch,
937 /*HasDynStack=*/RU.HasDynamicallySizedStack,
938 /*Sym=*/getSymbol(GV: &MF.getFunction())});
939 }
940
941 if (MFI->isModuleEntryFunction()) {
942 getSIProgramInfo(Out&: CurrentProgramInfo, MF);
943 }
944
945 if (STM.isAmdPalOS()) {
946 if (MFI->isEntryFunction())
947 EmitPALMetadata(MF, KernelInfo: CurrentProgramInfo);
948 else if (MFI->isModuleEntryFunction())
949 emitPALFunctionMetadata(MF);
950 } else if (!STM.isAmdHsaOS()) {
951 EmitProgramInfoSI(MF, KernelInfo: CurrentProgramInfo);
952 }
953
954 DumpCodeInstEmitter = nullptr;
955 if (STM.dumpCode()) {
956 // For -dumpcode, get the assembler out of the streamer. This only works
957 // with -filetype=obj.
958 MCAssembler *Assembler = OutStreamer->getAssemblerPtr();
959 if (Assembler)
960 DumpCodeInstEmitter = Assembler->getEmitterPtr();
961 }
962
963 DisasmLines.clear();
964 HexLines.clear();
965 DisasmLineMaxLen = 0;
966
967 emitFunctionBody();
968
969 emitResourceUsageRemarks(MF, CurrentProgramInfo, isModuleEntryFunction: MFI->isModuleEntryFunction(),
970 hasMAIInsts: STM.hasMAIInsts());
971
972 {
973 using RIK = MCResourceInfo::ResourceInfoKind;
974 getTargetStreamer()->EmitMCResourceInfo(
975 NumVGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumVGPR, OutContext),
976 NumAGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumAGPR, OutContext),
977 NumExplicitSGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumSGPR, OutContext),
978 NumNamedBarrier: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumNamedBarrier,
979 OutContext),
980 PrivateSegmentSize: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_PrivateSegSize,
981 OutContext),
982 UsesVCC: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_UsesVCC, OutContext),
983 UsesFlatScratch: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_UsesFlatScratch,
984 OutContext),
985 HasDynamicallySizedStack: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_HasDynSizedStack,
986 OutContext),
987 HasRecursion: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_HasRecursion,
988 OutContext),
989 HasIndirectCall: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_HasIndirectCall,
990 OutContext));
991 }
992
993 // Emit _dvgpr$ symbol when appropriate.
994 emitDVgprSymbol(MF);
995
996 if (isVerbose()) {
997 MCSectionELF *CommentSection =
998 Context.getELFSection(Section: ".AMDGPU.csdata", Type: ELF::SHT_PROGBITS, Flags: 0);
999 OutStreamer->switchSection(Section: CommentSection);
1000
1001 if (!MFI->isEntryFunction()) {
1002 using RIK = MCResourceInfo::ResourceInfoKind;
1003 OutStreamer->emitRawComment(T: " Function info:", TabPrefix: false);
1004
1005 emitCommonFunctionComments(
1006 NumVGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumVGPR, OutContext)
1007 ->getVariableValue(),
1008 NumAGPR: STM.hasMAIInsts() ? RI.getSymbol(FuncName: CurrentFnSym->getName(),
1009 RIK: RIK::RIK_NumAGPR, OutContext)
1010 ->getVariableValue()
1011 : nullptr,
1012 TotalNumVGPR: RI.createTotalNumVGPRs(MF, Ctx),
1013 NumSGPR: RI.createTotalNumSGPRs(
1014 MF,
1015 hasXnack: MF.getSubtarget<GCNSubtarget>().getTargetID().isXnackOnOrAny(),
1016 Ctx),
1017 ScratchSize: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_PrivateSegSize,
1018 OutContext)
1019 ->getVariableValue(),
1020 CodeSize: CurrentProgramInfo.getFunctionCodeSize(MF), MFI);
1021 return false;
1022 }
1023
1024 OutStreamer->emitRawComment(T: " Kernel info:", TabPrefix: false);
1025 emitCommonFunctionComments(
1026 NumVGPR: CurrentProgramInfo.NumArchVGPR,
1027 NumAGPR: STM.hasMAIInsts() ? CurrentProgramInfo.NumAccVGPR : nullptr,
1028 TotalNumVGPR: CurrentProgramInfo.NumVGPR, NumSGPR: CurrentProgramInfo.NumSGPR,
1029 ScratchSize: CurrentProgramInfo.ScratchSize,
1030 CodeSize: CurrentProgramInfo.getFunctionCodeSize(MF), MFI);
1031
1032 OutStreamer->emitRawComment(
1033 T: " FloatMode: " + Twine(CurrentProgramInfo.FloatMode), TabPrefix: false);
1034 OutStreamer->emitRawComment(
1035 T: " IeeeMode: " + Twine(CurrentProgramInfo.IEEEMode), TabPrefix: false);
1036 OutStreamer->emitRawComment(
1037 T: " LDSByteSize: " + Twine(CurrentProgramInfo.LDSSize) +
1038 " bytes/workgroup (compile time only)",
1039 TabPrefix: false);
1040
1041 OutStreamer->emitRawComment(
1042 T: " SGPRBlocks: " + getMCExprStr(Value: CurrentProgramInfo.SGPRBlocks), TabPrefix: false);
1043
1044 OutStreamer->emitRawComment(
1045 T: " VGPRBlocks: " + getMCExprStr(Value: CurrentProgramInfo.VGPRBlocks), TabPrefix: false);
1046
1047 OutStreamer->emitRawComment(
1048 T: " NumSGPRsForWavesPerEU: " +
1049 getMCExprStr(Value: CurrentProgramInfo.NumSGPRsForWavesPerEU),
1050 TabPrefix: false);
1051 OutStreamer->emitRawComment(
1052 T: " NumVGPRsForWavesPerEU: " +
1053 getMCExprStr(Value: CurrentProgramInfo.NumVGPRsForWavesPerEU),
1054 TabPrefix: false);
1055
1056 if (STM.hasGFX90AInsts()) {
1057 const MCExpr *AdjustedAccum = MCBinaryExpr::createAdd(
1058 LHS: CurrentProgramInfo.AccumOffset, RHS: MCConstantExpr::create(Value: 1, Ctx), Ctx);
1059 AdjustedAccum = MCBinaryExpr::createMul(
1060 LHS: AdjustedAccum, RHS: MCConstantExpr::create(Value: 4, Ctx), Ctx);
1061 OutStreamer->emitRawComment(
1062 T: " AccumOffset: " + getMCExprStr(Value: AdjustedAccum), TabPrefix: false);
1063 }
1064
1065 if (STM.hasGFX1250Insts())
1066 OutStreamer->emitRawComment(
1067 T: " NamedBarCnt: " + getMCExprStr(Value: CurrentProgramInfo.NamedBarCnt),
1068 TabPrefix: false);
1069
1070 OutStreamer->emitRawComment(
1071 T: " Occupancy: " + getMCExprStr(Value: CurrentProgramInfo.Occupancy), TabPrefix: false);
1072
1073 OutStreamer->emitRawComment(
1074 T: " WaveLimiterHint : " + Twine(MFI->needsWaveLimiter()), TabPrefix: false);
1075
1076 OutStreamer->emitRawComment(
1077 T: " COMPUTE_PGM_RSRC2:SCRATCH_EN: " +
1078 getMCExprStr(Value: CurrentProgramInfo.ScratchEnable),
1079 TabPrefix: false);
1080 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:USER_SGPR: " +
1081 Twine(CurrentProgramInfo.UserSGPR),
1082 TabPrefix: false);
1083 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TRAP_HANDLER: " +
1084 Twine(CurrentProgramInfo.TrapHandlerEnable),
1085 TabPrefix: false);
1086 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TGID_X_EN: " +
1087 Twine(CurrentProgramInfo.TGIdXEnable),
1088 TabPrefix: false);
1089 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TGID_Y_EN: " +
1090 Twine(CurrentProgramInfo.TGIdYEnable),
1091 TabPrefix: false);
1092 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TGID_Z_EN: " +
1093 Twine(CurrentProgramInfo.TGIdZEnable),
1094 TabPrefix: false);
1095 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TIDIG_COMP_CNT: " +
1096 Twine(CurrentProgramInfo.TIdIGCompCount),
1097 TabPrefix: false);
1098
1099 [[maybe_unused]] int64_t PGMRSrc3;
1100 assert(STM.getGeneration() >= AMDGPUSubtarget::GFX10 ||
1101 STM.hasGFX90AInsts() || STM.hasGFX1250Insts() ||
1102 (CurrentProgramInfo.ComputePGMRSrc3->evaluateAsAbsolute(PGMRSrc3) &&
1103 static_cast<uint64_t>(PGMRSrc3) == 0));
1104 if (STM.hasGFX90AInsts()) {
1105 OutStreamer->emitRawComment(
1106 T: " COMPUTE_PGM_RSRC3_GFX90A:ACCUM_OFFSET: " +
1107 getMCExprStr(Value: MCKernelDescriptor::bits_get(
1108 Src: CurrentProgramInfo.ComputePGMRSrc3,
1109 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
1110 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, Ctx)),
1111 TabPrefix: false);
1112 OutStreamer->emitRawComment(
1113 T: " COMPUTE_PGM_RSRC3_GFX90A:TG_SPLIT: " +
1114 getMCExprStr(Value: MCKernelDescriptor::bits_get(
1115 Src: CurrentProgramInfo.ComputePGMRSrc3,
1116 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT,
1117 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, Ctx)),
1118 TabPrefix: false);
1119 }
1120 }
1121
1122 if (DumpCodeInstEmitter) {
1123
1124 OutStreamer->switchSection(
1125 Section: Context.getELFSection(Section: ".AMDGPU.disasm", Type: ELF::SHT_PROGBITS, Flags: 0));
1126
1127 for (size_t i = 0; i < DisasmLines.size(); ++i) {
1128 std::string Comment = "\n";
1129 if (!HexLines[i].empty()) {
1130 Comment = std::string(DisasmLineMaxLen - DisasmLines[i].size(), ' ');
1131 Comment += " ; " + HexLines[i] + "\n";
1132 }
1133
1134 OutStreamer->emitBytes(Data: StringRef(DisasmLines[i]));
1135 OutStreamer->emitBytes(Data: StringRef(Comment));
1136 }
1137 }
1138
1139 return false;
1140}
1141
1142// When appropriate, add a _dvgpr$ symbol, with the value of the function
1143// symbol, plus an offset encoding one less than the number of VGPR blocks used
1144// by the function in bits 5..3 of the symbol value. A "VGPR block" can be
1145// either 16 VGPRs (for a max of 128), or 32 VGPRs (for a max of 256). This is
1146// used by a front-end to have functions that are chained rather than called,
1147// and a dispatcher that dynamically resizes the VGPR count before dispatching
1148// to a function.
1149void AMDGPUAsmPrinter::emitDVgprSymbol(MachineFunction &MF) {
1150 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
1151 if (MFI.isDynamicVGPREnabled() &&
1152 MF.getFunction().getCallingConv() == CallingConv::AMDGPU_CS_Chain) {
1153 MCContext &Ctx = MF.getContext();
1154 unsigned BlockSize = MFI.getDynamicVGPRBlockSize();
1155
1156 const MCExpr *EncodedBlocks;
1157 MCValue NumVGPRs;
1158 if (CurrentProgramInfo.NumVGPRsForWavesPerEU->evaluateAsRelocatable(
1159 Res&: NumVGPRs, Asm: nullptr) &&
1160 NumVGPRs.isAbsolute()) {
1161
1162 // Calculate number of VGPR blocks.
1163 // Treat 0 VGPRs as 1 VGPR to avoid underflowing.
1164 unsigned NumBlocks =
1165 divideCeil(Numerator: std::max(a: unsigned(NumVGPRs.getConstant()), b: 1U), Denominator: BlockSize);
1166
1167 if (NumBlocks > AMDGPU::IsaInfo::MaxDynamicVGPRBlocks) {
1168 OutContext.reportError(
1169 L: {}, Msg: "DVGPR block count " + Twine(NumBlocks) +
1170 " exceeds maximum of " +
1171 Twine(AMDGPU::IsaInfo::MaxDynamicVGPRBlocks) +
1172 " for __dvgpr$ symbol for '" +
1173 Twine(CurrentFnSym->getName()) + "'");
1174 return;
1175 }
1176 unsigned EncodedNumBlocks = (NumBlocks - 1) << 3;
1177 EncodedBlocks = MCConstantExpr::create(Value: EncodedNumBlocks, Ctx);
1178 } else {
1179 // Value not yet available so build a symbolic MCExpr:
1180 // ((alignTo(max(NumVGPRs, 1), BlockSize) / BlockSize - 1) << 3
1181 const MCExpr *One = MCConstantExpr::create(Value: 1, Ctx);
1182 const MCExpr *BlockSizeConst = MCConstantExpr::create(Value: BlockSize, Ctx);
1183 const MCExpr *MaxVGPRs = AMDGPUMCExpr::createMax(
1184 Args: {CurrentProgramInfo.NumVGPRsForWavesPerEU, One}, Ctx);
1185 const MCExpr *NumBlocks = MCBinaryExpr::createDiv(
1186 LHS: AMDGPUMCExpr::createAlignTo(Value: MaxVGPRs, Align: BlockSizeConst, Ctx),
1187 RHS: BlockSizeConst, Ctx);
1188 EncodedBlocks =
1189 MCBinaryExpr::createShl(LHS: MCBinaryExpr::createSub(LHS: NumBlocks, RHS: One, Ctx),
1190 RHS: MCConstantExpr::create(Value: 3, Ctx), Ctx);
1191 }
1192
1193 // Add to function symbol to create _dvgpr$ symbol.
1194 const MCExpr *DVgprFuncVal = MCBinaryExpr::createAdd(
1195 LHS: MCSymbolRefExpr::create(Symbol: CurrentFnSym, Ctx), RHS: EncodedBlocks, Ctx);
1196 MCSymbol *DVgprFuncSym =
1197 Ctx.getOrCreateSymbol(Name: Twine("_dvgpr$") + CurrentFnSym->getName());
1198 OutStreamer->emitAssignment(Symbol: DVgprFuncSym, Value: DVgprFuncVal);
1199 emitVisibility(Sym: DVgprFuncSym, Visibility: MF.getFunction().getVisibility());
1200 emitLinkage(GV: &MF.getFunction(), GVSym: DVgprFuncSym);
1201 }
1202}
1203
1204// TODO: Fold this into emitFunctionBodyStart.
1205void AMDGPUAsmPrinter::initializeTargetID(const Module &M) {
1206 getTargetStreamer()->initializeTargetID(STI: *getGlobalSTI());
1207
1208 auto &TSTargetID = getTargetStreamer()->getTargetID();
1209
1210 // Error if -mattr specified xnack or sramecc.
1211 // TODO: Remove this when subtarget features removed.
1212 StringRef FeatureString = getGlobalSTI()->getFeatureString();
1213 if (FeatureString.contains(Other: "xnack")) {
1214 M.getContext().diagnose(DI: DiagnosticInfoGeneric(
1215 "xnack/sramecc should be specified via module flags. "
1216 "Use module flag 'amdgpu.xnack' instead of subtarget feature",
1217 DS_Error));
1218 }
1219 if (FeatureString.contains(Other: "sramecc")) {
1220 M.getContext().diagnose(DI: DiagnosticInfoGeneric(
1221 "xnack/sramecc should be specified via module flags. "
1222 "Use module flag 'amdgpu.sramecc' instead of subtarget feature",
1223 DS_Error));
1224 }
1225
1226 // Apply xnack/sramecc settings from module flags.
1227 if (getGlobalSTI()->getFeatureBits().test(I: AMDGPU::FeatureXNACKOnOffModes)) {
1228 AMDGPU::TargetIDSetting Setting =
1229 GCNTargetMachine::getTargetIDSettingFromModuleFlag(M, FlagName: "amdgpu.xnack");
1230 if (Setting != AMDGPU::TargetIDSetting::Any)
1231 TSTargetID->setXnackSetting(Setting);
1232 }
1233
1234 if (getGlobalSTI()->getFeatureBits().test(I: AMDGPU::FeatureSRAMECCOnOffModes)) {
1235 AMDGPU::TargetIDSetting Setting =
1236 GCNTargetMachine::getTargetIDSettingFromModuleFlag(M, FlagName: "amdgpu.sramecc");
1237 if (Setting != AMDGPU::TargetIDSetting::Any)
1238 TSTargetID->setSramEccSetting(Setting);
1239 }
1240}
1241
1242// AccumOffset computed for the MCExpr equivalent of:
1243// alignTo(std::max(1, NumVGPR), 4) / 4 - 1;
1244static const MCExpr *computeAccumOffset(const MCExpr *NumVGPR, MCContext &Ctx) {
1245 const MCExpr *ConstFour = MCConstantExpr::create(Value: 4, Ctx);
1246 const MCExpr *ConstOne = MCConstantExpr::create(Value: 1, Ctx);
1247
1248 // Can't be lower than 1 for subsequent alignTo.
1249 const MCExpr *MaximumTaken =
1250 AMDGPUMCExpr::createMax(Args: {ConstOne, NumVGPR}, Ctx);
1251
1252 // Practically, it's computing divideCeil(MaximumTaken, 4).
1253 const MCExpr *DivCeil = MCBinaryExpr::createDiv(
1254 LHS: AMDGPUMCExpr::createAlignTo(Value: MaximumTaken, Align: ConstFour, Ctx), RHS: ConstFour,
1255 Ctx);
1256
1257 return MCBinaryExpr::createSub(LHS: DivCeil, RHS: ConstOne, Ctx);
1258}
1259
1260static unsigned getLDSEncodingGranule(const GCNSubtarget &ST) {
1261 unsigned Granule =
1262 AMDGPU::getLDSEncodingGranule(AK: ST.getTargetID().getGPUKind());
1263 // The legacy generic targets have no encoding granularity feature. Preserve
1264 // the default used for code generation when no GPU is specified.
1265 return Granule ? Granule : 256;
1266}
1267
1268void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
1269 const MachineFunction &MF) {
1270 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1271 MCContext &Ctx = MF.getContext();
1272
1273 auto CreateExpr = [&Ctx](int64_t Value) {
1274 return MCConstantExpr::create(Value, Ctx);
1275 };
1276
1277 auto TryGetMCExprValue = [](const MCExpr *Value, uint64_t &Res) -> bool {
1278 int64_t Val;
1279 if (Value->evaluateAsAbsolute(Res&: Val)) {
1280 Res = Val;
1281 return true;
1282 }
1283 return false;
1284 };
1285
1286 auto GetSymRefExpr =
1287 [&](MCResourceInfo::ResourceInfoKind RIK) -> const MCExpr * {
1288 MCSymbol *Sym = RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK, OutContext);
1289 return MCSymbolRefExpr::create(Symbol: Sym, Ctx);
1290 };
1291
1292 using RIK = MCResourceInfo::ResourceInfoKind;
1293 ProgInfo.NumArchVGPR = GetSymRefExpr(RIK::RIK_NumVGPR);
1294 ProgInfo.NumAccVGPR = GetSymRefExpr(RIK::RIK_NumAGPR);
1295 ProgInfo.NumVGPR = AMDGPUMCExpr::createTotalNumVGPR(
1296 NumAGPR: ProgInfo.NumAccVGPR, NumVGPR: ProgInfo.NumArchVGPR, Ctx);
1297
1298 ProgInfo.AccumOffset = computeAccumOffset(NumVGPR: ProgInfo.NumArchVGPR, Ctx);
1299 ProgInfo.TgSplit =
1300 STM.hasTgSplitSupport() && AMDGPU::isTgSplitEnabled(F: MF.getFunction());
1301 ProgInfo.NumSGPR = GetSymRefExpr(RIK::RIK_NumSGPR);
1302 ProgInfo.ScratchSize = GetSymRefExpr(RIK::RIK_PrivateSegSize);
1303 ProgInfo.VCCUsed = GetSymRefExpr(RIK::RIK_UsesVCC);
1304 ProgInfo.FlatUsed = GetSymRefExpr(RIK::RIK_UsesFlatScratch);
1305 ProgInfo.DynamicCallStack =
1306 MCBinaryExpr::createOr(LHS: GetSymRefExpr(RIK::RIK_HasDynSizedStack),
1307 RHS: GetSymRefExpr(RIK::RIK_HasRecursion), Ctx);
1308
1309 const MCExpr *BarBlkConst = MCConstantExpr::create(Value: 4, Ctx);
1310 const MCExpr *AlignToBlk = AMDGPUMCExpr::createAlignTo(
1311 Value: GetSymRefExpr(RIK::RIK_NumNamedBarrier), Align: BarBlkConst, Ctx);
1312 ProgInfo.NamedBarCnt = MCBinaryExpr::createDiv(LHS: AlignToBlk, RHS: BarBlkConst, Ctx);
1313
1314 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1315
1316 // The calculations related to SGPR/VGPR blocks are
1317 // duplicated in part in AMDGPUAsmParser::calculateGPRBlocks, and could be
1318 // unified.
1319 const MCExpr *ExtraSGPRs = AMDGPUMCExpr::createExtraSGPRs(
1320 VCCUsed: ProgInfo.VCCUsed, FlatScrUsed: ProgInfo.FlatUsed,
1321 XNACKUsed: getTargetStreamer()->getTargetID()->isXnackOnOrAny(), Ctx);
1322
1323 // Check the addressable register limit before we add ExtraSGPRs.
1324 if (STM.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
1325 !STM.hasSGPRInitBug()) {
1326 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
1327 uint64_t NumSgpr;
1328 if (TryGetMCExprValue(ProgInfo.NumSGPR, NumSgpr) &&
1329 NumSgpr > MaxAddressableNumSGPRs) {
1330 // This can happen due to a compiler bug or when using inline asm.
1331 LLVMContext &Ctx = MF.getFunction().getContext();
1332 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1333 MF.getFunction(), "addressable scalar registers", NumSgpr,
1334 MaxAddressableNumSGPRs, DS_Error, DK_ResourceLimit));
1335 ProgInfo.NumSGPR = CreateExpr(MaxAddressableNumSGPRs - 1);
1336 }
1337 }
1338
1339 // Account for extra SGPRs and VGPRs reserved for debugger use.
1340 ProgInfo.NumSGPR = MCBinaryExpr::createAdd(LHS: ProgInfo.NumSGPR, RHS: ExtraSGPRs, Ctx);
1341
1342 const Function &F = MF.getFunction();
1343
1344 // Ensure there are enough SGPRs and VGPRs for wave dispatch, where wave
1345 // dispatch registers as function args.
1346 unsigned WaveDispatchNumSGPR = MFI->getNumWaveDispatchSGPRs(),
1347 WaveDispatchNumVGPR = MFI->getNumWaveDispatchVGPRs();
1348
1349 if (WaveDispatchNumSGPR) {
1350 ProgInfo.NumSGPR = AMDGPUMCExpr::createMax(
1351 Args: {ProgInfo.NumSGPR,
1352 MCBinaryExpr::createAdd(LHS: CreateExpr(WaveDispatchNumSGPR), RHS: ExtraSGPRs,
1353 Ctx)},
1354 Ctx);
1355 }
1356
1357 if (WaveDispatchNumVGPR) {
1358 ProgInfo.NumArchVGPR = AMDGPUMCExpr::createMax(
1359 Args: {ProgInfo.NumArchVGPR, CreateExpr(WaveDispatchNumVGPR)}, Ctx);
1360
1361 ProgInfo.NumVGPR = AMDGPUMCExpr::createTotalNumVGPR(
1362 NumAGPR: ProgInfo.NumAccVGPR, NumVGPR: ProgInfo.NumArchVGPR, Ctx);
1363 }
1364
1365 // Adjust number of registers used to meet default/requested minimum/maximum
1366 // number of waves per execution unit request.
1367 unsigned MaxWaves = MFI->getMaxWavesPerEU();
1368 ProgInfo.NumSGPRsForWavesPerEU =
1369 AMDGPUMCExpr::createMax(Args: {ProgInfo.NumSGPR, CreateExpr(1ul),
1370 CreateExpr(STM.getMinNumSGPRs(WavesPerEU: MaxWaves))},
1371 Ctx);
1372 ProgInfo.NumVGPRsForWavesPerEU =
1373 AMDGPUMCExpr::createMax(Args: {ProgInfo.NumVGPR, CreateExpr(1ul),
1374 CreateExpr(STM.getMinNumVGPRs(
1375 WavesPerEU: MaxWaves, DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize()))},
1376 Ctx);
1377
1378 if (STM.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS ||
1379 STM.hasSGPRInitBug()) {
1380 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
1381 uint64_t NumSgpr;
1382 if (TryGetMCExprValue(ProgInfo.NumSGPR, NumSgpr) &&
1383 NumSgpr > MaxAddressableNumSGPRs) {
1384 // This can happen due to a compiler bug or when using inline asm to use
1385 // the registers which are usually reserved for vcc etc.
1386 LLVMContext &Ctx = MF.getFunction().getContext();
1387 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1388 MF.getFunction(), "scalar registers", NumSgpr, MaxAddressableNumSGPRs,
1389 DS_Error, DK_ResourceLimit));
1390 ProgInfo.NumSGPR = CreateExpr(MaxAddressableNumSGPRs);
1391 ProgInfo.NumSGPRsForWavesPerEU = CreateExpr(MaxAddressableNumSGPRs);
1392 }
1393 }
1394
1395 if (STM.hasSGPRInitBug()) {
1396 ProgInfo.NumSGPR =
1397 CreateExpr(AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG);
1398 ProgInfo.NumSGPRsForWavesPerEU =
1399 CreateExpr(AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG);
1400 }
1401
1402 if (MFI->getNumUserSGPRs() > STM.getMaxNumUserSGPRs()) {
1403 LLVMContext &Ctx = MF.getFunction().getContext();
1404 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1405 MF.getFunction(), "user SGPRs", MFI->getNumUserSGPRs(),
1406 STM.getMaxNumUserSGPRs(), DS_Error));
1407 }
1408
1409 if (MFI->getLDSSize() > STM.getAddressableLocalMemorySize()) {
1410 LLVMContext &Ctx = MF.getFunction().getContext();
1411 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1412 MF.getFunction(), "local memory", MFI->getLDSSize(),
1413 STM.getAddressableLocalMemorySize(), DS_Error));
1414 }
1415
1416 // When dynamic VGPRs are enabled, entry functions are launched with a single
1417 // VGPR block. The register allocator enforces this constraint, but we also
1418 // need to catch explicit physical registers in inline asm and wave dispatch
1419 // VGPR arguments.
1420 if (MFI->isDynamicVGPREnabled() &&
1421 AMDGPU::isEntryFunctionCC(CC: F.getCallingConv())) {
1422 unsigned BlockSize = MFI->getDynamicVGPRBlockSize();
1423 uint64_t NumVgpr;
1424 if (TryGetMCExprValue(ProgInfo.NumVGPRsForWavesPerEU, NumVgpr) &&
1425 NumVgpr > BlockSize) {
1426 LLVMContext &Ctx = F.getContext();
1427 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1428 F, "dynamic VGPR entry point vector registers", NumVgpr, BlockSize,
1429 DS_Warning, DK_ResourceLimit));
1430 }
1431 }
1432
1433 // The MCExpr equivalent of getNumSGPRBlocks/getNumVGPRBlocks:
1434 // (alignTo(max(1u, NumGPR), GPREncodingGranule) / GPREncodingGranule) - 1
1435 auto GetNumGPRBlocks = [&CreateExpr, &Ctx](const MCExpr *NumGPR,
1436 unsigned Granule) {
1437 const MCExpr *OneConst = CreateExpr(1ul);
1438 const MCExpr *GranuleConst = CreateExpr(Granule);
1439 const MCExpr *MaxNumGPR = AMDGPUMCExpr::createMax(Args: {NumGPR, OneConst}, Ctx);
1440 const MCExpr *AlignToGPR =
1441 AMDGPUMCExpr::createAlignTo(Value: MaxNumGPR, Align: GranuleConst, Ctx);
1442 const MCExpr *DivGPR =
1443 MCBinaryExpr::createDiv(LHS: AlignToGPR, RHS: GranuleConst, Ctx);
1444 const MCExpr *SubGPR = MCBinaryExpr::createSub(LHS: DivGPR, RHS: OneConst, Ctx);
1445 return SubGPR;
1446 };
1447 // GFX10+ will always allocate 128 SGPRs and this field must be 0
1448 if (STM.getGeneration() >= AMDGPUSubtarget::GFX10) {
1449 ProgInfo.SGPRBlocks = CreateExpr(0ul);
1450 } else {
1451 ProgInfo.SGPRBlocks = GetNumGPRBlocks(ProgInfo.NumSGPRsForWavesPerEU,
1452 IsaInfo::getSGPREncodingGranule(STI: STM));
1453 }
1454 ProgInfo.VGPRBlocks = GetNumGPRBlocks(ProgInfo.NumVGPRsForWavesPerEU,
1455 IsaInfo::getVGPREncodingGranule(STI: STM));
1456
1457 const SIModeRegisterDefaults Mode = MFI->getMode();
1458
1459 // Set the value to initialize FP_ROUND and FP_DENORM parts of the mode
1460 // register.
1461 ProgInfo.FloatMode = getFPMode(Mode);
1462
1463 ProgInfo.IEEEMode = Mode.IEEE;
1464
1465 // Make clamp modifier on NaN input returns 0.
1466 ProgInfo.DX10Clamp = Mode.DX10Clamp;
1467 ProgInfo.SGPRSpill = MFI->getNumSpilledSGPRs();
1468 ProgInfo.VGPRSpill = MFI->getNumSpilledVGPRs();
1469
1470 ProgInfo.LDSSize = MFI->getLDSSize();
1471
1472 unsigned LDSGranularityBytes = getLDSEncodingGranule(ST: STM);
1473 ProgInfo.LDSBlocks =
1474 alignTo(Value: ProgInfo.LDSSize, Align: LDSGranularityBytes) / LDSGranularityBytes;
1475
1476 // The MCExpr equivalent of divideCeil.
1477 auto DivideCeil = [&Ctx](const MCExpr *Numerator, const MCExpr *Denominator) {
1478 const MCExpr *Ceil =
1479 AMDGPUMCExpr::createAlignTo(Value: Numerator, Align: Denominator, Ctx);
1480 return MCBinaryExpr::createDiv(LHS: Ceil, RHS: Denominator, Ctx);
1481 };
1482
1483 // Scratch is allocated in 64-dword or 256-dword blocks.
1484 unsigned ScratchAlignShift =
1485 STM.getGeneration() >= AMDGPUSubtarget::GFX11 ? 8 : 10;
1486 // We need to program the hardware with the amount of scratch memory that
1487 // is used by the entire wave. ProgInfo.ScratchSize is the amount of
1488 // scratch memory used per thread.
1489 ProgInfo.ScratchBlocks = DivideCeil(
1490 MCBinaryExpr::createMul(LHS: ProgInfo.ScratchSize,
1491 RHS: CreateExpr(STM.getWavefrontSize()), Ctx),
1492 CreateExpr(1ULL << ScratchAlignShift));
1493
1494 if (STM.hasSupportsWGP()) {
1495 ProgInfo.WgpMode = STM.isCuModeEnabled() ? 0 : 1;
1496 }
1497
1498 if (getIsaVersion(GPU: getGlobalSTI()->getCPU()).Major >= 10) {
1499 ProgInfo.MemOrdered = 1;
1500 ProgInfo.FwdProgress = !F.hasFnAttribute(Kind: "amdgpu-no-fwd-progress");
1501 }
1502
1503 // 0 = X, 1 = XY, 2 = XYZ
1504 unsigned TIDIGCompCnt = 0;
1505 if (MFI->hasWorkItemIDZ())
1506 TIDIGCompCnt = 2;
1507 else if (MFI->hasWorkItemIDY())
1508 TIDIGCompCnt = 1;
1509
1510 // The private segment wave byte offset is the last of the system SGPRs. We
1511 // initially assumed it was allocated, and may have used it. It shouldn't harm
1512 // anything to disable it if we know the stack isn't used here. We may still
1513 // have emitted code reading it to initialize scratch, but if that's unused
1514 // reading garbage should be OK.
1515 ProgInfo.ScratchEnable = MCBinaryExpr::createLOr(
1516 LHS: MCBinaryExpr::createGT(LHS: ProgInfo.ScratchBlocks,
1517 RHS: MCConstantExpr::create(Value: 0, Ctx), Ctx),
1518 RHS: ProgInfo.DynamicCallStack, Ctx);
1519
1520 ProgInfo.UserSGPR = MFI->getNumUserSGPRs();
1521 // For AMDHSA, TRAP_HANDLER must be zero, as it is populated by the CP.
1522 ProgInfo.TrapHandlerEnable = STM.isAmdHsaOS() ? 0 : STM.hasTrapHandler();
1523 ProgInfo.TGIdXEnable = MFI->hasWorkGroupIDX();
1524 ProgInfo.TGIdYEnable = MFI->hasWorkGroupIDY();
1525 ProgInfo.TGIdZEnable = MFI->hasWorkGroupIDZ();
1526 ProgInfo.TGSizeEnable = MFI->hasWorkGroupInfo();
1527 ProgInfo.TIdIGCompCount = TIDIGCompCnt;
1528 ProgInfo.EXCPEnMSB = 0;
1529 // For AMDHSA, LDS_SIZE must be zero, as it is populated by the CP.
1530 ProgInfo.LdsSize = STM.isAmdHsaOS() ? 0 : ProgInfo.LDSBlocks;
1531 ProgInfo.EXCPEnable = 0;
1532
1533 if (STM.hasGFX90AInsts()) {
1534 ProgInfo.ComputePGMRSrc3 =
1535 setBits(Dst: ProgInfo.ComputePGMRSrc3, Value: ProgInfo.AccumOffset,
1536 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
1537 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT, Ctx);
1538 ProgInfo.ComputePGMRSrc3 =
1539 setBits(Dst: ProgInfo.ComputePGMRSrc3, Value: CreateExpr(ProgInfo.TgSplit),
1540 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT,
1541 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT, Ctx);
1542 }
1543
1544 if (STM.hasGFX1250Insts())
1545 ProgInfo.ComputePGMRSrc3 =
1546 setBits(Dst: ProgInfo.ComputePGMRSrc3, Value: ProgInfo.NamedBarCnt,
1547 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
1548 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT, Ctx);
1549
1550 ProgInfo.Occupancy = createOccupancy(
1551 InitOcc: STM.computeOccupancy(F, LDSSize: ProgInfo.LDSSize).second,
1552 NumSGPRs: ProgInfo.NumSGPRsForWavesPerEU, NumVGPRs: ProgInfo.NumVGPRsForWavesPerEU,
1553 DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize(), STM, Ctx);
1554
1555 const auto [MinWEU, MaxWEU] =
1556 AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-waves-per-eu", Default: {0, 0}, OnlyFirstRequired: true);
1557 uint64_t Occupancy;
1558 if (TryGetMCExprValue(ProgInfo.Occupancy, Occupancy) && Occupancy < MinWEU) {
1559 DiagnosticInfoOptimizationFailure Diag(
1560 F, F.getSubprogram(),
1561 "failed to meet occupancy target given by 'amdgpu-waves-per-eu' in "
1562 "'" +
1563 F.getName() + "': desired occupancy was " + Twine(MinWEU) +
1564 ", final occupancy is " + Twine(Occupancy));
1565 F.getContext().diagnose(DI: Diag);
1566 }
1567}
1568
1569static unsigned getRsrcReg(CallingConv::ID CallConv) {
1570 switch (CallConv) {
1571 default:
1572 [[fallthrough]];
1573 case CallingConv::AMDGPU_CS:
1574 return R_00B848_COMPUTE_PGM_RSRC1;
1575 case CallingConv::AMDGPU_LS:
1576 return R_00B528_SPI_SHADER_PGM_RSRC1_LS;
1577 case CallingConv::AMDGPU_HS:
1578 return R_00B428_SPI_SHADER_PGM_RSRC1_HS;
1579 case CallingConv::AMDGPU_ES:
1580 return R_00B328_SPI_SHADER_PGM_RSRC1_ES;
1581 case CallingConv::AMDGPU_GS:
1582 return R_00B228_SPI_SHADER_PGM_RSRC1_GS;
1583 case CallingConv::AMDGPU_VS:
1584 return R_00B128_SPI_SHADER_PGM_RSRC1_VS;
1585 case CallingConv::AMDGPU_PS:
1586 return R_00B028_SPI_SHADER_PGM_RSRC1_PS;
1587 }
1588}
1589
1590void AMDGPUAsmPrinter::EmitProgramInfoSI(
1591 const MachineFunction &MF, const SIProgramInfo &CurrentProgramInfo) {
1592 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1593 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1594 unsigned RsrcReg = getRsrcReg(CallConv: MF.getFunction().getCallingConv());
1595 MCContext &Ctx = MF.getContext();
1596
1597 // (((Value) & Mask) << Shift)
1598 auto SetBits = [&Ctx](const MCExpr *Value, uint32_t Mask, uint32_t Shift) {
1599 const MCExpr *msk = MCConstantExpr::create(Value: Mask, Ctx);
1600 const MCExpr *shft = MCConstantExpr::create(Value: Shift, Ctx);
1601 return MCBinaryExpr::createShl(LHS: MCBinaryExpr::createAnd(LHS: Value, RHS: msk, Ctx),
1602 RHS: shft, Ctx);
1603 };
1604
1605 auto EmitResolvedOrExpr = [this](const MCExpr *Value, unsigned Size) {
1606 int64_t Val;
1607 if (Value->evaluateAsAbsolute(Res&: Val))
1608 OutStreamer->emitIntValue(Value: static_cast<uint64_t>(Val), Size);
1609 else
1610 OutStreamer->emitValue(Value, Size);
1611 };
1612
1613 if (AMDGPU::isCompute(CC: MF.getFunction().getCallingConv())) {
1614 OutStreamer->emitInt32(R_00B848_COMPUTE_PGM_RSRC1);
1615
1616 EmitResolvedOrExpr(CurrentProgramInfo.getComputePGMRSrc1(ST: STM, Ctx),
1617 /*Size=*/4);
1618
1619 OutStreamer->emitInt32(R_00B84C_COMPUTE_PGM_RSRC2);
1620 EmitResolvedOrExpr(CurrentProgramInfo.getComputePGMRSrc2(ST: STM, Ctx),
1621 /*Size=*/4);
1622
1623 OutStreamer->emitInt32(R_00B860_COMPUTE_TMPRING_SIZE);
1624
1625 // Sets bits according to S_0286E8_WAVESIZE_* mask and shift values for the
1626 // appropriate generation.
1627 if (STM.getGeneration() >= AMDGPUSubtarget::GFX12) {
1628 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1629 /*Mask=*/0x3FFFF, /*Shift=*/12),
1630 /*Size=*/4);
1631 } else if (STM.getGeneration() == AMDGPUSubtarget::GFX11) {
1632 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1633 /*Mask=*/0x7FFF, /*Shift=*/12),
1634 /*Size=*/4);
1635 } else {
1636 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1637 /*Mask=*/0x1FFF, /*Shift=*/12),
1638 /*Size=*/4);
1639 }
1640
1641 // TODO: Should probably note flat usage somewhere. SC emits a "FlatPtr32 =
1642 // 0" comment but I don't see a corresponding field in the register spec.
1643 } else {
1644 OutStreamer->emitInt32(Value: RsrcReg);
1645
1646 const MCExpr *GPRBlocks = MCBinaryExpr::createOr(
1647 LHS: SetBits(CurrentProgramInfo.VGPRBlocks, /*Mask=*/0x3F, /*Shift=*/0),
1648 RHS: SetBits(CurrentProgramInfo.SGPRBlocks, /*Mask=*/0x0F, /*Shift=*/6),
1649 Ctx&: MF.getContext());
1650 EmitResolvedOrExpr(GPRBlocks, /*Size=*/4);
1651 OutStreamer->emitInt32(R_0286E8_SPI_TMPRING_SIZE);
1652
1653 // Sets bits according to S_0286E8_WAVESIZE_* mask and shift values for the
1654 // appropriate generation.
1655 if (STM.getGeneration() >= AMDGPUSubtarget::GFX12) {
1656 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1657 /*Mask=*/0x3FFFF, /*Shift=*/12),
1658 /*Size=*/4);
1659 } else if (STM.getGeneration() == AMDGPUSubtarget::GFX11) {
1660 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1661 /*Mask=*/0x7FFF, /*Shift=*/12),
1662 /*Size=*/4);
1663 } else {
1664 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1665 /*Mask=*/0x1FFF, /*Shift=*/12),
1666 /*Size=*/4);
1667 }
1668 }
1669
1670 if (MF.getFunction().getCallingConv() == CallingConv::AMDGPU_PS) {
1671 OutStreamer->emitInt32(R_00B02C_SPI_SHADER_PGM_RSRC2_PS);
1672 unsigned ExtraLDSSize = STM.getGeneration() >= AMDGPUSubtarget::GFX11
1673 ? divideCeil(Numerator: CurrentProgramInfo.LDSBlocks, Denominator: 2)
1674 : CurrentProgramInfo.LDSBlocks;
1675 OutStreamer->emitInt32(S_00B02C_EXTRA_LDS_SIZE(ExtraLDSSize));
1676 OutStreamer->emitInt32(R_0286CC_SPI_PS_INPUT_ENA);
1677 OutStreamer->emitInt32(Value: MFI->getPSInputEnable());
1678 OutStreamer->emitInt32(R_0286D0_SPI_PS_INPUT_ADDR);
1679 OutStreamer->emitInt32(Value: MFI->getPSInputAddr());
1680 }
1681
1682 OutStreamer->emitInt32(R_SPILLED_SGPRS);
1683 OutStreamer->emitInt32(Value: MFI->getNumSpilledSGPRs());
1684 OutStreamer->emitInt32(R_SPILLED_VGPRS);
1685 OutStreamer->emitInt32(Value: MFI->getNumSpilledVGPRs());
1686}
1687
1688// Helper function to add common PAL Metadata 3.0+
1689static void EmitPALMetadataCommon(AMDGPUPALMetadata *MD,
1690 const SIProgramInfo &CurrentProgramInfo,
1691 CallingConv::ID CC, const GCNSubtarget &ST,
1692 unsigned DynamicVGPRBlockSize) {
1693 if (ST.hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode))
1694 MD->setHwStage(CC, field: ".ieee_mode", Val: (bool)CurrentProgramInfo.IEEEMode);
1695
1696 MD->setHwStage(CC, field: ".wgp_mode", Val: (bool)CurrentProgramInfo.WgpMode);
1697 MD->setHwStage(CC, field: ".mem_ordered", Val: (bool)CurrentProgramInfo.MemOrdered);
1698 MD->setHwStage(CC, field: ".forward_progress", Val: (bool)CurrentProgramInfo.FwdProgress);
1699
1700 if (AMDGPU::isCompute(CC)) {
1701 MD->setHwStage(CC, field: ".trap_present",
1702 Val: (bool)CurrentProgramInfo.TrapHandlerEnable);
1703 MD->setHwStage(CC, field: ".excp_en", Val: CurrentProgramInfo.EXCPEnable);
1704
1705 if (DynamicVGPRBlockSize != 0)
1706 MD->setComputeRegisters(field: ".dynamic_vgpr_en", Val: true);
1707 }
1708
1709 MD->updateHwStageMaximum(
1710 CC, field: ".lds_size",
1711 Val: (unsigned)(CurrentProgramInfo.LdsSize * getLDSEncodingGranule(ST)));
1712}
1713
1714// This is the equivalent of EmitProgramInfoSI above, but for when the OS type
1715// is AMDPAL. It stores each compute/SPI register setting and other PAL
1716// metadata items into the PALMD::Metadata, combining with any provided by the
1717// frontend as LLVM metadata. Once all functions are written, the PAL metadata
1718// is then written as a single block in the .note section.
1719void AMDGPUAsmPrinter::EmitPALMetadata(
1720 const MachineFunction &MF, const SIProgramInfo &CurrentProgramInfo) {
1721 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1722 auto CC = MF.getFunction().getCallingConv();
1723 auto *MD = getTargetStreamer()->getPALMetadata();
1724 auto &Ctx = MF.getContext();
1725
1726 MD->setEntryPoint(CC, Name: MF.getFunction().getName());
1727 MD->setNumUsedVgprs(CC, Val: CurrentProgramInfo.NumVGPRsForWavesPerEU, Ctx);
1728
1729 // For targets that support dynamic VGPRs, set the number of saved dynamic
1730 // VGPRs (if any) in the PAL metadata.
1731 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1732 if (MFI->isDynamicVGPREnabled() &&
1733 MFI->getScratchReservedForDynamicVGPRs() > 0)
1734 MD->setHwStage(CC, field: ".dynamic_vgpr_saved_count",
1735 Val: MFI->getScratchReservedForDynamicVGPRs() / 4);
1736
1737 // Only set AGPRs for supported devices
1738 if (STM.hasMAIInsts()) {
1739 MD->setNumUsedAgprs(CC, Val: CurrentProgramInfo.NumAccVGPR);
1740 }
1741
1742 MD->setNumUsedSgprs(CC, Val: CurrentProgramInfo.NumSGPRsForWavesPerEU, Ctx);
1743 if (MD->getPALMajorVersion() < 3) {
1744 MD->setRsrc1(CC, Val: CurrentProgramInfo.getPGMRSrc1(CC, ST: STM, Ctx), Ctx);
1745 if (AMDGPU::isCompute(CC)) {
1746 MD->setRsrc2(CC, Val: CurrentProgramInfo.getComputePGMRSrc2(ST: STM, Ctx), Ctx);
1747 } else {
1748 const MCExpr *HasScratchBlocks =
1749 MCBinaryExpr::createGT(LHS: CurrentProgramInfo.ScratchBlocks,
1750 RHS: MCConstantExpr::create(Value: 0, Ctx), Ctx);
1751 auto [Shift, Mask] = getShiftMask(C_00B84C_SCRATCH_EN);
1752 MD->setRsrc2(CC, Val: maskShiftSet(Val: HasScratchBlocks, Mask, Shift, Ctx), Ctx);
1753 }
1754 } else {
1755 MD->setHwStage(CC, field: ".debug_mode", Val: (bool)CurrentProgramInfo.DebugMode);
1756 MD->setHwStage(CC, field: ".scratch_en", Type: msgpack::Type::Boolean,
1757 Val: CurrentProgramInfo.ScratchEnable);
1758 EmitPALMetadataCommon(MD, CurrentProgramInfo, CC, ST: STM,
1759 DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize());
1760 }
1761
1762 // ScratchSize is in bytes, 16 aligned.
1763 MD->setScratchSize(
1764 CC,
1765 Val: AMDGPUMCExpr::createAlignTo(Value: CurrentProgramInfo.ScratchSize,
1766 Align: MCConstantExpr::create(Value: 16, Ctx), Ctx),
1767 Ctx);
1768
1769 if (MF.getFunction().getCallingConv() == CallingConv::AMDGPU_PS) {
1770 unsigned ExtraLDSSize = STM.getGeneration() >= AMDGPUSubtarget::GFX11
1771 ? divideCeil(Numerator: CurrentProgramInfo.LDSBlocks, Denominator: 2)
1772 : CurrentProgramInfo.LDSBlocks;
1773 if (MD->getPALMajorVersion() < 3) {
1774 MD->setRsrc2(
1775 CC,
1776 Val: MCConstantExpr::create(S_00B02C_EXTRA_LDS_SIZE(ExtraLDSSize), Ctx),
1777 Ctx);
1778 MD->setSpiPsInputEna(MFI->getPSInputEnable());
1779 MD->setSpiPsInputAddr(MFI->getPSInputAddr());
1780 } else {
1781 // Graphics registers
1782 const unsigned ExtraLdsDwGranularity =
1783 STM.getGeneration() >= AMDGPUSubtarget::GFX11 ? 256 : 128;
1784 MD->setGraphicsRegisters(
1785 field: ".ps_extra_lds_size",
1786 Val: (unsigned)(ExtraLDSSize * ExtraLdsDwGranularity * sizeof(uint32_t)));
1787
1788 // Set PsInputEna and PsInputAddr .spi_ps_input_ena and .spi_ps_input_addr
1789 static StringLiteral const PsInputFields[] = {
1790 ".persp_sample_ena", ".persp_center_ena",
1791 ".persp_centroid_ena", ".persp_pull_model_ena",
1792 ".linear_sample_ena", ".linear_center_ena",
1793 ".linear_centroid_ena", ".line_stipple_tex_ena",
1794 ".pos_x_float_ena", ".pos_y_float_ena",
1795 ".pos_z_float_ena", ".pos_w_float_ena",
1796 ".front_face_ena", ".ancillary_ena",
1797 ".sample_coverage_ena", ".pos_fixed_pt_ena"};
1798 unsigned PSInputEna = MFI->getPSInputEnable();
1799 unsigned PSInputAddr = MFI->getPSInputAddr();
1800 for (auto [Idx, Field] : enumerate(First: PsInputFields)) {
1801 MD->setGraphicsRegisters(field1: ".spi_ps_input_ena", field2: Field,
1802 Val: (bool)((PSInputEna >> Idx) & 1));
1803 MD->setGraphicsRegisters(field1: ".spi_ps_input_addr", field2: Field,
1804 Val: (bool)((PSInputAddr >> Idx) & 1));
1805 }
1806 }
1807 }
1808
1809 // For version 3 and above the wave front size is already set in the metadata
1810 if (MD->getPALMajorVersion() < 3 && STM.isWave32())
1811 MD->setWave32(MF.getFunction().getCallingConv());
1812}
1813
1814void AMDGPUAsmPrinter::emitPALFunctionMetadata(const MachineFunction &MF) {
1815 auto *MD = getTargetStreamer()->getPALMetadata();
1816 const MachineFrameInfo &MFI = MF.getFrameInfo();
1817 StringRef FnName = MF.getFunction().getName();
1818 MD->setFunctionScratchSize(FnName, Val: MFI.getStackSize());
1819 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1820 MCContext &Ctx = MF.getContext();
1821
1822 if (MD->getPALMajorVersion() < 3) {
1823 // Set compute registers
1824 MD->setRsrc1(
1825 CC: CallingConv::AMDGPU_CS,
1826 Val: CurrentProgramInfo.getPGMRSrc1(CC: CallingConv::AMDGPU_CS, ST, Ctx), Ctx);
1827 MD->setRsrc2(CC: CallingConv::AMDGPU_CS,
1828 Val: CurrentProgramInfo.getComputePGMRSrc2(ST, Ctx), Ctx);
1829 } else {
1830 EmitPALMetadataCommon(
1831 MD, CurrentProgramInfo, CC: CallingConv::AMDGPU_CS, ST,
1832 DynamicVGPRBlockSize: MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize());
1833 }
1834
1835 // Set optional info
1836 MD->setFunctionLdsSize(FnName, Val: CurrentProgramInfo.LDSSize);
1837 MD->setFunctionNumUsedVgprs(FnName, Val: CurrentProgramInfo.NumVGPRsForWavesPerEU);
1838 MD->setFunctionNumUsedSgprs(FnName, Val: CurrentProgramInfo.NumSGPRsForWavesPerEU);
1839}
1840
1841// This is supposed to be log2(Size)
1842static amd_element_byte_size_t getElementByteSizeValue(unsigned Size) {
1843 switch (Size) {
1844 case 4:
1845 return AMD_ELEMENT_4_BYTES;
1846 case 8:
1847 return AMD_ELEMENT_8_BYTES;
1848 case 16:
1849 return AMD_ELEMENT_16_BYTES;
1850 default:
1851 llvm_unreachable("invalid private_element_size");
1852 }
1853}
1854
1855void AMDGPUAsmPrinter::getAmdKernelCode(AMDGPUMCKernelCodeT &Out,
1856 const SIProgramInfo &CurrentProgramInfo,
1857 const MachineFunction &MF) const {
1858 const Function &F = MF.getFunction();
1859 assert(F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
1860 F.getCallingConv() == CallingConv::SPIR_KERNEL);
1861
1862 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1863 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1864 MCContext &Ctx = MF.getContext();
1865
1866 Out.initDefault(STI: STM, Ctx, /*InitMCExpr=*/false);
1867
1868 Out.compute_pgm_resource1_registers =
1869 CurrentProgramInfo.getComputePGMRSrc1(ST: STM, Ctx);
1870 Out.compute_pgm_resource2_registers =
1871 CurrentProgramInfo.getComputePGMRSrc2(ST: STM, Ctx);
1872 Out.code_properties |= AMD_CODE_PROPERTY_IS_PTR64;
1873
1874 Out.is_dynamic_callstack = CurrentProgramInfo.DynamicCallStack;
1875
1876 AMD_HSA_BITS_SET(Out.code_properties, AMD_CODE_PROPERTY_PRIVATE_ELEMENT_SIZE,
1877 getElementByteSizeValue(STM.getMaxPrivateElementSize(true)));
1878
1879 const GCNUserSGPRUsageInfo &UserSGPRInfo = MFI->getUserSGPRInfo();
1880 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
1881 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER;
1882 }
1883
1884 if (UserSGPRInfo.hasDispatchPtr())
1885 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR;
1886
1887 if (UserSGPRInfo.hasQueuePtr())
1888 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR;
1889
1890 if (UserSGPRInfo.hasKernargSegmentPtr())
1891 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR;
1892
1893 if (UserSGPRInfo.hasDispatchID())
1894 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID;
1895
1896 if (UserSGPRInfo.hasFlatScratchInit())
1897 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT;
1898
1899 if (UserSGPRInfo.hasPrivateSegmentSize())
1900 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE;
1901
1902 if (STM.isXNACKEnabled())
1903 Out.code_properties |= AMD_CODE_PROPERTY_IS_XNACK_SUPPORTED;
1904
1905 Align MaxKernArgAlign;
1906 Out.kernarg_segment_byte_size = STM.getKernArgSegmentSize(F, MaxAlign&: MaxKernArgAlign);
1907 Out.wavefront_sgpr_count = CurrentProgramInfo.NumSGPR;
1908 Out.workitem_vgpr_count = CurrentProgramInfo.NumVGPR;
1909 Out.workitem_private_segment_byte_size = CurrentProgramInfo.ScratchSize;
1910 Out.workgroup_group_segment_byte_size = CurrentProgramInfo.LDSSize;
1911
1912 // kernarg_segment_alignment is specified as log of the alignment.
1913 // The minimum alignment is 16.
1914 // FIXME: The metadata treats the minimum as 4?
1915 Out.kernarg_segment_alignment = Log2(A: std::max(a: Align(16), b: MaxKernArgAlign));
1916}
1917
1918bool AMDGPUAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
1919 const char *ExtraCode, raw_ostream &O) {
1920 // First try the generic code, which knows about modifiers like 'c' and 'n'.
1921 if (!AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, OS&: O))
1922 return false;
1923
1924 if (ExtraCode && ExtraCode[0]) {
1925 if (ExtraCode[1] != 0)
1926 return true; // Unknown modifier.
1927
1928 switch (ExtraCode[0]) {
1929 case 'r':
1930 break;
1931 default:
1932 return true;
1933 }
1934 }
1935
1936 // TODO: Should be able to support other operand types like globals.
1937 const MachineOperand &MO = MI->getOperand(i: OpNo);
1938 if (MO.isReg()) {
1939 AMDGPUInstPrinter::printRegOperand(Reg: MO.getReg(), O,
1940 MRI: *MF->getSubtarget().getRegisterInfo());
1941 return false;
1942 }
1943 if (MO.isImm()) {
1944 int64_t Val = MO.getImm();
1945 if (AMDGPU::isInlinableIntLiteral(Literal: Val)) {
1946 O << Val;
1947 } else if (isUInt<16>(x: Val)) {
1948 O << format(Fmt: "0x%" PRIx16, Vals: static_cast<uint16_t>(Val));
1949 } else if (isUInt<32>(x: Val)) {
1950 O << format(Fmt: "0x%" PRIx32, Vals: static_cast<uint32_t>(Val));
1951 } else {
1952 O << format(Fmt: "0x%" PRIx64, Vals: static_cast<uint64_t>(Val));
1953 }
1954 return false;
1955 }
1956 return true;
1957}
1958
1959void AMDGPUAsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const {
1960 AU.addRequired<AMDGPUResourceUsageAnalysisWrapperPass>();
1961 AU.addPreserved<AMDGPUResourceUsageAnalysisWrapperPass>();
1962 AU.addRequired<MachineModuleInfoWrapperPass>();
1963 AU.addPreserved<MachineModuleInfoWrapperPass>();
1964 AsmPrinter::getAnalysisUsage(AU);
1965}
1966
1967void AMDGPUAsmPrinter::emitResourceUsageRemarks(
1968 const MachineFunction &MF, const SIProgramInfo &CurrentProgramInfo,
1969 bool isModuleEntryFunction, bool hasMAIInsts) {
1970 if (!ORE)
1971 return;
1972
1973 const char *Name = "kernel-resource-usage";
1974 const char *Indent = " ";
1975
1976 // If the remark is not specifically enabled, do not output to yaml
1977 LLVMContext &Ctx = MF.getFunction().getContext();
1978 if (!Ctx.getDiagHandlerPtr()->isAnalysisRemarkEnabled(PassName: Name))
1979 return;
1980
1981 // Currently non-kernel functions have no resources to emit.
1982 if (!isEntryFunctionCC(CC: MF.getFunction().getCallingConv()))
1983 return;
1984
1985 auto EmitResourceUsageRemark = [&](StringRef RemarkName,
1986 StringRef RemarkLabel, auto Argument) {
1987 // Add an indent for every line besides the line with the kernel name. This
1988 // makes it easier to tell which resource usage go with which kernel since
1989 // the kernel name will always be displayed first.
1990 std::string LabelStr = RemarkLabel.str() + ": ";
1991 if (RemarkName != "FunctionName")
1992 LabelStr = Indent + LabelStr;
1993
1994 ORE->emit([&]() {
1995 return MachineOptimizationRemarkAnalysis(Name, RemarkName,
1996 MF.getFunction().getSubprogram(),
1997 &MF.front())
1998 << LabelStr << ore::NV(RemarkName, Argument);
1999 });
2000 };
2001
2002 // FIXME: Formatting here is pretty nasty because clang does not accept
2003 // newlines from diagnostics. This forces us to emit multiple diagnostic
2004 // remarks to simulate newlines. If and when clang does accept newlines, this
2005 // formatting should be aggregated into one remark with newlines to avoid
2006 // printing multiple diagnostic location and diag opts.
2007 EmitResourceUsageRemark("FunctionName", "Function Name",
2008 MF.getFunction().getName());
2009 EmitResourceUsageRemark("NumSGPR", "TotalSGPRs",
2010 getMCExprStr(Value: CurrentProgramInfo.NumSGPR));
2011 EmitResourceUsageRemark("NumVGPR", "VGPRs",
2012 getMCExprStr(Value: CurrentProgramInfo.NumArchVGPR));
2013 if (hasMAIInsts) {
2014 EmitResourceUsageRemark("NumAGPR", "AGPRs",
2015 getMCExprStr(Value: CurrentProgramInfo.NumAccVGPR));
2016 }
2017 EmitResourceUsageRemark("ScratchSize", "ScratchSize [bytes/lane]",
2018 getMCExprStr(Value: CurrentProgramInfo.ScratchSize));
2019 int64_t DynStack;
2020 bool DynStackEvaluatable =
2021 CurrentProgramInfo.DynamicCallStack->evaluateAsAbsolute(Res&: DynStack);
2022 StringRef DynamicStackStr =
2023 DynStackEvaluatable && DynStack ? "True" : "False";
2024 EmitResourceUsageRemark("DynamicStack", "Dynamic Stack", DynamicStackStr);
2025 EmitResourceUsageRemark("Occupancy", "Occupancy [waves/SIMD]",
2026 getMCExprStr(Value: CurrentProgramInfo.Occupancy));
2027 EmitResourceUsageRemark("SGPRSpill", "SGPRs Spill",
2028 CurrentProgramInfo.SGPRSpill);
2029 EmitResourceUsageRemark("VGPRSpill", "VGPRs Spill",
2030 CurrentProgramInfo.VGPRSpill);
2031 if (isModuleEntryFunction)
2032 EmitResourceUsageRemark("BytesLDS", "LDS Size [bytes/block]",
2033 CurrentProgramInfo.LDSSize);
2034}
2035
2036PreservedAnalyses AMDGPUAsmPrinterBeginPass::run(Module &M,
2037 ModuleAnalysisManager &MAM) {
2038
2039 AMDGPUAsmPrinter &AsmPrinter = static_cast<AMDGPUAsmPrinter &>(
2040 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
2041 setupModuleAsmPrinter(M, MAM, AsmPrinter);
2042 AsmPrinter.doInitialization(M);
2043 return PreservedAnalyses::all();
2044}
2045
2046PreservedAnalyses
2047AMDGPUAsmPrinterPass::run(MachineFunction &MF,
2048 MachineFunctionAnalysisManager &MFAM) {
2049 AMDGPUAsmPrinter &AsmPrinter = static_cast<AMDGPUAsmPrinter &>(
2050 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF)
2051 .getCachedResult<AsmPrinterAnalysis>(IR&: *MF.getFunction().getParent())
2052 ->getPrinter());
2053 setupMachineFunctionAsmPrinter(MFAM, MF, AsmPrinter);
2054 AsmPrinter.GetResourceUsage = [&MFAM](MachineFunction &MF)
2055 -> const AMDGPUResourceUsageAnalysisImpl::SIFunctionResourceInfo * {
2056 return &MFAM.getResult<AMDGPUResourceUsageAnalysis>(IR&: MF);
2057 };
2058 AsmPrinter.runOnMachineFunction(MF);
2059 return PreservedAnalyses::all();
2060}
2061
2062PreservedAnalyses AMDGPUAsmPrinterEndPass::run(Module &M,
2063 ModuleAnalysisManager &MAM) {
2064 AMDGPUAsmPrinter &AsmPrinter = static_cast<AMDGPUAsmPrinter &>(
2065 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
2066 setupModuleAsmPrinter(M, MAM, AsmPrinter);
2067 AsmPrinter.doFinalization(M);
2068 return PreservedAnalyses::all();
2069}
2070
2071char AMDGPUAsmPrinter::ID = 0;
2072
2073INITIALIZE_PASS(AMDGPUAsmPrinter, "amdgpu-asm-printer",
2074 "AMDGPU Assembly Printer", false, false)
2075