1//===-- AMDGPUAsmPrinter.cpp - AMDGPU assembly printer --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10///
11/// The AMDGPUAsmPrinter is used to print both assembly string and also binary
12/// code. When passed an MCAsmStreamer it prints assembly and when passed
13/// an MCObjectStreamer it outputs binary code.
14//
15//===----------------------------------------------------------------------===//
16//
17
18#include "AMDGPUAsmPrinter.h"
19#include "AMDGPU.h"
20#include "AMDGPUHSAMetadataStreamer.h"
21#include "AMDGPUMCResourceInfo.h"
22#include "AMDGPUResourceUsageAnalysis.h"
23#include "AMDGPUTargetMachine.h"
24#include "GCNSubtarget.h"
25#include "MCTargetDesc/AMDGPUInstPrinter.h"
26#include "MCTargetDesc/AMDGPUMCExpr.h"
27#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h"
28#include "MCTargetDesc/AMDGPUTargetStreamer.h"
29#include "R600AsmPrinter.h"
30#include "SIMachineFunctionInfo.h"
31#include "TargetInfo/AMDGPUTargetInfo.h"
32#include "Utils/AMDGPUBaseInfo.h"
33#include "Utils/AMDKernelCodeTUtils.h"
34#include "Utils/SIDefinesUtils.h"
35#include "llvm/Analysis/OptimizationRemarkEmitter.h"
36#include "llvm/BinaryFormat/ELF.h"
37#include "llvm/CodeGen/AsmPrinterAnalysis.h"
38#include "llvm/CodeGen/AsmPrinterHandler.h"
39#include "llvm/CodeGen/MachineFrameInfo.h"
40#include "llvm/CodeGen/MachineModuleInfo.h"
41#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
42#include "llvm/IR/DiagnosticInfo.h"
43#include "llvm/MC/MCAssembler.h"
44#include "llvm/MC/MCContext.h"
45#include "llvm/MC/MCSectionELF.h"
46#include "llvm/MC/MCStreamer.h"
47#include "llvm/MC/MCValue.h"
48#include "llvm/MC/TargetRegistry.h"
49#include "llvm/Support/AMDHSAKernelDescriptor.h"
50#include "llvm/Support/Compiler.h"
51#include "llvm/Target/TargetLoweringObjectFile.h"
52#include "llvm/Target/TargetMachine.h"
53#include "llvm/TargetParser/AMDGPUTargetParser.h"
54
55using namespace llvm;
56using namespace llvm::AMDGPU;
57
58// This should get the default rounding mode from the kernel. We just set the
59// default here, but this could change if the OpenCL rounding mode pragmas are
60// used.
61//
62// The denormal mode here should match what is reported by the OpenCL runtime
63// for the CL_FP_DENORM bit from CL_DEVICE_{HALF|SINGLE|DOUBLE}_FP_CONFIG, but
64// can also be override to flush with the -cl-denorms-are-zero compiler flag.
65//
66// AMD OpenCL only sets flush none and reports CL_FP_DENORM for double
67// precision, and leaves single precision to flush all and does not report
68// CL_FP_DENORM for CL_DEVICE_SINGLE_FP_CONFIG. Mesa's OpenCL currently reports
69// CL_FP_DENORM for both.
70//
71// FIXME: It seems some instructions do not support single precision denormals
72// regardless of the mode (exp_*_f32, rcp_*_f32, rsq_*_f32, rsq_*f32, sqrt_f32,
73// and sin_f32, cos_f32 on most parts).
74
75// We want to use these instructions, and using fp32 denormals also causes
76// instructions to run at the double precision rate for the device so it's
77// probably best to just report no single precision denormals.
78static uint32_t getFPMode(SIModeRegisterDefaults Mode) {
79 return FP_ROUND_MODE_SP(FP_ROUND_ROUND_TO_NEAREST) |
80 FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_NEAREST) |
81 FP_DENORM_MODE_SP(Mode.fpDenormModeSPValue()) |
82 FP_DENORM_MODE_DP(Mode.fpDenormModeDPValue());
83}
84
85static AsmPrinter *
86createAMDGPUAsmPrinterPass(TargetMachine &tm,
87 std::unique_ptr<MCStreamer> &&Streamer) {
88 return new AMDGPUAsmPrinter(tm, std::move(Streamer));
89}
90
91extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
92LLVMInitializeAMDGPUAsmPrinter() {
93 TargetRegistry::RegisterAsmPrinter(T&: getTheR600Target(),
94 Fn: llvm::createR600AsmPrinterPass);
95 TargetRegistry::RegisterAsmPrinter(T&: getTheGCNTarget(),
96 Fn: createAMDGPUAsmPrinterPass);
97 TargetRegistry::RegisterAsmPrinter(T&: getTheGCNLegacyTarget(),
98 Fn: createAMDGPUAsmPrinterPass);
99}
100
101namespace {
102class AMDGPUAsmPrinterHandler : public AsmPrinterHandler {
103protected:
104 AMDGPUAsmPrinter *Asm;
105
106public:
107 AMDGPUAsmPrinterHandler(AMDGPUAsmPrinter *A) : Asm(A) {}
108
109 void beginFunction(const MachineFunction *MF) override {}
110
111 void endFunction(const MachineFunction *MF) override { Asm->endFunction(MF); }
112
113 void endModule() override {}
114};
115} // End anonymous namespace
116
117AMDGPUAsmPrinter::AMDGPUAsmPrinter(TargetMachine &TM,
118 std::unique_ptr<MCStreamer> Streamer)
119 : AsmPrinter(TM, std::move(Streamer)) {
120 assert(OutStreamer && "AsmPrinter constructed without streamer");
121 GetResourceUsage = [this](MachineFunction &MF)
122 -> const AMDGPUResourceUsageAnalysisImpl::SIFunctionResourceInfo * {
123 if (auto *ResourceUsageW =
124 getAnalysisIfAvailable<AMDGPUResourceUsageAnalysisWrapperPass>())
125 return &ResourceUsageW->getResourceInfo();
126 return nullptr;
127 };
128}
129
130StringRef AMDGPUAsmPrinter::getPassName() const {
131 return "AMDGPU Assembly Printer";
132}
133
134const MCSubtargetInfo *AMDGPUAsmPrinter::getGlobalSTI() const {
135 return &TM.getMCSubtargetInfo();
136}
137
138AMDGPUTargetStreamer *AMDGPUAsmPrinter::getTargetStreamer() const {
139 if (!OutStreamer)
140 return nullptr;
141 return static_cast<AMDGPUTargetStreamer *>(OutStreamer->getTargetStreamer());
142}
143
144void AMDGPUAsmPrinter::emitStartOfAsmFile(Module &M) {
145 IsTargetStreamerInitialized = false;
146}
147
148void AMDGPUAsmPrinter::initTargetStreamer(Module &M) {
149 IsTargetStreamerInitialized = true;
150
151 // TODO: Which one is called first, emitStartOfAsmFile or
152 // emitFunctionBodyStart?
153 if (getTargetStreamer() && !getTargetStreamer()->getTargetID())
154 initializeTargetID(M);
155
156 const Triple &TT = M.getTargetTriple();
157 if (TT.getOS() != Triple::AMDHSA && TT.getOS() != Triple::AMDPAL)
158 return;
159
160 getTargetStreamer()->EmitDirectiveAMDGCNTarget();
161
162 if (TT.getOS() == Triple::AMDHSA) {
163 getTargetStreamer()->EmitDirectiveAMDHSACodeObjectVersion(
164 COV: CodeObjectVersion);
165 HSAMetadataStream->begin(Mod: M, TargetID: *getTargetStreamer()->getTargetID());
166 }
167
168 if (TT.getOS() == Triple::AMDPAL)
169 getTargetStreamer()->getPALMetadata()->readFromIR(M);
170}
171
172void AMDGPUAsmPrinter::emitEndOfAsmFile(Module &M) {
173 // Init target streamer if it has not yet happened
174 if (!IsTargetStreamerInitialized)
175 initTargetStreamer(M);
176
177 const Triple &TT = M.getTargetTriple();
178 if (TT.getOS() != Triple::AMDHSA)
179 getTargetStreamer()->EmitISAVersion();
180
181 // Emit HSA Metadata (NT_AMD_AMDGPU_HSA_METADATA).
182 // Emit HSA Metadata (NT_AMD_HSA_METADATA).
183 if (TT.getOS() == Triple::AMDHSA) {
184 HSAMetadataStream->end();
185 bool Success = HSAMetadataStream->emitTo(TargetStreamer&: *getTargetStreamer());
186 (void)Success;
187 assert(Success && "Malformed HSA Metadata");
188 }
189}
190
191void AMDGPUAsmPrinter::emitFunctionBodyStart() {
192 const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
193 const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
194 const Function &F = MF->getFunction();
195
196 // TODO: We're checking this late, would be nice to check it earlier.
197 if (STM.requiresCodeObjectV6() && CodeObjectVersion < AMDGPU::AMDHSA_COV6) {
198 reportFatalUsageError(
199 reason: STM.getCPU() + " is only available on code object version 6 or better");
200 }
201
202 // TODO: Which one is called first, emitStartOfAsmFile or
203 // emitFunctionBodyStart?
204 if (!getTargetStreamer()->getTargetID())
205 initializeTargetID(M: *F.getParent());
206
207 if (!MFI.isEntryFunction())
208 return;
209
210 if (STM.isMesaKernel(F) &&
211 (F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
212 F.getCallingConv() == CallingConv::SPIR_KERNEL)) {
213 AMDGPUMCKernelCodeT KernelCode;
214 getAmdKernelCode(Out&: KernelCode, KernelInfo: CurrentProgramInfo, MF: *MF);
215 KernelCode.validate(STI: &STM, Ctx&: MF->getContext());
216 getTargetStreamer()->EmitAMDKernelCodeT(Header&: KernelCode);
217 }
218
219 if (STM.isAmdHsaOS())
220 HSAMetadataStream->emitKernel(MF: *MF, ProgramInfo: CurrentProgramInfo);
221}
222
223/// Set bits in a kernel descriptor MCExpr field:
224/// return ((Dst & ~Mask) | (Value << Shift))
225static const MCExpr *setBits(const MCExpr *Dst, const MCExpr *Value,
226 uint32_t Mask, uint32_t Shift, MCContext &Ctx) {
227 const auto *Shft = MCConstantExpr::create(Value: Shift, Ctx);
228 const auto *Msk = MCConstantExpr::create(Value: Mask, Ctx);
229 Dst = MCBinaryExpr::createAnd(LHS: Dst, RHS: MCUnaryExpr::createNot(Expr: Msk, Ctx), Ctx);
230 Dst = MCBinaryExpr::createOr(LHS: Dst, RHS: MCBinaryExpr::createShl(LHS: Value, RHS: Shft, Ctx),
231 Ctx);
232 return Dst;
233}
234
235void AMDGPUAsmPrinter::endFunction(const MachineFunction *MF) {
236 const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
237 if (!MFI.isEntryFunction())
238 return;
239
240 assert(TM.getTargetTriple().getOS() == Triple::AMDHSA);
241
242 const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
243 MCContext &Ctx = MF->getContext();
244
245 AMDGPU::MCKernelDescriptor KD =
246 getAmdhsaKernelDescriptor(MF: *MF, PI: CurrentProgramInfo);
247
248 // Compute inst_pref_size using MCExpr label subtraction for exact code
249 // size. At this point .Lfunc_end has been emitted (by the base AsmPrinter)
250 // right after the function code, so (Lfunc_end - func_sym) gives the
251 // exact function code size in bytes.
252 if (STM.hasInstPrefSize()) {
253 const MCExpr *CodeSizeExpr = MCBinaryExpr::createSub(
254 LHS: MCSymbolRefExpr::create(Symbol: getFunctionEnd(), Ctx&: OutContext),
255 RHS: MCSymbolRefExpr::create(Symbol: CurrentFnSym, Ctx&: OutContext), Ctx&: OutContext);
256
257 uint32_t Mask, Shift, Width, CacheLineSize;
258 STM.getInstPrefSizeArgs(Mask, Shift, Width, CacheLineSize);
259 const MCExpr *InstPrefSize =
260 AMDGPUMCExpr::createInstPrefSize(CodeSizeBytes: CodeSizeExpr, Ctx);
261 KD.compute_pgm_rsrc3 =
262 setBits(Dst: KD.compute_pgm_rsrc3, Value: InstPrefSize, Mask, Shift, Ctx);
263 }
264
265 auto &Streamer = getTargetStreamer()->getStreamer();
266 auto &Context = Streamer.getContext();
267 auto &ObjectFileInfo = *Context.getObjectFileInfo();
268 auto &ReadOnlySection = *ObjectFileInfo.getReadOnlySection();
269
270 Streamer.pushSection();
271 Streamer.switchSection(Section: &ReadOnlySection);
272
273 // CP microcode requires the kernel descriptor to be allocated on 64 byte
274 // alignment.
275 Streamer.emitValueToAlignment(Alignment: Align(64), Fill: 0, FillLen: 1, MaxBytesToEmit: 0);
276 ReadOnlySection.ensureMinAlignment(MinAlignment: Align(64));
277
278 SmallString<128> KernelName;
279 getNameWithPrefix(Name&: KernelName, GV: &MF->getFunction());
280 getTargetStreamer()->EmitAmdhsaKernelDescriptor(
281 STI: STM, KernelName, KernelDescriptor: KD, NextVGPR: CurrentProgramInfo.NumVGPRsForWavesPerEU,
282 NextSGPR: MCBinaryExpr::createSub(
283 LHS: CurrentProgramInfo.NumSGPRsForWavesPerEU,
284 RHS: AMDGPUMCExpr::createExtraSGPRs(
285 VCCUsed: CurrentProgramInfo.VCCUsed, FlatScrUsed: CurrentProgramInfo.FlatUsed,
286 XNACKUsed: getTargetStreamer()->getTargetID()->isXnackOnOrAny(), Ctx&: Context),
287 Ctx&: Context),
288 ReserveVCC: CurrentProgramInfo.VCCUsed, ReserveFlatScr: CurrentProgramInfo.FlatUsed);
289
290 Streamer.popSection();
291}
292
293void AMDGPUAsmPrinter::emitImplicitDef(const MachineInstr *MI) const {
294 Register RegNo = MI->getOperand(i: 0).getReg();
295
296 SmallString<128> Str;
297 raw_svector_ostream OS(Str);
298 OS << "implicit-def: "
299 << printReg(Reg: RegNo, TRI: MF->getSubtarget().getRegisterInfo());
300
301 if (MI->getAsmPrinterFlags() & AMDGPU::SGPR_SPILL)
302 OS << " : SGPR spill to VGPR lane";
303
304 OutStreamer->AddComment(T: OS.str());
305 OutStreamer->addBlankLine();
306}
307
308void AMDGPUAsmPrinter::emitFunctionEntryLabel() {
309 if (TM.getTargetTriple().getOS() == Triple::AMDHSA) {
310 AsmPrinter::emitFunctionEntryLabel();
311 return;
312 }
313
314 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
315 const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
316 if (MFI->isEntryFunction() && STM.isAmdHsaOrMesa(F: MF->getFunction())) {
317 SmallString<128> SymbolName;
318 getNameWithPrefix(Name&: SymbolName, GV: &MF->getFunction()),
319 getTargetStreamer()->EmitAMDGPUSymbolType(SymbolName,
320 Type: ELF::STT_AMDGPU_HSA_KERNEL);
321 }
322 if (DumpCodeInstEmitter) {
323 // Disassemble function name label to text.
324 DisasmLines.push_back(x: MF->getName().str() + ":");
325 DisasmLineMaxLen = std::max(a: DisasmLineMaxLen, b: DisasmLines.back().size());
326 HexLines.emplace_back(args: "");
327 }
328
329 AsmPrinter::emitFunctionEntryLabel();
330}
331
332void AMDGPUAsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) {
333 if (DumpCodeInstEmitter && !isBlockOnlyReachableByFallthrough(MBB: &MBB)) {
334 // Write a line for the basic block label if it is not only fallthrough.
335 DisasmLines.push_back(x: (Twine("BB") + Twine(getFunctionNumber()) + "_" +
336 Twine(MBB.getNumber()) + ":")
337 .str());
338 DisasmLineMaxLen = std::max(a: DisasmLineMaxLen, b: DisasmLines.back().size());
339 HexLines.emplace_back(args: "");
340 }
341 AsmPrinter::emitBasicBlockStart(MBB);
342}
343
344void AMDGPUAsmPrinter::emitGlobalVariable(const GlobalVariable *GV) {
345 if (GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
346 if (GV->hasInitializer() && !isa<UndefValue>(Val: GV->getInitializer())) {
347 OutContext.reportError(L: {},
348 Msg: Twine(GV->getName()) +
349 ": unsupported initializer for address space");
350 return;
351 }
352
353 const Triple::OSType OS = TM.getTargetTriple().getOS();
354 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL) {
355 if (!AMDGPUTargetMachine::EnableObjectLinking)
356 return;
357 // With object linking, LDS definitions should have been externalized
358 // by earlier passes (e.g. LDS lowering, named barrier lowering).
359 // Only declarations reach here, emitted as SHN_AMDGPU_LDS symbols
360 // so the linker can assign their offsets.
361 assert(GV->isDeclaration() &&
362 "LDS definitions should have been externalized when object "
363 "linking is enabled");
364 }
365
366 MCSymbol *GVSym = getSymbol(GV);
367
368 GVSym->redefineIfPossible();
369 if (GVSym->isDefined() || GVSym->isVariable())
370 report_fatal_error(reason: "symbol '" + Twine(GVSym->getName()) +
371 "' is already defined");
372
373 const DataLayout &DL = GV->getDataLayout();
374 uint64_t Size = GV->getGlobalSize(DL);
375 Align Alignment = GV->getAlign().value_or(u: Align(4));
376
377 emitVisibility(Sym: GVSym, Visibility: GV->getVisibility(), IsDefinition: !GV->isDeclaration());
378 emitLinkage(GV, GVSym);
379 auto *TS = getTargetStreamer();
380 TS->emitAMDGPULDS(Symbol: GVSym, Size, Alignment);
381 return;
382 }
383
384 AsmPrinter::emitGlobalVariable(GV);
385}
386
387bool AMDGPUAsmPrinter::doInitialization(Module &M) {
388 const Triple &TT = M.getTargetTriple();
389 if (TT.getSubArch() == Triple::NoSubArch) {
390 Triple::SubArchType SubArch =
391 AMDGPU::getSubArch(AK: AMDGPU::parseArchAMDGCN(CPU: getGlobalSTI()->getCPU()));
392 if (SubArch != Triple::NoSubArch) {
393 Triple Fixed(TT);
394 Fixed.setArch(Kind: Triple::amdgpu, SubArch);
395 M.getContext().diagnose(DI: DiagnosticInfoGeneric(
396 "codegen with no subarch in the target triple is deprecated and will "
397 "become an error; use the target triple '" +
398 Fixed.str() + "' instead",
399 DS_Warning));
400 } else {
401 M.getContext().diagnose(DI: DiagnosticInfoGeneric(
402 "codegen with no subarch in the target triple is deprecated and will "
403 "become an error",
404 DS_Warning));
405 }
406 }
407
408 CodeObjectVersion = AMDGPU::getAMDHSACodeObjectVersion(M);
409
410 if (TT.getOS() == Triple::AMDHSA) {
411 switch (CodeObjectVersion) {
412 case AMDGPU::AMDHSA_COV4:
413 HSAMetadataStream = std::make_unique<HSAMD::MetadataStreamerMsgPackV4>();
414 break;
415 case AMDGPU::AMDHSA_COV5:
416 HSAMetadataStream = std::make_unique<HSAMD::MetadataStreamerMsgPackV5>();
417 break;
418 case AMDGPU::AMDHSA_COV6:
419 HSAMetadataStream = std::make_unique<HSAMD::MetadataStreamerMsgPackV6>();
420 break;
421 default:
422 reportFatalUsageError(reason: "unsupported code object version");
423 }
424
425 addAsmPrinterHandler(Handler: std::make_unique<AMDGPUAsmPrinterHandler>(args: this));
426 }
427
428 return AsmPrinter::doInitialization(M);
429}
430
431/// Mimics GCNSubtarget::computeOccupancy for MCExpr.
432///
433/// Remove dependency on GCNSubtarget and depend only only the necessary values
434/// for said occupancy computation. Should match computeOccupancy implementation
435/// without passing \p STM on.
436const AMDGPUMCExpr *createOccupancy(unsigned InitOcc, const MCExpr *NumSGPRs,
437 const MCExpr *NumVGPRs,
438 unsigned DynamicVGPRBlockSize,
439 const GCNSubtarget &STM, MCContext &Ctx) {
440 unsigned MaxWaves = STM.getMaxWavesPerEU();
441 unsigned Granule = IsaInfo::getVGPRAllocGranule(STI: STM, DynamicVGPRBlockSize);
442 unsigned TargetTotalNumVGPRs = STM.getTotalNumVGPRs();
443
444 // Bake the per-function SGPR budget into the operands so the late-evaluated
445 // MCExpr stays arithmetic. The trap reservation in particular is implicit on
446 // amdhsa and lives on STM, not on the assembler's MCSubtargetInfo.
447 AMDGPU::GPUKind Kind = STM.getTargetID().getGPUKind();
448 unsigned SGPRTotal = AMDGPU::getTotalNumSGPRs(AK: Kind);
449 unsigned SGPRGranule = AMDGPU::getSGPRAllocGranule(AK: Kind);
450 unsigned SGPRTrapReserve = STM.hasTrapHandler() ? IsaInfo::TRAP_NUM_SGPRS : 0;
451
452 auto CreateExpr = [&Ctx](unsigned Value) {
453 return MCConstantExpr::create(Value, Ctx);
454 };
455
456 // Zero SGPR count when SGPRs don't limit occupancy, so the MCExpr skips the
457 // SGPR term without having to test the generation itself.
458 const MCExpr *SGPRArg =
459 IsaInfo::isSGPROccupancyLimited(STI: STM) ? NumSGPRs : CreateExpr(0);
460
461 return AMDGPUMCExpr::create(Kind: AMDGPUMCExpr::AGVK_Occupancy,
462 Args: {CreateExpr(MaxWaves), CreateExpr(Granule),
463 CreateExpr(TargetTotalNumVGPRs),
464 CreateExpr(InitOcc), CreateExpr(SGPRTotal),
465 CreateExpr(SGPRGranule),
466 CreateExpr(SGPRTrapReserve), SGPRArg, NumVGPRs},
467 Ctx);
468}
469
470void AMDGPUAsmPrinter::validateMCResourceInfo(Function &F) {
471 if (F.isDeclaration() || !AMDGPU::isModuleEntryFunctionCC(CC: F.getCallingConv()))
472 return;
473
474 using RIK = MCResourceInfo::ResourceInfoKind;
475 const GCNSubtarget &STM = TM.getSubtarget<GCNSubtarget>(F);
476 MCSymbol *FnSym = TM.getSymbol(GV: &F);
477
478 auto TryGetMCExprValue = [](const MCExpr *Value, uint64_t &Res) -> bool {
479 int64_t Val;
480 if (Value->evaluateAsAbsolute(Res&: Val)) {
481 Res = Val;
482 return true;
483 }
484 return false;
485 };
486
487 const uint64_t MaxScratchPerWorkitem =
488 STM.getMaxWaveScratchSize() / STM.getWavefrontSize();
489 MCSymbol *ScratchSizeSymbol =
490 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_PrivateSegSize, OutContext);
491 uint64_t ScratchSize;
492 if (ScratchSizeSymbol->isVariable() &&
493 TryGetMCExprValue(ScratchSizeSymbol->getVariableValue(), ScratchSize) &&
494 ScratchSize > MaxScratchPerWorkitem) {
495 DiagnosticInfoStackSize DiagStackSize(F, ScratchSize, MaxScratchPerWorkitem,
496 DS_Error);
497 F.getContext().diagnose(DI: DiagStackSize);
498 }
499
500 // Validate addressable scalar registers (i.e., prior to added implicit
501 // SGPRs).
502 MCSymbol *NumSGPRSymbol =
503 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_NumSGPR, OutContext);
504 if (STM.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
505 !STM.hasSGPRInitBug()) {
506 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
507 uint64_t NumSgpr;
508 if (NumSGPRSymbol->isVariable() &&
509 TryGetMCExprValue(NumSGPRSymbol->getVariableValue(), NumSgpr) &&
510 NumSgpr > MaxAddressableNumSGPRs) {
511 F.getContext().diagnose(DI: DiagnosticInfoResourceLimit(
512 F, "addressable scalar registers", NumSgpr, MaxAddressableNumSGPRs,
513 DS_Error, DK_ResourceLimit));
514 return;
515 }
516 }
517
518 MCSymbol *VCCUsedSymbol =
519 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_UsesVCC, OutContext);
520 MCSymbol *FlatUsedSymbol =
521 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_UsesFlatScratch, OutContext);
522 uint64_t VCCUsed, FlatUsed, NumSgpr;
523
524 if (NumSGPRSymbol->isVariable() && VCCUsedSymbol->isVariable() &&
525 FlatUsedSymbol->isVariable() &&
526 TryGetMCExprValue(NumSGPRSymbol->getVariableValue(), NumSgpr) &&
527 TryGetMCExprValue(VCCUsedSymbol->getVariableValue(), VCCUsed) &&
528 TryGetMCExprValue(FlatUsedSymbol->getVariableValue(), FlatUsed)) {
529
530 // Recomputes NumSgprs + implicit SGPRs but all symbols should now be
531 // resolvable.
532 NumSgpr += IsaInfo::getNumExtraSGPRs(
533 STI: STM, VCCUsed, FlatScrUsed: FlatUsed,
534 XNACKUsed: getTargetStreamer()->getTargetID()->isXnackOnOrAny());
535 if (STM.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS ||
536 STM.hasSGPRInitBug()) {
537 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
538 if (NumSgpr > MaxAddressableNumSGPRs) {
539 F.getContext().diagnose(DI: DiagnosticInfoResourceLimit(
540 F, "scalar registers", NumSgpr, MaxAddressableNumSGPRs, DS_Error,
541 DK_ResourceLimit));
542 return;
543 }
544 }
545
546 MCSymbol *NumVgprSymbol =
547 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_NumVGPR, OutContext);
548 MCSymbol *NumAgprSymbol =
549 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_NumAGPR, OutContext);
550 uint64_t NumVgpr, NumAgpr;
551
552 MachineModuleInfo &MMI = *GetMMI();
553 MachineFunction *MF = MMI.getMachineFunction(F);
554 if (MF && NumVgprSymbol->isVariable() && NumAgprSymbol->isVariable() &&
555 TryGetMCExprValue(NumVgprSymbol->getVariableValue(), NumVgpr) &&
556 TryGetMCExprValue(NumAgprSymbol->getVariableValue(), NumAgpr)) {
557 const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
558 unsigned MaxWaves = MFI.getMaxWavesPerEU();
559 uint64_t TotalNumVgpr =
560 getTotalNumVGPRs(has90AInsts: STM.hasGFX90AInsts(), ArgNumAGPR: NumAgpr, ArgNumVGPR: NumVgpr);
561 uint64_t NumVGPRsForWavesPerEU =
562 std::max(l: {TotalNumVgpr, (uint64_t)1,
563 (uint64_t)STM.getMinNumVGPRs(
564 WavesPerEU: MaxWaves, DynamicVGPRBlockSize: MFI.getDynamicVGPRBlockSize())});
565 uint64_t NumSGPRsForWavesPerEU = std::max(
566 l: {NumSgpr, (uint64_t)1, (uint64_t)STM.getMinNumSGPRs(WavesPerEU: MaxWaves)});
567 const MCExpr *OccupancyExpr = createOccupancy(
568 InitOcc: STM.getOccupancyWithWorkGroupSizes(MF: *MF).second,
569 NumSGPRs: MCConstantExpr::create(Value: NumSGPRsForWavesPerEU, Ctx&: OutContext),
570 NumVGPRs: MCConstantExpr::create(Value: NumVGPRsForWavesPerEU, Ctx&: OutContext),
571 DynamicVGPRBlockSize: MFI.getDynamicVGPRBlockSize(), STM, Ctx&: OutContext);
572 uint64_t Occupancy;
573
574 const auto [MinWEU, MaxWEU] = AMDGPU::getIntegerPairAttribute(
575 F, Name: "amdgpu-waves-per-eu", Default: {0, 0}, OnlyFirstRequired: true);
576
577 if (TryGetMCExprValue(OccupancyExpr, Occupancy) && Occupancy < MinWEU) {
578 DiagnosticInfoOptimizationFailure Diag(
579 F, F.getSubprogram(),
580 "failed to meet occupancy target given by 'amdgpu-waves-per-eu' in "
581 "'" +
582 F.getName() + "': desired occupancy was " + Twine(MinWEU) +
583 ", final occupancy is " + Twine(Occupancy));
584 F.getContext().diagnose(DI: Diag);
585 return;
586 }
587 }
588 }
589}
590
591static void appendTypeEncoding(std::string &Enc, Type *Ty, const DataLayout &DL,
592 bool IsReturnType) {
593 if (Ty->isVoidTy()) {
594 Enc += 'v';
595 return;
596 }
597 unsigned Bits = DL.getTypeSizeInBits(Ty);
598 // Zero-sized non-void types (e.g. `{}` or `[0 x i8]`) consume no ABI
599 // registers. For returns, emit the same no-result marker as void so the
600 // parameter encoding still has an explicit return-type prefix.
601 if (Bits == 0) {
602 if (IsReturnType)
603 Enc += 'v';
604 return;
605 }
606 if (Bits <= 32)
607 Enc += 'i';
608 else if (Bits <= 64)
609 Enc += 'l';
610 else
611 Enc.append(n: divideCeil(Numerator: Bits, Denominator: 32), c: 'i');
612}
613
614static std::string computeTypeId(const FunctionType *FTy,
615 const DataLayout &DL) {
616 std::string Enc;
617 appendTypeEncoding(Enc, Ty: FTy->getReturnType(), DL, /*IsReturnType=*/true);
618 for (Type *ParamTy : FTy->params())
619 appendTypeEncoding(Enc, Ty: ParamTy, DL, /*IsReturnType=*/false);
620 return Enc;
621}
622
623void AMDGPUAsmPrinter::collectCallEdge(const MachineInstr &MI) {
624 if (!AMDGPUTargetMachine::EnableObjectLinking)
625 return;
626 const SIInstrInfo *TII = MF->getSubtarget<GCNSubtarget>().getInstrInfo();
627 const MachineOperand *Callee =
628 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::callee);
629 if (!Callee || !Callee->isGlobal())
630 return;
631 DirectCallEdges.insert(
632 X: {getSymbol(GV: &MF->getFunction()), getSymbol(GV: Callee->getGlobal())});
633}
634
635void AMDGPUAsmPrinter::emitAMDGPUInfo(Module &M) {
636 if (!AMDGPUTargetMachine::EnableObjectLinking)
637 return;
638
639 const NamedMDNode *LDSMD = M.getNamedMetadata(Name: "amdgpu.lds.uses");
640 bool HasLDSUses = LDSMD && LDSMD->getNumOperands() > 0;
641
642 const NamedMDNode *BarMD = M.getNamedMetadata(Name: "amdgpu.named_barrier.uses");
643 bool HasNamedBarriers = BarMD && BarMD->getNumOperands() > 0;
644
645 // Collect address-taken functions (with type IDs) and indirect call sites.
646 DenseMap<const Function *, std::string> AddrTakenTypeIds;
647 using IndirectCallInfo = std::pair<const Function *, std::string>;
648 SmallVector<IndirectCallInfo, 8> IndirectCalls;
649
650 for (const Function &F : M) {
651 bool IsKernel = AMDGPU::isKernel(CC: F.getCallingConv());
652
653 if (!IsKernel && F.hasAddressTaken(/*PutOffender=*/nullptr,
654 /*IgnoreCallbackUses=*/false,
655 /*IgnoreAssumeLikeCalls=*/true,
656 /*IgnoreLLVMUsed=*/IngoreLLVMUsed: true)) {
657 AddrTakenTypeIds[&F] =
658 computeTypeId(FTy: F.getFunctionType(), DL: M.getDataLayout());
659 }
660
661 if (F.isDeclaration())
662 continue;
663
664 StringSet<> SeenTypeIds;
665 for (const BasicBlock &BB : F) {
666 for (const Instruction &I : BB) {
667 const auto *CB = dyn_cast<CallBase>(Val: &I);
668 if (!CB || !CB->isIndirectCall())
669 continue;
670 std::string TId =
671 computeTypeId(FTy: CB->getFunctionType(), DL: M.getDataLayout());
672 if (SeenTypeIds.insert(key: TId).second)
673 IndirectCalls.push_back(Elt: {&F, std::move(TId)});
674 }
675 }
676 }
677
678 if (FunctionInfos.empty() && DirectCallEdges.empty() && !HasLDSUses &&
679 !HasNamedBarriers && AddrTakenTypeIds.empty() && IndirectCalls.empty())
680 return;
681
682 AMDGPU::InfoSectionData Data;
683 Data.Funcs = std::move(FunctionInfos);
684
685 for (auto &[F, TypeId] : AddrTakenTypeIds) {
686 MCSymbol *Sym = getSymbol(GV: F);
687 Data.TypeIds.push_back(Elt: {Sym, TypeId});
688 }
689
690 for (auto &[CallerSym, CalleeSym] : DirectCallEdges)
691 Data.Calls.push_back(Elt: {CallerSym, CalleeSym});
692 DirectCallEdges.clear();
693
694 if (HasLDSUses) {
695 for (const MDNode *N : LDSMD->operands()) {
696 auto *Func = mdconst::extract<Function>(MD: N->getOperand(I: 0));
697 auto *LdsVar = mdconst::extract<GlobalVariable>(MD: N->getOperand(I: 1));
698 Data.Uses.push_back(Elt: {getSymbol(GV: Func), getSymbol(GV: LdsVar)});
699 }
700 }
701
702 if (HasNamedBarriers) {
703 for (const MDNode *N : BarMD->operands()) {
704 auto *BarVar = mdconst::extract<GlobalVariable>(MD: N->getOperand(I: 0));
705 MCSymbol *BarSym = getSymbol(GV: BarVar);
706 for (unsigned I = 1, E = N->getNumOperands(); I < E; ++I) {
707 auto *Func = mdconst::extract<Function>(MD: N->getOperand(I));
708 Data.Uses.push_back(Elt: {getSymbol(GV: Func), BarSym});
709 }
710 }
711 }
712
713 for (auto &[Caller, Enc] : IndirectCalls) {
714 MCSymbol *CallerSym = getSymbol(GV: Caller);
715 Data.IndirectCalls.push_back(Elt: {CallerSym, Enc});
716 }
717
718 getTargetStreamer()->emitAMDGPUInfo(Data);
719}
720
721bool AMDGPUAsmPrinter::doFinalization(Module &M) {
722 const Triple &TT = M.getTargetTriple();
723
724 // Pad with s_code_end to help tools and guard against instruction prefetch
725 // causing stale data in caches. Arguably this should be done by the linker,
726 // which is why this isn't done for Mesa.
727 // Don't do it if there is no code.
728 const MCSubtargetInfo &STI = *getGlobalSTI();
729 if ((AMDGPU::isGFX10Plus(STI) || AMDGPU::isGFX90A(STI)) &&
730 (TT.getOS() == Triple::AMDHSA || TT.getOS() == Triple::AMDPAL)) {
731 MCSection *TextSect = getObjFileLowering().getTextSection();
732 if (TextSect->hasInstructions()) {
733 OutStreamer->switchSection(Section: TextSect);
734 getTargetStreamer()->EmitCodeEnd(STI);
735 }
736 }
737
738 // Emit the unified .amdgpu.info section (per-function resources, call graph,
739 // LDS/named-barrier use edges, indirect calls, and address-taken type IDs).
740 emitAMDGPUInfo(M);
741
742 // Assign expressions which can only be resolved when all other functions are
743 // known.
744 RI.finalize(OutContext);
745
746 // Switch section and emit all GPR maximums within the processed module.
747 OutStreamer->pushSection();
748 MCSectionELF *MaxGPRSection =
749 OutContext.getELFSection(Section: ".AMDGPU.gpr_maximums", Type: ELF::SHT_PROGBITS, Flags: 0);
750 OutStreamer->switchSection(Section: MaxGPRSection);
751 getTargetStreamer()->EmitMCResourceMaximums(
752 MaxVGPR: RI.getMaxVGPRSymbol(OutContext), MaxAGPR: RI.getMaxAGPRSymbol(OutContext),
753 MaxSGPR: RI.getMaxSGPRSymbol(OutContext), MaxNamedBarrier: RI.getMaxNamedBarrierSymbol(OutContext));
754 OutStreamer->popSection();
755
756 // In the object-linking pipeline per-function resource MCExprs reference
757 // external callee symbols that cannot be evaluated here, so cross-TU limit
758 // checks would silently no-op for every non-leaf function. Defer resource
759 // sanity checking to the linker, which re-validates against the aggregated
760 // call graph in the combined .amdgpu.info metadata.
761 if (!AMDGPUTargetMachine::EnableObjectLinking) {
762 for (Function &F : M.functions())
763 validateMCResourceInfo(F);
764 }
765
766 RI.reset();
767
768 return AsmPrinter::doFinalization(M);
769}
770
771SmallString<128> AMDGPUAsmPrinter::getMCExprStr(const MCExpr *Value) {
772 SmallString<128> Str;
773 raw_svector_ostream OSS(Str);
774 auto &Streamer = getTargetStreamer()->getStreamer();
775 auto &Context = Streamer.getContext();
776 const MCExpr *New = foldAMDGPUMCExpr(Expr: Value, Ctx&: Context);
777 printAMDGPUMCExpr(Expr: New, OS&: OSS, MAI: &MAI);
778 return Str;
779}
780
781// Print comments that apply to both callable functions and entry points.
782void AMDGPUAsmPrinter::emitCommonFunctionComments(
783 const MCExpr *NumVGPR, const MCExpr *NumAGPR, const MCExpr *TotalNumVGPR,
784 const MCExpr *NumSGPR, const MCExpr *ScratchSize, uint64_t CodeSize,
785 const AMDGPUMachineFunctionInfo *MFI) {
786 OutStreamer->emitRawComment(T: " codeLenInByte = " + Twine(CodeSize), TabPrefix: false);
787 OutStreamer->emitRawComment(T: " TotalNumSgprs: " + getMCExprStr(Value: NumSGPR),
788 TabPrefix: false);
789 OutStreamer->emitRawComment(T: " NumVgprs: " + getMCExprStr(Value: NumVGPR), TabPrefix: false);
790 if (NumAGPR && TotalNumVGPR) {
791 OutStreamer->emitRawComment(T: " NumAgprs: " + getMCExprStr(Value: NumAGPR), TabPrefix: false);
792 OutStreamer->emitRawComment(T: " TotalNumVgprs: " + getMCExprStr(Value: TotalNumVGPR),
793 TabPrefix: false);
794 }
795 OutStreamer->emitRawComment(T: " ScratchSize: " + getMCExprStr(Value: ScratchSize),
796 TabPrefix: false);
797 OutStreamer->emitRawComment(T: " MemoryBound: " + Twine(MFI->isMemoryBound()),
798 TabPrefix: false);
799}
800
801const MCExpr *AMDGPUAsmPrinter::getAmdhsaKernelCodeProperties(
802 const MachineFunction &MF) const {
803 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
804 MCContext &Ctx = MF.getContext();
805 uint16_t KernelCodeProperties = 0;
806 const GCNUserSGPRUsageInfo &UserSGPRInfo = MFI.getUserSGPRInfo();
807
808 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
809 KernelCodeProperties |=
810 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER;
811 }
812 if (UserSGPRInfo.hasDispatchPtr()) {
813 KernelCodeProperties |=
814 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR;
815 }
816 if (UserSGPRInfo.hasQueuePtr()) {
817 KernelCodeProperties |= amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR;
818 }
819 if (UserSGPRInfo.hasKernargSegmentPtr()) {
820 KernelCodeProperties |=
821 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR;
822 }
823 if (UserSGPRInfo.hasDispatchID()) {
824 KernelCodeProperties |=
825 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID;
826 }
827 if (UserSGPRInfo.hasFlatScratchInit()) {
828 KernelCodeProperties |=
829 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT;
830 }
831 if (UserSGPRInfo.hasPrivateSegmentSize()) {
832 KernelCodeProperties |=
833 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE;
834 }
835 if (MF.getSubtarget<GCNSubtarget>().isWave32()) {
836 KernelCodeProperties |=
837 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;
838 }
839
840 // CurrentProgramInfo.DynamicCallStack is a MCExpr and could be
841 // un-evaluatable at this point so it cannot be conditionally checked here.
842 // Instead, we'll directly shift the possibly unknown MCExpr into its place
843 // and bitwise-or it into KernelCodeProperties.
844 const MCExpr *KernelCodePropExpr =
845 MCConstantExpr::create(Value: KernelCodeProperties, Ctx);
846 const MCExpr *OrValue = MCConstantExpr::create(
847 Value: amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK_SHIFT, Ctx);
848 OrValue = MCBinaryExpr::createShl(LHS: CurrentProgramInfo.DynamicCallStack,
849 RHS: OrValue, Ctx);
850 KernelCodePropExpr = MCBinaryExpr::createOr(LHS: KernelCodePropExpr, RHS: OrValue, Ctx);
851
852 return KernelCodePropExpr;
853}
854
855MCKernelDescriptor
856AMDGPUAsmPrinter::getAmdhsaKernelDescriptor(const MachineFunction &MF,
857 const SIProgramInfo &PI) const {
858 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
859 const Function &F = MF.getFunction();
860 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
861 MCContext &Ctx = MF.getContext();
862
863 MCKernelDescriptor KernelDescriptor;
864
865 KernelDescriptor.group_segment_fixed_size =
866 MCConstantExpr::create(Value: PI.LDSSize, Ctx);
867 KernelDescriptor.private_segment_fixed_size = PI.ScratchSize;
868
869 Align MaxKernArgAlign;
870 KernelDescriptor.kernarg_size = MCConstantExpr::create(
871 Value: STM.getKernArgSegmentSize(F, MaxAlign&: MaxKernArgAlign), Ctx);
872
873 KernelDescriptor.compute_pgm_rsrc1 = PI.getComputePGMRSrc1(ST: STM, Ctx);
874 KernelDescriptor.compute_pgm_rsrc2 = PI.getComputePGMRSrc2(ST: STM, Ctx);
875 KernelDescriptor.kernel_code_properties = getAmdhsaKernelCodeProperties(MF);
876
877 int64_t PGM_Rsrc3 = 1;
878 bool EvaluatableRsrc3 =
879 CurrentProgramInfo.ComputePGMRSrc3->evaluateAsAbsolute(Res&: PGM_Rsrc3);
880 (void)PGM_Rsrc3;
881 (void)EvaluatableRsrc3;
882 assert(STM.getGeneration() >= AMDGPUSubtarget::GFX10 ||
883 STM.hasGFX90AInsts() || STM.hasGFX1250Insts() || !EvaluatableRsrc3 ||
884 static_cast<uint64_t>(PGM_Rsrc3) == 0);
885 KernelDescriptor.compute_pgm_rsrc3 = CurrentProgramInfo.ComputePGMRSrc3;
886
887 KernelDescriptor.kernarg_preload = MCConstantExpr::create(
888 Value: AMDGPU::hasKernargPreload(STI: STM) ? Info->getNumKernargPreloadedSGPRs() : 0,
889 Ctx);
890
891 return KernelDescriptor;
892}
893
894bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
895 // Init target streamer lazily on the first function so that previous passes
896 // can set metadata.
897 if (!IsTargetStreamerInitialized)
898 initTargetStreamer(M&: *MF.getFunction().getParent());
899
900 ResourceUsage = GetResourceUsage(MF);
901 CurrentProgramInfo.reset(MF);
902
903 const AMDGPUMachineFunctionInfo *MFI =
904 MF.getInfo<AMDGPUMachineFunctionInfo>();
905 MCContext &Ctx = MF.getContext();
906
907 // The starting address of all shader programs must be 256 bytes aligned.
908 // Regular functions just need the basic required instruction alignment.
909 MF.ensureAlignment(A: MFI->isEntryFunction() ? Align(256) : Align(4));
910
911 SetupMachineFunction(MF);
912
913 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
914 MCContext &Context = getObjFileLowering().getContext();
915 // FIXME: This should be an explicit check for Mesa.
916 if (!STM.isAmdHsaOS() && !STM.isAmdPalOS()) {
917 MCSectionELF *ConfigSection =
918 Context.getELFSection(Section: ".AMDGPU.config", Type: ELF::SHT_PROGBITS, Flags: 0);
919 OutStreamer->switchSection(Section: ConfigSection);
920 }
921
922 RI.gatherResourceInfo(MF, FRI: *ResourceUsage, OutContext);
923
924 if (AMDGPUTargetMachine::EnableObjectLinking) {
925 const AMDGPUResourceUsageAnalysisWrapperPass::FunctionResourceInfo &RU =
926 *ResourceUsage;
927 FunctionInfos.push_back(
928 Elt: {/*NumSGPR=*/static_cast<uint32_t>(RU.NumExplicitSGPR),
929 /*NumArchVGPR=*/static_cast<uint32_t>(RU.NumVGPR),
930 /*NumAccVGPR=*/static_cast<uint32_t>(RU.NumAGPR),
931 /*PrivateSegmentSize=*/static_cast<uint32_t>(RU.PrivateSegmentSize),
932 /*UsesVCC=*/RU.UsesVCC,
933 /*UsesFlatScratch=*/RU.UsesFlatScratch,
934 /*HasDynStack=*/RU.HasDynamicallySizedStack,
935 /*Sym=*/getSymbol(GV: &MF.getFunction())});
936 }
937
938 if (MFI->isModuleEntryFunction()) {
939 getSIProgramInfo(Out&: CurrentProgramInfo, MF);
940 }
941
942 if (STM.isAmdPalOS()) {
943 if (MFI->isEntryFunction())
944 EmitPALMetadata(MF, KernelInfo: CurrentProgramInfo);
945 else if (MFI->isModuleEntryFunction())
946 emitPALFunctionMetadata(MF);
947 } else if (!STM.isAmdHsaOS()) {
948 EmitProgramInfoSI(MF, KernelInfo: CurrentProgramInfo);
949 }
950
951 DumpCodeInstEmitter = nullptr;
952 if (STM.dumpCode()) {
953 // For -dumpcode, get the assembler out of the streamer. This only works
954 // with -filetype=obj.
955 MCAssembler *Assembler = OutStreamer->getAssemblerPtr();
956 if (Assembler)
957 DumpCodeInstEmitter = Assembler->getEmitterPtr();
958 }
959
960 DisasmLines.clear();
961 HexLines.clear();
962 DisasmLineMaxLen = 0;
963
964 emitFunctionBody();
965
966 emitResourceUsageRemarks(MF, CurrentProgramInfo, isModuleEntryFunction: MFI->isModuleEntryFunction(),
967 hasMAIInsts: STM.hasMAIInsts());
968
969 {
970 using RIK = MCResourceInfo::ResourceInfoKind;
971 getTargetStreamer()->EmitMCResourceInfo(
972 NumVGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumVGPR, OutContext),
973 NumAGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumAGPR, OutContext),
974 NumExplicitSGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumSGPR, OutContext),
975 NumNamedBarrier: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumNamedBarrier,
976 OutContext),
977 PrivateSegmentSize: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_PrivateSegSize,
978 OutContext),
979 UsesVCC: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_UsesVCC, OutContext),
980 UsesFlatScratch: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_UsesFlatScratch,
981 OutContext),
982 HasDynamicallySizedStack: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_HasDynSizedStack,
983 OutContext),
984 HasRecursion: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_HasRecursion,
985 OutContext),
986 HasIndirectCall: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_HasIndirectCall,
987 OutContext));
988 }
989
990 // Emit _dvgpr$ symbol when appropriate.
991 emitDVgprSymbol(MF);
992
993 if (isVerbose()) {
994 MCSectionELF *CommentSection =
995 Context.getELFSection(Section: ".AMDGPU.csdata", Type: ELF::SHT_PROGBITS, Flags: 0);
996 OutStreamer->switchSection(Section: CommentSection);
997
998 if (!MFI->isEntryFunction()) {
999 using RIK = MCResourceInfo::ResourceInfoKind;
1000 OutStreamer->emitRawComment(T: " Function info:", TabPrefix: false);
1001
1002 emitCommonFunctionComments(
1003 NumVGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumVGPR, OutContext)
1004 ->getVariableValue(),
1005 NumAGPR: STM.hasMAIInsts() ? RI.getSymbol(FuncName: CurrentFnSym->getName(),
1006 RIK: RIK::RIK_NumAGPR, OutContext)
1007 ->getVariableValue()
1008 : nullptr,
1009 TotalNumVGPR: RI.createTotalNumVGPRs(MF, Ctx),
1010 NumSGPR: RI.createTotalNumSGPRs(
1011 MF,
1012 hasXnack: MF.getSubtarget<GCNSubtarget>().getTargetID().isXnackOnOrAny(),
1013 Ctx),
1014 ScratchSize: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_PrivateSegSize,
1015 OutContext)
1016 ->getVariableValue(),
1017 CodeSize: CurrentProgramInfo.getFunctionCodeSize(MF), MFI);
1018 return false;
1019 }
1020
1021 OutStreamer->emitRawComment(T: " Kernel info:", TabPrefix: false);
1022 emitCommonFunctionComments(
1023 NumVGPR: CurrentProgramInfo.NumArchVGPR,
1024 NumAGPR: STM.hasMAIInsts() ? CurrentProgramInfo.NumAccVGPR : nullptr,
1025 TotalNumVGPR: CurrentProgramInfo.NumVGPR, NumSGPR: CurrentProgramInfo.NumSGPR,
1026 ScratchSize: CurrentProgramInfo.ScratchSize,
1027 CodeSize: CurrentProgramInfo.getFunctionCodeSize(MF), MFI);
1028
1029 OutStreamer->emitRawComment(
1030 T: " FloatMode: " + Twine(CurrentProgramInfo.FloatMode), TabPrefix: false);
1031 OutStreamer->emitRawComment(
1032 T: " IeeeMode: " + Twine(CurrentProgramInfo.IEEEMode), TabPrefix: false);
1033 OutStreamer->emitRawComment(
1034 T: " LDSByteSize: " + Twine(CurrentProgramInfo.LDSSize) +
1035 " bytes/workgroup (compile time only)",
1036 TabPrefix: false);
1037
1038 OutStreamer->emitRawComment(
1039 T: " SGPRBlocks: " + getMCExprStr(Value: CurrentProgramInfo.SGPRBlocks), TabPrefix: false);
1040
1041 OutStreamer->emitRawComment(
1042 T: " VGPRBlocks: " + getMCExprStr(Value: CurrentProgramInfo.VGPRBlocks), TabPrefix: false);
1043
1044 OutStreamer->emitRawComment(
1045 T: " NumSGPRsForWavesPerEU: " +
1046 getMCExprStr(Value: CurrentProgramInfo.NumSGPRsForWavesPerEU),
1047 TabPrefix: false);
1048 OutStreamer->emitRawComment(
1049 T: " NumVGPRsForWavesPerEU: " +
1050 getMCExprStr(Value: CurrentProgramInfo.NumVGPRsForWavesPerEU),
1051 TabPrefix: false);
1052
1053 if (STM.hasGFX90AInsts()) {
1054 const MCExpr *AdjustedAccum = MCBinaryExpr::createAdd(
1055 LHS: CurrentProgramInfo.AccumOffset, RHS: MCConstantExpr::create(Value: 1, Ctx), Ctx);
1056 AdjustedAccum = MCBinaryExpr::createMul(
1057 LHS: AdjustedAccum, RHS: MCConstantExpr::create(Value: 4, Ctx), Ctx);
1058 OutStreamer->emitRawComment(
1059 T: " AccumOffset: " + getMCExprStr(Value: AdjustedAccum), TabPrefix: false);
1060 }
1061
1062 if (STM.hasGFX1250Insts())
1063 OutStreamer->emitRawComment(
1064 T: " NamedBarCnt: " + getMCExprStr(Value: CurrentProgramInfo.NamedBarCnt),
1065 TabPrefix: false);
1066
1067 OutStreamer->emitRawComment(
1068 T: " Occupancy: " + getMCExprStr(Value: CurrentProgramInfo.Occupancy), TabPrefix: false);
1069
1070 OutStreamer->emitRawComment(
1071 T: " WaveLimiterHint : " + Twine(MFI->needsWaveLimiter()), TabPrefix: false);
1072
1073 OutStreamer->emitRawComment(
1074 T: " COMPUTE_PGM_RSRC2:SCRATCH_EN: " +
1075 getMCExprStr(Value: CurrentProgramInfo.ScratchEnable),
1076 TabPrefix: false);
1077 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:USER_SGPR: " +
1078 Twine(CurrentProgramInfo.UserSGPR),
1079 TabPrefix: false);
1080 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TRAP_HANDLER: " +
1081 Twine(CurrentProgramInfo.TrapHandlerEnable),
1082 TabPrefix: false);
1083 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TGID_X_EN: " +
1084 Twine(CurrentProgramInfo.TGIdXEnable),
1085 TabPrefix: false);
1086 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TGID_Y_EN: " +
1087 Twine(CurrentProgramInfo.TGIdYEnable),
1088 TabPrefix: false);
1089 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TGID_Z_EN: " +
1090 Twine(CurrentProgramInfo.TGIdZEnable),
1091 TabPrefix: false);
1092 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TIDIG_COMP_CNT: " +
1093 Twine(CurrentProgramInfo.TIdIGCompCount),
1094 TabPrefix: false);
1095
1096 [[maybe_unused]] int64_t PGMRSrc3;
1097 assert(STM.getGeneration() >= AMDGPUSubtarget::GFX10 ||
1098 STM.hasGFX90AInsts() || STM.hasGFX1250Insts() ||
1099 (CurrentProgramInfo.ComputePGMRSrc3->evaluateAsAbsolute(PGMRSrc3) &&
1100 static_cast<uint64_t>(PGMRSrc3) == 0));
1101 if (STM.hasGFX90AInsts()) {
1102 OutStreamer->emitRawComment(
1103 T: " COMPUTE_PGM_RSRC3_GFX90A:ACCUM_OFFSET: " +
1104 getMCExprStr(Value: MCKernelDescriptor::bits_get(
1105 Src: CurrentProgramInfo.ComputePGMRSrc3,
1106 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
1107 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, Ctx)),
1108 TabPrefix: false);
1109 OutStreamer->emitRawComment(
1110 T: " COMPUTE_PGM_RSRC3_GFX90A:TG_SPLIT: " +
1111 getMCExprStr(Value: MCKernelDescriptor::bits_get(
1112 Src: CurrentProgramInfo.ComputePGMRSrc3,
1113 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT,
1114 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, Ctx)),
1115 TabPrefix: false);
1116 }
1117 }
1118
1119 if (DumpCodeInstEmitter) {
1120
1121 OutStreamer->switchSection(
1122 Section: Context.getELFSection(Section: ".AMDGPU.disasm", Type: ELF::SHT_PROGBITS, Flags: 0));
1123
1124 for (size_t i = 0; i < DisasmLines.size(); ++i) {
1125 std::string Comment = "\n";
1126 if (!HexLines[i].empty()) {
1127 Comment = std::string(DisasmLineMaxLen - DisasmLines[i].size(), ' ');
1128 Comment += " ; " + HexLines[i] + "\n";
1129 }
1130
1131 OutStreamer->emitBytes(Data: StringRef(DisasmLines[i]));
1132 OutStreamer->emitBytes(Data: StringRef(Comment));
1133 }
1134 }
1135
1136 return false;
1137}
1138
1139// When appropriate, add a _dvgpr$ symbol, with the value of the function
1140// symbol, plus an offset encoding one less than the number of VGPR blocks used
1141// by the function in bits 5..3 of the symbol value. A "VGPR block" can be
1142// either 16 VGPRs (for a max of 128), or 32 VGPRs (for a max of 256). This is
1143// used by a front-end to have functions that are chained rather than called,
1144// and a dispatcher that dynamically resizes the VGPR count before dispatching
1145// to a function.
1146void AMDGPUAsmPrinter::emitDVgprSymbol(MachineFunction &MF) {
1147 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
1148 if (MFI.isDynamicVGPREnabled() &&
1149 MF.getFunction().getCallingConv() == CallingConv::AMDGPU_CS_Chain) {
1150 MCContext &Ctx = MF.getContext();
1151 unsigned BlockSize = MFI.getDynamicVGPRBlockSize();
1152
1153 const MCExpr *EncodedBlocks;
1154 MCValue NumVGPRs;
1155 if (CurrentProgramInfo.NumVGPRsForWavesPerEU->evaluateAsRelocatable(
1156 Res&: NumVGPRs, Asm: nullptr) &&
1157 NumVGPRs.isAbsolute()) {
1158
1159 // Calculate number of VGPR blocks.
1160 // Treat 0 VGPRs as 1 VGPR to avoid underflowing.
1161 unsigned NumBlocks =
1162 divideCeil(Numerator: std::max(a: unsigned(NumVGPRs.getConstant()), b: 1U), Denominator: BlockSize);
1163
1164 if (NumBlocks > AMDGPU::IsaInfo::MaxDynamicVGPRBlocks) {
1165 OutContext.reportError(
1166 L: {}, Msg: "DVGPR block count " + Twine(NumBlocks) +
1167 " exceeds maximum of " +
1168 Twine(AMDGPU::IsaInfo::MaxDynamicVGPRBlocks) +
1169 " for __dvgpr$ symbol for '" +
1170 Twine(CurrentFnSym->getName()) + "'");
1171 return;
1172 }
1173 unsigned EncodedNumBlocks = (NumBlocks - 1) << 3;
1174 EncodedBlocks = MCConstantExpr::create(Value: EncodedNumBlocks, Ctx);
1175 } else {
1176 // Value not yet available so build a symbolic MCExpr:
1177 // ((alignTo(max(NumVGPRs, 1), BlockSize) / BlockSize - 1) << 3
1178 const MCExpr *One = MCConstantExpr::create(Value: 1, Ctx);
1179 const MCExpr *BlockSizeConst = MCConstantExpr::create(Value: BlockSize, Ctx);
1180 const MCExpr *MaxVGPRs = AMDGPUMCExpr::createMax(
1181 Args: {CurrentProgramInfo.NumVGPRsForWavesPerEU, One}, Ctx);
1182 const MCExpr *NumBlocks = MCBinaryExpr::createDiv(
1183 LHS: AMDGPUMCExpr::createAlignTo(Value: MaxVGPRs, Align: BlockSizeConst, Ctx),
1184 RHS: BlockSizeConst, Ctx);
1185 EncodedBlocks =
1186 MCBinaryExpr::createShl(LHS: MCBinaryExpr::createSub(LHS: NumBlocks, RHS: One, Ctx),
1187 RHS: MCConstantExpr::create(Value: 3, Ctx), Ctx);
1188 }
1189
1190 // Add to function symbol to create _dvgpr$ symbol.
1191 const MCExpr *DVgprFuncVal = MCBinaryExpr::createAdd(
1192 LHS: MCSymbolRefExpr::create(Symbol: CurrentFnSym, Ctx), RHS: EncodedBlocks, Ctx);
1193 MCSymbol *DVgprFuncSym =
1194 Ctx.getOrCreateSymbol(Name: Twine("_dvgpr$") + CurrentFnSym->getName());
1195 OutStreamer->emitAssignment(Symbol: DVgprFuncSym, Value: DVgprFuncVal);
1196 emitVisibility(Sym: DVgprFuncSym, Visibility: MF.getFunction().getVisibility());
1197 emitLinkage(GV: &MF.getFunction(), GVSym: DVgprFuncSym);
1198 }
1199}
1200
1201// TODO: Fold this into emitFunctionBodyStart.
1202void AMDGPUAsmPrinter::initializeTargetID(const Module &M) {
1203 getTargetStreamer()->initializeTargetID(STI: *getGlobalSTI());
1204
1205 auto &TSTargetID = getTargetStreamer()->getTargetID();
1206
1207 // Error if -mattr specified xnack or sramecc.
1208 // TODO: Remove this when subtarget features removed.
1209 StringRef FeatureString = getGlobalSTI()->getFeatureString();
1210 if (FeatureString.contains(Other: "xnack")) {
1211 M.getContext().diagnose(DI: DiagnosticInfoGeneric(
1212 "xnack/sramecc should be specified via module flags. "
1213 "Use module flag 'amdgpu.xnack' instead of subtarget feature",
1214 DS_Error));
1215 }
1216 if (FeatureString.contains(Other: "sramecc")) {
1217 M.getContext().diagnose(DI: DiagnosticInfoGeneric(
1218 "xnack/sramecc should be specified via module flags. "
1219 "Use module flag 'amdgpu.sramecc' instead of subtarget feature",
1220 DS_Error));
1221 }
1222
1223 // Apply xnack/sramecc settings from module flags.
1224 if (getGlobalSTI()->getFeatureBits().test(I: AMDGPU::FeatureXNACKOnOffModes)) {
1225 AMDGPU::TargetIDSetting Setting =
1226 GCNTargetMachine::getTargetIDSettingFromModuleFlag(M, FlagName: "amdgpu.xnack");
1227 if (Setting != AMDGPU::TargetIDSetting::Any)
1228 TSTargetID->setXnackSetting(Setting);
1229 }
1230
1231 if (getGlobalSTI()->getFeatureBits().test(I: AMDGPU::FeatureSRAMECCOnOffModes)) {
1232 AMDGPU::TargetIDSetting Setting =
1233 GCNTargetMachine::getTargetIDSettingFromModuleFlag(M, FlagName: "amdgpu.sramecc");
1234 if (Setting != AMDGPU::TargetIDSetting::Any)
1235 TSTargetID->setSramEccSetting(Setting);
1236 }
1237}
1238
1239// AccumOffset computed for the MCExpr equivalent of:
1240// alignTo(std::max(1, NumVGPR), 4) / 4 - 1;
1241static const MCExpr *computeAccumOffset(const MCExpr *NumVGPR, MCContext &Ctx) {
1242 const MCExpr *ConstFour = MCConstantExpr::create(Value: 4, Ctx);
1243 const MCExpr *ConstOne = MCConstantExpr::create(Value: 1, Ctx);
1244
1245 // Can't be lower than 1 for subsequent alignTo.
1246 const MCExpr *MaximumTaken =
1247 AMDGPUMCExpr::createMax(Args: {ConstOne, NumVGPR}, Ctx);
1248
1249 // Practically, it's computing divideCeil(MaximumTaken, 4).
1250 const MCExpr *DivCeil = MCBinaryExpr::createDiv(
1251 LHS: AMDGPUMCExpr::createAlignTo(Value: MaximumTaken, Align: ConstFour, Ctx), RHS: ConstFour,
1252 Ctx);
1253
1254 return MCBinaryExpr::createSub(LHS: DivCeil, RHS: ConstOne, Ctx);
1255}
1256
1257static unsigned getLDSEncodingGranule(const GCNSubtarget &ST) {
1258 unsigned Granule =
1259 AMDGPU::getLDSEncodingGranule(AK: ST.getTargetID().getGPUKind());
1260 // The legacy generic targets have no encoding granularity feature. Preserve
1261 // the default used for code generation when no GPU is specified.
1262 return Granule ? Granule : 256;
1263}
1264
1265void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
1266 const MachineFunction &MF) {
1267 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1268 MCContext &Ctx = MF.getContext();
1269
1270 auto CreateExpr = [&Ctx](int64_t Value) {
1271 return MCConstantExpr::create(Value, Ctx);
1272 };
1273
1274 auto TryGetMCExprValue = [](const MCExpr *Value, uint64_t &Res) -> bool {
1275 int64_t Val;
1276 if (Value->evaluateAsAbsolute(Res&: Val)) {
1277 Res = Val;
1278 return true;
1279 }
1280 return false;
1281 };
1282
1283 auto GetSymRefExpr =
1284 [&](MCResourceInfo::ResourceInfoKind RIK) -> const MCExpr * {
1285 MCSymbol *Sym = RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK, OutContext);
1286 return MCSymbolRefExpr::create(Symbol: Sym, Ctx);
1287 };
1288
1289 using RIK = MCResourceInfo::ResourceInfoKind;
1290 ProgInfo.NumArchVGPR = GetSymRefExpr(RIK::RIK_NumVGPR);
1291 ProgInfo.NumAccVGPR = GetSymRefExpr(RIK::RIK_NumAGPR);
1292 ProgInfo.NumVGPR = AMDGPUMCExpr::createTotalNumVGPR(
1293 NumAGPR: ProgInfo.NumAccVGPR, NumVGPR: ProgInfo.NumArchVGPR, Ctx);
1294
1295 ProgInfo.AccumOffset = computeAccumOffset(NumVGPR: ProgInfo.NumArchVGPR, Ctx);
1296 ProgInfo.TgSplit =
1297 STM.hasTgSplitSupport() && AMDGPU::isTgSplitEnabled(F: MF.getFunction());
1298 ProgInfo.NumSGPR = GetSymRefExpr(RIK::RIK_NumSGPR);
1299 ProgInfo.ScratchSize = GetSymRefExpr(RIK::RIK_PrivateSegSize);
1300 ProgInfo.VCCUsed = GetSymRefExpr(RIK::RIK_UsesVCC);
1301 ProgInfo.FlatUsed = GetSymRefExpr(RIK::RIK_UsesFlatScratch);
1302 ProgInfo.DynamicCallStack =
1303 MCBinaryExpr::createOr(LHS: GetSymRefExpr(RIK::RIK_HasDynSizedStack),
1304 RHS: GetSymRefExpr(RIK::RIK_HasRecursion), Ctx);
1305
1306 const MCExpr *BarBlkConst = MCConstantExpr::create(Value: 4, Ctx);
1307 const MCExpr *AlignToBlk = AMDGPUMCExpr::createAlignTo(
1308 Value: GetSymRefExpr(RIK::RIK_NumNamedBarrier), Align: BarBlkConst, Ctx);
1309 ProgInfo.NamedBarCnt = MCBinaryExpr::createDiv(LHS: AlignToBlk, RHS: BarBlkConst, Ctx);
1310
1311 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1312
1313 // The calculations related to SGPR/VGPR blocks are
1314 // duplicated in part in AMDGPUAsmParser::calculateGPRBlocks, and could be
1315 // unified.
1316 const MCExpr *ExtraSGPRs = AMDGPUMCExpr::createExtraSGPRs(
1317 VCCUsed: ProgInfo.VCCUsed, FlatScrUsed: ProgInfo.FlatUsed,
1318 XNACKUsed: getTargetStreamer()->getTargetID()->isXnackOnOrAny(), Ctx);
1319
1320 // Check the addressable register limit before we add ExtraSGPRs.
1321 if (STM.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
1322 !STM.hasSGPRInitBug()) {
1323 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
1324 uint64_t NumSgpr;
1325 if (TryGetMCExprValue(ProgInfo.NumSGPR, NumSgpr) &&
1326 NumSgpr > MaxAddressableNumSGPRs) {
1327 // This can happen due to a compiler bug or when using inline asm.
1328 LLVMContext &Ctx = MF.getFunction().getContext();
1329 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1330 MF.getFunction(), "addressable scalar registers", NumSgpr,
1331 MaxAddressableNumSGPRs, DS_Error, DK_ResourceLimit));
1332 ProgInfo.NumSGPR = CreateExpr(MaxAddressableNumSGPRs - 1);
1333 }
1334 }
1335
1336 // Account for extra SGPRs and VGPRs reserved for debugger use.
1337 ProgInfo.NumSGPR = MCBinaryExpr::createAdd(LHS: ProgInfo.NumSGPR, RHS: ExtraSGPRs, Ctx);
1338
1339 const Function &F = MF.getFunction();
1340
1341 // Ensure there are enough SGPRs and VGPRs for wave dispatch, where wave
1342 // dispatch registers as function args.
1343 unsigned WaveDispatchNumSGPR = MFI->getNumWaveDispatchSGPRs(),
1344 WaveDispatchNumVGPR = MFI->getNumWaveDispatchVGPRs();
1345
1346 if (WaveDispatchNumSGPR) {
1347 ProgInfo.NumSGPR = AMDGPUMCExpr::createMax(
1348 Args: {ProgInfo.NumSGPR,
1349 MCBinaryExpr::createAdd(LHS: CreateExpr(WaveDispatchNumSGPR), RHS: ExtraSGPRs,
1350 Ctx)},
1351 Ctx);
1352 }
1353
1354 if (WaveDispatchNumVGPR) {
1355 ProgInfo.NumArchVGPR = AMDGPUMCExpr::createMax(
1356 Args: {ProgInfo.NumArchVGPR, CreateExpr(WaveDispatchNumVGPR)}, Ctx);
1357
1358 ProgInfo.NumVGPR = AMDGPUMCExpr::createTotalNumVGPR(
1359 NumAGPR: ProgInfo.NumAccVGPR, NumVGPR: ProgInfo.NumArchVGPR, Ctx);
1360 }
1361
1362 // Adjust number of registers used to meet default/requested minimum/maximum
1363 // number of waves per execution unit request.
1364 unsigned MaxWaves = MFI->getMaxWavesPerEU();
1365 ProgInfo.NumSGPRsForWavesPerEU =
1366 AMDGPUMCExpr::createMax(Args: {ProgInfo.NumSGPR, CreateExpr(1ul),
1367 CreateExpr(STM.getMinNumSGPRs(WavesPerEU: MaxWaves))},
1368 Ctx);
1369 ProgInfo.NumVGPRsForWavesPerEU =
1370 AMDGPUMCExpr::createMax(Args: {ProgInfo.NumVGPR, CreateExpr(1ul),
1371 CreateExpr(STM.getMinNumVGPRs(
1372 WavesPerEU: MaxWaves, DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize()))},
1373 Ctx);
1374
1375 if (STM.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS ||
1376 STM.hasSGPRInitBug()) {
1377 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
1378 uint64_t NumSgpr;
1379 if (TryGetMCExprValue(ProgInfo.NumSGPR, NumSgpr) &&
1380 NumSgpr > MaxAddressableNumSGPRs) {
1381 // This can happen due to a compiler bug or when using inline asm to use
1382 // the registers which are usually reserved for vcc etc.
1383 LLVMContext &Ctx = MF.getFunction().getContext();
1384 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1385 MF.getFunction(), "scalar registers", NumSgpr, MaxAddressableNumSGPRs,
1386 DS_Error, DK_ResourceLimit));
1387 ProgInfo.NumSGPR = CreateExpr(MaxAddressableNumSGPRs);
1388 ProgInfo.NumSGPRsForWavesPerEU = CreateExpr(MaxAddressableNumSGPRs);
1389 }
1390 }
1391
1392 if (STM.hasSGPRInitBug()) {
1393 ProgInfo.NumSGPR =
1394 CreateExpr(AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG);
1395 ProgInfo.NumSGPRsForWavesPerEU =
1396 CreateExpr(AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG);
1397 }
1398
1399 if (MFI->getNumUserSGPRs() > STM.getMaxNumUserSGPRs()) {
1400 LLVMContext &Ctx = MF.getFunction().getContext();
1401 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1402 MF.getFunction(), "user SGPRs", MFI->getNumUserSGPRs(),
1403 STM.getMaxNumUserSGPRs(), DS_Error));
1404 }
1405
1406 if (MFI->getLDSSize() > STM.getAddressableLocalMemorySize()) {
1407 LLVMContext &Ctx = MF.getFunction().getContext();
1408 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1409 MF.getFunction(), "local memory", MFI->getLDSSize(),
1410 STM.getAddressableLocalMemorySize(), DS_Error));
1411 }
1412
1413 // When dynamic VGPRs are enabled, entry functions are launched with a single
1414 // VGPR block. The register allocator enforces this constraint, but we also
1415 // need to catch explicit physical registers in inline asm and wave dispatch
1416 // VGPR arguments.
1417 if (MFI->isDynamicVGPREnabled() &&
1418 AMDGPU::isEntryFunctionCC(CC: F.getCallingConv())) {
1419 unsigned BlockSize = MFI->getDynamicVGPRBlockSize();
1420 uint64_t NumVgpr;
1421 if (TryGetMCExprValue(ProgInfo.NumVGPRsForWavesPerEU, NumVgpr) &&
1422 NumVgpr > BlockSize) {
1423 LLVMContext &Ctx = F.getContext();
1424 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1425 F, "dynamic VGPR entry point vector registers", NumVgpr, BlockSize,
1426 DS_Warning, DK_ResourceLimit));
1427 }
1428 }
1429
1430 // The MCExpr equivalent of getNumSGPRBlocks/getNumVGPRBlocks:
1431 // (alignTo(max(1u, NumGPR), GPREncodingGranule) / GPREncodingGranule) - 1
1432 auto GetNumGPRBlocks = [&CreateExpr, &Ctx](const MCExpr *NumGPR,
1433 unsigned Granule) {
1434 const MCExpr *OneConst = CreateExpr(1ul);
1435 const MCExpr *GranuleConst = CreateExpr(Granule);
1436 const MCExpr *MaxNumGPR = AMDGPUMCExpr::createMax(Args: {NumGPR, OneConst}, Ctx);
1437 const MCExpr *AlignToGPR =
1438 AMDGPUMCExpr::createAlignTo(Value: MaxNumGPR, Align: GranuleConst, Ctx);
1439 const MCExpr *DivGPR =
1440 MCBinaryExpr::createDiv(LHS: AlignToGPR, RHS: GranuleConst, Ctx);
1441 const MCExpr *SubGPR = MCBinaryExpr::createSub(LHS: DivGPR, RHS: OneConst, Ctx);
1442 return SubGPR;
1443 };
1444 // GFX10+ will always allocate 128 SGPRs and this field must be 0
1445 if (STM.getGeneration() >= AMDGPUSubtarget::GFX10) {
1446 ProgInfo.SGPRBlocks = CreateExpr(0ul);
1447 } else {
1448 ProgInfo.SGPRBlocks = GetNumGPRBlocks(ProgInfo.NumSGPRsForWavesPerEU,
1449 IsaInfo::getSGPREncodingGranule(STI: STM));
1450 }
1451 ProgInfo.VGPRBlocks = GetNumGPRBlocks(ProgInfo.NumVGPRsForWavesPerEU,
1452 IsaInfo::getVGPREncodingGranule(STI: STM));
1453
1454 const SIModeRegisterDefaults Mode = MFI->getMode();
1455
1456 // Set the value to initialize FP_ROUND and FP_DENORM parts of the mode
1457 // register.
1458 ProgInfo.FloatMode = getFPMode(Mode);
1459
1460 ProgInfo.IEEEMode = Mode.IEEE;
1461
1462 // Make clamp modifier on NaN input returns 0.
1463 ProgInfo.DX10Clamp = Mode.DX10Clamp;
1464 ProgInfo.SGPRSpill = MFI->getNumSpilledSGPRs();
1465 ProgInfo.VGPRSpill = MFI->getNumSpilledVGPRs();
1466
1467 ProgInfo.LDSSize = MFI->getLDSSize();
1468
1469 unsigned LDSGranularityBytes = getLDSEncodingGranule(ST: STM);
1470 ProgInfo.LDSBlocks =
1471 alignTo(Value: ProgInfo.LDSSize, Align: LDSGranularityBytes) / LDSGranularityBytes;
1472
1473 // The MCExpr equivalent of divideCeil.
1474 auto DivideCeil = [&Ctx](const MCExpr *Numerator, const MCExpr *Denominator) {
1475 const MCExpr *Ceil =
1476 AMDGPUMCExpr::createAlignTo(Value: Numerator, Align: Denominator, Ctx);
1477 return MCBinaryExpr::createDiv(LHS: Ceil, RHS: Denominator, Ctx);
1478 };
1479
1480 // Scratch is allocated in 64-dword or 256-dword blocks.
1481 unsigned ScratchAlignShift =
1482 STM.getGeneration() >= AMDGPUSubtarget::GFX11 ? 8 : 10;
1483 // We need to program the hardware with the amount of scratch memory that
1484 // is used by the entire wave. ProgInfo.ScratchSize is the amount of
1485 // scratch memory used per thread.
1486 ProgInfo.ScratchBlocks = DivideCeil(
1487 MCBinaryExpr::createMul(LHS: ProgInfo.ScratchSize,
1488 RHS: CreateExpr(STM.getWavefrontSize()), Ctx),
1489 CreateExpr(1ULL << ScratchAlignShift));
1490
1491 if (STM.hasSupportsWGP()) {
1492 ProgInfo.WgpMode = STM.isCuModeEnabled() ? 0 : 1;
1493 }
1494
1495 if (getIsaVersion(GPU: getGlobalSTI()->getCPU()).Major >= 10) {
1496 ProgInfo.MemOrdered = 1;
1497 ProgInfo.FwdProgress = !F.hasFnAttribute(Kind: "amdgpu-no-fwd-progress");
1498 }
1499
1500 // 0 = X, 1 = XY, 2 = XYZ
1501 unsigned TIDIGCompCnt = 0;
1502 if (MFI->hasWorkItemIDZ())
1503 TIDIGCompCnt = 2;
1504 else if (MFI->hasWorkItemIDY())
1505 TIDIGCompCnt = 1;
1506
1507 // The private segment wave byte offset is the last of the system SGPRs. We
1508 // initially assumed it was allocated, and may have used it. It shouldn't harm
1509 // anything to disable it if we know the stack isn't used here. We may still
1510 // have emitted code reading it to initialize scratch, but if that's unused
1511 // reading garbage should be OK.
1512 ProgInfo.ScratchEnable = MCBinaryExpr::createLOr(
1513 LHS: MCBinaryExpr::createGT(LHS: ProgInfo.ScratchBlocks,
1514 RHS: MCConstantExpr::create(Value: 0, Ctx), Ctx),
1515 RHS: ProgInfo.DynamicCallStack, Ctx);
1516
1517 ProgInfo.UserSGPR = MFI->getNumUserSGPRs();
1518 // For AMDHSA, TRAP_HANDLER must be zero, as it is populated by the CP.
1519 ProgInfo.TrapHandlerEnable = STM.isAmdHsaOS() ? 0 : STM.hasTrapHandler();
1520 ProgInfo.TGIdXEnable = MFI->hasWorkGroupIDX();
1521 ProgInfo.TGIdYEnable = MFI->hasWorkGroupIDY();
1522 ProgInfo.TGIdZEnable = MFI->hasWorkGroupIDZ();
1523 ProgInfo.TGSizeEnable = MFI->hasWorkGroupInfo();
1524 ProgInfo.TIdIGCompCount = TIDIGCompCnt;
1525 ProgInfo.EXCPEnMSB = 0;
1526 // For AMDHSA, LDS_SIZE must be zero, as it is populated by the CP.
1527 ProgInfo.LdsSize = STM.isAmdHsaOS() ? 0 : ProgInfo.LDSBlocks;
1528 ProgInfo.EXCPEnable = 0;
1529
1530 if (STM.hasGFX90AInsts()) {
1531 ProgInfo.ComputePGMRSrc3 =
1532 setBits(Dst: ProgInfo.ComputePGMRSrc3, Value: ProgInfo.AccumOffset,
1533 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
1534 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT, Ctx);
1535 ProgInfo.ComputePGMRSrc3 =
1536 setBits(Dst: ProgInfo.ComputePGMRSrc3, Value: CreateExpr(ProgInfo.TgSplit),
1537 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT,
1538 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT, Ctx);
1539 }
1540
1541 if (STM.hasGFX1250Insts())
1542 ProgInfo.ComputePGMRSrc3 =
1543 setBits(Dst: ProgInfo.ComputePGMRSrc3, Value: ProgInfo.NamedBarCnt,
1544 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
1545 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT, Ctx);
1546
1547 ProgInfo.Occupancy = createOccupancy(
1548 InitOcc: STM.computeOccupancy(F, LDSSize: ProgInfo.LDSSize).second,
1549 NumSGPRs: ProgInfo.NumSGPRsForWavesPerEU, NumVGPRs: ProgInfo.NumVGPRsForWavesPerEU,
1550 DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize(), STM, Ctx);
1551
1552 const auto [MinWEU, MaxWEU] =
1553 AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-waves-per-eu", Default: {0, 0}, OnlyFirstRequired: true);
1554 uint64_t Occupancy;
1555 if (TryGetMCExprValue(ProgInfo.Occupancy, Occupancy) && Occupancy < MinWEU) {
1556 DiagnosticInfoOptimizationFailure Diag(
1557 F, F.getSubprogram(),
1558 "failed to meet occupancy target given by 'amdgpu-waves-per-eu' in "
1559 "'" +
1560 F.getName() + "': desired occupancy was " + Twine(MinWEU) +
1561 ", final occupancy is " + Twine(Occupancy));
1562 F.getContext().diagnose(DI: Diag);
1563 }
1564}
1565
1566static unsigned getRsrcReg(CallingConv::ID CallConv) {
1567 switch (CallConv) {
1568 default:
1569 [[fallthrough]];
1570 case CallingConv::AMDGPU_CS:
1571 return R_00B848_COMPUTE_PGM_RSRC1;
1572 case CallingConv::AMDGPU_LS:
1573 return R_00B528_SPI_SHADER_PGM_RSRC1_LS;
1574 case CallingConv::AMDGPU_HS:
1575 return R_00B428_SPI_SHADER_PGM_RSRC1_HS;
1576 case CallingConv::AMDGPU_ES:
1577 return R_00B328_SPI_SHADER_PGM_RSRC1_ES;
1578 case CallingConv::AMDGPU_GS:
1579 return R_00B228_SPI_SHADER_PGM_RSRC1_GS;
1580 case CallingConv::AMDGPU_VS:
1581 return R_00B128_SPI_SHADER_PGM_RSRC1_VS;
1582 case CallingConv::AMDGPU_PS:
1583 return R_00B028_SPI_SHADER_PGM_RSRC1_PS;
1584 }
1585}
1586
1587void AMDGPUAsmPrinter::EmitProgramInfoSI(
1588 const MachineFunction &MF, const SIProgramInfo &CurrentProgramInfo) {
1589 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1590 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1591 unsigned RsrcReg = getRsrcReg(CallConv: MF.getFunction().getCallingConv());
1592 MCContext &Ctx = MF.getContext();
1593
1594 // (((Value) & Mask) << Shift)
1595 auto SetBits = [&Ctx](const MCExpr *Value, uint32_t Mask, uint32_t Shift) {
1596 const MCExpr *msk = MCConstantExpr::create(Value: Mask, Ctx);
1597 const MCExpr *shft = MCConstantExpr::create(Value: Shift, Ctx);
1598 return MCBinaryExpr::createShl(LHS: MCBinaryExpr::createAnd(LHS: Value, RHS: msk, Ctx),
1599 RHS: shft, Ctx);
1600 };
1601
1602 auto EmitResolvedOrExpr = [this](const MCExpr *Value, unsigned Size) {
1603 int64_t Val;
1604 if (Value->evaluateAsAbsolute(Res&: Val))
1605 OutStreamer->emitIntValue(Value: static_cast<uint64_t>(Val), Size);
1606 else
1607 OutStreamer->emitValue(Value, Size);
1608 };
1609
1610 if (AMDGPU::isCompute(CC: MF.getFunction().getCallingConv())) {
1611 OutStreamer->emitInt32(R_00B848_COMPUTE_PGM_RSRC1);
1612
1613 EmitResolvedOrExpr(CurrentProgramInfo.getComputePGMRSrc1(ST: STM, Ctx),
1614 /*Size=*/4);
1615
1616 OutStreamer->emitInt32(R_00B84C_COMPUTE_PGM_RSRC2);
1617 EmitResolvedOrExpr(CurrentProgramInfo.getComputePGMRSrc2(ST: STM, Ctx),
1618 /*Size=*/4);
1619
1620 OutStreamer->emitInt32(R_00B860_COMPUTE_TMPRING_SIZE);
1621
1622 // Sets bits according to S_0286E8_WAVESIZE_* mask and shift values for the
1623 // appropriate generation.
1624 if (STM.getGeneration() >= AMDGPUSubtarget::GFX12) {
1625 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1626 /*Mask=*/0x3FFFF, /*Shift=*/12),
1627 /*Size=*/4);
1628 } else if (STM.getGeneration() == AMDGPUSubtarget::GFX11) {
1629 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1630 /*Mask=*/0x7FFF, /*Shift=*/12),
1631 /*Size=*/4);
1632 } else {
1633 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1634 /*Mask=*/0x1FFF, /*Shift=*/12),
1635 /*Size=*/4);
1636 }
1637
1638 // TODO: Should probably note flat usage somewhere. SC emits a "FlatPtr32 =
1639 // 0" comment but I don't see a corresponding field in the register spec.
1640 } else {
1641 OutStreamer->emitInt32(Value: RsrcReg);
1642
1643 const MCExpr *GPRBlocks = MCBinaryExpr::createOr(
1644 LHS: SetBits(CurrentProgramInfo.VGPRBlocks, /*Mask=*/0x3F, /*Shift=*/0),
1645 RHS: SetBits(CurrentProgramInfo.SGPRBlocks, /*Mask=*/0x0F, /*Shift=*/6),
1646 Ctx&: MF.getContext());
1647 EmitResolvedOrExpr(GPRBlocks, /*Size=*/4);
1648 OutStreamer->emitInt32(R_0286E8_SPI_TMPRING_SIZE);
1649
1650 // Sets bits according to S_0286E8_WAVESIZE_* mask and shift values for the
1651 // appropriate generation.
1652 if (STM.getGeneration() >= AMDGPUSubtarget::GFX12) {
1653 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1654 /*Mask=*/0x3FFFF, /*Shift=*/12),
1655 /*Size=*/4);
1656 } else if (STM.getGeneration() == AMDGPUSubtarget::GFX11) {
1657 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1658 /*Mask=*/0x7FFF, /*Shift=*/12),
1659 /*Size=*/4);
1660 } else {
1661 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1662 /*Mask=*/0x1FFF, /*Shift=*/12),
1663 /*Size=*/4);
1664 }
1665 }
1666
1667 if (MF.getFunction().getCallingConv() == CallingConv::AMDGPU_PS) {
1668 OutStreamer->emitInt32(R_00B02C_SPI_SHADER_PGM_RSRC2_PS);
1669 unsigned ExtraLDSSize = STM.getGeneration() >= AMDGPUSubtarget::GFX11
1670 ? divideCeil(Numerator: CurrentProgramInfo.LDSBlocks, Denominator: 2)
1671 : CurrentProgramInfo.LDSBlocks;
1672 OutStreamer->emitInt32(S_00B02C_EXTRA_LDS_SIZE(ExtraLDSSize));
1673 OutStreamer->emitInt32(R_0286CC_SPI_PS_INPUT_ENA);
1674 OutStreamer->emitInt32(Value: MFI->getPSInputEnable());
1675 OutStreamer->emitInt32(R_0286D0_SPI_PS_INPUT_ADDR);
1676 OutStreamer->emitInt32(Value: MFI->getPSInputAddr());
1677 }
1678
1679 OutStreamer->emitInt32(R_SPILLED_SGPRS);
1680 OutStreamer->emitInt32(Value: MFI->getNumSpilledSGPRs());
1681 OutStreamer->emitInt32(R_SPILLED_VGPRS);
1682 OutStreamer->emitInt32(Value: MFI->getNumSpilledVGPRs());
1683}
1684
1685// Helper function to add common PAL Metadata 3.0+
1686static void EmitPALMetadataCommon(AMDGPUPALMetadata *MD,
1687 const SIProgramInfo &CurrentProgramInfo,
1688 CallingConv::ID CC, const GCNSubtarget &ST,
1689 unsigned DynamicVGPRBlockSize) {
1690 if (ST.hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode))
1691 MD->setHwStage(CC, field: ".ieee_mode", Val: (bool)CurrentProgramInfo.IEEEMode);
1692
1693 MD->setHwStage(CC, field: ".wgp_mode", Val: (bool)CurrentProgramInfo.WgpMode);
1694 MD->setHwStage(CC, field: ".mem_ordered", Val: (bool)CurrentProgramInfo.MemOrdered);
1695 MD->setHwStage(CC, field: ".forward_progress", Val: (bool)CurrentProgramInfo.FwdProgress);
1696
1697 if (AMDGPU::isCompute(CC)) {
1698 MD->setHwStage(CC, field: ".trap_present",
1699 Val: (bool)CurrentProgramInfo.TrapHandlerEnable);
1700 MD->setHwStage(CC, field: ".excp_en", Val: CurrentProgramInfo.EXCPEnable);
1701
1702 if (DynamicVGPRBlockSize != 0)
1703 MD->setComputeRegisters(field: ".dynamic_vgpr_en", Val: true);
1704 }
1705
1706 MD->updateHwStageMaximum(
1707 CC, field: ".lds_size",
1708 Val: (unsigned)(CurrentProgramInfo.LdsSize * getLDSEncodingGranule(ST)));
1709}
1710
1711// This is the equivalent of EmitProgramInfoSI above, but for when the OS type
1712// is AMDPAL. It stores each compute/SPI register setting and other PAL
1713// metadata items into the PALMD::Metadata, combining with any provided by the
1714// frontend as LLVM metadata. Once all functions are written, the PAL metadata
1715// is then written as a single block in the .note section.
1716void AMDGPUAsmPrinter::EmitPALMetadata(
1717 const MachineFunction &MF, const SIProgramInfo &CurrentProgramInfo) {
1718 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1719 auto CC = MF.getFunction().getCallingConv();
1720 auto *MD = getTargetStreamer()->getPALMetadata();
1721 auto &Ctx = MF.getContext();
1722
1723 MD->setEntryPoint(CC, Name: MF.getFunction().getName());
1724 MD->setNumUsedVgprs(CC, Val: CurrentProgramInfo.NumVGPRsForWavesPerEU, Ctx);
1725
1726 // For targets that support dynamic VGPRs, set the number of saved dynamic
1727 // VGPRs (if any) in the PAL metadata.
1728 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1729 if (MFI->isDynamicVGPREnabled() &&
1730 MFI->getScratchReservedForDynamicVGPRs() > 0)
1731 MD->setHwStage(CC, field: ".dynamic_vgpr_saved_count",
1732 Val: MFI->getScratchReservedForDynamicVGPRs() / 4);
1733
1734 // Only set AGPRs for supported devices
1735 if (STM.hasMAIInsts()) {
1736 MD->setNumUsedAgprs(CC, Val: CurrentProgramInfo.NumAccVGPR);
1737 }
1738
1739 MD->setNumUsedSgprs(CC, Val: CurrentProgramInfo.NumSGPRsForWavesPerEU, Ctx);
1740 if (MD->getPALMajorVersion() < 3) {
1741 MD->setRsrc1(CC, Val: CurrentProgramInfo.getPGMRSrc1(CC, ST: STM, Ctx), Ctx);
1742 if (AMDGPU::isCompute(CC)) {
1743 MD->setRsrc2(CC, Val: CurrentProgramInfo.getComputePGMRSrc2(ST: STM, Ctx), Ctx);
1744 } else {
1745 const MCExpr *HasScratchBlocks =
1746 MCBinaryExpr::createGT(LHS: CurrentProgramInfo.ScratchBlocks,
1747 RHS: MCConstantExpr::create(Value: 0, Ctx), Ctx);
1748 auto [Shift, Mask] = getShiftMask(C_00B84C_SCRATCH_EN);
1749 MD->setRsrc2(CC, Val: maskShiftSet(Val: HasScratchBlocks, Mask, Shift, Ctx), Ctx);
1750 }
1751 } else {
1752 MD->setHwStage(CC, field: ".debug_mode", Val: (bool)CurrentProgramInfo.DebugMode);
1753 MD->setHwStage(CC, field: ".scratch_en", Type: msgpack::Type::Boolean,
1754 Val: CurrentProgramInfo.ScratchEnable);
1755 EmitPALMetadataCommon(MD, CurrentProgramInfo, CC, ST: STM,
1756 DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize());
1757 }
1758
1759 // ScratchSize is in bytes, 16 aligned.
1760 MD->setScratchSize(
1761 CC,
1762 Val: AMDGPUMCExpr::createAlignTo(Value: CurrentProgramInfo.ScratchSize,
1763 Align: MCConstantExpr::create(Value: 16, Ctx), Ctx),
1764 Ctx);
1765
1766 if (MF.getFunction().getCallingConv() == CallingConv::AMDGPU_PS) {
1767 unsigned ExtraLDSSize = STM.getGeneration() >= AMDGPUSubtarget::GFX11
1768 ? divideCeil(Numerator: CurrentProgramInfo.LDSBlocks, Denominator: 2)
1769 : CurrentProgramInfo.LDSBlocks;
1770 if (MD->getPALMajorVersion() < 3) {
1771 MD->setRsrc2(
1772 CC,
1773 Val: MCConstantExpr::create(S_00B02C_EXTRA_LDS_SIZE(ExtraLDSSize), Ctx),
1774 Ctx);
1775 MD->setSpiPsInputEna(MFI->getPSInputEnable());
1776 MD->setSpiPsInputAddr(MFI->getPSInputAddr());
1777 } else {
1778 // Graphics registers
1779 const unsigned ExtraLdsDwGranularity =
1780 STM.getGeneration() >= AMDGPUSubtarget::GFX11 ? 256 : 128;
1781 MD->setGraphicsRegisters(
1782 field: ".ps_extra_lds_size",
1783 Val: (unsigned)(ExtraLDSSize * ExtraLdsDwGranularity * sizeof(uint32_t)));
1784
1785 // Set PsInputEna and PsInputAddr .spi_ps_input_ena and .spi_ps_input_addr
1786 static StringLiteral const PsInputFields[] = {
1787 ".persp_sample_ena", ".persp_center_ena",
1788 ".persp_centroid_ena", ".persp_pull_model_ena",
1789 ".linear_sample_ena", ".linear_center_ena",
1790 ".linear_centroid_ena", ".line_stipple_tex_ena",
1791 ".pos_x_float_ena", ".pos_y_float_ena",
1792 ".pos_z_float_ena", ".pos_w_float_ena",
1793 ".front_face_ena", ".ancillary_ena",
1794 ".sample_coverage_ena", ".pos_fixed_pt_ena"};
1795 unsigned PSInputEna = MFI->getPSInputEnable();
1796 unsigned PSInputAddr = MFI->getPSInputAddr();
1797 for (auto [Idx, Field] : enumerate(First: PsInputFields)) {
1798 MD->setGraphicsRegisters(field1: ".spi_ps_input_ena", field2: Field,
1799 Val: (bool)((PSInputEna >> Idx) & 1));
1800 MD->setGraphicsRegisters(field1: ".spi_ps_input_addr", field2: Field,
1801 Val: (bool)((PSInputAddr >> Idx) & 1));
1802 }
1803 }
1804 }
1805
1806 // For version 3 and above the wave front size is already set in the metadata
1807 if (MD->getPALMajorVersion() < 3 && STM.isWave32())
1808 MD->setWave32(MF.getFunction().getCallingConv());
1809}
1810
1811void AMDGPUAsmPrinter::emitPALFunctionMetadata(const MachineFunction &MF) {
1812 auto *MD = getTargetStreamer()->getPALMetadata();
1813 const MachineFrameInfo &MFI = MF.getFrameInfo();
1814 StringRef FnName = MF.getFunction().getName();
1815 MD->setFunctionScratchSize(FnName, Val: MFI.getStackSize());
1816 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1817 MCContext &Ctx = MF.getContext();
1818
1819 if (MD->getPALMajorVersion() < 3) {
1820 // Set compute registers
1821 MD->setRsrc1(
1822 CC: CallingConv::AMDGPU_CS,
1823 Val: CurrentProgramInfo.getPGMRSrc1(CC: CallingConv::AMDGPU_CS, ST, Ctx), Ctx);
1824 MD->setRsrc2(CC: CallingConv::AMDGPU_CS,
1825 Val: CurrentProgramInfo.getComputePGMRSrc2(ST, Ctx), Ctx);
1826 } else {
1827 EmitPALMetadataCommon(
1828 MD, CurrentProgramInfo, CC: CallingConv::AMDGPU_CS, ST,
1829 DynamicVGPRBlockSize: MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize());
1830 }
1831
1832 // Set optional info
1833 MD->setFunctionLdsSize(FnName, Val: CurrentProgramInfo.LDSSize);
1834 MD->setFunctionNumUsedVgprs(FnName, Val: CurrentProgramInfo.NumVGPRsForWavesPerEU);
1835 MD->setFunctionNumUsedSgprs(FnName, Val: CurrentProgramInfo.NumSGPRsForWavesPerEU);
1836}
1837
1838// This is supposed to be log2(Size)
1839static amd_element_byte_size_t getElementByteSizeValue(unsigned Size) {
1840 switch (Size) {
1841 case 4:
1842 return AMD_ELEMENT_4_BYTES;
1843 case 8:
1844 return AMD_ELEMENT_8_BYTES;
1845 case 16:
1846 return AMD_ELEMENT_16_BYTES;
1847 default:
1848 llvm_unreachable("invalid private_element_size");
1849 }
1850}
1851
1852void AMDGPUAsmPrinter::getAmdKernelCode(AMDGPUMCKernelCodeT &Out,
1853 const SIProgramInfo &CurrentProgramInfo,
1854 const MachineFunction &MF) const {
1855 const Function &F = MF.getFunction();
1856 assert(F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
1857 F.getCallingConv() == CallingConv::SPIR_KERNEL);
1858
1859 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1860 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1861 MCContext &Ctx = MF.getContext();
1862
1863 Out.initDefault(STI: STM, Ctx, /*InitMCExpr=*/false);
1864
1865 Out.compute_pgm_resource1_registers =
1866 CurrentProgramInfo.getComputePGMRSrc1(ST: STM, Ctx);
1867 Out.compute_pgm_resource2_registers =
1868 CurrentProgramInfo.getComputePGMRSrc2(ST: STM, Ctx);
1869 Out.code_properties |= AMD_CODE_PROPERTY_IS_PTR64;
1870
1871 Out.is_dynamic_callstack = CurrentProgramInfo.DynamicCallStack;
1872
1873 AMD_HSA_BITS_SET(Out.code_properties, AMD_CODE_PROPERTY_PRIVATE_ELEMENT_SIZE,
1874 getElementByteSizeValue(STM.getMaxPrivateElementSize(true)));
1875
1876 const GCNUserSGPRUsageInfo &UserSGPRInfo = MFI->getUserSGPRInfo();
1877 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
1878 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER;
1879 }
1880
1881 if (UserSGPRInfo.hasDispatchPtr())
1882 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR;
1883
1884 if (UserSGPRInfo.hasQueuePtr())
1885 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR;
1886
1887 if (UserSGPRInfo.hasKernargSegmentPtr())
1888 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR;
1889
1890 if (UserSGPRInfo.hasDispatchID())
1891 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID;
1892
1893 if (UserSGPRInfo.hasFlatScratchInit())
1894 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT;
1895
1896 if (UserSGPRInfo.hasPrivateSegmentSize())
1897 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE;
1898
1899 if (STM.isXNACKEnabled())
1900 Out.code_properties |= AMD_CODE_PROPERTY_IS_XNACK_SUPPORTED;
1901
1902 Align MaxKernArgAlign;
1903 Out.kernarg_segment_byte_size = STM.getKernArgSegmentSize(F, MaxAlign&: MaxKernArgAlign);
1904 Out.wavefront_sgpr_count = CurrentProgramInfo.NumSGPR;
1905 Out.workitem_vgpr_count = CurrentProgramInfo.NumVGPR;
1906 Out.workitem_private_segment_byte_size = CurrentProgramInfo.ScratchSize;
1907 Out.workgroup_group_segment_byte_size = CurrentProgramInfo.LDSSize;
1908
1909 // kernarg_segment_alignment is specified as log of the alignment.
1910 // The minimum alignment is 16.
1911 // FIXME: The metadata treats the minimum as 4?
1912 Out.kernarg_segment_alignment = Log2(A: std::max(a: Align(16), b: MaxKernArgAlign));
1913}
1914
1915bool AMDGPUAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
1916 const char *ExtraCode, raw_ostream &O) {
1917 // First try the generic code, which knows about modifiers like 'c' and 'n'.
1918 if (!AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, OS&: O))
1919 return false;
1920
1921 if (ExtraCode && ExtraCode[0]) {
1922 if (ExtraCode[1] != 0)
1923 return true; // Unknown modifier.
1924
1925 switch (ExtraCode[0]) {
1926 case 'r':
1927 break;
1928 default:
1929 return true;
1930 }
1931 }
1932
1933 // TODO: Should be able to support other operand types like globals.
1934 const MachineOperand &MO = MI->getOperand(i: OpNo);
1935 if (MO.isReg()) {
1936 AMDGPUInstPrinter::printRegOperand(Reg: MO.getReg(), O,
1937 MRI: *MF->getSubtarget().getRegisterInfo());
1938 return false;
1939 }
1940 if (MO.isImm()) {
1941 int64_t Val = MO.getImm();
1942 if (AMDGPU::isInlinableIntLiteral(Literal: Val)) {
1943 O << Val;
1944 } else if (isUInt<16>(x: Val)) {
1945 O << format(Fmt: "0x%" PRIx16, Vals: static_cast<uint16_t>(Val));
1946 } else if (isUInt<32>(x: Val)) {
1947 O << format(Fmt: "0x%" PRIx32, Vals: static_cast<uint32_t>(Val));
1948 } else {
1949 O << format(Fmt: "0x%" PRIx64, Vals: static_cast<uint64_t>(Val));
1950 }
1951 return false;
1952 }
1953 return true;
1954}
1955
1956void AMDGPUAsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const {
1957 AU.addRequired<AMDGPUResourceUsageAnalysisWrapperPass>();
1958 AU.addPreserved<AMDGPUResourceUsageAnalysisWrapperPass>();
1959 AU.addRequired<MachineModuleInfoWrapperPass>();
1960 AU.addPreserved<MachineModuleInfoWrapperPass>();
1961 AsmPrinter::getAnalysisUsage(AU);
1962}
1963
1964void AMDGPUAsmPrinter::emitResourceUsageRemarks(
1965 const MachineFunction &MF, const SIProgramInfo &CurrentProgramInfo,
1966 bool isModuleEntryFunction, bool hasMAIInsts) {
1967 if (!ORE)
1968 return;
1969
1970 const char *Name = "kernel-resource-usage";
1971 const char *Indent = " ";
1972
1973 // If the remark is not specifically enabled, do not output to yaml
1974 LLVMContext &Ctx = MF.getFunction().getContext();
1975 if (!Ctx.getDiagHandlerPtr()->isAnalysisRemarkEnabled(PassName: Name))
1976 return;
1977
1978 // Currently non-kernel functions have no resources to emit.
1979 if (!isEntryFunctionCC(CC: MF.getFunction().getCallingConv()))
1980 return;
1981
1982 auto EmitResourceUsageRemark = [&](StringRef RemarkName,
1983 StringRef RemarkLabel, auto Argument) {
1984 // Add an indent for every line besides the line with the kernel name. This
1985 // makes it easier to tell which resource usage go with which kernel since
1986 // the kernel name will always be displayed first.
1987 std::string LabelStr = RemarkLabel.str() + ": ";
1988 if (RemarkName != "FunctionName")
1989 LabelStr = Indent + LabelStr;
1990
1991 ORE->emit([&]() {
1992 return MachineOptimizationRemarkAnalysis(Name, RemarkName,
1993 MF.getFunction().getSubprogram(),
1994 &MF.front())
1995 << LabelStr << ore::NV(RemarkName, Argument);
1996 });
1997 };
1998
1999 // FIXME: Formatting here is pretty nasty because clang does not accept
2000 // newlines from diagnostics. This forces us to emit multiple diagnostic
2001 // remarks to simulate newlines. If and when clang does accept newlines, this
2002 // formatting should be aggregated into one remark with newlines to avoid
2003 // printing multiple diagnostic location and diag opts.
2004 EmitResourceUsageRemark("FunctionName", "Function Name",
2005 MF.getFunction().getName());
2006 EmitResourceUsageRemark("NumSGPR", "TotalSGPRs",
2007 getMCExprStr(Value: CurrentProgramInfo.NumSGPR));
2008 EmitResourceUsageRemark("NumVGPR", "VGPRs",
2009 getMCExprStr(Value: CurrentProgramInfo.NumArchVGPR));
2010 if (hasMAIInsts) {
2011 EmitResourceUsageRemark("NumAGPR", "AGPRs",
2012 getMCExprStr(Value: CurrentProgramInfo.NumAccVGPR));
2013 }
2014 EmitResourceUsageRemark("ScratchSize", "ScratchSize [bytes/lane]",
2015 getMCExprStr(Value: CurrentProgramInfo.ScratchSize));
2016 int64_t DynStack;
2017 bool DynStackEvaluatable =
2018 CurrentProgramInfo.DynamicCallStack->evaluateAsAbsolute(Res&: DynStack);
2019 StringRef DynamicStackStr =
2020 DynStackEvaluatable && DynStack ? "True" : "False";
2021 EmitResourceUsageRemark("DynamicStack", "Dynamic Stack", DynamicStackStr);
2022 EmitResourceUsageRemark("Occupancy", "Occupancy [waves/SIMD]",
2023 getMCExprStr(Value: CurrentProgramInfo.Occupancy));
2024 EmitResourceUsageRemark("SGPRSpill", "SGPRs Spill",
2025 CurrentProgramInfo.SGPRSpill);
2026 EmitResourceUsageRemark("VGPRSpill", "VGPRs Spill",
2027 CurrentProgramInfo.VGPRSpill);
2028 if (isModuleEntryFunction)
2029 EmitResourceUsageRemark("BytesLDS", "LDS Size [bytes/block]",
2030 CurrentProgramInfo.LDSSize);
2031}
2032
2033PreservedAnalyses AMDGPUAsmPrinterBeginPass::run(Module &M,
2034 ModuleAnalysisManager &MAM) {
2035
2036 AMDGPUAsmPrinter &AsmPrinter = static_cast<AMDGPUAsmPrinter &>(
2037 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
2038 setupModuleAsmPrinter(M, MAM, AsmPrinter);
2039 AsmPrinter.doInitialization(M);
2040 return PreservedAnalyses::all();
2041}
2042
2043PreservedAnalyses
2044AMDGPUAsmPrinterPass::run(MachineFunction &MF,
2045 MachineFunctionAnalysisManager &MFAM) {
2046 AMDGPUAsmPrinter &AsmPrinter = static_cast<AMDGPUAsmPrinter &>(
2047 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF)
2048 .getCachedResult<AsmPrinterAnalysis>(IR&: *MF.getFunction().getParent())
2049 ->getPrinter());
2050 setupMachineFunctionAsmPrinter(MFAM, MF, AsmPrinter);
2051 AsmPrinter.GetResourceUsage = [&MFAM](MachineFunction &MF)
2052 -> const AMDGPUResourceUsageAnalysisImpl::SIFunctionResourceInfo * {
2053 return &MFAM.getResult<AMDGPUResourceUsageAnalysis>(IR&: MF);
2054 };
2055 AsmPrinter.runOnMachineFunction(MF);
2056 return PreservedAnalyses::all();
2057}
2058
2059PreservedAnalyses AMDGPUAsmPrinterEndPass::run(Module &M,
2060 ModuleAnalysisManager &MAM) {
2061 AMDGPUAsmPrinter &AsmPrinter = static_cast<AMDGPUAsmPrinter &>(
2062 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
2063 setupModuleAsmPrinter(M, MAM, AsmPrinter);
2064 AsmPrinter.doFinalization(M);
2065 return PreservedAnalyses::all();
2066}
2067
2068char AMDGPUAsmPrinter::ID = 0;
2069
2070INITIALIZE_PASS(AMDGPUAsmPrinter, "amdgpu-asm-printer",
2071 "AMDGPU Assembly Printer", false, false)
2072