1//===-- AMDGPUAsmPrinter.cpp - AMDGPU assembly printer --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10///
11/// The AMDGPUAsmPrinter is used to print both assembly string and also binary
12/// code. When passed an MCAsmStreamer it prints assembly and when passed
13/// an MCObjectStreamer it outputs binary code.
14//
15//===----------------------------------------------------------------------===//
16//
17
18#include "AMDGPUAsmPrinter.h"
19#include "AMDGPU.h"
20#include "AMDGPUHSAMetadataStreamer.h"
21#include "AMDGPUMCResourceInfo.h"
22#include "AMDGPUResourceUsageAnalysis.h"
23#include "AMDGPUTargetMachine.h"
24#include "GCNSubtarget.h"
25#include "MCTargetDesc/AMDGPUInstPrinter.h"
26#include "MCTargetDesc/AMDGPUMCExpr.h"
27#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h"
28#include "MCTargetDesc/AMDGPUTargetStreamer.h"
29#include "R600AsmPrinter.h"
30#include "SIMachineFunctionInfo.h"
31#include "TargetInfo/AMDGPUTargetInfo.h"
32#include "Utils/AMDGPUBaseInfo.h"
33#include "Utils/AMDKernelCodeTUtils.h"
34#include "Utils/SIDefinesUtils.h"
35#include "llvm/Analysis/OptimizationRemarkEmitter.h"
36#include "llvm/BinaryFormat/ELF.h"
37#include "llvm/CodeGen/AsmPrinterAnalysis.h"
38#include "llvm/CodeGen/AsmPrinterHandler.h"
39#include "llvm/CodeGen/MachineFrameInfo.h"
40#include "llvm/CodeGen/MachineModuleInfo.h"
41#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
42#include "llvm/IR/DiagnosticInfo.h"
43#include "llvm/MC/MCAssembler.h"
44#include "llvm/MC/MCContext.h"
45#include "llvm/MC/MCSectionELF.h"
46#include "llvm/MC/MCStreamer.h"
47#include "llvm/MC/MCValue.h"
48#include "llvm/MC/TargetRegistry.h"
49#include "llvm/Support/AMDHSAKernelDescriptor.h"
50#include "llvm/Support/Compiler.h"
51#include "llvm/Target/TargetLoweringObjectFile.h"
52#include "llvm/Target/TargetMachine.h"
53#include "llvm/TargetParser/AMDGPUTargetParser.h"
54
55using namespace llvm;
56using namespace llvm::AMDGPU;
57
58// This should get the default rounding mode from the kernel. We just set the
59// default here, but this could change if the OpenCL rounding mode pragmas are
60// used.
61//
62// The denormal mode here should match what is reported by the OpenCL runtime
63// for the CL_FP_DENORM bit from CL_DEVICE_{HALF|SINGLE|DOUBLE}_FP_CONFIG, but
64// can also be override to flush with the -cl-denorms-are-zero compiler flag.
65//
66// AMD OpenCL only sets flush none and reports CL_FP_DENORM for double
67// precision, and leaves single precision to flush all and does not report
68// CL_FP_DENORM for CL_DEVICE_SINGLE_FP_CONFIG. Mesa's OpenCL currently reports
69// CL_FP_DENORM for both.
70//
71// FIXME: It seems some instructions do not support single precision denormals
72// regardless of the mode (exp_*_f32, rcp_*_f32, rsq_*_f32, rsq_*f32, sqrt_f32,
73// and sin_f32, cos_f32 on most parts).
74
75// We want to use these instructions, and using fp32 denormals also causes
76// instructions to run at the double precision rate for the device so it's
77// probably best to just report no single precision denormals.
78static uint32_t getFPMode(SIModeRegisterDefaults Mode) {
79 return FP_ROUND_MODE_SP(FP_ROUND_ROUND_TO_NEAREST) |
80 FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_NEAREST) |
81 FP_DENORM_MODE_SP(Mode.fpDenormModeSPValue()) |
82 FP_DENORM_MODE_DP(Mode.fpDenormModeDPValue());
83}
84
85static AsmPrinter *
86createAMDGPUAsmPrinterPass(TargetMachine &tm,
87 std::unique_ptr<MCStreamer> &&Streamer) {
88 return new AMDGPUAsmPrinter(tm, std::move(Streamer));
89}
90
91extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
92LLVMInitializeAMDGPUAsmPrinter() {
93 TargetRegistry::RegisterAsmPrinter(T&: getTheR600Target(),
94 Fn: llvm::createR600AsmPrinterPass);
95 TargetRegistry::RegisterAsmPrinter(T&: getTheGCNTarget(),
96 Fn: createAMDGPUAsmPrinterPass);
97 TargetRegistry::RegisterAsmPrinter(T&: getTheGCNLegacyTarget(),
98 Fn: createAMDGPUAsmPrinterPass);
99}
100
101namespace {
102class AMDGPUAsmPrinterHandler : public AsmPrinterHandler {
103protected:
104 AMDGPUAsmPrinter *Asm;
105
106public:
107 AMDGPUAsmPrinterHandler(AMDGPUAsmPrinter *A) : Asm(A) {}
108
109 void beginFunction(const MachineFunction *MF) override {}
110
111 void endFunction(const MachineFunction *MF) override { Asm->endFunction(MF); }
112
113 void endModule() override {}
114};
115} // End anonymous namespace
116
117AMDGPUAsmPrinter::AMDGPUAsmPrinter(TargetMachine &TM,
118 std::unique_ptr<MCStreamer> Streamer)
119 : AsmPrinter(TM, std::move(Streamer)) {
120 assert(OutStreamer && "AsmPrinter constructed without streamer");
121 GetResourceUsage = [this](MachineFunction &MF)
122 -> const AMDGPUResourceUsageAnalysisImpl::SIFunctionResourceInfo * {
123 if (auto *ResourceUsageW =
124 getAnalysisIfAvailable<AMDGPUResourceUsageAnalysisWrapperPass>())
125 return &ResourceUsageW->getResourceInfo();
126 return nullptr;
127 };
128}
129
130StringRef AMDGPUAsmPrinter::getPassName() const {
131 return "AMDGPU Assembly Printer";
132}
133
134const MCSubtargetInfo *AMDGPUAsmPrinter::getGlobalSTI() const {
135 return &TM.getMCSubtargetInfo();
136}
137
138AMDGPUTargetStreamer *AMDGPUAsmPrinter::getTargetStreamer() const {
139 if (!OutStreamer)
140 return nullptr;
141 return static_cast<AMDGPUTargetStreamer *>(OutStreamer->getTargetStreamer());
142}
143
144void AMDGPUAsmPrinter::emitStartOfAsmFile(Module &M) {
145 IsTargetStreamerInitialized = false;
146}
147
148void AMDGPUAsmPrinter::initTargetStreamer(Module &M) {
149 IsTargetStreamerInitialized = true;
150
151 // TODO: Which one is called first, emitStartOfAsmFile or
152 // emitFunctionBodyStart?
153 if (getTargetStreamer() && !getTargetStreamer()->getTargetID())
154 initializeTargetID(M);
155
156 const Triple &TT = M.getTargetTriple();
157 if (TT.getOS() != Triple::AMDHSA && TT.getOS() != Triple::AMDPAL)
158 return;
159
160 getTargetStreamer()->EmitDirectiveAMDGCNTarget();
161
162 if (TT.getOS() == Triple::AMDHSA) {
163 getTargetStreamer()->EmitDirectiveAMDHSACodeObjectVersion(
164 COV: CodeObjectVersion);
165 HSAMetadataStream->begin(Mod: M, TargetID: *getTargetStreamer()->getTargetID());
166 }
167
168 if (TT.getOS() == Triple::AMDPAL)
169 getTargetStreamer()->getPALMetadata()->readFromIR(M);
170}
171
172void AMDGPUAsmPrinter::emitEndOfAsmFile(Module &M) {
173 // Init target streamer if it has not yet happened
174 if (!IsTargetStreamerInitialized)
175 initTargetStreamer(M);
176
177 const Triple &TT = M.getTargetTriple();
178 if (TT.getOS() != Triple::AMDHSA)
179 getTargetStreamer()->EmitISAVersion();
180
181 // Emit HSA Metadata (NT_AMD_AMDGPU_HSA_METADATA).
182 // Emit HSA Metadata (NT_AMD_HSA_METADATA).
183 if (TT.getOS() == Triple::AMDHSA) {
184 HSAMetadataStream->end();
185 bool Success = HSAMetadataStream->emitTo(TargetStreamer&: *getTargetStreamer());
186 (void)Success;
187 assert(Success && "Malformed HSA Metadata");
188 }
189}
190
191void AMDGPUAsmPrinter::emitFunctionBodyStart() {
192 const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
193 const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
194 const Function &F = MF->getFunction();
195
196 // TODO: We're checking this late, would be nice to check it earlier.
197 if (STM.requiresCodeObjectV6() && CodeObjectVersion < AMDGPU::AMDHSA_COV6) {
198 reportFatalUsageError(
199 reason: STM.getCPU() + " is only available on code object version 6 or better");
200 }
201
202 // TODO: Which one is called first, emitStartOfAsmFile or
203 // emitFunctionBodyStart?
204 if (!getTargetStreamer()->getTargetID())
205 initializeTargetID(M: *F.getParent());
206
207 if (!MFI.isEntryFunction())
208 return;
209
210 if (STM.isMesaKernel(F) &&
211 (F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
212 F.getCallingConv() == CallingConv::SPIR_KERNEL)) {
213 AMDGPUMCKernelCodeT KernelCode;
214 getAmdKernelCode(Out&: KernelCode, KernelInfo: CurrentProgramInfo, MF: *MF);
215 KernelCode.validate(STI: &STM, Ctx&: MF->getContext());
216 getTargetStreamer()->EmitAMDKernelCodeT(Header&: KernelCode);
217 }
218
219 if (STM.isAmdHsaOS())
220 HSAMetadataStream->emitKernel(MF: *MF, ProgramInfo: CurrentProgramInfo);
221}
222
223/// Set bits in a kernel descriptor MCExpr field:
224/// return ((Dst & ~Mask) | (Value << Shift))
225static const MCExpr *setBits(const MCExpr *Dst, const MCExpr *Value,
226 uint32_t Mask, uint32_t Shift, MCContext &Ctx) {
227 const auto *Shft = MCConstantExpr::create(Value: Shift, Ctx);
228 const auto *Msk = MCConstantExpr::create(Value: Mask, Ctx);
229 Dst = MCBinaryExpr::createAnd(LHS: Dst, RHS: MCUnaryExpr::createNot(Expr: Msk, Ctx), Ctx);
230 Dst = MCBinaryExpr::createOr(LHS: Dst, RHS: MCBinaryExpr::createShl(LHS: Value, RHS: Shft, Ctx),
231 Ctx);
232 return Dst;
233}
234
235void AMDGPUAsmPrinter::endFunction(const MachineFunction *MF) {
236 const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
237 if (!MFI.isEntryFunction())
238 return;
239
240 assert(TM.getTargetTriple().getOS() == Triple::AMDHSA);
241
242 const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
243 MCContext &Ctx = MF->getContext();
244
245 AMDGPU::MCKernelDescriptor KD =
246 getAmdhsaKernelDescriptor(MF: *MF, PI: CurrentProgramInfo);
247
248 // Compute inst_pref_size using MCExpr label subtraction for exact code
249 // size. At this point .Lfunc_end has been emitted (by the base AsmPrinter)
250 // right after the function code, so (Lfunc_end - func_sym) gives the
251 // exact function code size in bytes.
252 if (STM.hasInstPrefSize()) {
253 const MCExpr *CodeSizeExpr = MCBinaryExpr::createSub(
254 LHS: MCSymbolRefExpr::create(Symbol: getFunctionEnd(), Ctx&: OutContext),
255 RHS: MCSymbolRefExpr::create(Symbol: CurrentFnSym, Ctx&: OutContext), Ctx&: OutContext);
256
257 uint32_t Mask, Shift, Width, CacheLineSize;
258 STM.getInstPrefSizeArgs(Mask, Shift, Width, CacheLineSize);
259 const MCExpr *InstPrefSize =
260 AMDGPUMCExpr::createInstPrefSize(CodeSizeBytes: CodeSizeExpr, Ctx);
261 KD.compute_pgm_rsrc3 =
262 setBits(Dst: KD.compute_pgm_rsrc3, Value: InstPrefSize, Mask, Shift, Ctx);
263 }
264
265 auto &Streamer = getTargetStreamer()->getStreamer();
266 auto &Context = Streamer.getContext();
267 auto &ObjectFileInfo = *Context.getObjectFileInfo();
268 auto &ReadOnlySection = *ObjectFileInfo.getReadOnlySection();
269
270 Streamer.pushSection();
271 Streamer.switchSection(Section: &ReadOnlySection);
272
273 // CP microcode requires the kernel descriptor to be allocated on 64 byte
274 // alignment.
275 Streamer.emitValueToAlignment(Alignment: Align(64), Fill: 0, FillLen: 1, MaxBytesToEmit: 0);
276 ReadOnlySection.ensureMinAlignment(MinAlignment: Align(64));
277
278 SmallString<128> KernelName;
279 getNameWithPrefix(Name&: KernelName, GV: &MF->getFunction());
280 getTargetStreamer()->EmitAmdhsaKernelDescriptor(
281 STI: STM, KernelName, KernelDescriptor: KD, NextVGPR: CurrentProgramInfo.NumVGPRsForWavesPerEU,
282 NextSGPR: MCBinaryExpr::createSub(
283 LHS: CurrentProgramInfo.NumSGPRsForWavesPerEU,
284 RHS: AMDGPUMCExpr::createExtraSGPRs(
285 VCCUsed: CurrentProgramInfo.VCCUsed, FlatScrUsed: CurrentProgramInfo.FlatUsed,
286 XNACKUsed: getTargetStreamer()->getTargetID()->isXnackOnOrAny(), Ctx&: Context),
287 Ctx&: Context),
288 ReserveVCC: CurrentProgramInfo.VCCUsed, ReserveFlatScr: CurrentProgramInfo.FlatUsed);
289
290 Streamer.popSection();
291}
292
293void AMDGPUAsmPrinter::emitImplicitDef(const MachineInstr *MI) const {
294 Register RegNo = MI->getOperand(i: 0).getReg();
295
296 SmallString<128> Str;
297 raw_svector_ostream OS(Str);
298 OS << "implicit-def: "
299 << printReg(Reg: RegNo, TRI: MF->getSubtarget().getRegisterInfo());
300
301 if (MI->getAsmPrinterFlags() & AMDGPU::SGPR_SPILL)
302 OS << " : SGPR spill to VGPR lane";
303
304 OutStreamer->AddComment(T: OS.str());
305 OutStreamer->addBlankLine();
306}
307
308void AMDGPUAsmPrinter::emitFunctionEntryLabel() {
309 if (TM.getTargetTriple().getOS() == Triple::AMDHSA) {
310 AsmPrinter::emitFunctionEntryLabel();
311 return;
312 }
313
314 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
315 const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
316 if (MFI->isEntryFunction() && STM.isAmdHsaOrMesa(F: MF->getFunction())) {
317 SmallString<128> SymbolName;
318 getNameWithPrefix(Name&: SymbolName, GV: &MF->getFunction()),
319 getTargetStreamer()->EmitAMDGPUSymbolType(SymbolName,
320 Type: ELF::STT_AMDGPU_HSA_KERNEL);
321 }
322 if (DumpCodeInstEmitter) {
323 // Disassemble function name label to text.
324 DisasmLines.push_back(x: MF->getName().str() + ":");
325 DisasmLineMaxLen = std::max(a: DisasmLineMaxLen, b: DisasmLines.back().size());
326 HexLines.emplace_back(args: "");
327 }
328
329 AsmPrinter::emitFunctionEntryLabel();
330}
331
332void AMDGPUAsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) {
333 if (DumpCodeInstEmitter && !isBlockOnlyReachableByFallthrough(MBB: &MBB)) {
334 // Write a line for the basic block label if it is not only fallthrough.
335 DisasmLines.push_back(x: (Twine("BB") + Twine(getFunctionNumber()) + "_" +
336 Twine(MBB.getNumber()) + ":")
337 .str());
338 DisasmLineMaxLen = std::max(a: DisasmLineMaxLen, b: DisasmLines.back().size());
339 HexLines.emplace_back(args: "");
340 }
341 AsmPrinter::emitBasicBlockStart(MBB);
342}
343
344void AMDGPUAsmPrinter::emitGlobalVariable(const GlobalVariable *GV) {
345 if (GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
346 if (GV->hasInitializer() && !isa<UndefValue>(Val: GV->getInitializer())) {
347 OutContext.reportError(L: {},
348 Msg: Twine(GV->getName()) +
349 ": unsupported initializer for address space");
350 return;
351 }
352
353 const Triple::OSType OS = TM.getTargetTriple().getOS();
354 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL) {
355 if (!AMDGPUTargetMachine::EnableObjectLinking)
356 return;
357 // With object linking, LDS definitions should have been externalized
358 // by earlier passes (e.g. LDS lowering, named barrier lowering).
359 // Only declarations reach here, emitted as SHN_AMDGPU_LDS symbols
360 // so the linker can assign their offsets.
361 assert(GV->isDeclaration() &&
362 "LDS definitions should have been externalized when object "
363 "linking is enabled");
364 }
365
366 MCSymbol *GVSym = getSymbol(GV);
367
368 GVSym->redefineIfPossible();
369 if (GVSym->isDefined() || GVSym->isVariable())
370 report_fatal_error(reason: "symbol '" + Twine(GVSym->getName()) +
371 "' is already defined");
372
373 const DataLayout &DL = GV->getDataLayout();
374 uint64_t Size = GV->getGlobalSize(DL);
375 Align Alignment = GV->getAlign().value_or(u: Align(4));
376
377 emitVisibility(Sym: GVSym, Visibility: GV->getVisibility(), IsDefinition: !GV->isDeclaration());
378 emitLinkage(GV, GVSym);
379 auto *TS = getTargetStreamer();
380 TS->emitAMDGPULDS(Symbol: GVSym, Size, Alignment);
381 return;
382 }
383
384 AsmPrinter::emitGlobalVariable(GV);
385}
386
387bool AMDGPUAsmPrinter::doInitialization(Module &M) {
388 const Triple &TT = M.getTargetTriple();
389 if (TT.getSubArch() == Triple::NoSubArch) {
390 Triple::SubArchType SubArch =
391 AMDGPU::getSubArch(AK: AMDGPU::parseArchAMDGCN(CPU: getGlobalSTI()->getCPU()));
392 if (SubArch != Triple::NoSubArch) {
393 Triple Fixed(TT);
394 Fixed.setArch(Kind: Triple::amdgpu, SubArch);
395 M.getContext().diagnose(DI: DiagnosticInfoGeneric(
396 "codegen with no subarch in the target triple is deprecated and will "
397 "become an error; use the target triple '" +
398 Fixed.str() + "' instead",
399 DS_Warning));
400 } else {
401 M.getContext().diagnose(DI: DiagnosticInfoGeneric(
402 "codegen with no subarch in the target triple is deprecated and will "
403 "become an error",
404 DS_Warning));
405 }
406 }
407
408 CodeObjectVersion = AMDGPU::getAMDHSACodeObjectVersion(M);
409
410 if (TT.getOS() == Triple::AMDHSA) {
411 switch (CodeObjectVersion) {
412 case AMDGPU::AMDHSA_COV4:
413 HSAMetadataStream = std::make_unique<HSAMD::MetadataStreamerMsgPackV4>();
414 break;
415 case AMDGPU::AMDHSA_COV5:
416 HSAMetadataStream = std::make_unique<HSAMD::MetadataStreamerMsgPackV5>();
417 break;
418 case AMDGPU::AMDHSA_COV6:
419 HSAMetadataStream = std::make_unique<HSAMD::MetadataStreamerMsgPackV6>();
420 break;
421 default:
422 reportFatalUsageError(reason: "unsupported code object version");
423 }
424
425 addAsmPrinterHandler(Handler: std::make_unique<AMDGPUAsmPrinterHandler>(args: this));
426 }
427
428 return AsmPrinter::doInitialization(M);
429}
430
431/// Mimics GCNSubtarget::computeOccupancy for MCExpr.
432///
433/// Remove dependency on GCNSubtarget and depend only only the necessary values
434/// for said occupancy computation. Should match computeOccupancy implementation
435/// without passing \p STM on.
436const AMDGPUMCExpr *createOccupancy(unsigned InitOcc, const MCExpr *NumSGPRs,
437 const MCExpr *NumVGPRs,
438 unsigned DynamicVGPRBlockSize,
439 const GCNSubtarget &STM, MCContext &Ctx) {
440 unsigned MaxWaves = STM.getMaxWavesPerEU();
441 unsigned Granule = IsaInfo::getVGPRAllocGranule(STI: STM, DynamicVGPRBlockSize);
442 unsigned TargetTotalNumVGPRs = STM.getTotalNumVGPRs();
443
444 // Bake the per-function SGPR budget into the operands so the late-evaluated
445 // MCExpr stays arithmetic. The trap reservation in particular is implicit on
446 // amdhsa and lives on STM, not on the assembler's MCSubtargetInfo.
447 AMDGPU::GPUKind Kind = STM.getTargetID().getGPUKind();
448 unsigned SGPRTotal = AMDGPU::getTotalNumSGPRs(AK: Kind);
449 unsigned SGPRGranule = AMDGPU::getSGPRAllocGranule(AK: Kind);
450 unsigned SGPRTrapReserve = STM.hasTrapHandler() ? IsaInfo::TRAP_NUM_SGPRS : 0;
451
452 auto CreateExpr = [&Ctx](unsigned Value) {
453 return MCConstantExpr::create(Value, Ctx);
454 };
455
456 // Zero SGPR count when SGPRs don't limit occupancy, so the MCExpr skips the
457 // SGPR term without having to test the generation itself.
458 const MCExpr *SGPRArg =
459 IsaInfo::isSGPROccupancyLimited(STI: STM) ? NumSGPRs : CreateExpr(0);
460
461 return AMDGPUMCExpr::create(Kind: AMDGPUMCExpr::AGVK_Occupancy,
462 Args: {CreateExpr(MaxWaves), CreateExpr(Granule),
463 CreateExpr(TargetTotalNumVGPRs),
464 CreateExpr(InitOcc), CreateExpr(SGPRTotal),
465 CreateExpr(SGPRGranule),
466 CreateExpr(SGPRTrapReserve), SGPRArg, NumVGPRs},
467 Ctx);
468}
469
470void AMDGPUAsmPrinter::validateMCResourceInfo(Function &F) {
471 if (F.isDeclaration() || !AMDGPU::isModuleEntryFunctionCC(CC: F.getCallingConv()))
472 return;
473
474 using RIK = MCResourceInfo::ResourceInfoKind;
475 const GCNSubtarget &STM = TM.getSubtarget<GCNSubtarget>(F);
476 MCSymbol *FnSym = TM.getSymbol(GV: &F);
477
478 auto TryGetMCExprValue = [](const MCExpr *Value, uint64_t &Res) -> bool {
479 int64_t Val;
480 if (Value->evaluateAsAbsolute(Res&: Val)) {
481 Res = Val;
482 return true;
483 }
484 return false;
485 };
486
487 const uint64_t MaxScratchPerWorkitem =
488 STM.getMaxWaveScratchSize() / STM.getWavefrontSize();
489 MCSymbol *ScratchSizeSymbol =
490 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_PrivateSegSize, OutContext);
491 uint64_t ScratchSize;
492 if (ScratchSizeSymbol->isVariable() &&
493 TryGetMCExprValue(ScratchSizeSymbol->getVariableValue(), ScratchSize) &&
494 ScratchSize > MaxScratchPerWorkitem) {
495 DiagnosticInfoStackSize DiagStackSize(F, ScratchSize, MaxScratchPerWorkitem,
496 DS_Error);
497 F.getContext().diagnose(DI: DiagStackSize);
498 }
499
500 // Validate addressable scalar registers (i.e., prior to added implicit
501 // SGPRs).
502 MCSymbol *NumSGPRSymbol =
503 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_NumSGPR, OutContext);
504 if (STM.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
505 !STM.hasSGPRInitBug()) {
506 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
507 uint64_t NumSgpr;
508 if (NumSGPRSymbol->isVariable() &&
509 TryGetMCExprValue(NumSGPRSymbol->getVariableValue(), NumSgpr) &&
510 NumSgpr > MaxAddressableNumSGPRs) {
511 F.getContext().diagnose(DI: DiagnosticInfoResourceLimit(
512 F, "addressable scalar registers", NumSgpr, MaxAddressableNumSGPRs,
513 DS_Error, DK_ResourceLimit));
514 return;
515 }
516 }
517
518 MCSymbol *VCCUsedSymbol =
519 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_UsesVCC, OutContext);
520 MCSymbol *FlatUsedSymbol =
521 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_UsesFlatScratch, OutContext);
522 uint64_t VCCUsed, FlatUsed, NumSgpr;
523
524 if (NumSGPRSymbol->isVariable() && VCCUsedSymbol->isVariable() &&
525 FlatUsedSymbol->isVariable() &&
526 TryGetMCExprValue(NumSGPRSymbol->getVariableValue(), NumSgpr) &&
527 TryGetMCExprValue(VCCUsedSymbol->getVariableValue(), VCCUsed) &&
528 TryGetMCExprValue(FlatUsedSymbol->getVariableValue(), FlatUsed)) {
529
530 // Recomputes NumSgprs + implicit SGPRs but all symbols should now be
531 // resolvable.
532 NumSgpr += IsaInfo::getNumExtraSGPRs(
533 STI: STM, VCCUsed, FlatScrUsed: FlatUsed,
534 XNACKUsed: getTargetStreamer()->getTargetID()->isXnackOnOrAny());
535 if (STM.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS ||
536 STM.hasSGPRInitBug()) {
537 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
538 if (NumSgpr > MaxAddressableNumSGPRs) {
539 F.getContext().diagnose(DI: DiagnosticInfoResourceLimit(
540 F, "scalar registers", NumSgpr, MaxAddressableNumSGPRs, DS_Error,
541 DK_ResourceLimit));
542 return;
543 }
544 }
545
546 MCSymbol *NumVgprSymbol =
547 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_NumVGPR, OutContext);
548 MCSymbol *NumAgprSymbol =
549 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_NumAGPR, OutContext);
550 uint64_t NumVgpr, NumAgpr;
551
552 MachineModuleInfo &MMI = *GetMMI();
553 MachineFunction *MF = MMI.getMachineFunction(F);
554 if (MF && NumVgprSymbol->isVariable() && NumAgprSymbol->isVariable() &&
555 TryGetMCExprValue(NumVgprSymbol->getVariableValue(), NumVgpr) &&
556 TryGetMCExprValue(NumAgprSymbol->getVariableValue(), NumAgpr)) {
557 const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
558 unsigned MaxWaves = MFI.getMaxWavesPerEU();
559 uint64_t TotalNumVgpr =
560 getTotalNumVGPRs(has90AInsts: STM.hasGFX90AInsts(), ArgNumAGPR: NumAgpr, ArgNumVGPR: NumVgpr);
561 uint64_t NumVGPRsForWavesPerEU =
562 std::max(l: {TotalNumVgpr, (uint64_t)1,
563 (uint64_t)STM.getMinNumVGPRs(
564 WavesPerEU: MaxWaves, DynamicVGPRBlockSize: MFI.getDynamicVGPRBlockSize())});
565 uint64_t NumSGPRsForWavesPerEU = std::max(
566 l: {NumSgpr, (uint64_t)1, (uint64_t)STM.getMinNumSGPRs(WavesPerEU: MaxWaves)});
567 const MCExpr *OccupancyExpr = createOccupancy(
568 InitOcc: STM.getOccupancyWithWorkGroupSizes(MF: *MF).second,
569 NumSGPRs: MCConstantExpr::create(Value: NumSGPRsForWavesPerEU, Ctx&: OutContext),
570 NumVGPRs: MCConstantExpr::create(Value: NumVGPRsForWavesPerEU, Ctx&: OutContext),
571 DynamicVGPRBlockSize: MFI.getDynamicVGPRBlockSize(), STM, Ctx&: OutContext);
572 uint64_t Occupancy;
573
574 const auto [MinWEU, MaxWEU] = AMDGPU::getIntegerPairAttribute(
575 F, Name: "amdgpu-waves-per-eu", Default: {0, 0}, OnlyFirstRequired: true);
576
577 if (TryGetMCExprValue(OccupancyExpr, Occupancy) && Occupancy < MinWEU) {
578 DiagnosticInfoOptimizationFailure Diag(
579 F, F.getSubprogram(),
580 "failed to meet occupancy target given by 'amdgpu-waves-per-eu' in "
581 "'" +
582 F.getName() + "': desired occupancy was " + Twine(MinWEU) +
583 ", final occupancy is " + Twine(Occupancy));
584 F.getContext().diagnose(DI: Diag);
585 return;
586 }
587 }
588 }
589}
590
591static void appendTypeEncoding(std::string &Enc, Type *Ty, const DataLayout &DL,
592 bool IsReturnType) {
593 if (Ty->isVoidTy()) {
594 Enc += 'v';
595 return;
596 }
597 unsigned Bits = DL.getTypeSizeInBits(Ty);
598 // Zero-sized non-void types (e.g. `{}` or `[0 x i8]`) consume no ABI
599 // registers. For returns, emit the same no-result marker as void so the
600 // parameter encoding still has an explicit return-type prefix.
601 if (Bits == 0) {
602 if (IsReturnType)
603 Enc += 'v';
604 return;
605 }
606 if (Bits <= 32)
607 Enc += 'i';
608 else if (Bits <= 64)
609 Enc += 'l';
610 else
611 Enc.append(n: divideCeil(Numerator: Bits, Denominator: 32), c: 'i');
612}
613
614static std::string computeTypeId(const FunctionType *FTy,
615 const DataLayout &DL) {
616 std::string Enc;
617 appendTypeEncoding(Enc, Ty: FTy->getReturnType(), DL, /*IsReturnType=*/true);
618 for (Type *ParamTy : FTy->params())
619 appendTypeEncoding(Enc, Ty: ParamTy, DL, /*IsReturnType=*/false);
620 return Enc;
621}
622
623void AMDGPUAsmPrinter::collectCallEdge(const MachineInstr &MI) {
624 if (!AMDGPUTargetMachine::EnableObjectLinking)
625 return;
626 const SIInstrInfo *TII = MF->getSubtarget<GCNSubtarget>().getInstrInfo();
627 const MachineOperand *Callee =
628 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::callee);
629 if (!Callee || !Callee->isGlobal())
630 return;
631 DirectCallEdges.insert(
632 X: {getSymbol(GV: &MF->getFunction()), getSymbol(GV: Callee->getGlobal())});
633}
634
635void AMDGPUAsmPrinter::emitAMDGPUInfo(Module &M) {
636 if (!AMDGPUTargetMachine::EnableObjectLinking)
637 return;
638
639 const NamedMDNode *LDSMD = M.getNamedMetadata(Name: "amdgpu.lds.uses");
640 bool HasLDSUses = LDSMD && LDSMD->getNumOperands() > 0;
641
642 const NamedMDNode *BarMD = M.getNamedMetadata(Name: "amdgpu.named_barrier.uses");
643 bool HasNamedBarriers = BarMD && BarMD->getNumOperands() > 0;
644
645 // Collect address-taken functions (with type IDs) and indirect call sites.
646 DenseMap<const Function *, std::string> AddrTakenTypeIds;
647 using IndirectCallInfo = std::pair<const Function *, std::string>;
648 SmallVector<IndirectCallInfo, 8> IndirectCalls;
649
650 for (const Function &F : M) {
651 bool IsKernel = AMDGPU::isKernel(CC: F.getCallingConv());
652
653 if (!IsKernel && F.hasAddressTaken(/*PutOffender=*/nullptr,
654 /*IgnoreCallbackUses=*/false,
655 /*IgnoreAssumeLikeCalls=*/true,
656 /*IgnoreLLVMUsed=*/IngoreLLVMUsed: true)) {
657 AddrTakenTypeIds[&F] =
658 computeTypeId(FTy: F.getFunctionType(), DL: M.getDataLayout());
659 }
660
661 if (F.isDeclaration())
662 continue;
663
664 StringSet<> SeenTypeIds;
665 for (const BasicBlock &BB : F) {
666 for (const Instruction &I : BB) {
667 const auto *CB = dyn_cast<CallBase>(Val: &I);
668 if (!CB || !CB->isIndirectCall())
669 continue;
670 std::string TId =
671 computeTypeId(FTy: CB->getFunctionType(), DL: M.getDataLayout());
672 if (SeenTypeIds.insert(key: TId).second)
673 IndirectCalls.push_back(Elt: {&F, std::move(TId)});
674 }
675 }
676 }
677
678 if (FunctionInfos.empty() && DirectCallEdges.empty() && !HasLDSUses &&
679 !HasNamedBarriers && AddrTakenTypeIds.empty() && IndirectCalls.empty())
680 return;
681
682 AMDGPU::InfoSectionData Data;
683 Data.Funcs = std::move(FunctionInfos);
684
685 for (auto &[F, TypeId] : AddrTakenTypeIds) {
686 MCSymbol *Sym = getSymbol(GV: F);
687 Data.TypeIds.push_back(Elt: {Sym, TypeId});
688 }
689
690 for (auto &[CallerSym, CalleeSym] : DirectCallEdges)
691 Data.Calls.push_back(Elt: {CallerSym, CalleeSym});
692 DirectCallEdges.clear();
693
694 if (HasLDSUses) {
695 for (const MDNode *N : LDSMD->operands()) {
696 auto *Func = mdconst::extract<Function>(MD: N->getOperand(I: 0));
697 auto *LdsVar = mdconst::extract<GlobalVariable>(MD: N->getOperand(I: 1));
698 Data.Uses.push_back(Elt: {getSymbol(GV: Func), getSymbol(GV: LdsVar)});
699 }
700 }
701
702 if (HasNamedBarriers) {
703 for (const MDNode *N : BarMD->operands()) {
704 auto *BarVar = mdconst::extract<GlobalVariable>(MD: N->getOperand(I: 0));
705 MCSymbol *BarSym = getSymbol(GV: BarVar);
706 for (unsigned I = 1, E = N->getNumOperands(); I < E; ++I) {
707 auto *Func = mdconst::extract<Function>(MD: N->getOperand(I));
708 Data.Uses.push_back(Elt: {getSymbol(GV: Func), BarSym});
709 }
710 }
711 }
712
713 for (auto &[Caller, Enc] : IndirectCalls) {
714 MCSymbol *CallerSym = getSymbol(GV: Caller);
715 Data.IndirectCalls.push_back(Elt: {CallerSym, Enc});
716 }
717
718 getTargetStreamer()->emitAMDGPUInfo(Data);
719}
720
721bool AMDGPUAsmPrinter::doFinalization(Module &M) {
722 const Triple &TT = M.getTargetTriple();
723
724 // Pad with s_code_end to help tools and guard against instruction prefetch
725 // causing stale data in caches. Arguably this should be done by the linker,
726 // which is why this isn't done for Mesa.
727 // Don't do it if there is no code.
728 const MCSubtargetInfo &STI = *getGlobalSTI();
729 if ((AMDGPU::isGFX10Plus(STI) || AMDGPU::isGFX90A(STI)) &&
730 (TT.getOS() == Triple::AMDHSA || TT.getOS() == Triple::AMDPAL)) {
731 MCSection *TextSect = getObjFileLowering().getTextSection();
732 if (TextSect->hasInstructions()) {
733 OutStreamer->switchSection(Section: TextSect);
734 getTargetStreamer()->EmitCodeEnd(STI);
735 }
736 }
737
738 // Emit the unified .amdgpu.info section (per-function resources, call graph,
739 // LDS/named-barrier use edges, indirect calls, and address-taken type IDs).
740 emitAMDGPUInfo(M);
741
742 // Assign expressions which can only be resolved when all other functions are
743 // known.
744 RI.finalize(OutContext);
745
746 // Switch section and emit all GPR maximums within the processed module.
747 OutStreamer->pushSection();
748 MCSectionELF *MaxGPRSection =
749 OutContext.getELFSection(Section: ".AMDGPU.gpr_maximums", Type: ELF::SHT_PROGBITS, Flags: 0);
750 OutStreamer->switchSection(Section: MaxGPRSection);
751 getTargetStreamer()->EmitMCResourceMaximums(
752 MaxVGPR: RI.getMaxVGPRSymbol(OutContext), MaxAGPR: RI.getMaxAGPRSymbol(OutContext),
753 MaxSGPR: RI.getMaxSGPRSymbol(OutContext), MaxNamedBarrier: RI.getMaxNamedBarrierSymbol(OutContext));
754 OutStreamer->popSection();
755
756 // In the object-linking pipeline per-function resource MCExprs reference
757 // external callee symbols that cannot be evaluated here, so cross-TU limit
758 // checks would silently no-op for every non-leaf function. Defer resource
759 // sanity checking to the linker, which re-validates against the aggregated
760 // call graph in the combined .amdgpu.info metadata.
761 if (!AMDGPUTargetMachine::EnableObjectLinking) {
762 for (Function &F : M.functions())
763 validateMCResourceInfo(F);
764 }
765
766 RI.reset();
767
768 return AsmPrinter::doFinalization(M);
769}
770
771SmallString<128> AMDGPUAsmPrinter::getMCExprStr(const MCExpr *Value) {
772 SmallString<128> Str;
773 raw_svector_ostream OSS(Str);
774 auto &Streamer = getTargetStreamer()->getStreamer();
775 auto &Context = Streamer.getContext();
776 const MCExpr *New = foldAMDGPUMCExpr(Expr: Value, Ctx&: Context);
777 printAMDGPUMCExpr(Expr: New, OS&: OSS, MAI: &MAI);
778 return Str;
779}
780
781// Print comments that apply to both callable functions and entry points.
782void AMDGPUAsmPrinter::emitCommonFunctionComments(
783 const MCExpr *NumVGPR, const MCExpr *NumAGPR, const MCExpr *TotalNumVGPR,
784 const MCExpr *NumSGPR, const MCExpr *ScratchSize, uint64_t CodeSize,
785 const AMDGPUMachineFunctionInfo *MFI) {
786 OutStreamer->emitRawComment(T: " codeLenInByte = " + Twine(CodeSize), TabPrefix: false);
787 OutStreamer->emitRawComment(T: " TotalNumSgprs: " + getMCExprStr(Value: NumSGPR),
788 TabPrefix: false);
789 OutStreamer->emitRawComment(T: " NumVgprs: " + getMCExprStr(Value: NumVGPR), TabPrefix: false);
790 if (NumAGPR && TotalNumVGPR) {
791 OutStreamer->emitRawComment(T: " NumAgprs: " + getMCExprStr(Value: NumAGPR), TabPrefix: false);
792 OutStreamer->emitRawComment(T: " TotalNumVgprs: " + getMCExprStr(Value: TotalNumVGPR),
793 TabPrefix: false);
794 }
795 OutStreamer->emitRawComment(T: " ScratchSize: " + getMCExprStr(Value: ScratchSize),
796 TabPrefix: false);
797 OutStreamer->emitRawComment(T: " MemoryBound: " + Twine(MFI->isMemoryBound()),
798 TabPrefix: false);
799}
800
801const MCExpr *AMDGPUAsmPrinter::getAmdhsaKernelCodeProperties(
802 const MachineFunction &MF) const {
803 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
804 MCContext &Ctx = MF.getContext();
805 uint16_t KernelCodeProperties = 0;
806 const GCNUserSGPRUsageInfo &UserSGPRInfo = MFI.getUserSGPRInfo();
807
808 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
809 KernelCodeProperties |=
810 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER;
811 }
812 if (UserSGPRInfo.hasDispatchPtr()) {
813 KernelCodeProperties |=
814 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR;
815 }
816 if (UserSGPRInfo.hasQueuePtr()) {
817 KernelCodeProperties |= amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR;
818 }
819 if (UserSGPRInfo.hasKernargSegmentPtr()) {
820 KernelCodeProperties |=
821 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR;
822 }
823 if (UserSGPRInfo.hasDispatchID()) {
824 KernelCodeProperties |=
825 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID;
826 }
827 if (UserSGPRInfo.hasFlatScratchInit()) {
828 KernelCodeProperties |=
829 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT;
830 }
831 if (UserSGPRInfo.hasPrivateSegmentSize()) {
832 KernelCodeProperties |=
833 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE;
834 }
835 if (MF.getSubtarget<GCNSubtarget>().isWave32()) {
836 KernelCodeProperties |=
837 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;
838 }
839
840 // CurrentProgramInfo.DynamicCallStack is a MCExpr and could be
841 // un-evaluatable at this point so it cannot be conditionally checked here.
842 // Instead, we'll directly shift the possibly unknown MCExpr into its place
843 // and bitwise-or it into KernelCodeProperties.
844 const MCExpr *KernelCodePropExpr =
845 MCConstantExpr::create(Value: KernelCodeProperties, Ctx);
846 const MCExpr *OrValue = MCConstantExpr::create(
847 Value: amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK_SHIFT, Ctx);
848 OrValue = MCBinaryExpr::createShl(LHS: CurrentProgramInfo.DynamicCallStack,
849 RHS: OrValue, Ctx);
850 KernelCodePropExpr = MCBinaryExpr::createOr(LHS: KernelCodePropExpr, RHS: OrValue, Ctx);
851
852 return KernelCodePropExpr;
853}
854
855MCKernelDescriptor
856AMDGPUAsmPrinter::getAmdhsaKernelDescriptor(const MachineFunction &MF,
857 const SIProgramInfo &PI) const {
858 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
859 const Function &F = MF.getFunction();
860 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
861 MCContext &Ctx = MF.getContext();
862
863 MCKernelDescriptor KernelDescriptor;
864
865 KernelDescriptor.group_segment_fixed_size =
866 MCConstantExpr::create(Value: PI.LDSSize, Ctx);
867 KernelDescriptor.private_segment_fixed_size = PI.ScratchSize;
868
869 Align MaxKernArgAlign;
870 KernelDescriptor.kernarg_size = MCConstantExpr::create(
871 Value: STM.getKernArgSegmentSize(F, MaxAlign&: MaxKernArgAlign), Ctx);
872
873 KernelDescriptor.compute_pgm_rsrc1 = PI.getComputePGMRSrc1(ST: STM, Ctx);
874 KernelDescriptor.compute_pgm_rsrc2 = PI.getComputePGMRSrc2(ST: STM, Ctx);
875 KernelDescriptor.kernel_code_properties = getAmdhsaKernelCodeProperties(MF);
876
877 int64_t PGM_Rsrc3 = 1;
878 bool EvaluatableRsrc3 =
879 CurrentProgramInfo.ComputePGMRSrc3->evaluateAsAbsolute(Res&: PGM_Rsrc3);
880 (void)PGM_Rsrc3;
881 (void)EvaluatableRsrc3;
882 assert(STM.getGeneration() >= AMDGPUSubtarget::GFX10 ||
883 STM.hasGFX90AInsts() || STM.hasGFX1250Insts() || !EvaluatableRsrc3 ||
884 static_cast<uint64_t>(PGM_Rsrc3) == 0);
885 KernelDescriptor.compute_pgm_rsrc3 = CurrentProgramInfo.ComputePGMRSrc3;
886
887 KernelDescriptor.kernarg_preload = MCConstantExpr::create(
888 Value: AMDGPU::hasKernargPreload(STI: STM) ? Info->getNumKernargPreloadedSGPRs() : 0,
889 Ctx);
890
891 return KernelDescriptor;
892}
893
894bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
895 // Init target streamer lazily on the first function so that previous passes
896 // can set metadata.
897 if (!IsTargetStreamerInitialized)
898 initTargetStreamer(M&: *MF.getFunction().getParent());
899
900 ResourceUsage = GetResourceUsage(MF);
901 CurrentProgramInfo.reset(MF);
902
903 const AMDGPUMachineFunctionInfo *MFI =
904 MF.getInfo<AMDGPUMachineFunctionInfo>();
905 MCContext &Ctx = MF.getContext();
906
907 // The starting address of all shader programs must be 256 bytes aligned.
908 // Regular functions just need the basic required instruction alignment.
909 MF.ensureAlignment(A: MFI->isEntryFunction() ? Align(256) : Align(4));
910
911 SetupMachineFunction(MF);
912
913 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
914 MCContext &Context = getObjFileLowering().getContext();
915 // FIXME: This should be an explicit check for Mesa.
916 if (!STM.isAmdHsaOS() && !STM.isAmdPalOS()) {
917 MCSectionELF *ConfigSection =
918 Context.getELFSection(Section: ".AMDGPU.config", Type: ELF::SHT_PROGBITS, Flags: 0);
919 OutStreamer->switchSection(Section: ConfigSection);
920 }
921
922 RI.gatherResourceInfo(MF, FRI: *ResourceUsage, OutContext);
923
924 if (AMDGPUTargetMachine::EnableObjectLinking) {
925 const AMDGPUResourceUsageAnalysisWrapperPass::FunctionResourceInfo &RU =
926 *ResourceUsage;
927 FunctionInfos.push_back(
928 Elt: {/*NumSGPR=*/static_cast<uint32_t>(RU.NumExplicitSGPR),
929 /*NumArchVGPR=*/static_cast<uint32_t>(RU.NumVGPR),
930 /*NumAccVGPR=*/static_cast<uint32_t>(RU.NumAGPR),
931 /*PrivateSegmentSize=*/static_cast<uint32_t>(RU.PrivateSegmentSize),
932 /*UsesVCC=*/RU.UsesVCC,
933 /*UsesFlatScratch=*/RU.UsesFlatScratch,
934 /*HasDynStack=*/RU.HasDynamicallySizedStack,
935 /*Sym=*/getSymbol(GV: &MF.getFunction())});
936 }
937
938 if (MFI->isModuleEntryFunction()) {
939 getSIProgramInfo(Out&: CurrentProgramInfo, MF);
940 }
941
942 if (STM.isAmdPalOS()) {
943 if (MFI->isEntryFunction())
944 EmitPALMetadata(MF, KernelInfo: CurrentProgramInfo);
945 else if (MFI->isModuleEntryFunction())
946 emitPALFunctionMetadata(MF);
947 } else if (!STM.isAmdHsaOS()) {
948 EmitProgramInfoSI(MF, KernelInfo: CurrentProgramInfo);
949 }
950
951 DumpCodeInstEmitter = nullptr;
952 if (STM.dumpCode()) {
953 // For -dumpcode, get the assembler out of the streamer. This only works
954 // with -filetype=obj.
955 MCAssembler *Assembler = OutStreamer->getAssemblerPtr();
956 if (Assembler)
957 DumpCodeInstEmitter = Assembler->getEmitterPtr();
958 }
959
960 DisasmLines.clear();
961 HexLines.clear();
962 DisasmLineMaxLen = 0;
963
964 emitFunctionBody();
965
966 emitResourceUsageRemarks(MF, CurrentProgramInfo, isModuleEntryFunction: MFI->isModuleEntryFunction(),
967 hasMAIInsts: STM.hasMAIInsts());
968
969 {
970 using RIK = MCResourceInfo::ResourceInfoKind;
971 getTargetStreamer()->EmitMCResourceInfo(
972 NumVGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumVGPR, OutContext),
973 NumAGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumAGPR, OutContext),
974 NumExplicitSGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumSGPR, OutContext),
975 NumNamedBarrier: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumNamedBarrier,
976 OutContext),
977 PrivateSegmentSize: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_PrivateSegSize,
978 OutContext),
979 UsesVCC: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_UsesVCC, OutContext),
980 UsesFlatScratch: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_UsesFlatScratch,
981 OutContext),
982 HasDynamicallySizedStack: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_HasDynSizedStack,
983 OutContext),
984 HasRecursion: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_HasRecursion,
985 OutContext),
986 HasIndirectCall: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_HasIndirectCall,
987 OutContext));
988 }
989
990 // Emit _dvgpr$ symbol when appropriate.
991 emitDVgprSymbol(MF);
992
993 if (isVerbose()) {
994 MCSectionELF *CommentSection =
995 Context.getELFSection(Section: ".AMDGPU.csdata", Type: ELF::SHT_PROGBITS, Flags: 0);
996 OutStreamer->switchSection(Section: CommentSection);
997
998 if (!MFI->isEntryFunction()) {
999 using RIK = MCResourceInfo::ResourceInfoKind;
1000 OutStreamer->emitRawComment(T: " Function info:", TabPrefix: false);
1001
1002 emitCommonFunctionComments(
1003 NumVGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumVGPR, OutContext)
1004 ->getVariableValue(),
1005 NumAGPR: STM.hasMAIInsts() ? RI.getSymbol(FuncName: CurrentFnSym->getName(),
1006 RIK: RIK::RIK_NumAGPR, OutContext)
1007 ->getVariableValue()
1008 : nullptr,
1009 TotalNumVGPR: RI.createTotalNumVGPRs(MF, Ctx),
1010 NumSGPR: RI.createTotalNumSGPRs(
1011 MF,
1012 hasXnack: MF.getSubtarget<GCNSubtarget>().getTargetID().isXnackOnOrAny(),
1013 Ctx),
1014 ScratchSize: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_PrivateSegSize,
1015 OutContext)
1016 ->getVariableValue(),
1017 CodeSize: CurrentProgramInfo.getFunctionCodeSize(MF), MFI);
1018 return false;
1019 }
1020
1021 OutStreamer->emitRawComment(T: " Kernel info:", TabPrefix: false);
1022 emitCommonFunctionComments(
1023 NumVGPR: CurrentProgramInfo.NumArchVGPR,
1024 NumAGPR: STM.hasMAIInsts() ? CurrentProgramInfo.NumAccVGPR : nullptr,
1025 TotalNumVGPR: CurrentProgramInfo.NumVGPR, NumSGPR: CurrentProgramInfo.NumSGPR,
1026 ScratchSize: CurrentProgramInfo.ScratchSize,
1027 CodeSize: CurrentProgramInfo.getFunctionCodeSize(MF), MFI);
1028
1029 OutStreamer->emitRawComment(
1030 T: " FloatMode: " + Twine(CurrentProgramInfo.FloatMode), TabPrefix: false);
1031 OutStreamer->emitRawComment(
1032 T: " IeeeMode: " + Twine(CurrentProgramInfo.IEEEMode), TabPrefix: false);
1033 OutStreamer->emitRawComment(
1034 T: " LDSByteSize: " + Twine(CurrentProgramInfo.LDSSize) +
1035 " bytes/workgroup (compile time only)",
1036 TabPrefix: false);
1037
1038 OutStreamer->emitRawComment(
1039 T: " SGPRBlocks: " + getMCExprStr(Value: CurrentProgramInfo.SGPRBlocks), TabPrefix: false);
1040
1041 OutStreamer->emitRawComment(
1042 T: " VGPRBlocks: " + getMCExprStr(Value: CurrentProgramInfo.VGPRBlocks), TabPrefix: false);
1043
1044 OutStreamer->emitRawComment(
1045 T: " NumSGPRsForWavesPerEU: " +
1046 getMCExprStr(Value: CurrentProgramInfo.NumSGPRsForWavesPerEU),
1047 TabPrefix: false);
1048 OutStreamer->emitRawComment(
1049 T: " NumVGPRsForWavesPerEU: " +
1050 getMCExprStr(Value: CurrentProgramInfo.NumVGPRsForWavesPerEU),
1051 TabPrefix: false);
1052
1053 if (STM.hasGFX90AInsts()) {
1054 const MCExpr *AdjustedAccum = MCBinaryExpr::createAdd(
1055 LHS: CurrentProgramInfo.AccumOffset, RHS: MCConstantExpr::create(Value: 1, Ctx), Ctx);
1056 AdjustedAccum = MCBinaryExpr::createMul(
1057 LHS: AdjustedAccum, RHS: MCConstantExpr::create(Value: 4, Ctx), Ctx);
1058 OutStreamer->emitRawComment(
1059 T: " AccumOffset: " + getMCExprStr(Value: AdjustedAccum), TabPrefix: false);
1060 }
1061
1062 if (STM.hasGFX1250Insts())
1063 OutStreamer->emitRawComment(
1064 T: " NamedBarCnt: " + getMCExprStr(Value: CurrentProgramInfo.NamedBarCnt),
1065 TabPrefix: false);
1066
1067 OutStreamer->emitRawComment(
1068 T: " Occupancy: " + getMCExprStr(Value: CurrentProgramInfo.Occupancy), TabPrefix: false);
1069
1070 OutStreamer->emitRawComment(
1071 T: " WaveLimiterHint : " + Twine(MFI->needsWaveLimiter()), TabPrefix: false);
1072
1073 OutStreamer->emitRawComment(
1074 T: " COMPUTE_PGM_RSRC2:SCRATCH_EN: " +
1075 getMCExprStr(Value: CurrentProgramInfo.ScratchEnable),
1076 TabPrefix: false);
1077 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:USER_SGPR: " +
1078 Twine(CurrentProgramInfo.UserSGPR),
1079 TabPrefix: false);
1080 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TRAP_HANDLER: " +
1081 Twine(CurrentProgramInfo.TrapHandlerEnable),
1082 TabPrefix: false);
1083 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TGID_X_EN: " +
1084 Twine(CurrentProgramInfo.TGIdXEnable),
1085 TabPrefix: false);
1086 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TGID_Y_EN: " +
1087 Twine(CurrentProgramInfo.TGIdYEnable),
1088 TabPrefix: false);
1089 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TGID_Z_EN: " +
1090 Twine(CurrentProgramInfo.TGIdZEnable),
1091 TabPrefix: false);
1092 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TIDIG_COMP_CNT: " +
1093 Twine(CurrentProgramInfo.TIdIGCompCount),
1094 TabPrefix: false);
1095
1096 [[maybe_unused]] int64_t PGMRSrc3;
1097 assert(STM.getGeneration() >= AMDGPUSubtarget::GFX10 ||
1098 STM.hasGFX90AInsts() || STM.hasGFX1250Insts() ||
1099 (CurrentProgramInfo.ComputePGMRSrc3->evaluateAsAbsolute(PGMRSrc3) &&
1100 static_cast<uint64_t>(PGMRSrc3) == 0));
1101 if (STM.hasGFX90AInsts()) {
1102 OutStreamer->emitRawComment(
1103 T: " COMPUTE_PGM_RSRC3_GFX90A:ACCUM_OFFSET: " +
1104 getMCExprStr(Value: MCKernelDescriptor::bits_get(
1105 Src: CurrentProgramInfo.ComputePGMRSrc3,
1106 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
1107 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, Ctx)),
1108 TabPrefix: false);
1109 OutStreamer->emitRawComment(
1110 T: " COMPUTE_PGM_RSRC3_GFX90A:TG_SPLIT: " +
1111 getMCExprStr(Value: MCKernelDescriptor::bits_get(
1112 Src: CurrentProgramInfo.ComputePGMRSrc3,
1113 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT,
1114 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, Ctx)),
1115 TabPrefix: false);
1116 }
1117 }
1118
1119 if (DumpCodeInstEmitter) {
1120
1121 OutStreamer->switchSection(
1122 Section: Context.getELFSection(Section: ".AMDGPU.disasm", Type: ELF::SHT_PROGBITS, Flags: 0));
1123
1124 for (size_t i = 0; i < DisasmLines.size(); ++i) {
1125 std::string Comment = "\n";
1126 if (!HexLines[i].empty()) {
1127 Comment = std::string(DisasmLineMaxLen - DisasmLines[i].size(), ' ');
1128 Comment += " ; " + HexLines[i] + "\n";
1129 }
1130
1131 OutStreamer->emitBytes(Data: StringRef(DisasmLines[i]));
1132 OutStreamer->emitBytes(Data: StringRef(Comment));
1133 }
1134 }
1135
1136 return false;
1137}
1138
1139// When appropriate, add a _dvgpr$ symbol, with the value of the function
1140// symbol, plus an offset encoding one less than the number of VGPR blocks used
1141// by the function in bits 5..3 of the symbol value. A "VGPR block" can be
1142// either 16 VGPRs (for a max of 128), or 32 VGPRs (for a max of 256). This is
1143// used by a front-end to have functions that are chained rather than called,
1144// and a dispatcher that dynamically resizes the VGPR count before dispatching
1145// to a function.
1146void AMDGPUAsmPrinter::emitDVgprSymbol(MachineFunction &MF) {
1147 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
1148 if (MFI.isDynamicVGPREnabled() &&
1149 MF.getFunction().getCallingConv() == CallingConv::AMDGPU_CS_Chain) {
1150 MCContext &Ctx = MF.getContext();
1151 unsigned BlockSize = MFI.getDynamicVGPRBlockSize();
1152
1153 const MCExpr *EncodedBlocks;
1154 MCValue NumVGPRs;
1155 if (CurrentProgramInfo.NumVGPRsForWavesPerEU->evaluateAsRelocatable(
1156 Res&: NumVGPRs, Asm: nullptr) &&
1157 NumVGPRs.isAbsolute()) {
1158
1159 // Calculate number of VGPR blocks.
1160 // Treat 0 VGPRs as 1 VGPR to avoid underflowing.
1161 unsigned NumBlocks =
1162 divideCeil(Numerator: std::max(a: unsigned(NumVGPRs.getConstant()), b: 1U), Denominator: BlockSize);
1163
1164 if (NumBlocks > AMDGPU::IsaInfo::MaxDynamicVGPRBlocks) {
1165 OutContext.reportError(
1166 L: {}, Msg: "DVGPR block count " + Twine(NumBlocks) +
1167 " exceeds maximum of " +
1168 Twine(AMDGPU::IsaInfo::MaxDynamicVGPRBlocks) +
1169 " for __dvgpr$ symbol for '" +
1170 Twine(CurrentFnSym->getName()) + "'");
1171 return;
1172 }
1173 unsigned EncodedNumBlocks = (NumBlocks - 1) << 3;
1174 EncodedBlocks = MCConstantExpr::create(Value: EncodedNumBlocks, Ctx);
1175 } else {
1176 // Value not yet available so build a symbolic MCExpr:
1177 // ((alignTo(max(NumVGPRs, 1), BlockSize) / BlockSize - 1) << 3
1178 const MCExpr *One = MCConstantExpr::create(Value: 1, Ctx);
1179 const MCExpr *BlockSizeConst = MCConstantExpr::create(Value: BlockSize, Ctx);
1180 const MCExpr *MaxVGPRs = AMDGPUMCExpr::createMax(
1181 Args: {CurrentProgramInfo.NumVGPRsForWavesPerEU, One}, Ctx);
1182 const MCExpr *NumBlocks = MCBinaryExpr::createDiv(
1183 LHS: AMDGPUMCExpr::createAlignTo(Value: MaxVGPRs, Align: BlockSizeConst, Ctx),
1184 RHS: BlockSizeConst, Ctx);
1185 EncodedBlocks =
1186 MCBinaryExpr::createShl(LHS: MCBinaryExpr::createSub(LHS: NumBlocks, RHS: One, Ctx),
1187 RHS: MCConstantExpr::create(Value: 3, Ctx), Ctx);
1188 }
1189
1190 // Add to function symbol to create _dvgpr$ symbol.
1191 const MCExpr *DVgprFuncVal = MCBinaryExpr::createAdd(
1192 LHS: MCSymbolRefExpr::create(Symbol: CurrentFnSym, Ctx), RHS: EncodedBlocks, Ctx);
1193 MCSymbol *DVgprFuncSym =
1194 Ctx.getOrCreateSymbol(Name: Twine("_dvgpr$") + CurrentFnSym->getName());
1195 OutStreamer->emitAssignment(Symbol: DVgprFuncSym, Value: DVgprFuncVal);
1196 emitVisibility(Sym: DVgprFuncSym, Visibility: MF.getFunction().getVisibility());
1197 emitLinkage(GV: &MF.getFunction(), GVSym: DVgprFuncSym);
1198 }
1199}
1200
1201// TODO: Fold this into emitFunctionBodyStart.
1202void AMDGPUAsmPrinter::initializeTargetID(const Module &M) {
1203 getTargetStreamer()->initializeTargetID(STI: *getGlobalSTI());
1204
1205 auto &TSTargetID = getTargetStreamer()->getTargetID();
1206
1207 // Error if -mattr specified xnack or sramecc.
1208 // TODO: Remove this when subtarget features removed.
1209 StringRef FeatureString = getGlobalSTI()->getFeatureString();
1210 if (FeatureString.contains(Other: "xnack")) {
1211 M.getContext().diagnose(DI: DiagnosticInfoGeneric(
1212 "xnack/sramecc should be specified via module flags. "
1213 "Use module flag 'amdgpu.xnack' instead of subtarget feature",
1214 DS_Error));
1215 }
1216 if (FeatureString.contains(Other: "sramecc")) {
1217 M.getContext().diagnose(DI: DiagnosticInfoGeneric(
1218 "xnack/sramecc should be specified via module flags. "
1219 "Use module flag 'amdgpu.sramecc' instead of subtarget feature",
1220 DS_Error));
1221 }
1222
1223 // Apply xnack/sramecc settings from module flags.
1224 if (getGlobalSTI()->getFeatureBits().test(I: AMDGPU::FeatureXNACKOnOffModes)) {
1225 AMDGPU::TargetIDSetting Setting =
1226 GCNTargetMachine::getTargetIDSettingFromModuleFlag(M, FlagName: "amdgpu.xnack");
1227 if (Setting != AMDGPU::TargetIDSetting::Any)
1228 TSTargetID->setXnackSetting(Setting);
1229 }
1230
1231 if (getGlobalSTI()->getFeatureBits().test(I: AMDGPU::FeatureSupportsSRAMECC)) {
1232 AMDGPU::TargetIDSetting Setting =
1233 GCNTargetMachine::getTargetIDSettingFromModuleFlag(M, FlagName: "amdgpu.sramecc");
1234 if (Setting != AMDGPU::TargetIDSetting::Any)
1235 TSTargetID->setSramEccSetting(Setting);
1236 }
1237}
1238
1239// AccumOffset computed for the MCExpr equivalent of:
1240// alignTo(std::max(1, NumVGPR), 4) / 4 - 1;
1241static const MCExpr *computeAccumOffset(const MCExpr *NumVGPR, MCContext &Ctx) {
1242 const MCExpr *ConstFour = MCConstantExpr::create(Value: 4, Ctx);
1243 const MCExpr *ConstOne = MCConstantExpr::create(Value: 1, Ctx);
1244
1245 // Can't be lower than 1 for subsequent alignTo.
1246 const MCExpr *MaximumTaken =
1247 AMDGPUMCExpr::createMax(Args: {ConstOne, NumVGPR}, Ctx);
1248
1249 // Practically, it's computing divideCeil(MaximumTaken, 4).
1250 const MCExpr *DivCeil = MCBinaryExpr::createDiv(
1251 LHS: AMDGPUMCExpr::createAlignTo(Value: MaximumTaken, Align: ConstFour, Ctx), RHS: ConstFour,
1252 Ctx);
1253
1254 return MCBinaryExpr::createSub(LHS: DivCeil, RHS: ConstOne, Ctx);
1255}
1256
1257void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
1258 const MachineFunction &MF) {
1259 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1260 MCContext &Ctx = MF.getContext();
1261
1262 auto CreateExpr = [&Ctx](int64_t Value) {
1263 return MCConstantExpr::create(Value, Ctx);
1264 };
1265
1266 auto TryGetMCExprValue = [](const MCExpr *Value, uint64_t &Res) -> bool {
1267 int64_t Val;
1268 if (Value->evaluateAsAbsolute(Res&: Val)) {
1269 Res = Val;
1270 return true;
1271 }
1272 return false;
1273 };
1274
1275 auto GetSymRefExpr =
1276 [&](MCResourceInfo::ResourceInfoKind RIK) -> const MCExpr * {
1277 MCSymbol *Sym = RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK, OutContext);
1278 return MCSymbolRefExpr::create(Symbol: Sym, Ctx);
1279 };
1280
1281 using RIK = MCResourceInfo::ResourceInfoKind;
1282 ProgInfo.NumArchVGPR = GetSymRefExpr(RIK::RIK_NumVGPR);
1283 ProgInfo.NumAccVGPR = GetSymRefExpr(RIK::RIK_NumAGPR);
1284 ProgInfo.NumVGPR = AMDGPUMCExpr::createTotalNumVGPR(
1285 NumAGPR: ProgInfo.NumAccVGPR, NumVGPR: ProgInfo.NumArchVGPR, Ctx);
1286
1287 ProgInfo.AccumOffset = computeAccumOffset(NumVGPR: ProgInfo.NumArchVGPR, Ctx);
1288 ProgInfo.TgSplit =
1289 STM.hasTgSplitSupport() && AMDGPU::isTgSplitEnabled(F: MF.getFunction());
1290 ProgInfo.NumSGPR = GetSymRefExpr(RIK::RIK_NumSGPR);
1291 ProgInfo.ScratchSize = GetSymRefExpr(RIK::RIK_PrivateSegSize);
1292 ProgInfo.VCCUsed = GetSymRefExpr(RIK::RIK_UsesVCC);
1293 ProgInfo.FlatUsed = GetSymRefExpr(RIK::RIK_UsesFlatScratch);
1294 ProgInfo.DynamicCallStack =
1295 MCBinaryExpr::createOr(LHS: GetSymRefExpr(RIK::RIK_HasDynSizedStack),
1296 RHS: GetSymRefExpr(RIK::RIK_HasRecursion), Ctx);
1297
1298 const MCExpr *BarBlkConst = MCConstantExpr::create(Value: 4, Ctx);
1299 const MCExpr *AlignToBlk = AMDGPUMCExpr::createAlignTo(
1300 Value: GetSymRefExpr(RIK::RIK_NumNamedBarrier), Align: BarBlkConst, Ctx);
1301 ProgInfo.NamedBarCnt = MCBinaryExpr::createDiv(LHS: AlignToBlk, RHS: BarBlkConst, Ctx);
1302
1303 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1304
1305 // The calculations related to SGPR/VGPR blocks are
1306 // duplicated in part in AMDGPUAsmParser::calculateGPRBlocks, and could be
1307 // unified.
1308 const MCExpr *ExtraSGPRs = AMDGPUMCExpr::createExtraSGPRs(
1309 VCCUsed: ProgInfo.VCCUsed, FlatScrUsed: ProgInfo.FlatUsed,
1310 XNACKUsed: getTargetStreamer()->getTargetID()->isXnackOnOrAny(), Ctx);
1311
1312 // Check the addressable register limit before we add ExtraSGPRs.
1313 if (STM.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
1314 !STM.hasSGPRInitBug()) {
1315 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
1316 uint64_t NumSgpr;
1317 if (TryGetMCExprValue(ProgInfo.NumSGPR, NumSgpr) &&
1318 NumSgpr > MaxAddressableNumSGPRs) {
1319 // This can happen due to a compiler bug or when using inline asm.
1320 LLVMContext &Ctx = MF.getFunction().getContext();
1321 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1322 MF.getFunction(), "addressable scalar registers", NumSgpr,
1323 MaxAddressableNumSGPRs, DS_Error, DK_ResourceLimit));
1324 ProgInfo.NumSGPR = CreateExpr(MaxAddressableNumSGPRs - 1);
1325 }
1326 }
1327
1328 // Account for extra SGPRs and VGPRs reserved for debugger use.
1329 ProgInfo.NumSGPR = MCBinaryExpr::createAdd(LHS: ProgInfo.NumSGPR, RHS: ExtraSGPRs, Ctx);
1330
1331 const Function &F = MF.getFunction();
1332
1333 // Ensure there are enough SGPRs and VGPRs for wave dispatch, where wave
1334 // dispatch registers as function args.
1335 unsigned WaveDispatchNumSGPR = MFI->getNumWaveDispatchSGPRs(),
1336 WaveDispatchNumVGPR = MFI->getNumWaveDispatchVGPRs();
1337
1338 if (WaveDispatchNumSGPR) {
1339 ProgInfo.NumSGPR = AMDGPUMCExpr::createMax(
1340 Args: {ProgInfo.NumSGPR,
1341 MCBinaryExpr::createAdd(LHS: CreateExpr(WaveDispatchNumSGPR), RHS: ExtraSGPRs,
1342 Ctx)},
1343 Ctx);
1344 }
1345
1346 if (WaveDispatchNumVGPR) {
1347 ProgInfo.NumArchVGPR = AMDGPUMCExpr::createMax(
1348 Args: {ProgInfo.NumVGPR, CreateExpr(WaveDispatchNumVGPR)}, Ctx);
1349
1350 ProgInfo.NumVGPR = AMDGPUMCExpr::createTotalNumVGPR(
1351 NumAGPR: ProgInfo.NumAccVGPR, NumVGPR: ProgInfo.NumArchVGPR, Ctx);
1352 }
1353
1354 // Adjust number of registers used to meet default/requested minimum/maximum
1355 // number of waves per execution unit request.
1356 unsigned MaxWaves = MFI->getMaxWavesPerEU();
1357 ProgInfo.NumSGPRsForWavesPerEU =
1358 AMDGPUMCExpr::createMax(Args: {ProgInfo.NumSGPR, CreateExpr(1ul),
1359 CreateExpr(STM.getMinNumSGPRs(WavesPerEU: MaxWaves))},
1360 Ctx);
1361 ProgInfo.NumVGPRsForWavesPerEU =
1362 AMDGPUMCExpr::createMax(Args: {ProgInfo.NumVGPR, CreateExpr(1ul),
1363 CreateExpr(STM.getMinNumVGPRs(
1364 WavesPerEU: MaxWaves, DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize()))},
1365 Ctx);
1366
1367 if (STM.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS ||
1368 STM.hasSGPRInitBug()) {
1369 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
1370 uint64_t NumSgpr;
1371 if (TryGetMCExprValue(ProgInfo.NumSGPR, NumSgpr) &&
1372 NumSgpr > MaxAddressableNumSGPRs) {
1373 // This can happen due to a compiler bug or when using inline asm to use
1374 // the registers which are usually reserved for vcc etc.
1375 LLVMContext &Ctx = MF.getFunction().getContext();
1376 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1377 MF.getFunction(), "scalar registers", NumSgpr, MaxAddressableNumSGPRs,
1378 DS_Error, DK_ResourceLimit));
1379 ProgInfo.NumSGPR = CreateExpr(MaxAddressableNumSGPRs);
1380 ProgInfo.NumSGPRsForWavesPerEU = CreateExpr(MaxAddressableNumSGPRs);
1381 }
1382 }
1383
1384 if (STM.hasSGPRInitBug()) {
1385 ProgInfo.NumSGPR =
1386 CreateExpr(AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG);
1387 ProgInfo.NumSGPRsForWavesPerEU =
1388 CreateExpr(AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG);
1389 }
1390
1391 if (MFI->getNumUserSGPRs() > STM.getMaxNumUserSGPRs()) {
1392 LLVMContext &Ctx = MF.getFunction().getContext();
1393 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1394 MF.getFunction(), "user SGPRs", MFI->getNumUserSGPRs(),
1395 STM.getMaxNumUserSGPRs(), DS_Error));
1396 }
1397
1398 if (MFI->getLDSSize() > STM.getAddressableLocalMemorySize()) {
1399 LLVMContext &Ctx = MF.getFunction().getContext();
1400 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1401 MF.getFunction(), "local memory", MFI->getLDSSize(),
1402 STM.getAddressableLocalMemorySize(), DS_Error));
1403 }
1404 // The MCExpr equivalent of getNumSGPRBlocks/getNumVGPRBlocks:
1405 // (alignTo(max(1u, NumGPR), GPREncodingGranule) / GPREncodingGranule) - 1
1406 auto GetNumGPRBlocks = [&CreateExpr, &Ctx](const MCExpr *NumGPR,
1407 unsigned Granule) {
1408 const MCExpr *OneConst = CreateExpr(1ul);
1409 const MCExpr *GranuleConst = CreateExpr(Granule);
1410 const MCExpr *MaxNumGPR = AMDGPUMCExpr::createMax(Args: {NumGPR, OneConst}, Ctx);
1411 const MCExpr *AlignToGPR =
1412 AMDGPUMCExpr::createAlignTo(Value: MaxNumGPR, Align: GranuleConst, Ctx);
1413 const MCExpr *DivGPR =
1414 MCBinaryExpr::createDiv(LHS: AlignToGPR, RHS: GranuleConst, Ctx);
1415 const MCExpr *SubGPR = MCBinaryExpr::createSub(LHS: DivGPR, RHS: OneConst, Ctx);
1416 return SubGPR;
1417 };
1418 // GFX10+ will always allocate 128 SGPRs and this field must be 0
1419 if (STM.getGeneration() >= AMDGPUSubtarget::GFX10) {
1420 ProgInfo.SGPRBlocks = CreateExpr(0ul);
1421 } else {
1422 ProgInfo.SGPRBlocks = GetNumGPRBlocks(ProgInfo.NumSGPRsForWavesPerEU,
1423 IsaInfo::getSGPREncodingGranule(STI: STM));
1424 }
1425 ProgInfo.VGPRBlocks = GetNumGPRBlocks(ProgInfo.NumVGPRsForWavesPerEU,
1426 IsaInfo::getVGPREncodingGranule(STI: STM));
1427
1428 const SIModeRegisterDefaults Mode = MFI->getMode();
1429
1430 // Set the value to initialize FP_ROUND and FP_DENORM parts of the mode
1431 // register.
1432 ProgInfo.FloatMode = getFPMode(Mode);
1433
1434 ProgInfo.IEEEMode = Mode.IEEE;
1435
1436 // Make clamp modifier on NaN input returns 0.
1437 ProgInfo.DX10Clamp = Mode.DX10Clamp;
1438 ProgInfo.SGPRSpill = MFI->getNumSpilledSGPRs();
1439 ProgInfo.VGPRSpill = MFI->getNumSpilledVGPRs();
1440
1441 ProgInfo.LDSSize = MFI->getLDSSize();
1442
1443 unsigned LDSGranularityBytes = getLdsDwGranularity(ST: STM) * 4;
1444 ProgInfo.LDSBlocks =
1445 alignTo(Value: ProgInfo.LDSSize, Align: LDSGranularityBytes) / LDSGranularityBytes;
1446
1447 // The MCExpr equivalent of divideCeil.
1448 auto DivideCeil = [&Ctx](const MCExpr *Numerator, const MCExpr *Denominator) {
1449 const MCExpr *Ceil =
1450 AMDGPUMCExpr::createAlignTo(Value: Numerator, Align: Denominator, Ctx);
1451 return MCBinaryExpr::createDiv(LHS: Ceil, RHS: Denominator, Ctx);
1452 };
1453
1454 // Scratch is allocated in 64-dword or 256-dword blocks.
1455 unsigned ScratchAlignShift =
1456 STM.getGeneration() >= AMDGPUSubtarget::GFX11 ? 8 : 10;
1457 // We need to program the hardware with the amount of scratch memory that
1458 // is used by the entire wave. ProgInfo.ScratchSize is the amount of
1459 // scratch memory used per thread.
1460 ProgInfo.ScratchBlocks = DivideCeil(
1461 MCBinaryExpr::createMul(LHS: ProgInfo.ScratchSize,
1462 RHS: CreateExpr(STM.getWavefrontSize()), Ctx),
1463 CreateExpr(1ULL << ScratchAlignShift));
1464
1465 if (STM.hasSupportsWGP()) {
1466 ProgInfo.WgpMode = STM.isCuModeEnabled() ? 0 : 1;
1467 }
1468
1469 if (getIsaVersion(GPU: getGlobalSTI()->getCPU()).Major >= 10) {
1470 ProgInfo.MemOrdered = 1;
1471 ProgInfo.FwdProgress = !F.hasFnAttribute(Kind: "amdgpu-no-fwd-progress");
1472 }
1473
1474 // 0 = X, 1 = XY, 2 = XYZ
1475 unsigned TIDIGCompCnt = 0;
1476 if (MFI->hasWorkItemIDZ())
1477 TIDIGCompCnt = 2;
1478 else if (MFI->hasWorkItemIDY())
1479 TIDIGCompCnt = 1;
1480
1481 // The private segment wave byte offset is the last of the system SGPRs. We
1482 // initially assumed it was allocated, and may have used it. It shouldn't harm
1483 // anything to disable it if we know the stack isn't used here. We may still
1484 // have emitted code reading it to initialize scratch, but if that's unused
1485 // reading garbage should be OK.
1486 ProgInfo.ScratchEnable = MCBinaryExpr::createLOr(
1487 LHS: MCBinaryExpr::createGT(LHS: ProgInfo.ScratchBlocks,
1488 RHS: MCConstantExpr::create(Value: 0, Ctx), Ctx),
1489 RHS: ProgInfo.DynamicCallStack, Ctx);
1490
1491 ProgInfo.UserSGPR = MFI->getNumUserSGPRs();
1492 // For AMDHSA, TRAP_HANDLER must be zero, as it is populated by the CP.
1493 ProgInfo.TrapHandlerEnable = STM.isAmdHsaOS() ? 0 : STM.hasTrapHandler();
1494 ProgInfo.TGIdXEnable = MFI->hasWorkGroupIDX();
1495 ProgInfo.TGIdYEnable = MFI->hasWorkGroupIDY();
1496 ProgInfo.TGIdZEnable = MFI->hasWorkGroupIDZ();
1497 ProgInfo.TGSizeEnable = MFI->hasWorkGroupInfo();
1498 ProgInfo.TIdIGCompCount = TIDIGCompCnt;
1499 ProgInfo.EXCPEnMSB = 0;
1500 // For AMDHSA, LDS_SIZE must be zero, as it is populated by the CP.
1501 ProgInfo.LdsSize = STM.isAmdHsaOS() ? 0 : ProgInfo.LDSBlocks;
1502 ProgInfo.EXCPEnable = 0;
1503
1504 if (STM.hasGFX90AInsts()) {
1505 ProgInfo.ComputePGMRSrc3 =
1506 setBits(Dst: ProgInfo.ComputePGMRSrc3, Value: ProgInfo.AccumOffset,
1507 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
1508 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT, Ctx);
1509 ProgInfo.ComputePGMRSrc3 =
1510 setBits(Dst: ProgInfo.ComputePGMRSrc3, Value: CreateExpr(ProgInfo.TgSplit),
1511 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT,
1512 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT, Ctx);
1513 }
1514
1515 if (STM.hasGFX1250Insts())
1516 ProgInfo.ComputePGMRSrc3 =
1517 setBits(Dst: ProgInfo.ComputePGMRSrc3, Value: ProgInfo.NamedBarCnt,
1518 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
1519 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT, Ctx);
1520
1521 ProgInfo.Occupancy = createOccupancy(
1522 InitOcc: STM.computeOccupancy(F, LDSSize: ProgInfo.LDSSize).second,
1523 NumSGPRs: ProgInfo.NumSGPRsForWavesPerEU, NumVGPRs: ProgInfo.NumVGPRsForWavesPerEU,
1524 DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize(), STM, Ctx);
1525
1526 const auto [MinWEU, MaxWEU] =
1527 AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-waves-per-eu", Default: {0, 0}, OnlyFirstRequired: true);
1528 uint64_t Occupancy;
1529 if (TryGetMCExprValue(ProgInfo.Occupancy, Occupancy) && Occupancy < MinWEU) {
1530 DiagnosticInfoOptimizationFailure Diag(
1531 F, F.getSubprogram(),
1532 "failed to meet occupancy target given by 'amdgpu-waves-per-eu' in "
1533 "'" +
1534 F.getName() + "': desired occupancy was " + Twine(MinWEU) +
1535 ", final occupancy is " + Twine(Occupancy));
1536 F.getContext().diagnose(DI: Diag);
1537 }
1538}
1539
1540static unsigned getRsrcReg(CallingConv::ID CallConv) {
1541 switch (CallConv) {
1542 default:
1543 [[fallthrough]];
1544 case CallingConv::AMDGPU_CS:
1545 return R_00B848_COMPUTE_PGM_RSRC1;
1546 case CallingConv::AMDGPU_LS:
1547 return R_00B528_SPI_SHADER_PGM_RSRC1_LS;
1548 case CallingConv::AMDGPU_HS:
1549 return R_00B428_SPI_SHADER_PGM_RSRC1_HS;
1550 case CallingConv::AMDGPU_ES:
1551 return R_00B328_SPI_SHADER_PGM_RSRC1_ES;
1552 case CallingConv::AMDGPU_GS:
1553 return R_00B228_SPI_SHADER_PGM_RSRC1_GS;
1554 case CallingConv::AMDGPU_VS:
1555 return R_00B128_SPI_SHADER_PGM_RSRC1_VS;
1556 case CallingConv::AMDGPU_PS:
1557 return R_00B028_SPI_SHADER_PGM_RSRC1_PS;
1558 }
1559}
1560
1561void AMDGPUAsmPrinter::EmitProgramInfoSI(
1562 const MachineFunction &MF, const SIProgramInfo &CurrentProgramInfo) {
1563 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1564 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1565 unsigned RsrcReg = getRsrcReg(CallConv: MF.getFunction().getCallingConv());
1566 MCContext &Ctx = MF.getContext();
1567
1568 // (((Value) & Mask) << Shift)
1569 auto SetBits = [&Ctx](const MCExpr *Value, uint32_t Mask, uint32_t Shift) {
1570 const MCExpr *msk = MCConstantExpr::create(Value: Mask, Ctx);
1571 const MCExpr *shft = MCConstantExpr::create(Value: Shift, Ctx);
1572 return MCBinaryExpr::createShl(LHS: MCBinaryExpr::createAnd(LHS: Value, RHS: msk, Ctx),
1573 RHS: shft, Ctx);
1574 };
1575
1576 auto EmitResolvedOrExpr = [this](const MCExpr *Value, unsigned Size) {
1577 int64_t Val;
1578 if (Value->evaluateAsAbsolute(Res&: Val))
1579 OutStreamer->emitIntValue(Value: static_cast<uint64_t>(Val), Size);
1580 else
1581 OutStreamer->emitValue(Value, Size);
1582 };
1583
1584 if (AMDGPU::isCompute(CC: MF.getFunction().getCallingConv())) {
1585 OutStreamer->emitInt32(R_00B848_COMPUTE_PGM_RSRC1);
1586
1587 EmitResolvedOrExpr(CurrentProgramInfo.getComputePGMRSrc1(ST: STM, Ctx),
1588 /*Size=*/4);
1589
1590 OutStreamer->emitInt32(R_00B84C_COMPUTE_PGM_RSRC2);
1591 EmitResolvedOrExpr(CurrentProgramInfo.getComputePGMRSrc2(ST: STM, Ctx),
1592 /*Size=*/4);
1593
1594 OutStreamer->emitInt32(R_00B860_COMPUTE_TMPRING_SIZE);
1595
1596 // Sets bits according to S_0286E8_WAVESIZE_* mask and shift values for the
1597 // appropriate generation.
1598 if (STM.getGeneration() >= AMDGPUSubtarget::GFX12) {
1599 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1600 /*Mask=*/0x3FFFF, /*Shift=*/12),
1601 /*Size=*/4);
1602 } else if (STM.getGeneration() == AMDGPUSubtarget::GFX11) {
1603 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1604 /*Mask=*/0x7FFF, /*Shift=*/12),
1605 /*Size=*/4);
1606 } else {
1607 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1608 /*Mask=*/0x1FFF, /*Shift=*/12),
1609 /*Size=*/4);
1610 }
1611
1612 // TODO: Should probably note flat usage somewhere. SC emits a "FlatPtr32 =
1613 // 0" comment but I don't see a corresponding field in the register spec.
1614 } else {
1615 OutStreamer->emitInt32(Value: RsrcReg);
1616
1617 const MCExpr *GPRBlocks = MCBinaryExpr::createOr(
1618 LHS: SetBits(CurrentProgramInfo.VGPRBlocks, /*Mask=*/0x3F, /*Shift=*/0),
1619 RHS: SetBits(CurrentProgramInfo.SGPRBlocks, /*Mask=*/0x0F, /*Shift=*/6),
1620 Ctx&: MF.getContext());
1621 EmitResolvedOrExpr(GPRBlocks, /*Size=*/4);
1622 OutStreamer->emitInt32(R_0286E8_SPI_TMPRING_SIZE);
1623
1624 // Sets bits according to S_0286E8_WAVESIZE_* mask and shift values for the
1625 // appropriate generation.
1626 if (STM.getGeneration() >= AMDGPUSubtarget::GFX12) {
1627 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1628 /*Mask=*/0x3FFFF, /*Shift=*/12),
1629 /*Size=*/4);
1630 } else if (STM.getGeneration() == AMDGPUSubtarget::GFX11) {
1631 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1632 /*Mask=*/0x7FFF, /*Shift=*/12),
1633 /*Size=*/4);
1634 } else {
1635 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1636 /*Mask=*/0x1FFF, /*Shift=*/12),
1637 /*Size=*/4);
1638 }
1639 }
1640
1641 if (MF.getFunction().getCallingConv() == CallingConv::AMDGPU_PS) {
1642 OutStreamer->emitInt32(R_00B02C_SPI_SHADER_PGM_RSRC2_PS);
1643 unsigned ExtraLDSSize = STM.getGeneration() >= AMDGPUSubtarget::GFX11
1644 ? divideCeil(Numerator: CurrentProgramInfo.LDSBlocks, Denominator: 2)
1645 : CurrentProgramInfo.LDSBlocks;
1646 OutStreamer->emitInt32(S_00B02C_EXTRA_LDS_SIZE(ExtraLDSSize));
1647 OutStreamer->emitInt32(R_0286CC_SPI_PS_INPUT_ENA);
1648 OutStreamer->emitInt32(Value: MFI->getPSInputEnable());
1649 OutStreamer->emitInt32(R_0286D0_SPI_PS_INPUT_ADDR);
1650 OutStreamer->emitInt32(Value: MFI->getPSInputAddr());
1651 }
1652
1653 OutStreamer->emitInt32(R_SPILLED_SGPRS);
1654 OutStreamer->emitInt32(Value: MFI->getNumSpilledSGPRs());
1655 OutStreamer->emitInt32(R_SPILLED_VGPRS);
1656 OutStreamer->emitInt32(Value: MFI->getNumSpilledVGPRs());
1657}
1658
1659// Helper function to add common PAL Metadata 3.0+
1660static void EmitPALMetadataCommon(AMDGPUPALMetadata *MD,
1661 const SIProgramInfo &CurrentProgramInfo,
1662 CallingConv::ID CC, const GCNSubtarget &ST,
1663 unsigned DynamicVGPRBlockSize) {
1664 if (ST.hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode))
1665 MD->setHwStage(CC, field: ".ieee_mode", Val: (bool)CurrentProgramInfo.IEEEMode);
1666
1667 MD->setHwStage(CC, field: ".wgp_mode", Val: (bool)CurrentProgramInfo.WgpMode);
1668 MD->setHwStage(CC, field: ".mem_ordered", Val: (bool)CurrentProgramInfo.MemOrdered);
1669 MD->setHwStage(CC, field: ".forward_progress", Val: (bool)CurrentProgramInfo.FwdProgress);
1670
1671 if (AMDGPU::isCompute(CC)) {
1672 MD->setHwStage(CC, field: ".trap_present",
1673 Val: (bool)CurrentProgramInfo.TrapHandlerEnable);
1674 MD->setHwStage(CC, field: ".excp_en", Val: CurrentProgramInfo.EXCPEnable);
1675
1676 if (DynamicVGPRBlockSize != 0)
1677 MD->setComputeRegisters(field: ".dynamic_vgpr_en", Val: true);
1678 }
1679
1680 MD->updateHwStageMaximum(
1681 CC, field: ".lds_size",
1682 Val: (unsigned)(CurrentProgramInfo.LdsSize * getLdsDwGranularity(ST) *
1683 sizeof(uint32_t)));
1684}
1685
1686// This is the equivalent of EmitProgramInfoSI above, but for when the OS type
1687// is AMDPAL. It stores each compute/SPI register setting and other PAL
1688// metadata items into the PALMD::Metadata, combining with any provided by the
1689// frontend as LLVM metadata. Once all functions are written, the PAL metadata
1690// is then written as a single block in the .note section.
1691void AMDGPUAsmPrinter::EmitPALMetadata(
1692 const MachineFunction &MF, const SIProgramInfo &CurrentProgramInfo) {
1693 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1694 auto CC = MF.getFunction().getCallingConv();
1695 auto *MD = getTargetStreamer()->getPALMetadata();
1696 auto &Ctx = MF.getContext();
1697
1698 MD->setEntryPoint(CC, Name: MF.getFunction().getName());
1699 MD->setNumUsedVgprs(CC, Val: CurrentProgramInfo.NumVGPRsForWavesPerEU, Ctx);
1700
1701 // For targets that support dynamic VGPRs, set the number of saved dynamic
1702 // VGPRs (if any) in the PAL metadata.
1703 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1704 if (MFI->isDynamicVGPREnabled() &&
1705 MFI->getScratchReservedForDynamicVGPRs() > 0)
1706 MD->setHwStage(CC, field: ".dynamic_vgpr_saved_count",
1707 Val: MFI->getScratchReservedForDynamicVGPRs() / 4);
1708
1709 // Only set AGPRs for supported devices
1710 if (STM.hasMAIInsts()) {
1711 MD->setNumUsedAgprs(CC, Val: CurrentProgramInfo.NumAccVGPR);
1712 }
1713
1714 MD->setNumUsedSgprs(CC, Val: CurrentProgramInfo.NumSGPRsForWavesPerEU, Ctx);
1715 if (MD->getPALMajorVersion() < 3) {
1716 MD->setRsrc1(CC, Val: CurrentProgramInfo.getPGMRSrc1(CC, ST: STM, Ctx), Ctx);
1717 if (AMDGPU::isCompute(CC)) {
1718 MD->setRsrc2(CC, Val: CurrentProgramInfo.getComputePGMRSrc2(ST: STM, Ctx), Ctx);
1719 } else {
1720 const MCExpr *HasScratchBlocks =
1721 MCBinaryExpr::createGT(LHS: CurrentProgramInfo.ScratchBlocks,
1722 RHS: MCConstantExpr::create(Value: 0, Ctx), Ctx);
1723 auto [Shift, Mask] = getShiftMask(C_00B84C_SCRATCH_EN);
1724 MD->setRsrc2(CC, Val: maskShiftSet(Val: HasScratchBlocks, Mask, Shift, Ctx), Ctx);
1725 }
1726 } else {
1727 MD->setHwStage(CC, field: ".debug_mode", Val: (bool)CurrentProgramInfo.DebugMode);
1728 MD->setHwStage(CC, field: ".scratch_en", Type: msgpack::Type::Boolean,
1729 Val: CurrentProgramInfo.ScratchEnable);
1730 EmitPALMetadataCommon(MD, CurrentProgramInfo, CC, ST: STM,
1731 DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize());
1732 }
1733
1734 // ScratchSize is in bytes, 16 aligned.
1735 MD->setScratchSize(
1736 CC,
1737 Val: AMDGPUMCExpr::createAlignTo(Value: CurrentProgramInfo.ScratchSize,
1738 Align: MCConstantExpr::create(Value: 16, Ctx), Ctx),
1739 Ctx);
1740
1741 if (MF.getFunction().getCallingConv() == CallingConv::AMDGPU_PS) {
1742 unsigned ExtraLDSSize = STM.getGeneration() >= AMDGPUSubtarget::GFX11
1743 ? divideCeil(Numerator: CurrentProgramInfo.LDSBlocks, Denominator: 2)
1744 : CurrentProgramInfo.LDSBlocks;
1745 if (MD->getPALMajorVersion() < 3) {
1746 MD->setRsrc2(
1747 CC,
1748 Val: MCConstantExpr::create(S_00B02C_EXTRA_LDS_SIZE(ExtraLDSSize), Ctx),
1749 Ctx);
1750 MD->setSpiPsInputEna(MFI->getPSInputEnable());
1751 MD->setSpiPsInputAddr(MFI->getPSInputAddr());
1752 } else {
1753 // Graphics registers
1754 const unsigned ExtraLdsDwGranularity =
1755 STM.getGeneration() >= AMDGPUSubtarget::GFX11 ? 256 : 128;
1756 MD->setGraphicsRegisters(
1757 field: ".ps_extra_lds_size",
1758 Val: (unsigned)(ExtraLDSSize * ExtraLdsDwGranularity * sizeof(uint32_t)));
1759
1760 // Set PsInputEna and PsInputAddr .spi_ps_input_ena and .spi_ps_input_addr
1761 static StringLiteral const PsInputFields[] = {
1762 ".persp_sample_ena", ".persp_center_ena",
1763 ".persp_centroid_ena", ".persp_pull_model_ena",
1764 ".linear_sample_ena", ".linear_center_ena",
1765 ".linear_centroid_ena", ".line_stipple_tex_ena",
1766 ".pos_x_float_ena", ".pos_y_float_ena",
1767 ".pos_z_float_ena", ".pos_w_float_ena",
1768 ".front_face_ena", ".ancillary_ena",
1769 ".sample_coverage_ena", ".pos_fixed_pt_ena"};
1770 unsigned PSInputEna = MFI->getPSInputEnable();
1771 unsigned PSInputAddr = MFI->getPSInputAddr();
1772 for (auto [Idx, Field] : enumerate(First: PsInputFields)) {
1773 MD->setGraphicsRegisters(field1: ".spi_ps_input_ena", field2: Field,
1774 Val: (bool)((PSInputEna >> Idx) & 1));
1775 MD->setGraphicsRegisters(field1: ".spi_ps_input_addr", field2: Field,
1776 Val: (bool)((PSInputAddr >> Idx) & 1));
1777 }
1778 }
1779 }
1780
1781 // For version 3 and above the wave front size is already set in the metadata
1782 if (MD->getPALMajorVersion() < 3 && STM.isWave32())
1783 MD->setWave32(MF.getFunction().getCallingConv());
1784}
1785
1786void AMDGPUAsmPrinter::emitPALFunctionMetadata(const MachineFunction &MF) {
1787 auto *MD = getTargetStreamer()->getPALMetadata();
1788 const MachineFrameInfo &MFI = MF.getFrameInfo();
1789 StringRef FnName = MF.getFunction().getName();
1790 MD->setFunctionScratchSize(FnName, Val: MFI.getStackSize());
1791 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1792 MCContext &Ctx = MF.getContext();
1793
1794 if (MD->getPALMajorVersion() < 3) {
1795 // Set compute registers
1796 MD->setRsrc1(
1797 CC: CallingConv::AMDGPU_CS,
1798 Val: CurrentProgramInfo.getPGMRSrc1(CC: CallingConv::AMDGPU_CS, ST, Ctx), Ctx);
1799 MD->setRsrc2(CC: CallingConv::AMDGPU_CS,
1800 Val: CurrentProgramInfo.getComputePGMRSrc2(ST, Ctx), Ctx);
1801 } else {
1802 EmitPALMetadataCommon(
1803 MD, CurrentProgramInfo, CC: CallingConv::AMDGPU_CS, ST,
1804 DynamicVGPRBlockSize: MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize());
1805 }
1806
1807 // Set optional info
1808 MD->setFunctionLdsSize(FnName, Val: CurrentProgramInfo.LDSSize);
1809 MD->setFunctionNumUsedVgprs(FnName, Val: CurrentProgramInfo.NumVGPRsForWavesPerEU);
1810 MD->setFunctionNumUsedSgprs(FnName, Val: CurrentProgramInfo.NumSGPRsForWavesPerEU);
1811}
1812
1813// This is supposed to be log2(Size)
1814static amd_element_byte_size_t getElementByteSizeValue(unsigned Size) {
1815 switch (Size) {
1816 case 4:
1817 return AMD_ELEMENT_4_BYTES;
1818 case 8:
1819 return AMD_ELEMENT_8_BYTES;
1820 case 16:
1821 return AMD_ELEMENT_16_BYTES;
1822 default:
1823 llvm_unreachable("invalid private_element_size");
1824 }
1825}
1826
1827void AMDGPUAsmPrinter::getAmdKernelCode(AMDGPUMCKernelCodeT &Out,
1828 const SIProgramInfo &CurrentProgramInfo,
1829 const MachineFunction &MF) const {
1830 const Function &F = MF.getFunction();
1831 assert(F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
1832 F.getCallingConv() == CallingConv::SPIR_KERNEL);
1833
1834 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1835 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1836 MCContext &Ctx = MF.getContext();
1837
1838 Out.initDefault(STI: STM, Ctx, /*InitMCExpr=*/false);
1839
1840 Out.compute_pgm_resource1_registers =
1841 CurrentProgramInfo.getComputePGMRSrc1(ST: STM, Ctx);
1842 Out.compute_pgm_resource2_registers =
1843 CurrentProgramInfo.getComputePGMRSrc2(ST: STM, Ctx);
1844 Out.code_properties |= AMD_CODE_PROPERTY_IS_PTR64;
1845
1846 Out.is_dynamic_callstack = CurrentProgramInfo.DynamicCallStack;
1847
1848 AMD_HSA_BITS_SET(Out.code_properties, AMD_CODE_PROPERTY_PRIVATE_ELEMENT_SIZE,
1849 getElementByteSizeValue(STM.getMaxPrivateElementSize(true)));
1850
1851 const GCNUserSGPRUsageInfo &UserSGPRInfo = MFI->getUserSGPRInfo();
1852 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
1853 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER;
1854 }
1855
1856 if (UserSGPRInfo.hasDispatchPtr())
1857 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR;
1858
1859 if (UserSGPRInfo.hasQueuePtr())
1860 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR;
1861
1862 if (UserSGPRInfo.hasKernargSegmentPtr())
1863 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR;
1864
1865 if (UserSGPRInfo.hasDispatchID())
1866 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID;
1867
1868 if (UserSGPRInfo.hasFlatScratchInit())
1869 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT;
1870
1871 if (UserSGPRInfo.hasPrivateSegmentSize())
1872 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE;
1873
1874 if (STM.isXNACKEnabled())
1875 Out.code_properties |= AMD_CODE_PROPERTY_IS_XNACK_SUPPORTED;
1876
1877 Align MaxKernArgAlign;
1878 Out.kernarg_segment_byte_size = STM.getKernArgSegmentSize(F, MaxAlign&: MaxKernArgAlign);
1879 Out.wavefront_sgpr_count = CurrentProgramInfo.NumSGPR;
1880 Out.workitem_vgpr_count = CurrentProgramInfo.NumVGPR;
1881 Out.workitem_private_segment_byte_size = CurrentProgramInfo.ScratchSize;
1882 Out.workgroup_group_segment_byte_size = CurrentProgramInfo.LDSSize;
1883
1884 // kernarg_segment_alignment is specified as log of the alignment.
1885 // The minimum alignment is 16.
1886 // FIXME: The metadata treats the minimum as 4?
1887 Out.kernarg_segment_alignment = Log2(A: std::max(a: Align(16), b: MaxKernArgAlign));
1888}
1889
1890bool AMDGPUAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
1891 const char *ExtraCode, raw_ostream &O) {
1892 // First try the generic code, which knows about modifiers like 'c' and 'n'.
1893 if (!AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, OS&: O))
1894 return false;
1895
1896 if (ExtraCode && ExtraCode[0]) {
1897 if (ExtraCode[1] != 0)
1898 return true; // Unknown modifier.
1899
1900 switch (ExtraCode[0]) {
1901 case 'r':
1902 break;
1903 default:
1904 return true;
1905 }
1906 }
1907
1908 // TODO: Should be able to support other operand types like globals.
1909 const MachineOperand &MO = MI->getOperand(i: OpNo);
1910 if (MO.isReg()) {
1911 AMDGPUInstPrinter::printRegOperand(Reg: MO.getReg(), O,
1912 MRI: *MF->getSubtarget().getRegisterInfo());
1913 return false;
1914 }
1915 if (MO.isImm()) {
1916 int64_t Val = MO.getImm();
1917 if (AMDGPU::isInlinableIntLiteral(Literal: Val)) {
1918 O << Val;
1919 } else if (isUInt<16>(x: Val)) {
1920 O << format(Fmt: "0x%" PRIx16, Vals: static_cast<uint16_t>(Val));
1921 } else if (isUInt<32>(x: Val)) {
1922 O << format(Fmt: "0x%" PRIx32, Vals: static_cast<uint32_t>(Val));
1923 } else {
1924 O << format(Fmt: "0x%" PRIx64, Vals: static_cast<uint64_t>(Val));
1925 }
1926 return false;
1927 }
1928 return true;
1929}
1930
1931void AMDGPUAsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const {
1932 AU.addRequired<AMDGPUResourceUsageAnalysisWrapperPass>();
1933 AU.addPreserved<AMDGPUResourceUsageAnalysisWrapperPass>();
1934 AU.addRequired<MachineModuleInfoWrapperPass>();
1935 AU.addPreserved<MachineModuleInfoWrapperPass>();
1936 AsmPrinter::getAnalysisUsage(AU);
1937}
1938
1939void AMDGPUAsmPrinter::emitResourceUsageRemarks(
1940 const MachineFunction &MF, const SIProgramInfo &CurrentProgramInfo,
1941 bool isModuleEntryFunction, bool hasMAIInsts) {
1942 if (!ORE)
1943 return;
1944
1945 const char *Name = "kernel-resource-usage";
1946 const char *Indent = " ";
1947
1948 // If the remark is not specifically enabled, do not output to yaml
1949 LLVMContext &Ctx = MF.getFunction().getContext();
1950 if (!Ctx.getDiagHandlerPtr()->isAnalysisRemarkEnabled(PassName: Name))
1951 return;
1952
1953 // Currently non-kernel functions have no resources to emit.
1954 if (!isEntryFunctionCC(CC: MF.getFunction().getCallingConv()))
1955 return;
1956
1957 auto EmitResourceUsageRemark = [&](StringRef RemarkName,
1958 StringRef RemarkLabel, auto Argument) {
1959 // Add an indent for every line besides the line with the kernel name. This
1960 // makes it easier to tell which resource usage go with which kernel since
1961 // the kernel name will always be displayed first.
1962 std::string LabelStr = RemarkLabel.str() + ": ";
1963 if (RemarkName != "FunctionName")
1964 LabelStr = Indent + LabelStr;
1965
1966 ORE->emit([&]() {
1967 return MachineOptimizationRemarkAnalysis(Name, RemarkName,
1968 MF.getFunction().getSubprogram(),
1969 &MF.front())
1970 << LabelStr << ore::NV(RemarkName, Argument);
1971 });
1972 };
1973
1974 // FIXME: Formatting here is pretty nasty because clang does not accept
1975 // newlines from diagnostics. This forces us to emit multiple diagnostic
1976 // remarks to simulate newlines. If and when clang does accept newlines, this
1977 // formatting should be aggregated into one remark with newlines to avoid
1978 // printing multiple diagnostic location and diag opts.
1979 EmitResourceUsageRemark("FunctionName", "Function Name",
1980 MF.getFunction().getName());
1981 EmitResourceUsageRemark("NumSGPR", "TotalSGPRs",
1982 getMCExprStr(Value: CurrentProgramInfo.NumSGPR));
1983 EmitResourceUsageRemark("NumVGPR", "VGPRs",
1984 getMCExprStr(Value: CurrentProgramInfo.NumArchVGPR));
1985 if (hasMAIInsts) {
1986 EmitResourceUsageRemark("NumAGPR", "AGPRs",
1987 getMCExprStr(Value: CurrentProgramInfo.NumAccVGPR));
1988 }
1989 EmitResourceUsageRemark("ScratchSize", "ScratchSize [bytes/lane]",
1990 getMCExprStr(Value: CurrentProgramInfo.ScratchSize));
1991 int64_t DynStack;
1992 bool DynStackEvaluatable =
1993 CurrentProgramInfo.DynamicCallStack->evaluateAsAbsolute(Res&: DynStack);
1994 StringRef DynamicStackStr =
1995 DynStackEvaluatable && DynStack ? "True" : "False";
1996 EmitResourceUsageRemark("DynamicStack", "Dynamic Stack", DynamicStackStr);
1997 EmitResourceUsageRemark("Occupancy", "Occupancy [waves/SIMD]",
1998 getMCExprStr(Value: CurrentProgramInfo.Occupancy));
1999 EmitResourceUsageRemark("SGPRSpill", "SGPRs Spill",
2000 CurrentProgramInfo.SGPRSpill);
2001 EmitResourceUsageRemark("VGPRSpill", "VGPRs Spill",
2002 CurrentProgramInfo.VGPRSpill);
2003 if (isModuleEntryFunction)
2004 EmitResourceUsageRemark("BytesLDS", "LDS Size [bytes/block]",
2005 CurrentProgramInfo.LDSSize);
2006}
2007
2008PreservedAnalyses AMDGPUAsmPrinterBeginPass::run(Module &M,
2009 ModuleAnalysisManager &MAM) {
2010
2011 AMDGPUAsmPrinter &AsmPrinter = static_cast<AMDGPUAsmPrinter &>(
2012 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
2013 setupModuleAsmPrinter(M, MAM, AsmPrinter);
2014 AsmPrinter.doInitialization(M);
2015 return PreservedAnalyses::all();
2016}
2017
2018PreservedAnalyses
2019AMDGPUAsmPrinterPass::run(MachineFunction &MF,
2020 MachineFunctionAnalysisManager &MFAM) {
2021 AMDGPUAsmPrinter &AsmPrinter = static_cast<AMDGPUAsmPrinter &>(
2022 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF)
2023 .getCachedResult<AsmPrinterAnalysis>(IR&: *MF.getFunction().getParent())
2024 ->getPrinter());
2025 setupMachineFunctionAsmPrinter(MFAM, MF, AsmPrinter);
2026 AsmPrinter.GetResourceUsage = [&MFAM](MachineFunction &MF)
2027 -> const AMDGPUResourceUsageAnalysisImpl::SIFunctionResourceInfo * {
2028 return &MFAM.getResult<AMDGPUResourceUsageAnalysis>(IR&: MF);
2029 };
2030 AsmPrinter.runOnMachineFunction(MF);
2031 return PreservedAnalyses::all();
2032}
2033
2034PreservedAnalyses AMDGPUAsmPrinterEndPass::run(Module &M,
2035 ModuleAnalysisManager &MAM) {
2036 AMDGPUAsmPrinter &AsmPrinter = static_cast<AMDGPUAsmPrinter &>(
2037 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
2038 setupModuleAsmPrinter(M, MAM, AsmPrinter);
2039 AsmPrinter.doFinalization(M);
2040 return PreservedAnalyses::all();
2041}
2042
2043char AMDGPUAsmPrinter::ID = 0;
2044
2045INITIALIZE_PASS(AMDGPUAsmPrinter, "amdgpu-asm-printer",
2046 "AMDGPU Assembly Printer", false, false)
2047