1//===-- AMDGPUAsmPrinter.cpp - AMDGPU assembly printer --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10///
11/// The AMDGPUAsmPrinter is used to print both assembly string and also binary
12/// code. When passed an MCAsmStreamer it prints assembly and when passed
13/// an MCObjectStreamer it outputs binary code.
14//
15//===----------------------------------------------------------------------===//
16//
17
18#include "AMDGPUAsmPrinter.h"
19#include "AMDGPU.h"
20#include "AMDGPUHSAMetadataStreamer.h"
21#include "AMDGPUMCResourceInfo.h"
22#include "AMDGPUResourceUsageAnalysis.h"
23#include "AMDGPUTargetMachine.h"
24#include "GCNSubtarget.h"
25#include "MCTargetDesc/AMDGPUInstPrinter.h"
26#include "MCTargetDesc/AMDGPUMCExpr.h"
27#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h"
28#include "MCTargetDesc/AMDGPUTargetStreamer.h"
29#include "R600AsmPrinter.h"
30#include "SIMachineFunctionInfo.h"
31#include "TargetInfo/AMDGPUTargetInfo.h"
32#include "Utils/AMDGPUBaseInfo.h"
33#include "Utils/AMDKernelCodeTUtils.h"
34#include "Utils/SIDefinesUtils.h"
35#include "llvm/Analysis/OptimizationRemarkEmitter.h"
36#include "llvm/BinaryFormat/ELF.h"
37#include "llvm/CodeGen/AsmPrinterAnalysis.h"
38#include "llvm/CodeGen/AsmPrinterHandler.h"
39#include "llvm/CodeGen/MachineFrameInfo.h"
40#include "llvm/CodeGen/MachineModuleInfo.h"
41#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
42#include "llvm/IR/DiagnosticInfo.h"
43#include "llvm/MC/MCAssembler.h"
44#include "llvm/MC/MCContext.h"
45#include "llvm/MC/MCSectionELF.h"
46#include "llvm/MC/MCStreamer.h"
47#include "llvm/MC/MCValue.h"
48#include "llvm/MC/TargetRegistry.h"
49#include "llvm/Support/AMDHSAKernelDescriptor.h"
50#include "llvm/Support/Compiler.h"
51#include "llvm/Target/TargetLoweringObjectFile.h"
52#include "llvm/Target/TargetMachine.h"
53#include "llvm/TargetParser/AMDGPUTargetParser.h"
54
55using namespace llvm;
56using namespace llvm::AMDGPU;
57
58// This should get the default rounding mode from the kernel. We just set the
59// default here, but this could change if the OpenCL rounding mode pragmas are
60// used.
61//
62// The denormal mode here should match what is reported by the OpenCL runtime
63// for the CL_FP_DENORM bit from CL_DEVICE_{HALF|SINGLE|DOUBLE}_FP_CONFIG, but
64// can also be override to flush with the -cl-denorms-are-zero compiler flag.
65//
66// AMD OpenCL only sets flush none and reports CL_FP_DENORM for double
67// precision, and leaves single precision to flush all and does not report
68// CL_FP_DENORM for CL_DEVICE_SINGLE_FP_CONFIG. Mesa's OpenCL currently reports
69// CL_FP_DENORM for both.
70//
71// FIXME: It seems some instructions do not support single precision denormals
72// regardless of the mode (exp_*_f32, rcp_*_f32, rsq_*_f32, rsq_*f32, sqrt_f32,
73// and sin_f32, cos_f32 on most parts).
74
75// We want to use these instructions, and using fp32 denormals also causes
76// instructions to run at the double precision rate for the device so it's
77// probably best to just report no single precision denormals.
78static uint32_t getFPMode(SIModeRegisterDefaults Mode) {
79 return FP_ROUND_MODE_SP(FP_ROUND_ROUND_TO_NEAREST) |
80 FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_NEAREST) |
81 FP_DENORM_MODE_SP(Mode.fpDenormModeSPValue()) |
82 FP_DENORM_MODE_DP(Mode.fpDenormModeDPValue());
83}
84
85static AsmPrinter *
86createAMDGPUAsmPrinterPass(TargetMachine &tm,
87 std::unique_ptr<MCStreamer> &&Streamer) {
88 return new AMDGPUAsmPrinter(tm, std::move(Streamer));
89}
90
91extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
92LLVMInitializeAMDGPUAsmPrinter() {
93 TargetRegistry::RegisterAsmPrinter(T&: getTheR600Target(),
94 Fn: llvm::createR600AsmPrinterPass);
95 TargetRegistry::RegisterAsmPrinter(T&: getTheGCNTarget(),
96 Fn: createAMDGPUAsmPrinterPass);
97 TargetRegistry::RegisterAsmPrinter(T&: getTheGCNLegacyTarget(),
98 Fn: createAMDGPUAsmPrinterPass);
99}
100
101namespace {
102class AMDGPUAsmPrinterHandler : public AsmPrinterHandler {
103protected:
104 AMDGPUAsmPrinter *Asm;
105
106public:
107 AMDGPUAsmPrinterHandler(AMDGPUAsmPrinter *A) : Asm(A) {}
108
109 void beginFunction(const MachineFunction *MF) override {}
110
111 void endFunction(const MachineFunction *MF) override { Asm->endFunction(MF); }
112
113 void endModule() override {}
114};
115} // End anonymous namespace
116
117AMDGPUAsmPrinter::AMDGPUAsmPrinter(TargetMachine &TM,
118 std::unique_ptr<MCStreamer> Streamer)
119 : AsmPrinter(TM, std::move(Streamer)) {
120 assert(OutStreamer && "AsmPrinter constructed without streamer");
121 // AMDGPU describes a pointer's address space with its own attribute rather
122 // than DW_AT_address_class.
123 TypeAddressSpaceAttribute = dwarf::DW_AT_LLVM_address_space;
124 GetResourceUsage = [this](MachineFunction &MF)
125 -> const AMDGPUResourceUsageAnalysisImpl::SIFunctionResourceInfo * {
126 if (auto *ResourceUsageW =
127 getAnalysisIfAvailable<AMDGPUResourceUsageAnalysisWrapperPass>())
128 return &ResourceUsageW->getResourceInfo();
129 return nullptr;
130 };
131}
132
133StringRef AMDGPUAsmPrinter::getPassName() const {
134 return "AMDGPU Assembly Printer";
135}
136
137const MCSubtargetInfo *AMDGPUAsmPrinter::getGlobalSTI() const {
138 return &TM.getMCSubtargetInfo();
139}
140
141AMDGPUTargetStreamer *AMDGPUAsmPrinter::getTargetStreamer() const {
142 if (!OutStreamer)
143 return nullptr;
144 return static_cast<AMDGPUTargetStreamer *>(OutStreamer->getTargetStreamer());
145}
146
147void AMDGPUAsmPrinter::emitStartOfAsmFile(Module &M) {
148 IsTargetStreamerInitialized = false;
149}
150
151void AMDGPUAsmPrinter::initTargetStreamer(Module &M) {
152 IsTargetStreamerInitialized = true;
153
154 // TODO: Which one is called first, emitStartOfAsmFile or
155 // emitFunctionBodyStart?
156 if (getTargetStreamer() && !getTargetStreamer()->getTargetID())
157 initializeTargetID(M);
158
159 const Triple &TT = M.getTargetTriple();
160 if (TT.getOS() != Triple::AMDHSA && TT.getOS() != Triple::AMDPAL)
161 return;
162
163 getTargetStreamer()->emitDirectiveAMDGCNTargetOnce();
164
165 if (TT.getOS() == Triple::AMDHSA) {
166 getTargetStreamer()->EmitDirectiveAMDHSACodeObjectVersion(
167 COV: CodeObjectVersion);
168 HSAMetadataStream->begin(Mod: M, TargetID: *getTargetStreamer()->getTargetID());
169 }
170
171 if (TT.getOS() == Triple::AMDPAL)
172 getTargetStreamer()->getPALMetadata()->readFromIR(M);
173}
174
175void AMDGPUAsmPrinter::emitEndOfAsmFile(Module &M) {
176 // Init target streamer if it has not yet happened
177 if (!IsTargetStreamerInitialized)
178 initTargetStreamer(M);
179
180 const Triple &TT = M.getTargetTriple();
181 if (TT.getOS() != Triple::AMDHSA)
182 getTargetStreamer()->EmitISAVersion();
183
184 // Emit HSA Metadata (NT_AMD_AMDGPU_HSA_METADATA).
185 // Emit HSA Metadata (NT_AMD_HSA_METADATA).
186 if (TT.getOS() == Triple::AMDHSA) {
187 HSAMetadataStream->end();
188 bool Success = HSAMetadataStream->emitTo(TargetStreamer&: *getTargetStreamer());
189 (void)Success;
190 assert(Success && "Malformed HSA Metadata");
191 }
192}
193
194void AMDGPUAsmPrinter::emitFunctionBodyStart() {
195 const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
196 const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
197 const Function &F = MF->getFunction();
198
199 // TODO: We're checking this late, would be nice to check it earlier.
200 if (STM.requiresCodeObjectV6() && CodeObjectVersion < AMDGPU::AMDHSA_COV6) {
201 reportFatalUsageError(
202 reason: STM.getCPU() + " is only available on code object version 6 or better");
203 }
204
205 // TODO: Which one is called first, emitStartOfAsmFile or
206 // emitFunctionBodyStart?
207 if (!getTargetStreamer()->getTargetID())
208 initializeTargetID(M: *F.getParent());
209
210 if (!MFI.isEntryFunction())
211 return;
212
213 if (STM.isMesaKernel(F) &&
214 (F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
215 F.getCallingConv() == CallingConv::SPIR_KERNEL)) {
216 AMDGPUMCKernelCodeT KernelCode;
217 getAmdKernelCode(Out&: KernelCode, KernelInfo: CurrentProgramInfo, MF: *MF);
218 KernelCode.validate(STI: &STM, Ctx&: MF->getContext());
219 getTargetStreamer()->EmitAMDKernelCodeT(Header&: KernelCode);
220 }
221
222 if (STM.isAmdHsaOS())
223 HSAMetadataStream->emitKernel(MF: *MF, ProgramInfo: CurrentProgramInfo);
224}
225
226/// Set bits in a kernel descriptor MCExpr field:
227/// return ((Dst & ~Mask) | (Value << Shift))
228static const MCExpr *setBits(const MCExpr *Dst, const MCExpr *Value,
229 uint32_t Mask, uint32_t Shift, MCContext &Ctx) {
230 const auto *Shft = MCConstantExpr::create(Value: Shift, Ctx);
231 const auto *Msk = MCConstantExpr::create(Value: Mask, Ctx);
232 Dst = MCBinaryExpr::createAnd(LHS: Dst, RHS: MCUnaryExpr::createNot(Expr: Msk, Ctx), Ctx);
233 Dst = MCBinaryExpr::createOr(LHS: Dst, RHS: MCBinaryExpr::createShl(LHS: Value, RHS: Shft, Ctx),
234 Ctx);
235 return Dst;
236}
237
238void AMDGPUAsmPrinter::endFunction(const MachineFunction *MF) {
239 const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
240 if (!MFI.isEntryFunction())
241 return;
242
243 assert(TM.getTargetTriple().getOS() == Triple::AMDHSA);
244
245 const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
246 MCContext &Ctx = MF->getContext();
247
248 AMDGPU::MCKernelDescriptor KD =
249 getAmdhsaKernelDescriptor(MF: *MF, PI: CurrentProgramInfo);
250
251 // Compute inst_pref_size using MCExpr label subtraction for exact code
252 // size. At this point .Lfunc_end has been emitted (by the base AsmPrinter)
253 // right after the function code, so (Lfunc_end - func_sym) gives the
254 // exact function code size in bytes.
255 if (STM.hasInstPrefSize()) {
256 const MCExpr *CodeSizeExpr = MCBinaryExpr::createSub(
257 LHS: MCSymbolRefExpr::create(Symbol: getFunctionEnd(), Ctx&: OutContext),
258 RHS: MCSymbolRefExpr::create(Symbol: CurrentFnSym, Ctx&: OutContext), Ctx&: OutContext);
259
260 uint32_t Mask, Shift, Width, CacheLineSize;
261 STM.getInstPrefSizeArgs(Mask, Shift, Width, CacheLineSize);
262 const MCExpr *InstPrefSize =
263 AMDGPUMCExpr::createInstPrefSize(CodeSizeBytes: CodeSizeExpr, Ctx);
264 KD.compute_pgm_rsrc3 =
265 setBits(Dst: KD.compute_pgm_rsrc3, Value: InstPrefSize, Mask, Shift, Ctx);
266 }
267
268 auto &Streamer = getTargetStreamer()->getStreamer();
269 auto &Context = Streamer.getContext();
270 auto &ObjectFileInfo = *Context.getObjectFileInfo();
271 auto &ReadOnlySection = *ObjectFileInfo.getReadOnlySection();
272
273 Streamer.pushSection();
274 Streamer.switchSection(Section: &ReadOnlySection);
275
276 // CP microcode requires the kernel descriptor to be allocated on 64 byte
277 // alignment.
278 Streamer.emitValueToAlignment(Alignment: Align(64), Fill: 0, FillLen: 1, MaxBytesToEmit: 0);
279 ReadOnlySection.ensureMinAlignment(MinAlignment: Align(64));
280
281 SmallString<128> KernelName;
282 getNameWithPrefix(Name&: KernelName, GV: &MF->getFunction());
283 getTargetStreamer()->EmitAmdhsaKernelDescriptor(
284 STI: STM, KernelName, KernelDescriptor: KD, NextVGPR: CurrentProgramInfo.NumVGPRsForWavesPerEU,
285 NextSGPR: MCBinaryExpr::createSub(
286 LHS: CurrentProgramInfo.NumSGPRsForWavesPerEU,
287 RHS: AMDGPUMCExpr::createExtraSGPRs(
288 VCCUsed: CurrentProgramInfo.VCCUsed, FlatScrUsed: CurrentProgramInfo.FlatUsed,
289 XNACKUsed: getTargetStreamer()->getTargetID()->isXnackOnOrAny(), Ctx&: Context),
290 Ctx&: Context),
291 ReserveVCC: CurrentProgramInfo.VCCUsed, ReserveFlatScr: CurrentProgramInfo.FlatUsed);
292
293 Streamer.popSection();
294}
295
296void AMDGPUAsmPrinter::emitImplicitDef(const MachineInstr *MI) const {
297 Register RegNo = MI->getOperand(i: 0).getReg();
298
299 SmallString<128> Str;
300 raw_svector_ostream OS(Str);
301 OS << "implicit-def: "
302 << printReg(Reg: RegNo, TRI: MF->getSubtarget().getRegisterInfo());
303
304 if (MI->getAsmPrinterFlags() & AMDGPU::SGPR_SPILL)
305 OS << " : SGPR spill to VGPR lane";
306
307 OutStreamer->AddComment(T: OS.str());
308 OutStreamer->addBlankLine();
309}
310
311void AMDGPUAsmPrinter::emitFunctionEntryLabel() {
312 if (TM.getTargetTriple().getOS() == Triple::AMDHSA) {
313 AsmPrinter::emitFunctionEntryLabel();
314 return;
315 }
316
317 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
318 const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
319 if (MFI->isEntryFunction() && STM.isAmdHsaOrMesa(F: MF->getFunction())) {
320 SmallString<128> SymbolName;
321 getNameWithPrefix(Name&: SymbolName, GV: &MF->getFunction()),
322 getTargetStreamer()->EmitAMDGPUSymbolType(SymbolName,
323 Type: ELF::STT_AMDGPU_HSA_KERNEL);
324 }
325 if (DumpCodeInstEmitter) {
326 // Disassemble function name label to text.
327 DisasmLines.push_back(x: MF->getName().str() + ":");
328 DisasmLineMaxLen = std::max(a: DisasmLineMaxLen, b: DisasmLines.back().size());
329 HexLines.emplace_back(args: "");
330 }
331
332 AsmPrinter::emitFunctionEntryLabel();
333}
334
335void AMDGPUAsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) {
336 if (DumpCodeInstEmitter && !isBlockOnlyReachableByFallthrough(MBB: &MBB)) {
337 // Write a line for the basic block label if it is not only fallthrough.
338 DisasmLines.push_back(x: (Twine("BB") + Twine(getFunctionNumber()) + "_" +
339 Twine(MBB.getNumber()) + ":")
340 .str());
341 DisasmLineMaxLen = std::max(a: DisasmLineMaxLen, b: DisasmLines.back().size());
342 HexLines.emplace_back(args: "");
343 }
344 AsmPrinter::emitBasicBlockStart(MBB);
345}
346
347void AMDGPUAsmPrinter::emitGlobalVariable(const GlobalVariable *GV) {
348 if (GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
349 if (GV->hasInitializer() && !isa<UndefValue>(Val: GV->getInitializer())) {
350 OutContext.reportError(L: {},
351 Msg: Twine(GV->getName()) +
352 ": unsupported initializer for address space");
353 return;
354 }
355
356 const Triple::OSType OS = TM.getTargetTriple().getOS();
357 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL) {
358 if (!AMDGPUTargetMachine::EnableObjectLinking)
359 return;
360 // With object linking, LDS definitions should have been externalized
361 // by earlier passes (e.g. LDS lowering, named barrier lowering).
362 // Only declarations reach here, emitted as SHN_AMDGPU_LDS symbols
363 // so the linker can assign their offsets.
364 assert(GV->isDeclaration() &&
365 "LDS definitions should have been externalized when object "
366 "linking is enabled");
367 }
368
369 MCSymbol *GVSym = getSymbol(GV);
370
371 GVSym->redefineIfPossible();
372 if (GVSym->isDefined() || GVSym->isVariable())
373 report_fatal_error(reason: "symbol '" + Twine(GVSym->getName()) +
374 "' is already defined");
375
376 const DataLayout &DL = GV->getDataLayout();
377 uint64_t Size = GV->getGlobalSize(DL);
378 Align Alignment = GV->getAlign().value_or(u: Align(4));
379
380 emitVisibility(Sym: GVSym, Visibility: GV->getVisibility(), IsDefinition: !GV->isDeclaration());
381 emitLinkage(GV, GVSym);
382 auto *TS = getTargetStreamer();
383 TS->emitAMDGPULDS(Symbol: GVSym, Size, Alignment);
384 return;
385 }
386
387 AsmPrinter::emitGlobalVariable(GV);
388}
389
390bool AMDGPUAsmPrinter::doInitialization(Module &M) {
391 const Triple &TT = M.getTargetTriple();
392 if (TT.getSubArch() == Triple::NoSubArch) {
393 Triple::SubArchType SubArch =
394 AMDGPU::getSubArch(AK: AMDGPU::parseArchAMDGCN(CPU: getGlobalSTI()->getCPU()));
395 if (SubArch != Triple::NoSubArch) {
396 Triple Fixed(TT);
397 Fixed.setArch(Kind: Triple::amdgpu, SubArch);
398 M.getContext().diagnose(DI: DiagnosticInfoGeneric(
399 "codegen with no subarch in the target triple is deprecated and will "
400 "become an error; use the target triple '" +
401 Fixed.str() + "' instead",
402 DS_Warning));
403 } else {
404 M.getContext().diagnose(DI: DiagnosticInfoGeneric(
405 "codegen with no subarch in the target triple is deprecated and will "
406 "become an error",
407 DS_Warning));
408 }
409 }
410
411 CodeObjectVersion = AMDGPU::getAMDHSACodeObjectVersion(M);
412
413 if (TT.getOS() == Triple::AMDHSA) {
414 const AMDGPUOptions &CLOpts =
415 static_cast<const AMDGPUTargetMachine &>(TM).getCLOpts();
416 switch (CodeObjectVersion) {
417 case AMDGPU::AMDHSA_COV4:
418 HSAMetadataStream =
419 std::make_unique<HSAMD::MetadataStreamerMsgPackV4>(args: CLOpts);
420 break;
421 case AMDGPU::AMDHSA_COV5:
422 HSAMetadataStream =
423 std::make_unique<HSAMD::MetadataStreamerMsgPackV5>(args: CLOpts);
424 break;
425 case AMDGPU::AMDHSA_COV6:
426 HSAMetadataStream =
427 std::make_unique<HSAMD::MetadataStreamerMsgPackV6>(args: CLOpts);
428 break;
429 default:
430 reportFatalUsageError(reason: "unsupported code object version");
431 }
432
433 addAsmPrinterHandler(Handler: std::make_unique<AMDGPUAsmPrinterHandler>(args: this));
434 }
435
436 return AsmPrinter::doInitialization(M);
437}
438
439/// Mimics GCNSubtarget::computeOccupancy for MCExpr.
440///
441/// Remove dependency on GCNSubtarget and depend only only the necessary values
442/// for said occupancy computation. Should match computeOccupancy implementation
443/// without passing \p STM on.
444const AMDGPUMCExpr *createOccupancy(unsigned InitOcc, const MCExpr *NumSGPRs,
445 const MCExpr *NumVGPRs,
446 unsigned DynamicVGPRBlockSize,
447 const GCNSubtarget &STM, MCContext &Ctx) {
448 unsigned MaxWaves = STM.getMaxWavesPerEU();
449 unsigned Granule = IsaInfo::getVGPRAllocGranule(STI: STM, DynamicVGPRBlockSize);
450 unsigned TargetTotalNumVGPRs = STM.getTotalNumVGPRs();
451
452 // Bake the per-function SGPR budget into the operands so the late-evaluated
453 // MCExpr stays arithmetic. The trap reservation in particular is implicit on
454 // amdhsa and lives on STM, not on the assembler's MCSubtargetInfo.
455 AMDGPU::GPUKind Kind = STM.getTargetID().getGPUKind();
456 unsigned SGPRTotal = AMDGPU::getTotalNumSGPRs(AK: Kind);
457 unsigned SGPRGranule = AMDGPU::getSGPRAllocGranule(AK: Kind);
458 unsigned SGPRTrapReserve = STM.hasTrapHandler() ? IsaInfo::TRAP_NUM_SGPRS : 0;
459
460 auto CreateExpr = [&Ctx](unsigned Value) {
461 return MCConstantExpr::create(Value, Ctx);
462 };
463
464 // Zero SGPR count when SGPRs don't limit occupancy, so the MCExpr skips the
465 // SGPR term without having to test the generation itself.
466 const MCExpr *SGPRArg =
467 IsaInfo::isSGPROccupancyLimited(STI: STM) ? NumSGPRs : CreateExpr(0);
468
469 return AMDGPUMCExpr::create(Kind: AMDGPUMCExpr::AGVK_Occupancy,
470 Args: {CreateExpr(MaxWaves), CreateExpr(Granule),
471 CreateExpr(TargetTotalNumVGPRs),
472 CreateExpr(InitOcc), CreateExpr(SGPRTotal),
473 CreateExpr(SGPRGranule),
474 CreateExpr(SGPRTrapReserve), SGPRArg, NumVGPRs},
475 Ctx);
476}
477
478// Diagnoses an "amdgpu-waves-per-eu" request that the final occupancy does not
479// meet. Returns true if a diagnostic was emitted.
480static bool diagnoseUnmetWavesPerEU(const Function &F,
481 const SIMachineFunctionInfo &MFI,
482 uint64_t Occupancy) {
483 const auto [MinWEU, MaxWEU] =
484 AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-waves-per-eu", Default: {0, 0}, OnlyFirstRequired: true);
485 std::string Desired;
486 if (Occupancy < MinWEU) {
487 Desired = std::to_string(val: MinWEU);
488 } else if (MaxWEU != 0 && Occupancy > MaxWEU &&
489 MFI.getMaxWavesPerEU() > MaxWEU) {
490 // The requested maximum was not applied: a request whose minimum is below
491 // what the workgroup size requires is ignored in favor of the default
492 // range. An applied maximum always caps the effective maximum, so this
493 // does not fire when the occupancy merely exceeds a requested maximum that
494 // register usage cannot enforce.
495 Desired = "at most " + std::to_string(val: MaxWEU);
496 } else {
497 return false;
498 }
499 DiagnosticInfoOptimizationFailure Diag(
500 F, F.getSubprogram(),
501 "failed to meet occupancy target given by 'amdgpu-waves-per-eu' in '" +
502 F.getName() + "': desired occupancy was " + Desired +
503 ", final occupancy is " + Twine(Occupancy));
504 F.getContext().diagnose(DI: Diag);
505 return true;
506}
507
508void AMDGPUAsmPrinter::validateMCResourceInfo(Function &F) {
509 if (F.isDeclaration() || !AMDGPU::isModuleEntryFunctionCC(CC: F.getCallingConv()))
510 return;
511
512 using RIK = MCResourceInfo::ResourceInfoKind;
513 const GCNSubtarget &STM = TM.getSubtarget<GCNSubtarget>(F);
514 MCSymbol *FnSym = TM.getSymbol(GV: &F);
515
516 auto TryGetMCExprValue = [](const MCExpr *Value, uint64_t &Res) -> bool {
517 int64_t Val;
518 if (Value->evaluateAsAbsolute(Res&: Val)) {
519 Res = Val;
520 return true;
521 }
522 return false;
523 };
524
525 const uint64_t MaxScratchPerWorkitem =
526 STM.getMaxWaveScratchSize() / STM.getWavefrontSize();
527 MCSymbol *ScratchSizeSymbol =
528 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_PrivateSegSize, OutContext);
529 uint64_t ScratchSize;
530 if (ScratchSizeSymbol->isVariable() &&
531 TryGetMCExprValue(ScratchSizeSymbol->getVariableValue(), ScratchSize) &&
532 ScratchSize > MaxScratchPerWorkitem) {
533 DiagnosticInfoStackSize DiagStackSize(F, ScratchSize, MaxScratchPerWorkitem,
534 DS_Error);
535 F.getContext().diagnose(DI: DiagStackSize);
536 }
537
538 // Validate addressable scalar registers (i.e., prior to added implicit
539 // SGPRs).
540 MCSymbol *NumSGPRSymbol =
541 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_NumSGPR, OutContext);
542 if (STM.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
543 !STM.hasSGPRInitBug()) {
544 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
545 uint64_t NumSgpr;
546 if (NumSGPRSymbol->isVariable() &&
547 TryGetMCExprValue(NumSGPRSymbol->getVariableValue(), NumSgpr) &&
548 NumSgpr > MaxAddressableNumSGPRs) {
549 F.getContext().diagnose(DI: DiagnosticInfoResourceLimit(
550 F, "addressable scalar registers", NumSgpr, MaxAddressableNumSGPRs,
551 DS_Error, DK_ResourceLimit));
552 return;
553 }
554 }
555
556 MCSymbol *VCCUsedSymbol =
557 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_UsesVCC, OutContext);
558 MCSymbol *FlatUsedSymbol =
559 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_UsesFlatScratch, OutContext);
560 uint64_t VCCUsed, FlatUsed, NumSgpr;
561
562 if (NumSGPRSymbol->isVariable() && VCCUsedSymbol->isVariable() &&
563 FlatUsedSymbol->isVariable() &&
564 TryGetMCExprValue(NumSGPRSymbol->getVariableValue(), NumSgpr) &&
565 TryGetMCExprValue(VCCUsedSymbol->getVariableValue(), VCCUsed) &&
566 TryGetMCExprValue(FlatUsedSymbol->getVariableValue(), FlatUsed)) {
567
568 // Recomputes NumSgprs + implicit SGPRs but all symbols should now be
569 // resolvable.
570 NumSgpr += IsaInfo::getNumExtraSGPRs(
571 STI: STM, VCCUsed, FlatScrUsed: FlatUsed,
572 XNACKUsed: getTargetStreamer()->getTargetID()->isXnackOnOrAny());
573 if (STM.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS ||
574 STM.hasSGPRInitBug()) {
575 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
576 if (NumSgpr > MaxAddressableNumSGPRs) {
577 F.getContext().diagnose(DI: DiagnosticInfoResourceLimit(
578 F, "scalar registers", NumSgpr, MaxAddressableNumSGPRs, DS_Error,
579 DK_ResourceLimit));
580 return;
581 }
582 }
583
584 MCSymbol *NumVgprSymbol =
585 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_NumVGPR, OutContext);
586 MCSymbol *NumAgprSymbol =
587 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_NumAGPR, OutContext);
588 uint64_t NumVgpr, NumAgpr;
589
590 MachineModuleInfo &MMI = *GetMMI();
591 MachineFunction *MF = MMI.getMachineFunction(F);
592 if (MF && NumVgprSymbol->isVariable() && NumAgprSymbol->isVariable() &&
593 TryGetMCExprValue(NumVgprSymbol->getVariableValue(), NumVgpr) &&
594 TryGetMCExprValue(NumAgprSymbol->getVariableValue(), NumAgpr)) {
595 const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
596 unsigned MaxWaves = MFI.getMaxWavesPerEU();
597 uint64_t TotalNumVgpr =
598 getTotalNumVGPRs(has90AInsts: STM.hasGFX90AInsts(), ArgNumAGPR: static_cast<uint32_t>(NumAgpr),
599 ArgNumVGPR: static_cast<uint32_t>(NumVgpr));
600 uint64_t NumVGPRsForWavesPerEU =
601 std::max(l: {TotalNumVgpr, (uint64_t)1,
602 (uint64_t)STM.getMinNumVGPRs(
603 WavesPerEU: MaxWaves, DynamicVGPRBlockSize: MFI.getDynamicVGPRBlockSize())});
604 uint64_t NumSGPRsForWavesPerEU = std::max(
605 l: {NumSgpr, (uint64_t)1, (uint64_t)STM.getMinNumSGPRs(WavesPerEU: MaxWaves)});
606 const MCExpr *OccupancyExpr = createOccupancy(
607 InitOcc: STM.getOccupancyWithWorkGroupSizes(MF: *MF).second,
608 NumSGPRs: MCConstantExpr::create(Value: NumSGPRsForWavesPerEU, Ctx&: OutContext),
609 NumVGPRs: MCConstantExpr::create(Value: NumVGPRsForWavesPerEU, Ctx&: OutContext),
610 DynamicVGPRBlockSize: MFI.getDynamicVGPRBlockSize(), STM, Ctx&: OutContext);
611 uint64_t Occupancy;
612 if (TryGetMCExprValue(OccupancyExpr, Occupancy) &&
613 diagnoseUnmetWavesPerEU(F, MFI, Occupancy))
614 return;
615 }
616 }
617}
618
619static void appendTypeEncoding(std::string &Enc, Type *Ty, const DataLayout &DL,
620 bool IsReturnType) {
621 if (Ty->isVoidTy()) {
622 Enc += 'v';
623 return;
624 }
625 uint64_t Bits = DL.getTypeSizeInBits(Ty);
626 // Zero-sized non-void types (e.g. `{}` or `[0 x i8]`) consume no ABI
627 // registers. For returns, emit the same no-result marker as void so the
628 // parameter encoding still has an explicit return-type prefix.
629 if (Bits == 0) {
630 if (IsReturnType)
631 Enc += 'v';
632 return;
633 }
634 if (Bits <= 32)
635 Enc += 'i';
636 else if (Bits <= 64)
637 Enc += 'l';
638 else
639 Enc.append(n: divideCeil(Numerator: Bits, Denominator: 32), c: 'i');
640}
641
642static std::string computeTypeId(const FunctionType *FTy,
643 const DataLayout &DL) {
644 std::string Enc;
645 appendTypeEncoding(Enc, Ty: FTy->getReturnType(), DL, /*IsReturnType=*/true);
646 for (Type *ParamTy : FTy->params())
647 appendTypeEncoding(Enc, Ty: ParamTy, DL, /*IsReturnType=*/false);
648 return Enc;
649}
650
651void AMDGPUAsmPrinter::collectCallEdge(const MachineInstr &MI) {
652 if (!AMDGPUTargetMachine::EnableObjectLinking)
653 return;
654 const SIInstrInfo *TII = MF->getSubtarget<GCNSubtarget>().getInstrInfo();
655 const MachineOperand *Callee =
656 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::callee);
657 if (!Callee || !Callee->isGlobal())
658 return;
659 DirectCallEdges.insert(
660 X: {getSymbol(GV: &MF->getFunction()), getSymbol(GV: Callee->getGlobal())});
661}
662
663void AMDGPUAsmPrinter::emitAMDGPUInfo(Module &M) {
664 if (!AMDGPUTargetMachine::EnableObjectLinking)
665 return;
666
667 const NamedMDNode *LDSMD = M.getNamedMetadata(Name: "amdgpu.lds.uses");
668 bool HasLDSUses = LDSMD && LDSMD->getNumOperands() > 0;
669
670 const NamedMDNode *BarMD = M.getNamedMetadata(Name: "amdgpu.named_barrier.uses");
671 bool HasNamedBarriers = BarMD && BarMD->getNumOperands() > 0;
672
673 // Collect address-taken functions (with type IDs) and indirect call sites.
674 DenseMap<const Function *, std::string> AddrTakenTypeIds;
675 using IndirectCallInfo = std::pair<const Function *, std::string>;
676 SmallVector<IndirectCallInfo, 8> IndirectCalls;
677
678 for (const Function &F : M) {
679 bool IsKernel = AMDGPU::isKernel(CC: F.getCallingConv());
680
681 if (!IsKernel && F.hasAddressTaken(/*PutOffender=*/nullptr,
682 /*IgnoreCallbackUses=*/false,
683 /*IgnoreAssumeLikeCalls=*/true,
684 /*IgnoreLLVMUsed=*/IngoreLLVMUsed: true)) {
685 AddrTakenTypeIds[&F] =
686 computeTypeId(FTy: F.getFunctionType(), DL: M.getDataLayout());
687 }
688
689 if (F.isDeclaration())
690 continue;
691
692 StringSet<> SeenTypeIds;
693 for (const BasicBlock &BB : F) {
694 for (const Instruction &I : BB) {
695 const auto *CB = dyn_cast<CallBase>(Val: &I);
696 if (!CB || !CB->isIndirectCall())
697 continue;
698 std::string TId =
699 computeTypeId(FTy: CB->getFunctionType(), DL: M.getDataLayout());
700 if (SeenTypeIds.insert(key: TId).second)
701 IndirectCalls.push_back(Elt: {&F, std::move(TId)});
702 }
703 }
704 }
705
706 if (FunctionInfos.empty() && DirectCallEdges.empty() && !HasLDSUses &&
707 !HasNamedBarriers && AddrTakenTypeIds.empty() && IndirectCalls.empty())
708 return;
709
710 AMDGPU::InfoSectionData Data;
711 Data.Funcs = std::move(FunctionInfos);
712
713 for (auto &[F, TypeId] : AddrTakenTypeIds) {
714 MCSymbol *Sym = getSymbol(GV: F);
715 Data.TypeIds.push_back(Elt: {Sym, TypeId});
716 }
717
718 for (auto &[CallerSym, CalleeSym] : DirectCallEdges)
719 Data.Calls.push_back(Elt: {CallerSym, CalleeSym});
720 DirectCallEdges.clear();
721
722 if (HasLDSUses) {
723 for (const MDNode *N : LDSMD->operands()) {
724 auto *Func = mdconst::extract<Function>(MD: N->getOperand(I: 0));
725 auto *LdsVar = mdconst::extract<GlobalVariable>(MD: N->getOperand(I: 1));
726 Data.Uses.push_back(Elt: {getSymbol(GV: Func), getSymbol(GV: LdsVar)});
727 }
728 }
729
730 if (HasNamedBarriers) {
731 for (const MDNode *N : BarMD->operands()) {
732 auto *BarVar = mdconst::extract<GlobalVariable>(MD: N->getOperand(I: 0));
733 MCSymbol *BarSym = getSymbol(GV: BarVar);
734 for (unsigned I = 1, E = N->getNumOperands(); I < E; ++I) {
735 auto *Func = mdconst::extract<Function>(MD: N->getOperand(I));
736 Data.Uses.push_back(Elt: {getSymbol(GV: Func), BarSym});
737 }
738 }
739 }
740
741 for (auto &[Caller, Enc] : IndirectCalls) {
742 MCSymbol *CallerSym = getSymbol(GV: Caller);
743 Data.IndirectCalls.push_back(Elt: {CallerSym, Enc});
744 }
745
746 getTargetStreamer()->emitAMDGPUInfo(Data);
747}
748
749bool AMDGPUAsmPrinter::doFinalization(Module &M) {
750 const Triple &TT = M.getTargetTriple();
751
752 // Pad with s_code_end to help tools and guard against instruction prefetch
753 // causing stale data in caches. Arguably this should be done by the linker,
754 // which is why this isn't done for Mesa.
755 // Don't do it if there is no code.
756 const MCSubtargetInfo &STI = *getGlobalSTI();
757 if ((AMDGPU::isGFX10Plus(STI) || AMDGPU::isGFX90A(STI)) &&
758 (TT.getOS() == Triple::AMDHSA || TT.getOS() == Triple::AMDPAL)) {
759 MCSection *TextSect = getObjFileLowering().getTextSection();
760 if (TextSect->hasInstructions()) {
761 OutStreamer->switchSection(Section: TextSect);
762 getTargetStreamer()->EmitCodeEnd(STI);
763 }
764 }
765
766 // Emit the unified .amdgpu.info section (per-function resources, call graph,
767 // LDS/named-barrier use edges, indirect calls, and address-taken type IDs).
768 emitAMDGPUInfo(M);
769
770 // Assign expressions which can only be resolved when all other functions are
771 // known.
772 RI.finalize(OutContext);
773
774 // Switch section and emit all GPR maximums within the processed module.
775 OutStreamer->pushSection();
776 MCSectionELF *MaxGPRSection =
777 OutContext.getELFSection(Section: ".AMDGPU.gpr_maximums", Type: ELF::SHT_PROGBITS, Flags: 0);
778 OutStreamer->switchSection(Section: MaxGPRSection);
779 getTargetStreamer()->EmitMCResourceMaximums(
780 MaxVGPR: RI.getMaxVGPRSymbol(OutContext), MaxAGPR: RI.getMaxAGPRSymbol(OutContext),
781 MaxSGPR: RI.getMaxSGPRSymbol(OutContext), MaxNamedBarrier: RI.getMaxNamedBarrierSymbol(OutContext));
782 OutStreamer->popSection();
783
784 // In the object-linking pipeline per-function resource MCExprs reference
785 // external callee symbols that cannot be evaluated here, so cross-TU limit
786 // checks would silently no-op for every non-leaf function. Defer resource
787 // sanity checking to the linker, which re-validates against the aggregated
788 // call graph in the combined .amdgpu.info metadata.
789 if (!AMDGPUTargetMachine::EnableObjectLinking) {
790 for (Function &F : M.functions())
791 validateMCResourceInfo(F);
792 }
793
794 RI.reset();
795
796 return AsmPrinter::doFinalization(M);
797}
798
799SmallString<128> AMDGPUAsmPrinter::getMCExprStr(const MCExpr *Value) {
800 SmallString<128> Str;
801 raw_svector_ostream OSS(Str);
802 auto &Streamer = getTargetStreamer()->getStreamer();
803 auto &Context = Streamer.getContext();
804 const MCExpr *New = foldAMDGPUMCExpr(Expr: Value, Ctx&: Context);
805 printAMDGPUMCExpr(Expr: New, OS&: OSS, MAI: &MAI);
806 return Str;
807}
808
809// Print comments that apply to both callable functions and entry points.
810void AMDGPUAsmPrinter::emitCommonFunctionComments(
811 const MCExpr *NumVGPR, const MCExpr *NumAGPR, const MCExpr *TotalNumVGPR,
812 const MCExpr *NumSGPR, const MCExpr *ScratchSize, uint64_t CodeSize,
813 const AMDGPUMachineFunctionInfo *MFI) {
814 OutStreamer->emitRawComment(T: " codeLenInByte = " + Twine(CodeSize), TabPrefix: false);
815 OutStreamer->emitRawComment(T: " TotalNumSgprs: " + getMCExprStr(Value: NumSGPR),
816 TabPrefix: false);
817 OutStreamer->emitRawComment(T: " NumVgprs: " + getMCExprStr(Value: NumVGPR), TabPrefix: false);
818 if (NumAGPR && TotalNumVGPR) {
819 OutStreamer->emitRawComment(T: " NumAgprs: " + getMCExprStr(Value: NumAGPR), TabPrefix: false);
820 OutStreamer->emitRawComment(T: " TotalNumVgprs: " + getMCExprStr(Value: TotalNumVGPR),
821 TabPrefix: false);
822 }
823 OutStreamer->emitRawComment(T: " ScratchSize: " + getMCExprStr(Value: ScratchSize),
824 TabPrefix: false);
825 OutStreamer->emitRawComment(T: " MemoryBound: " + Twine(MFI->isMemoryBound()),
826 TabPrefix: false);
827}
828
829const MCExpr *AMDGPUAsmPrinter::getAmdhsaKernelCodeProperties(
830 const MachineFunction &MF) const {
831 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
832 MCContext &Ctx = MF.getContext();
833 uint16_t KernelCodeProperties = 0;
834 const GCNUserSGPRUsageInfo &UserSGPRInfo = MFI.getUserSGPRInfo();
835
836 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
837 KernelCodeProperties |=
838 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER;
839 }
840 if (UserSGPRInfo.hasDispatchPtr()) {
841 KernelCodeProperties |=
842 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR;
843 }
844 if (UserSGPRInfo.hasQueuePtr()) {
845 KernelCodeProperties |= amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR;
846 }
847 if (UserSGPRInfo.hasKernargSegmentPtr()) {
848 KernelCodeProperties |=
849 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR;
850 }
851 if (UserSGPRInfo.hasDispatchID()) {
852 KernelCodeProperties |=
853 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID;
854 }
855 if (UserSGPRInfo.hasFlatScratchInit()) {
856 KernelCodeProperties |=
857 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT;
858 }
859 if (UserSGPRInfo.hasPrivateSegmentSize()) {
860 KernelCodeProperties |=
861 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE;
862 }
863 if (MF.getSubtarget<GCNSubtarget>().isWave32()) {
864 KernelCodeProperties |=
865 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;
866 }
867
868 // CurrentProgramInfo.DynamicCallStack is a MCExpr and could be
869 // un-evaluatable at this point so it cannot be conditionally checked here.
870 // Instead, we'll directly shift the possibly unknown MCExpr into its place
871 // and bitwise-or it into KernelCodeProperties.
872 const MCExpr *KernelCodePropExpr =
873 MCConstantExpr::create(Value: KernelCodeProperties, Ctx);
874 const MCExpr *OrValue = MCConstantExpr::create(
875 Value: amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK_SHIFT, Ctx);
876 OrValue = MCBinaryExpr::createShl(LHS: CurrentProgramInfo.DynamicCallStack,
877 RHS: OrValue, Ctx);
878 KernelCodePropExpr = MCBinaryExpr::createOr(LHS: KernelCodePropExpr, RHS: OrValue, Ctx);
879
880 return KernelCodePropExpr;
881}
882
883MCKernelDescriptor
884AMDGPUAsmPrinter::getAmdhsaKernelDescriptor(const MachineFunction &MF,
885 const SIProgramInfo &PI) const {
886 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
887 const Function &F = MF.getFunction();
888 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
889 MCContext &Ctx = MF.getContext();
890
891 MCKernelDescriptor KernelDescriptor;
892
893 KernelDescriptor.group_segment_fixed_size =
894 MCConstantExpr::create(Value: PI.LDSSize, Ctx);
895 KernelDescriptor.private_segment_fixed_size = PI.ScratchSize;
896
897 Align MaxKernArgAlign;
898 KernelDescriptor.kernarg_size = MCConstantExpr::create(
899 Value: STM.getKernArgSegmentSize(F, MaxAlign&: MaxKernArgAlign), Ctx);
900
901 KernelDescriptor.compute_pgm_rsrc1 = PI.getComputePGMRSrc1(ST: STM, Ctx);
902 KernelDescriptor.compute_pgm_rsrc2 = PI.getComputePGMRSrc2(ST: STM, Ctx);
903 KernelDescriptor.kernel_code_properties = getAmdhsaKernelCodeProperties(MF);
904
905 int64_t PGM_Rsrc3 = 1;
906 bool EvaluatableRsrc3 =
907 CurrentProgramInfo.ComputePGMRSrc3->evaluateAsAbsolute(Res&: PGM_Rsrc3);
908 (void)PGM_Rsrc3;
909 (void)EvaluatableRsrc3;
910 assert(STM.getGeneration() >= AMDGPUSubtarget::GFX10 ||
911 STM.hasGFX90AInsts() || STM.hasGFX1250Insts() || !EvaluatableRsrc3 ||
912 static_cast<uint64_t>(PGM_Rsrc3) == 0);
913 KernelDescriptor.compute_pgm_rsrc3 = CurrentProgramInfo.ComputePGMRSrc3;
914
915 KernelDescriptor.kernarg_preload = MCConstantExpr::create(
916 Value: AMDGPU::hasKernargPreload(STI: STM) ? Info->getNumKernargPreloadedSGPRs() : 0,
917 Ctx);
918
919 return KernelDescriptor;
920}
921
922bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
923 // Init target streamer lazily on the first function so that previous passes
924 // can set metadata.
925 if (!IsTargetStreamerInitialized)
926 initTargetStreamer(M&: *MF.getFunction().getParent());
927
928 ResourceUsage = GetResourceUsage(MF);
929 CurrentProgramInfo.reset(MF);
930
931 const AMDGPUMachineFunctionInfo *MFI =
932 MF.getInfo<AMDGPUMachineFunctionInfo>();
933 MCContext &Ctx = MF.getContext();
934
935 // The starting address of all shader programs must be 256 bytes aligned.
936 // Regular functions just need the basic required instruction alignment.
937 MF.ensureAlignment(A: MFI->isEntryFunction() ? Align(256) : Align(4));
938
939 SetupMachineFunction(MF);
940
941 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
942 MCContext &Context = getObjFileLowering().getContext();
943 // FIXME: This should be an explicit check for Mesa.
944 if (!STM.isAmdHsaOS() && !STM.isAmdPalOS()) {
945 MCSectionELF *ConfigSection =
946 Context.getELFSection(Section: ".AMDGPU.config", Type: ELF::SHT_PROGBITS, Flags: 0);
947 OutStreamer->switchSection(Section: ConfigSection);
948 }
949
950 RI.gatherResourceInfo(MF, FRI: *ResourceUsage, OutContext);
951
952 if (AMDGPUTargetMachine::EnableObjectLinking) {
953 const AMDGPUResourceUsageAnalysisWrapperPass::FunctionResourceInfo &RU =
954 *ResourceUsage;
955 FunctionInfos.push_back(
956 Elt: {/*NumSGPR=*/static_cast<uint32_t>(RU.NumExplicitSGPR),
957 /*NumArchVGPR=*/static_cast<uint32_t>(RU.NumVGPR),
958 /*NumAccVGPR=*/static_cast<uint32_t>(RU.NumAGPR),
959 /*PrivateSegmentSize=*/static_cast<uint32_t>(RU.PrivateSegmentSize),
960 /*UsesVCC=*/RU.UsesVCC,
961 /*UsesFlatScratch=*/RU.UsesFlatScratch,
962 /*HasDynStack=*/RU.HasDynamicallySizedStack,
963 /*Sym=*/getSymbol(GV: &MF.getFunction())});
964 }
965
966 if (MFI->isModuleEntryFunction()) {
967 getSIProgramInfo(Out&: CurrentProgramInfo, MF);
968 }
969
970 if (STM.isAmdPalOS()) {
971 if (MFI->isEntryFunction())
972 EmitPALMetadata(MF, KernelInfo: CurrentProgramInfo);
973 else if (MFI->isModuleEntryFunction())
974 emitPALFunctionMetadata(MF);
975 } else if (!STM.isAmdHsaOS()) {
976 EmitProgramInfoSI(MF, KernelInfo: CurrentProgramInfo);
977 }
978
979 DumpCodeInstEmitter = nullptr;
980 if (STM.dumpCode()) {
981 // For -dumpcode, get the assembler out of the streamer. This only works
982 // with -filetype=obj.
983 MCAssembler *Assembler = OutStreamer->getAssemblerPtr();
984 if (Assembler)
985 DumpCodeInstEmitter = Assembler->getEmitterPtr();
986 }
987
988 DisasmLines.clear();
989 HexLines.clear();
990 DisasmLineMaxLen = 0;
991
992 emitFunctionBody();
993
994 emitResourceUsageRemarks(MF, CurrentProgramInfo, isModuleEntryFunction: MFI->isModuleEntryFunction(),
995 hasMAIInsts: STM.hasMAIInsts());
996
997 {
998 using RIK = MCResourceInfo::ResourceInfoKind;
999 getTargetStreamer()->EmitMCResourceInfo(
1000 NumVGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumVGPR, OutContext),
1001 NumAGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumAGPR, OutContext),
1002 NumExplicitSGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumSGPR, OutContext),
1003 NumNamedBarrier: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumNamedBarrier,
1004 OutContext),
1005 PrivateSegmentSize: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_PrivateSegSize,
1006 OutContext),
1007 UsesVCC: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_UsesVCC, OutContext),
1008 UsesFlatScratch: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_UsesFlatScratch,
1009 OutContext),
1010 HasDynamicallySizedStack: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_HasDynSizedStack,
1011 OutContext),
1012 HasRecursion: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_HasRecursion,
1013 OutContext),
1014 HasIndirectCall: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_HasIndirectCall,
1015 OutContext));
1016 }
1017
1018 // Emit _dvgpr$ symbol when appropriate.
1019 emitDVgprSymbol(MF);
1020
1021 if (isVerbose()) {
1022 MCSectionELF *CommentSection =
1023 Context.getELFSection(Section: ".AMDGPU.csdata", Type: ELF::SHT_PROGBITS, Flags: 0);
1024 OutStreamer->switchSection(Section: CommentSection);
1025
1026 if (!MFI->isEntryFunction()) {
1027 using RIK = MCResourceInfo::ResourceInfoKind;
1028 OutStreamer->emitRawComment(T: " Function info:", TabPrefix: false);
1029
1030 emitCommonFunctionComments(
1031 NumVGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumVGPR, OutContext)
1032 ->getVariableValue(),
1033 NumAGPR: STM.hasMAIInsts() ? RI.getSymbol(FuncName: CurrentFnSym->getName(),
1034 RIK: RIK::RIK_NumAGPR, OutContext)
1035 ->getVariableValue()
1036 : nullptr,
1037 TotalNumVGPR: RI.createTotalNumVGPRs(MF, Ctx),
1038 NumSGPR: RI.createTotalNumSGPRs(
1039 MF,
1040 hasXnack: MF.getSubtarget<GCNSubtarget>().getTargetID().isXnackOnOrAny(),
1041 Ctx),
1042 ScratchSize: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_PrivateSegSize,
1043 OutContext)
1044 ->getVariableValue(),
1045 CodeSize: CurrentProgramInfo.getFunctionCodeSize(MF), MFI);
1046 return false;
1047 }
1048
1049 OutStreamer->emitRawComment(T: " Kernel info:", TabPrefix: false);
1050 emitCommonFunctionComments(
1051 NumVGPR: CurrentProgramInfo.NumArchVGPR,
1052 NumAGPR: STM.hasMAIInsts() ? CurrentProgramInfo.NumAccVGPR : nullptr,
1053 TotalNumVGPR: CurrentProgramInfo.NumVGPR, NumSGPR: CurrentProgramInfo.NumSGPR,
1054 ScratchSize: CurrentProgramInfo.ScratchSize,
1055 CodeSize: CurrentProgramInfo.getFunctionCodeSize(MF), MFI);
1056
1057 OutStreamer->emitRawComment(
1058 T: " FloatMode: " + Twine(CurrentProgramInfo.FloatMode), TabPrefix: false);
1059 OutStreamer->emitRawComment(
1060 T: " IeeeMode: " + Twine(CurrentProgramInfo.IEEEMode), TabPrefix: false);
1061 OutStreamer->emitRawComment(
1062 T: " LDSByteSize: " + Twine(CurrentProgramInfo.LDSSize) +
1063 " bytes/workgroup (compile time only)",
1064 TabPrefix: false);
1065
1066 OutStreamer->emitRawComment(
1067 T: " SGPRBlocks: " + getMCExprStr(Value: CurrentProgramInfo.SGPRBlocks), TabPrefix: false);
1068
1069 OutStreamer->emitRawComment(
1070 T: " VGPRBlocks: " + getMCExprStr(Value: CurrentProgramInfo.VGPRBlocks), TabPrefix: false);
1071
1072 OutStreamer->emitRawComment(
1073 T: " NumSGPRsForWavesPerEU: " +
1074 getMCExprStr(Value: CurrentProgramInfo.NumSGPRsForWavesPerEU),
1075 TabPrefix: false);
1076 OutStreamer->emitRawComment(
1077 T: " NumVGPRsForWavesPerEU: " +
1078 getMCExprStr(Value: CurrentProgramInfo.NumVGPRsForWavesPerEU),
1079 TabPrefix: false);
1080
1081 if (STM.hasGFX90AInsts()) {
1082 const MCExpr *AdjustedAccum = MCBinaryExpr::createAdd(
1083 LHS: CurrentProgramInfo.AccumOffset, RHS: MCConstantExpr::create(Value: 1, Ctx), Ctx);
1084 AdjustedAccum = MCBinaryExpr::createMul(
1085 LHS: AdjustedAccum, RHS: MCConstantExpr::create(Value: 4, Ctx), Ctx);
1086 OutStreamer->emitRawComment(
1087 T: " AccumOffset: " + getMCExprStr(Value: AdjustedAccum), TabPrefix: false);
1088 }
1089
1090 if (STM.hasGFX1250Insts())
1091 OutStreamer->emitRawComment(
1092 T: " NamedBarCnt: " + getMCExprStr(Value: CurrentProgramInfo.NamedBarCnt),
1093 TabPrefix: false);
1094
1095 OutStreamer->emitRawComment(
1096 T: " Occupancy: " + getMCExprStr(Value: CurrentProgramInfo.Occupancy), TabPrefix: false);
1097
1098 OutStreamer->emitRawComment(
1099 T: " WaveLimiterHint : " + Twine(MFI->needsWaveLimiter()), TabPrefix: false);
1100
1101 OutStreamer->emitRawComment(
1102 T: " COMPUTE_PGM_RSRC2:SCRATCH_EN: " +
1103 getMCExprStr(Value: CurrentProgramInfo.ScratchEnable),
1104 TabPrefix: false);
1105 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:USER_SGPR: " +
1106 Twine(CurrentProgramInfo.UserSGPR),
1107 TabPrefix: false);
1108 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TRAP_HANDLER: " +
1109 Twine(CurrentProgramInfo.TrapHandlerEnable),
1110 TabPrefix: false);
1111 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TGID_X_EN: " +
1112 Twine(CurrentProgramInfo.TGIdXEnable),
1113 TabPrefix: false);
1114 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TGID_Y_EN: " +
1115 Twine(CurrentProgramInfo.TGIdYEnable),
1116 TabPrefix: false);
1117 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TGID_Z_EN: " +
1118 Twine(CurrentProgramInfo.TGIdZEnable),
1119 TabPrefix: false);
1120 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TIDIG_COMP_CNT: " +
1121 Twine(CurrentProgramInfo.TIdIGCompCount),
1122 TabPrefix: false);
1123
1124 [[maybe_unused]] int64_t PGMRSrc3;
1125 assert(STM.getGeneration() >= AMDGPUSubtarget::GFX10 ||
1126 STM.hasGFX90AInsts() || STM.hasGFX1250Insts() ||
1127 (CurrentProgramInfo.ComputePGMRSrc3->evaluateAsAbsolute(PGMRSrc3) &&
1128 static_cast<uint64_t>(PGMRSrc3) == 0));
1129 if (STM.hasGFX90AInsts()) {
1130 OutStreamer->emitRawComment(
1131 T: " COMPUTE_PGM_RSRC3_GFX90A:ACCUM_OFFSET: " +
1132 getMCExprStr(Value: MCKernelDescriptor::bits_get(
1133 Src: CurrentProgramInfo.ComputePGMRSrc3,
1134 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
1135 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, Ctx)),
1136 TabPrefix: false);
1137 OutStreamer->emitRawComment(
1138 T: " COMPUTE_PGM_RSRC3_GFX90A:TG_SPLIT: " +
1139 getMCExprStr(Value: MCKernelDescriptor::bits_get(
1140 Src: CurrentProgramInfo.ComputePGMRSrc3,
1141 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT,
1142 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, Ctx)),
1143 TabPrefix: false);
1144 }
1145 }
1146
1147 if (DumpCodeInstEmitter) {
1148
1149 OutStreamer->switchSection(
1150 Section: Context.getELFSection(Section: ".AMDGPU.disasm", Type: ELF::SHT_PROGBITS, Flags: 0));
1151
1152 for (size_t i = 0; i < DisasmLines.size(); ++i) {
1153 std::string Comment = "\n";
1154 if (!HexLines[i].empty()) {
1155 Comment = std::string(DisasmLineMaxLen - DisasmLines[i].size(), ' ');
1156 Comment += " ; " + HexLines[i] + "\n";
1157 }
1158
1159 OutStreamer->emitBytes(Data: StringRef(DisasmLines[i]));
1160 OutStreamer->emitBytes(Data: StringRef(Comment));
1161 }
1162 }
1163
1164 return false;
1165}
1166
1167// When appropriate, add a _dvgpr$ symbol, with the value of the function
1168// symbol, plus an offset encoding one less than the number of VGPR blocks used
1169// by the function in bits 5..3 of the symbol value. A "VGPR block" can be
1170// either 16 VGPRs (for a max of 128), or 32 VGPRs (for a max of 256). This is
1171// used by a front-end to have functions that are chained rather than called,
1172// and a dispatcher that dynamically resizes the VGPR count before dispatching
1173// to a function.
1174void AMDGPUAsmPrinter::emitDVgprSymbol(MachineFunction &MF) {
1175 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
1176 if (MFI.isDynamicVGPREnabled() &&
1177 MF.getFunction().getCallingConv() == CallingConv::AMDGPU_CS_Chain) {
1178 MCContext &Ctx = MF.getContext();
1179 unsigned BlockSize = MFI.getDynamicVGPRBlockSize();
1180
1181 const MCExpr *EncodedBlocks;
1182 MCValue NumVGPRs;
1183 if (CurrentProgramInfo.NumVGPRsForWavesPerEU->evaluateAsRelocatable(
1184 Res&: NumVGPRs, Asm: nullptr) &&
1185 NumVGPRs.isAbsolute()) {
1186
1187 // Calculate number of VGPR blocks.
1188 // Treat 0 VGPRs as 1 VGPR to avoid underflowing.
1189 unsigned NumBlocks =
1190 divideCeil(Numerator: std::max(a: unsigned(NumVGPRs.getConstant()), b: 1U), Denominator: BlockSize);
1191
1192 if (NumBlocks > AMDGPU::IsaInfo::MaxDynamicVGPRBlocks) {
1193 OutContext.reportError(
1194 L: {}, Msg: "DVGPR block count " + Twine(NumBlocks) +
1195 " exceeds maximum of " +
1196 Twine(AMDGPU::IsaInfo::MaxDynamicVGPRBlocks) +
1197 " for __dvgpr$ symbol for '" +
1198 Twine(CurrentFnSym->getName()) + "'");
1199 return;
1200 }
1201 unsigned EncodedNumBlocks = (NumBlocks - 1) << 3;
1202 EncodedBlocks = MCConstantExpr::create(Value: EncodedNumBlocks, Ctx);
1203 } else {
1204 // Value not yet available so build a symbolic MCExpr:
1205 // ((alignTo(max(NumVGPRs, 1), BlockSize) / BlockSize - 1) << 3
1206 const MCExpr *One = MCConstantExpr::create(Value: 1, Ctx);
1207 const MCExpr *BlockSizeConst = MCConstantExpr::create(Value: BlockSize, Ctx);
1208 const MCExpr *MaxVGPRs = AMDGPUMCExpr::createMax(
1209 Args: {CurrentProgramInfo.NumVGPRsForWavesPerEU, One}, Ctx);
1210 const MCExpr *NumBlocks = MCBinaryExpr::createDiv(
1211 LHS: AMDGPUMCExpr::createAlignTo(Value: MaxVGPRs, Align: BlockSizeConst, Ctx),
1212 RHS: BlockSizeConst, Ctx);
1213 EncodedBlocks =
1214 MCBinaryExpr::createShl(LHS: MCBinaryExpr::createSub(LHS: NumBlocks, RHS: One, Ctx),
1215 RHS: MCConstantExpr::create(Value: 3, Ctx), Ctx);
1216 }
1217
1218 // Add to function symbol to create _dvgpr$ symbol.
1219 const MCExpr *DVgprFuncVal = MCBinaryExpr::createAdd(
1220 LHS: MCSymbolRefExpr::create(Symbol: CurrentFnSym, Ctx), RHS: EncodedBlocks, Ctx);
1221 MCSymbol *DVgprFuncSym =
1222 Ctx.getOrCreateSymbol(Name: Twine("_dvgpr$") + CurrentFnSym->getName());
1223 OutStreamer->emitAssignment(Symbol: DVgprFuncSym, Value: DVgprFuncVal);
1224 emitVisibility(Sym: DVgprFuncSym, Visibility: MF.getFunction().getVisibility());
1225 emitLinkage(GV: &MF.getFunction(), GVSym: DVgprFuncSym);
1226 }
1227}
1228
1229// TODO: Fold this into emitFunctionBodyStart.
1230void AMDGPUAsmPrinter::initializeTargetID(const Module &M) {
1231 getTargetStreamer()->initializeTargetID(STI: *getGlobalSTI());
1232
1233 auto &TSTargetID = getTargetStreamer()->getTargetID();
1234
1235 // Error if -mattr specified xnack or sramecc.
1236 // TODO: Remove this when subtarget features removed.
1237 StringRef FeatureString = getGlobalSTI()->getFeatureString();
1238 if (FeatureString.contains(Other: "xnack")) {
1239 M.getContext().diagnose(DI: DiagnosticInfoGeneric(
1240 "xnack/sramecc should be specified via module flags. "
1241 "Use module flag 'amdgpu.xnack' instead of subtarget feature",
1242 DS_Error));
1243 }
1244 if (FeatureString.contains(Other: "sramecc")) {
1245 M.getContext().diagnose(DI: DiagnosticInfoGeneric(
1246 "xnack/sramecc should be specified via module flags. "
1247 "Use module flag 'amdgpu.sramecc' instead of subtarget feature",
1248 DS_Error));
1249 }
1250
1251 // Apply xnack/sramecc settings from module flags.
1252 const auto &GCNTM = static_cast<const GCNTargetMachine &>(TM);
1253 if (getGlobalSTI()->getFeatureBits().test(I: AMDGPU::FeatureXNACKOnOffModes)) {
1254 AMDGPU::TargetIDSetting Setting =
1255 GCNTM.getTargetIDSettingFromModuleFlag(M, FlagName: "amdgpu.xnack");
1256 if (Setting != AMDGPU::TargetIDSetting::Any)
1257 TSTargetID->setXnackSetting(Setting);
1258 }
1259
1260 if (getGlobalSTI()->getFeatureBits().test(I: AMDGPU::FeatureSRAMECCOnOffModes)) {
1261 AMDGPU::TargetIDSetting Setting =
1262 GCNTM.getTargetIDSettingFromModuleFlag(M, FlagName: "amdgpu.sramecc");
1263 if (Setting != AMDGPU::TargetIDSetting::Any)
1264 TSTargetID->setSramEccSetting(Setting);
1265 }
1266}
1267
1268// AccumOffset computed for the MCExpr equivalent of:
1269// alignTo(std::max(1, NumVGPR), 4) / 4 - 1;
1270static const MCExpr *computeAccumOffset(const MCExpr *NumVGPR, MCContext &Ctx) {
1271 const MCExpr *ConstFour = MCConstantExpr::create(Value: 4, Ctx);
1272 const MCExpr *ConstOne = MCConstantExpr::create(Value: 1, Ctx);
1273
1274 // Can't be lower than 1 for subsequent alignTo.
1275 const MCExpr *MaximumTaken =
1276 AMDGPUMCExpr::createMax(Args: {ConstOne, NumVGPR}, Ctx);
1277
1278 // Practically, it's computing divideCeil(MaximumTaken, 4).
1279 const MCExpr *DivCeil = MCBinaryExpr::createDiv(
1280 LHS: AMDGPUMCExpr::createAlignTo(Value: MaximumTaken, Align: ConstFour, Ctx), RHS: ConstFour,
1281 Ctx);
1282
1283 return MCBinaryExpr::createSub(LHS: DivCeil, RHS: ConstOne, Ctx);
1284}
1285
1286static unsigned getLDSEncodingGranule(const GCNSubtarget &ST) {
1287 unsigned Granule =
1288 AMDGPU::getLDSEncodingGranule(AK: ST.getTargetID().getGPUKind());
1289 // The legacy generic targets have no encoding granularity feature. Preserve
1290 // the default used for code generation when no GPU is specified.
1291 return Granule ? Granule : 256;
1292}
1293
1294void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
1295 const MachineFunction &MF) {
1296 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1297 MCContext &Ctx = MF.getContext();
1298
1299 auto CreateExpr = [&Ctx](int64_t Value) {
1300 return MCConstantExpr::create(Value, Ctx);
1301 };
1302
1303 auto TryGetMCExprValue = [](const MCExpr *Value, uint64_t &Res) -> bool {
1304 int64_t Val;
1305 if (Value->evaluateAsAbsolute(Res&: Val)) {
1306 Res = Val;
1307 return true;
1308 }
1309 return false;
1310 };
1311
1312 auto GetSymRefExpr =
1313 [&](MCResourceInfo::ResourceInfoKind RIK) -> const MCExpr * {
1314 MCSymbol *Sym = RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK, OutContext);
1315 return MCSymbolRefExpr::create(Symbol: Sym, Ctx);
1316 };
1317
1318 using RIK = MCResourceInfo::ResourceInfoKind;
1319 ProgInfo.NumArchVGPR = GetSymRefExpr(RIK::RIK_NumVGPR);
1320 ProgInfo.NumAccVGPR = GetSymRefExpr(RIK::RIK_NumAGPR);
1321 ProgInfo.NumVGPR = AMDGPUMCExpr::createTotalNumVGPR(
1322 NumAGPR: ProgInfo.NumAccVGPR, NumVGPR: ProgInfo.NumArchVGPR, Ctx);
1323
1324 ProgInfo.AccumOffset = computeAccumOffset(NumVGPR: ProgInfo.NumArchVGPR, Ctx);
1325 ProgInfo.TgSplit =
1326 STM.hasTgSplitSupport() && AMDGPU::isTgSplitEnabled(F: MF.getFunction());
1327 ProgInfo.NumSGPR = GetSymRefExpr(RIK::RIK_NumSGPR);
1328 ProgInfo.ScratchSize = GetSymRefExpr(RIK::RIK_PrivateSegSize);
1329 ProgInfo.VCCUsed = GetSymRefExpr(RIK::RIK_UsesVCC);
1330 ProgInfo.FlatUsed = GetSymRefExpr(RIK::RIK_UsesFlatScratch);
1331 ProgInfo.DynamicCallStack =
1332 MCBinaryExpr::createOr(LHS: GetSymRefExpr(RIK::RIK_HasDynSizedStack),
1333 RHS: GetSymRefExpr(RIK::RIK_HasRecursion), Ctx);
1334
1335 const MCExpr *BarBlkConst = MCConstantExpr::create(Value: 4, Ctx);
1336 const MCExpr *AlignToBlk = AMDGPUMCExpr::createAlignTo(
1337 Value: GetSymRefExpr(RIK::RIK_NumNamedBarrier), Align: BarBlkConst, Ctx);
1338 ProgInfo.NamedBarCnt = MCBinaryExpr::createDiv(LHS: AlignToBlk, RHS: BarBlkConst, Ctx);
1339
1340 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1341
1342 // The calculations related to SGPR/VGPR blocks are
1343 // duplicated in part in AMDGPUAsmParser::calculateGPRBlocks, and could be
1344 // unified.
1345 const MCExpr *ExtraSGPRs = AMDGPUMCExpr::createExtraSGPRs(
1346 VCCUsed: ProgInfo.VCCUsed, FlatScrUsed: ProgInfo.FlatUsed,
1347 XNACKUsed: getTargetStreamer()->getTargetID()->isXnackOnOrAny(), Ctx);
1348
1349 // Check the addressable register limit before we add ExtraSGPRs.
1350 if (STM.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
1351 !STM.hasSGPRInitBug()) {
1352 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
1353 uint64_t NumSgpr;
1354 if (TryGetMCExprValue(ProgInfo.NumSGPR, NumSgpr) &&
1355 NumSgpr > MaxAddressableNumSGPRs) {
1356 // This can happen due to a compiler bug or when using inline asm.
1357 LLVMContext &Ctx = MF.getFunction().getContext();
1358 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1359 MF.getFunction(), "addressable scalar registers", NumSgpr,
1360 MaxAddressableNumSGPRs, DS_Error, DK_ResourceLimit));
1361 ProgInfo.NumSGPR = CreateExpr(MaxAddressableNumSGPRs - 1);
1362 }
1363 }
1364
1365 // Account for extra SGPRs and VGPRs reserved for debugger use.
1366 ProgInfo.NumSGPR = MCBinaryExpr::createAdd(LHS: ProgInfo.NumSGPR, RHS: ExtraSGPRs, Ctx);
1367
1368 const Function &F = MF.getFunction();
1369
1370 // Ensure there are enough SGPRs and VGPRs for wave dispatch, where wave
1371 // dispatch registers as function args.
1372 unsigned WaveDispatchNumSGPR = MFI->getNumWaveDispatchSGPRs(),
1373 WaveDispatchNumVGPR = MFI->getNumWaveDispatchVGPRs();
1374
1375 if (WaveDispatchNumSGPR) {
1376 ProgInfo.NumSGPR = AMDGPUMCExpr::createMax(
1377 Args: {ProgInfo.NumSGPR,
1378 MCBinaryExpr::createAdd(LHS: CreateExpr(WaveDispatchNumSGPR), RHS: ExtraSGPRs,
1379 Ctx)},
1380 Ctx);
1381 }
1382
1383 if (WaveDispatchNumVGPR) {
1384 ProgInfo.NumArchVGPR = AMDGPUMCExpr::createMax(
1385 Args: {ProgInfo.NumArchVGPR, CreateExpr(WaveDispatchNumVGPR)}, Ctx);
1386
1387 ProgInfo.NumVGPR = AMDGPUMCExpr::createTotalNumVGPR(
1388 NumAGPR: ProgInfo.NumAccVGPR, NumVGPR: ProgInfo.NumArchVGPR, Ctx);
1389 }
1390
1391 // Adjust number of registers used to meet default/requested minimum/maximum
1392 // number of waves per execution unit request.
1393 unsigned MaxWaves = MFI->getMaxWavesPerEU();
1394 ProgInfo.NumSGPRsForWavesPerEU =
1395 AMDGPUMCExpr::createMax(Args: {ProgInfo.NumSGPR, CreateExpr(1ul),
1396 CreateExpr(STM.getMinNumSGPRs(WavesPerEU: MaxWaves))},
1397 Ctx);
1398 ProgInfo.NumVGPRsForWavesPerEU =
1399 AMDGPUMCExpr::createMax(Args: {ProgInfo.NumVGPR, CreateExpr(1ul),
1400 CreateExpr(STM.getMinNumVGPRs(
1401 WavesPerEU: MaxWaves, DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize()))},
1402 Ctx);
1403
1404 if (STM.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS ||
1405 STM.hasSGPRInitBug()) {
1406 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
1407 uint64_t NumSgpr;
1408 if (TryGetMCExprValue(ProgInfo.NumSGPR, NumSgpr) &&
1409 NumSgpr > MaxAddressableNumSGPRs) {
1410 // This can happen due to a compiler bug or when using inline asm to use
1411 // the registers which are usually reserved for vcc etc.
1412 LLVMContext &Ctx = MF.getFunction().getContext();
1413 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1414 MF.getFunction(), "scalar registers", NumSgpr, MaxAddressableNumSGPRs,
1415 DS_Error, DK_ResourceLimit));
1416 ProgInfo.NumSGPR = CreateExpr(MaxAddressableNumSGPRs);
1417 ProgInfo.NumSGPRsForWavesPerEU = CreateExpr(MaxAddressableNumSGPRs);
1418 }
1419 }
1420
1421 if (STM.hasSGPRInitBug()) {
1422 ProgInfo.NumSGPR =
1423 CreateExpr(AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG);
1424 ProgInfo.NumSGPRsForWavesPerEU =
1425 CreateExpr(AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG);
1426 }
1427
1428 if (MFI->getNumUserSGPRs() > STM.getMaxNumUserSGPRs()) {
1429 LLVMContext &Ctx = MF.getFunction().getContext();
1430 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1431 MF.getFunction(), "user SGPRs", MFI->getNumUserSGPRs(),
1432 STM.getMaxNumUserSGPRs(), DS_Error));
1433 }
1434
1435 if (MFI->getLDSSize() > STM.getAddressableLocalMemorySize()) {
1436 LLVMContext &Ctx = MF.getFunction().getContext();
1437 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1438 MF.getFunction(), "local memory", MFI->getLDSSize(),
1439 STM.getAddressableLocalMemorySize(), DS_Error));
1440 }
1441
1442 // When dynamic VGPRs are enabled, entry functions are launched with a single
1443 // VGPR block. The register allocator enforces this constraint, but we also
1444 // need to catch explicit physical registers in inline asm and wave dispatch
1445 // VGPR arguments.
1446 if (MFI->isDynamicVGPREnabled() &&
1447 AMDGPU::isEntryFunctionCC(CC: F.getCallingConv())) {
1448 unsigned BlockSize = MFI->getDynamicVGPRBlockSize();
1449 uint64_t NumVgpr;
1450 if (TryGetMCExprValue(ProgInfo.NumVGPRsForWavesPerEU, NumVgpr) &&
1451 NumVgpr > BlockSize) {
1452 LLVMContext &Ctx = F.getContext();
1453 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1454 F, "dynamic VGPR entry point vector registers", NumVgpr, BlockSize,
1455 DS_Warning, DK_ResourceLimit));
1456 }
1457 }
1458
1459 // The MCExpr equivalent of getNumSGPRBlocks/getNumVGPRBlocks:
1460 // (alignTo(max(1u, NumGPR), GPREncodingGranule) / GPREncodingGranule) - 1
1461 auto GetNumGPRBlocks = [&CreateExpr, &Ctx](const MCExpr *NumGPR,
1462 unsigned Granule) {
1463 const MCExpr *OneConst = CreateExpr(1ul);
1464 const MCExpr *GranuleConst = CreateExpr(Granule);
1465 const MCExpr *MaxNumGPR = AMDGPUMCExpr::createMax(Args: {NumGPR, OneConst}, Ctx);
1466 const MCExpr *AlignToGPR =
1467 AMDGPUMCExpr::createAlignTo(Value: MaxNumGPR, Align: GranuleConst, Ctx);
1468 const MCExpr *DivGPR =
1469 MCBinaryExpr::createDiv(LHS: AlignToGPR, RHS: GranuleConst, Ctx);
1470 const MCExpr *SubGPR = MCBinaryExpr::createSub(LHS: DivGPR, RHS: OneConst, Ctx);
1471 return SubGPR;
1472 };
1473 // GFX10+ will always allocate 128 SGPRs and this field must be 0
1474 if (STM.getGeneration() >= AMDGPUSubtarget::GFX10) {
1475 ProgInfo.SGPRBlocks = CreateExpr(0ul);
1476 } else {
1477 ProgInfo.SGPRBlocks = GetNumGPRBlocks(ProgInfo.NumSGPRsForWavesPerEU,
1478 IsaInfo::getSGPREncodingGranule(STI: STM));
1479 }
1480 ProgInfo.VGPRBlocks = GetNumGPRBlocks(ProgInfo.NumVGPRsForWavesPerEU,
1481 IsaInfo::getVGPREncodingGranule(STI: STM));
1482
1483 const SIModeRegisterDefaults Mode = MFI->getMode();
1484
1485 // Set the value to initialize FP_ROUND and FP_DENORM parts of the mode
1486 // register.
1487 ProgInfo.FloatMode = getFPMode(Mode);
1488
1489 ProgInfo.IEEEMode = Mode.IEEE;
1490
1491 // Make clamp modifier on NaN input returns 0.
1492 ProgInfo.DX10Clamp = Mode.DX10Clamp;
1493 ProgInfo.SGPRSpill = MFI->getNumSpilledSGPRs();
1494 ProgInfo.VGPRSpill = MFI->getNumSpilledVGPRs();
1495
1496 ProgInfo.LDSSize = MFI->getLDSSize();
1497
1498 unsigned LDSGranularityBytes = getLDSEncodingGranule(ST: STM);
1499 ProgInfo.LDSBlocks =
1500 alignTo(Value: ProgInfo.LDSSize, Align: LDSGranularityBytes) / LDSGranularityBytes;
1501
1502 // The MCExpr equivalent of divideCeil.
1503 auto DivideCeil = [&Ctx](const MCExpr *Numerator, const MCExpr *Denominator) {
1504 const MCExpr *Ceil =
1505 AMDGPUMCExpr::createAlignTo(Value: Numerator, Align: Denominator, Ctx);
1506 return MCBinaryExpr::createDiv(LHS: Ceil, RHS: Denominator, Ctx);
1507 };
1508
1509 // Scratch is allocated in 64-dword or 256-dword blocks.
1510 unsigned ScratchAlignShift =
1511 STM.getGeneration() >= AMDGPUSubtarget::GFX11 ? 8 : 10;
1512 // We need to program the hardware with the amount of scratch memory that
1513 // is used by the entire wave. ProgInfo.ScratchSize is the amount of
1514 // scratch memory used per thread.
1515 ProgInfo.ScratchBlocks = DivideCeil(
1516 MCBinaryExpr::createMul(LHS: ProgInfo.ScratchSize,
1517 RHS: CreateExpr(STM.getWavefrontSize()), Ctx),
1518 CreateExpr(1ULL << ScratchAlignShift));
1519
1520 if (STM.hasSupportsWGP()) {
1521 ProgInfo.WgpMode = STM.isCuModeEnabled() ? 0 : 1;
1522 }
1523
1524 if (getIsaVersion(GPU: getGlobalSTI()->getCPU()).Major >= 10) {
1525 ProgInfo.MemOrdered = 1;
1526 ProgInfo.FwdProgress = !F.hasFnAttribute(Kind: "amdgpu-no-fwd-progress");
1527 }
1528
1529 // 0 = X, 1 = XY, 2 = XYZ
1530 unsigned TIDIGCompCnt = 0;
1531 if (MFI->hasWorkItemIDZ())
1532 TIDIGCompCnt = 2;
1533 else if (MFI->hasWorkItemIDY())
1534 TIDIGCompCnt = 1;
1535
1536 // The private segment wave byte offset is the last of the system SGPRs. We
1537 // initially assumed it was allocated, and may have used it. It shouldn't harm
1538 // anything to disable it if we know the stack isn't used here. We may still
1539 // have emitted code reading it to initialize scratch, but if that's unused
1540 // reading garbage should be OK.
1541 ProgInfo.ScratchEnable = MCBinaryExpr::createLOr(
1542 LHS: MCBinaryExpr::createGT(LHS: ProgInfo.ScratchBlocks,
1543 RHS: MCConstantExpr::create(Value: 0, Ctx), Ctx),
1544 RHS: ProgInfo.DynamicCallStack, Ctx);
1545
1546 ProgInfo.UserSGPR = MFI->getNumUserSGPRs();
1547 // For AMDHSA, TRAP_HANDLER must be zero, as it is populated by the CP.
1548 ProgInfo.TrapHandlerEnable = STM.isAmdHsaOS() ? 0 : STM.hasTrapHandler();
1549 ProgInfo.TGIdXEnable = MFI->hasWorkGroupIDX();
1550 ProgInfo.TGIdYEnable = MFI->hasWorkGroupIDY();
1551 ProgInfo.TGIdZEnable = MFI->hasWorkGroupIDZ();
1552 ProgInfo.TGSizeEnable = MFI->hasWorkGroupInfo();
1553 ProgInfo.TIdIGCompCount = TIDIGCompCnt;
1554 ProgInfo.EXCPEnMSB = 0;
1555 // For AMDHSA, LDS_SIZE must be zero, as it is populated by the CP.
1556 ProgInfo.LdsSize = STM.isAmdHsaOS() ? 0 : ProgInfo.LDSBlocks;
1557 ProgInfo.EXCPEnable = 0;
1558
1559 if (STM.hasGFX90AInsts()) {
1560 ProgInfo.ComputePGMRSrc3 =
1561 setBits(Dst: ProgInfo.ComputePGMRSrc3, Value: ProgInfo.AccumOffset,
1562 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
1563 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT, Ctx);
1564 ProgInfo.ComputePGMRSrc3 =
1565 setBits(Dst: ProgInfo.ComputePGMRSrc3, Value: CreateExpr(ProgInfo.TgSplit),
1566 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT,
1567 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT, Ctx);
1568 }
1569
1570 if (STM.hasGFX1250Insts())
1571 ProgInfo.ComputePGMRSrc3 =
1572 setBits(Dst: ProgInfo.ComputePGMRSrc3, Value: ProgInfo.NamedBarCnt,
1573 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
1574 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT, Ctx);
1575
1576 ProgInfo.Occupancy = createOccupancy(
1577 InitOcc: STM.getOccupancyWithWorkGroupSizes(MF).second,
1578 NumSGPRs: ProgInfo.NumSGPRsForWavesPerEU, NumVGPRs: ProgInfo.NumVGPRsForWavesPerEU,
1579 DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize(), STM, Ctx);
1580
1581 uint64_t Occupancy;
1582 if (TryGetMCExprValue(ProgInfo.Occupancy, Occupancy))
1583 diagnoseUnmetWavesPerEU(F, MFI: *MFI, Occupancy);
1584}
1585
1586static unsigned getRsrcReg(CallingConv::ID CallConv) {
1587 switch (CallConv) {
1588 default:
1589 [[fallthrough]];
1590 case CallingConv::AMDGPU_CS:
1591 return R_00B848_COMPUTE_PGM_RSRC1;
1592 case CallingConv::AMDGPU_LS:
1593 return R_00B528_SPI_SHADER_PGM_RSRC1_LS;
1594 case CallingConv::AMDGPU_HS:
1595 return R_00B428_SPI_SHADER_PGM_RSRC1_HS;
1596 case CallingConv::AMDGPU_ES:
1597 return R_00B328_SPI_SHADER_PGM_RSRC1_ES;
1598 case CallingConv::AMDGPU_GS:
1599 return R_00B228_SPI_SHADER_PGM_RSRC1_GS;
1600 case CallingConv::AMDGPU_VS:
1601 return R_00B128_SPI_SHADER_PGM_RSRC1_VS;
1602 case CallingConv::AMDGPU_PS:
1603 return R_00B028_SPI_SHADER_PGM_RSRC1_PS;
1604 }
1605}
1606
1607void AMDGPUAsmPrinter::EmitProgramInfoSI(
1608 const MachineFunction &MF, const SIProgramInfo &CurrentProgramInfo) {
1609 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1610 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1611 unsigned RsrcReg = getRsrcReg(CallConv: MF.getFunction().getCallingConv());
1612 MCContext &Ctx = MF.getContext();
1613
1614 // (((Value) & Mask) << Shift)
1615 auto SetBits = [&Ctx](const MCExpr *Value, uint32_t Mask, uint32_t Shift) {
1616 const MCExpr *msk = MCConstantExpr::create(Value: Mask, Ctx);
1617 const MCExpr *shft = MCConstantExpr::create(Value: Shift, Ctx);
1618 return MCBinaryExpr::createShl(LHS: MCBinaryExpr::createAnd(LHS: Value, RHS: msk, Ctx),
1619 RHS: shft, Ctx);
1620 };
1621
1622 auto EmitResolvedOrExpr = [this](const MCExpr *Value, unsigned Size) {
1623 int64_t Val;
1624 if (Value->evaluateAsAbsolute(Res&: Val))
1625 OutStreamer->emitIntValue(Value: static_cast<uint64_t>(Val), Size);
1626 else
1627 OutStreamer->emitValue(Value, Size);
1628 };
1629
1630 if (AMDGPU::isCompute(CC: MF.getFunction().getCallingConv())) {
1631 OutStreamer->emitInt32(R_00B848_COMPUTE_PGM_RSRC1);
1632
1633 EmitResolvedOrExpr(CurrentProgramInfo.getComputePGMRSrc1(ST: STM, Ctx),
1634 /*Size=*/4);
1635
1636 OutStreamer->emitInt32(R_00B84C_COMPUTE_PGM_RSRC2);
1637 EmitResolvedOrExpr(CurrentProgramInfo.getComputePGMRSrc2(ST: STM, Ctx),
1638 /*Size=*/4);
1639
1640 OutStreamer->emitInt32(R_00B860_COMPUTE_TMPRING_SIZE);
1641
1642 // Sets bits according to S_0286E8_WAVESIZE_* mask and shift values for the
1643 // appropriate generation.
1644 if (STM.getGeneration() >= AMDGPUSubtarget::GFX12) {
1645 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1646 /*Mask=*/0x3FFFF, /*Shift=*/12),
1647 /*Size=*/4);
1648 } else if (STM.getGeneration() == AMDGPUSubtarget::GFX11) {
1649 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1650 /*Mask=*/0x7FFF, /*Shift=*/12),
1651 /*Size=*/4);
1652 } else {
1653 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1654 /*Mask=*/0x1FFF, /*Shift=*/12),
1655 /*Size=*/4);
1656 }
1657
1658 // TODO: Should probably note flat usage somewhere. SC emits a "FlatPtr32 =
1659 // 0" comment but I don't see a corresponding field in the register spec.
1660 } else {
1661 OutStreamer->emitInt32(Value: RsrcReg);
1662
1663 const MCExpr *GPRBlocks = MCBinaryExpr::createOr(
1664 LHS: SetBits(CurrentProgramInfo.VGPRBlocks, /*Mask=*/0x3F, /*Shift=*/0),
1665 RHS: SetBits(CurrentProgramInfo.SGPRBlocks, /*Mask=*/0x0F, /*Shift=*/6),
1666 Ctx&: MF.getContext());
1667 EmitResolvedOrExpr(GPRBlocks, /*Size=*/4);
1668 OutStreamer->emitInt32(R_0286E8_SPI_TMPRING_SIZE);
1669
1670 // Sets bits according to S_0286E8_WAVESIZE_* mask and shift values for the
1671 // appropriate generation.
1672 if (STM.getGeneration() >= AMDGPUSubtarget::GFX12) {
1673 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1674 /*Mask=*/0x3FFFF, /*Shift=*/12),
1675 /*Size=*/4);
1676 } else if (STM.getGeneration() == AMDGPUSubtarget::GFX11) {
1677 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1678 /*Mask=*/0x7FFF, /*Shift=*/12),
1679 /*Size=*/4);
1680 } else {
1681 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1682 /*Mask=*/0x1FFF, /*Shift=*/12),
1683 /*Size=*/4);
1684 }
1685 }
1686
1687 if (MF.getFunction().getCallingConv() == CallingConv::AMDGPU_PS) {
1688 OutStreamer->emitInt32(R_00B02C_SPI_SHADER_PGM_RSRC2_PS);
1689 unsigned ExtraLDSSize = STM.getGeneration() >= AMDGPUSubtarget::GFX11
1690 ? divideCeil(Numerator: CurrentProgramInfo.LDSBlocks, Denominator: 2)
1691 : CurrentProgramInfo.LDSBlocks;
1692 OutStreamer->emitInt32(S_00B02C_EXTRA_LDS_SIZE(ExtraLDSSize));
1693 OutStreamer->emitInt32(R_0286CC_SPI_PS_INPUT_ENA);
1694 OutStreamer->emitInt32(Value: MFI->getPSInputEnable());
1695 OutStreamer->emitInt32(R_0286D0_SPI_PS_INPUT_ADDR);
1696 OutStreamer->emitInt32(Value: MFI->getPSInputAddr());
1697 }
1698
1699 OutStreamer->emitInt32(R_SPILLED_SGPRS);
1700 OutStreamer->emitInt32(Value: MFI->getNumSpilledSGPRs());
1701 OutStreamer->emitInt32(R_SPILLED_VGPRS);
1702 OutStreamer->emitInt32(Value: MFI->getNumSpilledVGPRs());
1703}
1704
1705// Helper function to add common PAL Metadata 3.0+
1706static void EmitPALMetadataCommon(AMDGPUPALMetadata *MD,
1707 const SIProgramInfo &CurrentProgramInfo,
1708 CallingConv::ID CC, const GCNSubtarget &ST,
1709 unsigned DynamicVGPRBlockSize) {
1710 if (ST.hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode))
1711 MD->setHwStage(CC, field: ".ieee_mode", Val: (bool)CurrentProgramInfo.IEEEMode);
1712
1713 MD->setHwStage(CC, field: ".wgp_mode", Val: (bool)CurrentProgramInfo.WgpMode);
1714 MD->setHwStage(CC, field: ".mem_ordered", Val: (bool)CurrentProgramInfo.MemOrdered);
1715 MD->setHwStage(CC, field: ".forward_progress", Val: (bool)CurrentProgramInfo.FwdProgress);
1716
1717 if (AMDGPU::isCompute(CC)) {
1718 MD->setHwStage(CC, field: ".trap_present",
1719 Val: (bool)CurrentProgramInfo.TrapHandlerEnable);
1720 MD->setHwStage(CC, field: ".excp_en", Val: CurrentProgramInfo.EXCPEnable);
1721
1722 if (DynamicVGPRBlockSize != 0)
1723 MD->setComputeRegisters(field: ".dynamic_vgpr_en", Val: true);
1724 }
1725
1726 MD->updateHwStageMaximum(
1727 CC, field: ".lds_size",
1728 Val: (unsigned)(CurrentProgramInfo.LdsSize * getLDSEncodingGranule(ST)));
1729}
1730
1731// This is the equivalent of EmitProgramInfoSI above, but for when the OS type
1732// is AMDPAL. It stores each compute/SPI register setting and other PAL
1733// metadata items into the PALMD::Metadata, combining with any provided by the
1734// frontend as LLVM metadata. Once all functions are written, the PAL metadata
1735// is then written as a single block in the .note section.
1736void AMDGPUAsmPrinter::EmitPALMetadata(
1737 const MachineFunction &MF, const SIProgramInfo &CurrentProgramInfo) {
1738 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1739 auto CC = MF.getFunction().getCallingConv();
1740 auto *MD = getTargetStreamer()->getPALMetadata();
1741 auto &Ctx = MF.getContext();
1742
1743 MD->setEntryPoint(CC, Name: MF.getFunction().getName());
1744 MD->setNumUsedVgprs(CC, Val: CurrentProgramInfo.NumVGPRsForWavesPerEU, Ctx);
1745
1746 // For targets that support dynamic VGPRs, set the number of saved dynamic
1747 // VGPRs (if any) in the PAL metadata.
1748 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1749 if (MFI->isDynamicVGPREnabled() &&
1750 MFI->getScratchReservedForDynamicVGPRs() > 0)
1751 MD->setHwStage(CC, field: ".dynamic_vgpr_saved_count",
1752 Val: MFI->getScratchReservedForDynamicVGPRs() / 4);
1753
1754 // Only set AGPRs for supported devices
1755 if (STM.hasMAIInsts()) {
1756 MD->setNumUsedAgprs(CC, Val: CurrentProgramInfo.NumAccVGPR);
1757 }
1758
1759 MD->setNumUsedSgprs(CC, Val: CurrentProgramInfo.NumSGPRsForWavesPerEU, Ctx);
1760 if (MD->getPALMajorVersion() < 3) {
1761 MD->setRsrc1(CC, Val: CurrentProgramInfo.getPGMRSrc1(CC, ST: STM, Ctx), Ctx);
1762 if (AMDGPU::isCompute(CC)) {
1763 MD->setRsrc2(CC, Val: CurrentProgramInfo.getComputePGMRSrc2(ST: STM, Ctx), Ctx);
1764 } else {
1765 const MCExpr *HasScratchBlocks =
1766 MCBinaryExpr::createGT(LHS: CurrentProgramInfo.ScratchBlocks,
1767 RHS: MCConstantExpr::create(Value: 0, Ctx), Ctx);
1768 auto [Shift, Mask] = getShiftMask(C_00B84C_SCRATCH_EN);
1769 MD->setRsrc2(CC, Val: maskShiftSet(Val: HasScratchBlocks, Mask, Shift, Ctx), Ctx);
1770 }
1771 } else {
1772 MD->setHwStage(CC, field: ".debug_mode", Val: (bool)CurrentProgramInfo.DebugMode);
1773 MD->setHwStage(CC, field: ".scratch_en", Type: msgpack::Type::Boolean,
1774 Val: CurrentProgramInfo.ScratchEnable);
1775 EmitPALMetadataCommon(MD, CurrentProgramInfo, CC, ST: STM,
1776 DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize());
1777 }
1778
1779 // ScratchSize is in bytes, 16 aligned.
1780 MD->setScratchSize(
1781 CC,
1782 Val: AMDGPUMCExpr::createAlignTo(Value: CurrentProgramInfo.ScratchSize,
1783 Align: MCConstantExpr::create(Value: 16, Ctx), Ctx),
1784 Ctx);
1785
1786 MD->setBackendStackSize(CC, Val: CurrentProgramInfo.ScratchSize);
1787
1788 if (MF.getFunction().getCallingConv() == CallingConv::AMDGPU_PS) {
1789 unsigned ExtraLDSSize = STM.getGeneration() >= AMDGPUSubtarget::GFX11
1790 ? divideCeil(Numerator: CurrentProgramInfo.LDSBlocks, Denominator: 2)
1791 : CurrentProgramInfo.LDSBlocks;
1792 if (MD->getPALMajorVersion() < 3) {
1793 MD->setRsrc2(
1794 CC,
1795 Val: MCConstantExpr::create(S_00B02C_EXTRA_LDS_SIZE(ExtraLDSSize), Ctx),
1796 Ctx);
1797 MD->setSpiPsInputEna(MFI->getPSInputEnable());
1798 MD->setSpiPsInputAddr(MFI->getPSInputAddr());
1799 } else {
1800 // Graphics registers
1801 const unsigned ExtraLdsDwGranularity =
1802 STM.getGeneration() >= AMDGPUSubtarget::GFX11 ? 256 : 128;
1803 MD->setGraphicsRegisters(
1804 field: ".ps_extra_lds_size",
1805 Val: (unsigned)(ExtraLDSSize * ExtraLdsDwGranularity * sizeof(uint32_t)));
1806
1807 // Set PsInputEna and PsInputAddr .spi_ps_input_ena and .spi_ps_input_addr
1808 static StringLiteral const PsInputFields[] = {
1809 ".persp_sample_ena", ".persp_center_ena",
1810 ".persp_centroid_ena", ".persp_pull_model_ena",
1811 ".linear_sample_ena", ".linear_center_ena",
1812 ".linear_centroid_ena", ".line_stipple_tex_ena",
1813 ".pos_x_float_ena", ".pos_y_float_ena",
1814 ".pos_z_float_ena", ".pos_w_float_ena",
1815 ".front_face_ena", ".ancillary_ena",
1816 ".sample_coverage_ena", ".pos_fixed_pt_ena"};
1817 unsigned PSInputEna = MFI->getPSInputEnable();
1818 unsigned PSInputAddr = MFI->getPSInputAddr();
1819 for (auto [Idx, Field] : enumerate(First: PsInputFields)) {
1820 MD->setGraphicsRegisters(field1: ".spi_ps_input_ena", field2: Field,
1821 Val: (bool)((PSInputEna >> Idx) & 1));
1822 MD->setGraphicsRegisters(field1: ".spi_ps_input_addr", field2: Field,
1823 Val: (bool)((PSInputAddr >> Idx) & 1));
1824 }
1825 }
1826 }
1827
1828 // For version 3 and above the wave front size is already set in the metadata
1829 if (MD->getPALMajorVersion() < 3 && STM.isWave32())
1830 MD->setWave32(MF.getFunction().getCallingConv());
1831}
1832
1833void AMDGPUAsmPrinter::emitPALFunctionMetadata(const MachineFunction &MF) {
1834 auto *MD = getTargetStreamer()->getPALMetadata();
1835 const MachineFrameInfo &MFI = MF.getFrameInfo();
1836 StringRef FnName = MF.getFunction().getName();
1837 MD->setFunctionStackFrameSize(FnName, Val: MFI.getStackSize());
1838 MD->setFunctionBackendStackSize(FnName, Val: CurrentProgramInfo.ScratchSize);
1839 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1840 MCContext &Ctx = MF.getContext();
1841
1842 if (MD->getPALMajorVersion() < 3) {
1843 // Set compute registers
1844 MD->setRsrc1(
1845 CC: CallingConv::AMDGPU_CS,
1846 Val: CurrentProgramInfo.getPGMRSrc1(CC: CallingConv::AMDGPU_CS, ST, Ctx), Ctx);
1847 MD->setRsrc2(CC: CallingConv::AMDGPU_CS,
1848 Val: CurrentProgramInfo.getComputePGMRSrc2(ST, Ctx), Ctx);
1849 } else {
1850 EmitPALMetadataCommon(
1851 MD, CurrentProgramInfo, CC: CallingConv::AMDGPU_CS, ST,
1852 DynamicVGPRBlockSize: MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize());
1853 }
1854
1855 // Set optional info
1856 MD->setFunctionLdsSize(FnName, Val: CurrentProgramInfo.LDSSize);
1857 MD->setFunctionNumUsedVgprs(FnName, Val: CurrentProgramInfo.NumVGPRsForWavesPerEU);
1858 MD->setFunctionNumUsedSgprs(FnName, Val: CurrentProgramInfo.NumSGPRsForWavesPerEU);
1859}
1860
1861// This is supposed to be log2(Size)
1862static amd_element_byte_size_t getElementByteSizeValue(unsigned Size) {
1863 switch (Size) {
1864 case 4:
1865 return AMD_ELEMENT_4_BYTES;
1866 case 8:
1867 return AMD_ELEMENT_8_BYTES;
1868 case 16:
1869 return AMD_ELEMENT_16_BYTES;
1870 default:
1871 llvm_unreachable("invalid private_element_size");
1872 }
1873}
1874
1875void AMDGPUAsmPrinter::getAmdKernelCode(AMDGPUMCKernelCodeT &Out,
1876 const SIProgramInfo &CurrentProgramInfo,
1877 const MachineFunction &MF) const {
1878 const Function &F = MF.getFunction();
1879 assert(F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
1880 F.getCallingConv() == CallingConv::SPIR_KERNEL);
1881
1882 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1883 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1884 MCContext &Ctx = MF.getContext();
1885
1886 Out.initDefault(STI: STM, Ctx, /*InitMCExpr=*/false);
1887
1888 Out.compute_pgm_resource1_registers =
1889 CurrentProgramInfo.getComputePGMRSrc1(ST: STM, Ctx);
1890 Out.compute_pgm_resource2_registers =
1891 CurrentProgramInfo.getComputePGMRSrc2(ST: STM, Ctx);
1892 Out.code_properties |= AMD_CODE_PROPERTY_IS_PTR64;
1893
1894 Out.is_dynamic_callstack = CurrentProgramInfo.DynamicCallStack;
1895
1896 AMD_HSA_BITS_SET(Out.code_properties, AMD_CODE_PROPERTY_PRIVATE_ELEMENT_SIZE,
1897 getElementByteSizeValue(STM.getMaxPrivateElementSize(true)));
1898
1899 const GCNUserSGPRUsageInfo &UserSGPRInfo = MFI->getUserSGPRInfo();
1900 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
1901 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER;
1902 }
1903
1904 if (UserSGPRInfo.hasDispatchPtr())
1905 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR;
1906
1907 if (UserSGPRInfo.hasQueuePtr())
1908 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR;
1909
1910 if (UserSGPRInfo.hasKernargSegmentPtr())
1911 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR;
1912
1913 if (UserSGPRInfo.hasDispatchID())
1914 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID;
1915
1916 if (UserSGPRInfo.hasFlatScratchInit())
1917 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT;
1918
1919 if (UserSGPRInfo.hasPrivateSegmentSize())
1920 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE;
1921
1922 if (STM.isXNACKEnabled())
1923 Out.code_properties |= AMD_CODE_PROPERTY_IS_XNACK_SUPPORTED;
1924
1925 Align MaxKernArgAlign;
1926 Out.kernarg_segment_byte_size = STM.getKernArgSegmentSize(F, MaxAlign&: MaxKernArgAlign);
1927 Out.wavefront_sgpr_count = CurrentProgramInfo.NumSGPR;
1928 Out.workitem_vgpr_count = CurrentProgramInfo.NumVGPR;
1929 Out.workitem_private_segment_byte_size = CurrentProgramInfo.ScratchSize;
1930 Out.workgroup_group_segment_byte_size = CurrentProgramInfo.LDSSize;
1931
1932 // kernarg_segment_alignment is specified as log of the alignment.
1933 // The minimum alignment is 16.
1934 // FIXME: The metadata treats the minimum as 4?
1935 Out.kernarg_segment_alignment =
1936 static_cast<uint8_t>(Log2(A: std::max(a: Align(16), b: MaxKernArgAlign)));
1937}
1938
1939bool AMDGPUAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
1940 const char *ExtraCode, raw_ostream &O) {
1941 // First try the generic code, which knows about modifiers like 'c' and 'n'.
1942 if (!AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, OS&: O))
1943 return false;
1944
1945 if (ExtraCode && ExtraCode[0]) {
1946 if (ExtraCode[1] != 0)
1947 return true; // Unknown modifier.
1948
1949 switch (ExtraCode[0]) {
1950 case 'r':
1951 break;
1952 default:
1953 return true;
1954 }
1955 }
1956
1957 // TODO: Should be able to support other operand types like globals.
1958 const MachineOperand &MO = MI->getOperand(i: OpNo);
1959 if (MO.isReg()) {
1960 AMDGPUInstPrinter::printRegOperand(Reg: MO.getReg(), O,
1961 MRI: *MF->getSubtarget().getRegisterInfo());
1962 return false;
1963 }
1964 if (MO.isImm()) {
1965 int64_t Val = MO.getImm();
1966 if (AMDGPU::isInlinableIntLiteral(Literal: Val)) {
1967 O << Val;
1968 } else if (isUInt<16>(x: Val)) {
1969 O << format(Fmt: "0x%" PRIx16, Vals: static_cast<uint16_t>(Val));
1970 } else if (isUInt<32>(x: Val)) {
1971 O << format(Fmt: "0x%" PRIx32, Vals: static_cast<uint32_t>(Val));
1972 } else {
1973 O << format(Fmt: "0x%" PRIx64, Vals: static_cast<uint64_t>(Val));
1974 }
1975 return false;
1976 }
1977 return true;
1978}
1979
1980bool AMDGPUAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
1981 unsigned OpNo,
1982 const char *ExtraCode,
1983 raw_ostream &OS) {
1984 const MachineOperand &MO = MI->getOperand(i: OpNo);
1985 AMDGPUInstPrinter::printRegOperand(Reg: MO.getReg(), O&: OS,
1986 MRI: *MF->getSubtarget().getRegisterInfo());
1987 return false;
1988}
1989
1990void AMDGPUAsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const {
1991 AU.addRequired<AMDGPUResourceUsageAnalysisWrapperPass>();
1992 AU.addPreserved<AMDGPUResourceUsageAnalysisWrapperPass>();
1993 AU.addRequired<MachineModuleInfoWrapperPass>();
1994 AU.addPreserved<MachineModuleInfoWrapperPass>();
1995 AsmPrinter::getAnalysisUsage(AU);
1996}
1997
1998void AMDGPUAsmPrinter::emitResourceUsageRemarks(
1999 const MachineFunction &MF, const SIProgramInfo &CurrentProgramInfo,
2000 bool isModuleEntryFunction, bool hasMAIInsts) {
2001 if (!ORE)
2002 return;
2003
2004 const char *Name = "kernel-resource-usage";
2005 const char *Indent = " ";
2006
2007 // If the remark is not specifically enabled, do not output to yaml
2008 LLVMContext &Ctx = MF.getFunction().getContext();
2009 if (!Ctx.getDiagHandlerPtr()->isAnalysisRemarkEnabled(PassName: Name))
2010 return;
2011
2012 // Currently non-kernel functions have no resources to emit.
2013 if (!isEntryFunctionCC(CC: MF.getFunction().getCallingConv()))
2014 return;
2015
2016 auto EmitResourceUsageRemark = [&](StringRef RemarkName,
2017 StringRef RemarkLabel, auto Argument) {
2018 // Add an indent for every line besides the line with the kernel name. This
2019 // makes it easier to tell which resource usage go with which kernel since
2020 // the kernel name will always be displayed first.
2021 std::string LabelStr = RemarkLabel.str() + ": ";
2022 if (RemarkName != "FunctionName")
2023 LabelStr = Indent + LabelStr;
2024
2025 ORE->emit([&]() {
2026 return MachineOptimizationRemarkAnalysis(Name, RemarkName,
2027 MF.getFunction().getSubprogram(),
2028 &MF.front())
2029 << LabelStr << ore::NV(RemarkName, Argument);
2030 });
2031 };
2032
2033 // FIXME: Formatting here is pretty nasty because clang does not accept
2034 // newlines from diagnostics. This forces us to emit multiple diagnostic
2035 // remarks to simulate newlines. If and when clang does accept newlines, this
2036 // formatting should be aggregated into one remark with newlines to avoid
2037 // printing multiple diagnostic location and diag opts.
2038 EmitResourceUsageRemark("FunctionName", "Function Name",
2039 MF.getFunction().getName());
2040 EmitResourceUsageRemark("NumSGPR", "TotalSGPRs",
2041 getMCExprStr(Value: CurrentProgramInfo.NumSGPR));
2042 EmitResourceUsageRemark("NumVGPR", "VGPRs",
2043 getMCExprStr(Value: CurrentProgramInfo.NumArchVGPR));
2044 if (hasMAIInsts) {
2045 EmitResourceUsageRemark("NumAGPR", "AGPRs",
2046 getMCExprStr(Value: CurrentProgramInfo.NumAccVGPR));
2047 }
2048 EmitResourceUsageRemark("ScratchSize", "ScratchSize [bytes/lane]",
2049 getMCExprStr(Value: CurrentProgramInfo.ScratchSize));
2050 int64_t DynStack;
2051 bool DynStackEvaluatable =
2052 CurrentProgramInfo.DynamicCallStack->evaluateAsAbsolute(Res&: DynStack);
2053 StringRef DynamicStackStr =
2054 DynStackEvaluatable && DynStack ? "True" : "False";
2055 EmitResourceUsageRemark("DynamicStack", "Dynamic Stack", DynamicStackStr);
2056 EmitResourceUsageRemark("Occupancy", "Occupancy [waves/SIMD]",
2057 getMCExprStr(Value: CurrentProgramInfo.Occupancy));
2058 EmitResourceUsageRemark("SGPRSpill", "SGPRs Spill",
2059 CurrentProgramInfo.SGPRSpill);
2060 EmitResourceUsageRemark("VGPRSpill", "VGPRs Spill",
2061 CurrentProgramInfo.VGPRSpill);
2062 if (isModuleEntryFunction)
2063 EmitResourceUsageRemark("BytesLDS", "LDS Size [bytes/block]",
2064 CurrentProgramInfo.LDSSize);
2065}
2066
2067PreservedAnalyses AMDGPUAsmPrinterBeginPass::run(Module &M,
2068 ModuleAnalysisManager &MAM) {
2069
2070 AMDGPUAsmPrinter &AsmPrinter = static_cast<AMDGPUAsmPrinter &>(
2071 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
2072 setupModuleAsmPrinter(M, MAM, AsmPrinter);
2073 AsmPrinter.doInitialization(M);
2074 return PreservedAnalyses::all();
2075}
2076
2077PreservedAnalyses
2078AMDGPUAsmPrinterPass::run(MachineFunction &MF,
2079 MachineFunctionAnalysisManager &MFAM) {
2080 AMDGPUAsmPrinter &AsmPrinter = static_cast<AMDGPUAsmPrinter &>(
2081 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF)
2082 .getCachedResult<AsmPrinterAnalysis>(IR&: *MF.getFunction().getParent())
2083 ->getPrinter());
2084 setupMachineFunctionAsmPrinter(MFAM, MF, AsmPrinter);
2085 AsmPrinter.GetResourceUsage = [&MFAM](MachineFunction &MF)
2086 -> const AMDGPUResourceUsageAnalysisImpl::SIFunctionResourceInfo * {
2087 return &MFAM.getResult<AMDGPUResourceUsageAnalysis>(IR&: MF);
2088 };
2089 AsmPrinter.runOnMachineFunction(MF);
2090 return PreservedAnalyses::all();
2091}
2092
2093PreservedAnalyses AMDGPUAsmPrinterEndPass::run(Module &M,
2094 ModuleAnalysisManager &MAM) {
2095 AMDGPUAsmPrinter &AsmPrinter = static_cast<AMDGPUAsmPrinter &>(
2096 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
2097 setupModuleAsmPrinter(M, MAM, AsmPrinter);
2098 AsmPrinter.doFinalization(M);
2099 return PreservedAnalyses::all();
2100}
2101
2102char AMDGPUAsmPrinter::ID = 0;
2103
2104INITIALIZE_PASS(AMDGPUAsmPrinter, "amdgpu-asm-printer",
2105 "AMDGPU Assembly Printer", false, false)
2106