1//===-- AMDGPUAsmPrinter.cpp - AMDGPU assembly printer --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10///
11/// The AMDGPUAsmPrinter is used to print both assembly string and also binary
12/// code. When passed an MCAsmStreamer it prints assembly and when passed
13/// an MCObjectStreamer it outputs binary code.
14//
15//===----------------------------------------------------------------------===//
16//
17
18#include "AMDGPUAsmPrinter.h"
19#include "AMDGPU.h"
20#include "AMDGPUHSAMetadataStreamer.h"
21#include "AMDGPUMCResourceInfo.h"
22#include "AMDGPUResourceUsageAnalysis.h"
23#include "AMDGPUTargetMachine.h"
24#include "GCNSubtarget.h"
25#include "MCTargetDesc/AMDGPUInstPrinter.h"
26#include "MCTargetDesc/AMDGPUMCExpr.h"
27#include "MCTargetDesc/AMDGPUMCKernelDescriptor.h"
28#include "MCTargetDesc/AMDGPUTargetStreamer.h"
29#include "R600AsmPrinter.h"
30#include "SIMachineFunctionInfo.h"
31#include "TargetInfo/AMDGPUTargetInfo.h"
32#include "Utils/AMDGPUBaseInfo.h"
33#include "Utils/AMDKernelCodeTUtils.h"
34#include "Utils/SIDefinesUtils.h"
35#include "llvm/ADT/StringSet.h"
36#include "llvm/Analysis/OptimizationRemarkEmitter.h"
37#include "llvm/BinaryFormat/ELF.h"
38#include "llvm/CodeGen/AsmPrinterAnalysis.h"
39#include "llvm/CodeGen/AsmPrinterHandler.h"
40#include "llvm/CodeGen/MachineFrameInfo.h"
41#include "llvm/CodeGen/MachineModuleInfo.h"
42#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
43#include "llvm/IR/DiagnosticInfo.h"
44#include "llvm/MC/MCAssembler.h"
45#include "llvm/MC/MCContext.h"
46#include "llvm/MC/MCSectionELF.h"
47#include "llvm/MC/MCStreamer.h"
48#include "llvm/MC/MCValue.h"
49#include "llvm/MC/TargetRegistry.h"
50#include "llvm/Support/AMDHSAKernelDescriptor.h"
51#include "llvm/Support/Compiler.h"
52#include "llvm/Target/TargetLoweringObjectFile.h"
53#include "llvm/Target/TargetMachine.h"
54#include "llvm/TargetParser/AMDGPUTargetParser.h"
55
56using namespace llvm;
57using namespace llvm::AMDGPU;
58
59// This should get the default rounding mode from the kernel. We just set the
60// default here, but this could change if the OpenCL rounding mode pragmas are
61// used.
62//
63// The denormal mode here should match what is reported by the OpenCL runtime
64// for the CL_FP_DENORM bit from CL_DEVICE_{HALF|SINGLE|DOUBLE}_FP_CONFIG, but
65// can also be override to flush with the -cl-denorms-are-zero compiler flag.
66//
67// AMD OpenCL only sets flush none and reports CL_FP_DENORM for double
68// precision, and leaves single precision to flush all and does not report
69// CL_FP_DENORM for CL_DEVICE_SINGLE_FP_CONFIG. Mesa's OpenCL currently reports
70// CL_FP_DENORM for both.
71//
72// FIXME: It seems some instructions do not support single precision denormals
73// regardless of the mode (exp_*_f32, rcp_*_f32, rsq_*_f32, rsq_*f32, sqrt_f32,
74// and sin_f32, cos_f32 on most parts).
75
76// We want to use these instructions, and using fp32 denormals also causes
77// instructions to run at the double precision rate for the device so it's
78// probably best to just report no single precision denormals.
79static uint32_t getFPMode(SIModeRegisterDefaults Mode) {
80 return FP_ROUND_MODE_SP(FP_ROUND_ROUND_TO_NEAREST) |
81 FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_NEAREST) |
82 FP_DENORM_MODE_SP(Mode.fpDenormModeSPValue()) |
83 FP_DENORM_MODE_DP(Mode.fpDenormModeDPValue());
84}
85
86static AsmPrinter *
87createAMDGPUAsmPrinterPass(TargetMachine &tm,
88 std::unique_ptr<MCStreamer> &&Streamer) {
89 return new AMDGPUAsmPrinter(tm, std::move(Streamer));
90}
91
92extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
93LLVMInitializeAMDGPUAsmPrinter() {
94 TargetRegistry::RegisterAsmPrinter(T&: getTheR600Target(),
95 Fn: llvm::createR600AsmPrinterPass);
96 TargetRegistry::RegisterAsmPrinter(T&: getTheGCNTarget(),
97 Fn: createAMDGPUAsmPrinterPass);
98 TargetRegistry::RegisterAsmPrinter(T&: getTheGCNLegacyTarget(),
99 Fn: createAMDGPUAsmPrinterPass);
100}
101
102namespace {
103class AMDGPUAsmPrinterHandler : public AsmPrinterHandler {
104protected:
105 AMDGPUAsmPrinter *Asm;
106
107public:
108 AMDGPUAsmPrinterHandler(AMDGPUAsmPrinter *A) : Asm(A) {}
109
110 void beginFunction(const MachineFunction *MF) override {}
111
112 void endFunction(const MachineFunction *MF) override { Asm->endFunction(MF); }
113
114 void endModule() override {}
115};
116} // End anonymous namespace
117
118AMDGPUAsmPrinter::AMDGPUAsmPrinter(TargetMachine &TM,
119 std::unique_ptr<MCStreamer> Streamer)
120 : AsmPrinter(TM, std::move(Streamer)) {
121 assert(OutStreamer && "AsmPrinter constructed without streamer");
122 GetResourceUsage = [this](MachineFunction &MF)
123 -> const AMDGPUResourceUsageAnalysisImpl::SIFunctionResourceInfo * {
124 if (auto *ResourceUsageW =
125 getAnalysisIfAvailable<AMDGPUResourceUsageAnalysisWrapperPass>())
126 return &ResourceUsageW->getResourceInfo();
127 return nullptr;
128 };
129}
130
131StringRef AMDGPUAsmPrinter::getPassName() const {
132 return "AMDGPU Assembly Printer";
133}
134
135const MCSubtargetInfo *AMDGPUAsmPrinter::getGlobalSTI() const {
136 return &TM.getMCSubtargetInfo();
137}
138
139AMDGPUTargetStreamer *AMDGPUAsmPrinter::getTargetStreamer() const {
140 if (!OutStreamer)
141 return nullptr;
142 return static_cast<AMDGPUTargetStreamer *>(OutStreamer->getTargetStreamer());
143}
144
145void AMDGPUAsmPrinter::emitStartOfAsmFile(Module &M) {
146 IsTargetStreamerInitialized = false;
147}
148
149void AMDGPUAsmPrinter::initTargetStreamer(Module &M) {
150 IsTargetStreamerInitialized = true;
151
152 // TODO: Which one is called first, emitStartOfAsmFile or
153 // emitFunctionBodyStart?
154 if (getTargetStreamer() && !getTargetStreamer()->getTargetID())
155 initializeTargetID(M);
156
157 const Triple &TT = M.getTargetTriple();
158 if (TT.getOS() != Triple::AMDHSA && TT.getOS() != Triple::AMDPAL)
159 return;
160
161 getTargetStreamer()->EmitDirectiveAMDGCNTarget();
162
163 if (TT.getOS() == Triple::AMDHSA) {
164 getTargetStreamer()->EmitDirectiveAMDHSACodeObjectVersion(
165 COV: CodeObjectVersion);
166 HSAMetadataStream->begin(Mod: M, TargetID: *getTargetStreamer()->getTargetID());
167 }
168
169 if (TT.getOS() == Triple::AMDPAL)
170 getTargetStreamer()->getPALMetadata()->readFromIR(M);
171}
172
173void AMDGPUAsmPrinter::emitEndOfAsmFile(Module &M) {
174 // Init target streamer if it has not yet happened
175 if (!IsTargetStreamerInitialized)
176 initTargetStreamer(M);
177
178 const Triple &TT = M.getTargetTriple();
179 if (TT.getOS() != Triple::AMDHSA)
180 getTargetStreamer()->EmitISAVersion();
181
182 // Emit HSA Metadata (NT_AMD_AMDGPU_HSA_METADATA).
183 // Emit HSA Metadata (NT_AMD_HSA_METADATA).
184 if (TT.getOS() == Triple::AMDHSA) {
185 HSAMetadataStream->end();
186 bool Success = HSAMetadataStream->emitTo(TargetStreamer&: *getTargetStreamer());
187 (void)Success;
188 assert(Success && "Malformed HSA Metadata");
189 }
190}
191
192void AMDGPUAsmPrinter::emitFunctionBodyStart() {
193 const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
194 const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
195 const Function &F = MF->getFunction();
196
197 // TODO: We're checking this late, would be nice to check it earlier.
198 if (STM.requiresCodeObjectV6() && CodeObjectVersion < AMDGPU::AMDHSA_COV6) {
199 reportFatalUsageError(
200 reason: STM.getCPU() + " is only available on code object version 6 or better");
201 }
202
203 // TODO: Which one is called first, emitStartOfAsmFile or
204 // emitFunctionBodyStart?
205 if (!getTargetStreamer()->getTargetID())
206 initializeTargetID(M: *F.getParent());
207
208 const auto &FunctionTargetID = STM.getTargetID();
209 // Make sure function's xnack settings are compatible with module's
210 // xnack settings.
211 if (FunctionTargetID.isXnackSupported() &&
212 FunctionTargetID.getXnackSetting() != AMDGPU::TargetIDSetting::Any &&
213 FunctionTargetID.getXnackSetting() !=
214 getTargetStreamer()->getTargetID()->getXnackSetting()) {
215 OutContext.reportError(
216 L: {}, Msg: "xnack setting of '" + Twine(MF->getName()) +
217 "' function does not match module xnack setting");
218 return;
219 }
220 // Make sure function's sramecc settings are compatible with module's
221 // sramecc settings.
222 if (FunctionTargetID.isSramEccSupported() &&
223 FunctionTargetID.getSramEccSetting() != AMDGPU::TargetIDSetting::Any &&
224 FunctionTargetID.getSramEccSetting() !=
225 getTargetStreamer()->getTargetID()->getSramEccSetting()) {
226 OutContext.reportError(
227 L: {}, Msg: "sramecc setting of '" + Twine(MF->getName()) +
228 "' function does not match module sramecc setting");
229 return;
230 }
231
232 if (!MFI.isEntryFunction())
233 return;
234
235 if (STM.isMesaKernel(F) &&
236 (F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
237 F.getCallingConv() == CallingConv::SPIR_KERNEL)) {
238 AMDGPUMCKernelCodeT KernelCode;
239 getAmdKernelCode(Out&: KernelCode, KernelInfo: CurrentProgramInfo, MF: *MF);
240 KernelCode.validate(STI: &STM, Ctx&: MF->getContext());
241 getTargetStreamer()->EmitAMDKernelCodeT(Header&: KernelCode);
242 }
243
244 if (STM.isAmdHsaOS())
245 HSAMetadataStream->emitKernel(MF: *MF, ProgramInfo: CurrentProgramInfo);
246}
247
248/// Set bits in a kernel descriptor MCExpr field:
249/// return ((Dst & ~Mask) | (Value << Shift))
250static const MCExpr *setBits(const MCExpr *Dst, const MCExpr *Value,
251 uint32_t Mask, uint32_t Shift, MCContext &Ctx) {
252 const auto *Shft = MCConstantExpr::create(Value: Shift, Ctx);
253 const auto *Msk = MCConstantExpr::create(Value: Mask, Ctx);
254 Dst = MCBinaryExpr::createAnd(LHS: Dst, RHS: MCUnaryExpr::createNot(Expr: Msk, Ctx), Ctx);
255 Dst = MCBinaryExpr::createOr(LHS: Dst, RHS: MCBinaryExpr::createShl(LHS: Value, RHS: Shft, Ctx),
256 Ctx);
257 return Dst;
258}
259
260void AMDGPUAsmPrinter::endFunction(const MachineFunction *MF) {
261 const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
262 if (!MFI.isEntryFunction())
263 return;
264
265 assert(TM.getTargetTriple().getOS() == Triple::AMDHSA);
266
267 const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
268 MCContext &Ctx = MF->getContext();
269
270 AMDGPU::MCKernelDescriptor KD =
271 getAmdhsaKernelDescriptor(MF: *MF, PI: CurrentProgramInfo);
272
273 // Compute inst_pref_size using MCExpr label subtraction for exact code
274 // size. At this point .Lfunc_end has been emitted (by the base AsmPrinter)
275 // right after the function code, so (Lfunc_end - func_sym) gives the
276 // exact function code size in bytes.
277 if (STM.hasInstPrefSize()) {
278 const MCExpr *CodeSizeExpr = MCBinaryExpr::createSub(
279 LHS: MCSymbolRefExpr::create(Symbol: getFunctionEnd(), Ctx&: OutContext),
280 RHS: MCSymbolRefExpr::create(Symbol: CurrentFnSym, Ctx&: OutContext), Ctx&: OutContext);
281
282 uint32_t Mask, Shift, Width, CacheLineSize;
283 STM.getInstPrefSizeArgs(Mask, Shift, Width, CacheLineSize);
284 const MCExpr *InstPrefSize =
285 AMDGPUMCExpr::createInstPrefSize(CodeSizeBytes: CodeSizeExpr, Ctx);
286 KD.compute_pgm_rsrc3 =
287 setBits(Dst: KD.compute_pgm_rsrc3, Value: InstPrefSize, Mask, Shift, Ctx);
288 }
289
290 auto &Streamer = getTargetStreamer()->getStreamer();
291 auto &Context = Streamer.getContext();
292 auto &ObjectFileInfo = *Context.getObjectFileInfo();
293 auto &ReadOnlySection = *ObjectFileInfo.getReadOnlySection();
294
295 Streamer.pushSection();
296 Streamer.switchSection(Section: &ReadOnlySection);
297
298 // CP microcode requires the kernel descriptor to be allocated on 64 byte
299 // alignment.
300 Streamer.emitValueToAlignment(Alignment: Align(64), Fill: 0, FillLen: 1, MaxBytesToEmit: 0);
301 ReadOnlySection.ensureMinAlignment(MinAlignment: Align(64));
302
303 SmallString<128> KernelName;
304 getNameWithPrefix(Name&: KernelName, GV: &MF->getFunction());
305 getTargetStreamer()->EmitAmdhsaKernelDescriptor(
306 STI: STM, KernelName, KernelDescriptor: KD, NextVGPR: CurrentProgramInfo.NumVGPRsForWavesPerEU,
307 NextSGPR: MCBinaryExpr::createSub(
308 LHS: CurrentProgramInfo.NumSGPRsForWavesPerEU,
309 RHS: AMDGPUMCExpr::createExtraSGPRs(
310 VCCUsed: CurrentProgramInfo.VCCUsed, FlatScrUsed: CurrentProgramInfo.FlatUsed,
311 XNACKUsed: getTargetStreamer()->getTargetID()->isXnackOnOrAny(), Ctx&: Context),
312 Ctx&: Context),
313 ReserveVCC: CurrentProgramInfo.VCCUsed, ReserveFlatScr: CurrentProgramInfo.FlatUsed);
314
315 Streamer.popSection();
316}
317
318void AMDGPUAsmPrinter::emitImplicitDef(const MachineInstr *MI) const {
319 Register RegNo = MI->getOperand(i: 0).getReg();
320
321 SmallString<128> Str;
322 raw_svector_ostream OS(Str);
323 OS << "implicit-def: "
324 << printReg(Reg: RegNo, TRI: MF->getSubtarget().getRegisterInfo());
325
326 if (MI->getAsmPrinterFlags() & AMDGPU::SGPR_SPILL)
327 OS << " : SGPR spill to VGPR lane";
328
329 OutStreamer->AddComment(T: OS.str());
330 OutStreamer->addBlankLine();
331}
332
333void AMDGPUAsmPrinter::emitFunctionEntryLabel() {
334 if (TM.getTargetTriple().getOS() == Triple::AMDHSA) {
335 AsmPrinter::emitFunctionEntryLabel();
336 return;
337 }
338
339 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
340 const GCNSubtarget &STM = MF->getSubtarget<GCNSubtarget>();
341 if (MFI->isEntryFunction() && STM.isAmdHsaOrMesa(F: MF->getFunction())) {
342 SmallString<128> SymbolName;
343 getNameWithPrefix(Name&: SymbolName, GV: &MF->getFunction()),
344 getTargetStreamer()->EmitAMDGPUSymbolType(SymbolName,
345 Type: ELF::STT_AMDGPU_HSA_KERNEL);
346 }
347 if (DumpCodeInstEmitter) {
348 // Disassemble function name label to text.
349 DisasmLines.push_back(x: MF->getName().str() + ":");
350 DisasmLineMaxLen = std::max(a: DisasmLineMaxLen, b: DisasmLines.back().size());
351 HexLines.emplace_back(args: "");
352 }
353
354 AsmPrinter::emitFunctionEntryLabel();
355}
356
357void AMDGPUAsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) {
358 if (DumpCodeInstEmitter && !isBlockOnlyReachableByFallthrough(MBB: &MBB)) {
359 // Write a line for the basic block label if it is not only fallthrough.
360 DisasmLines.push_back(x: (Twine("BB") + Twine(getFunctionNumber()) + "_" +
361 Twine(MBB.getNumber()) + ":")
362 .str());
363 DisasmLineMaxLen = std::max(a: DisasmLineMaxLen, b: DisasmLines.back().size());
364 HexLines.emplace_back(args: "");
365 }
366 AsmPrinter::emitBasicBlockStart(MBB);
367}
368
369void AMDGPUAsmPrinter::emitGlobalVariable(const GlobalVariable *GV) {
370 if (GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
371 if (GV->hasInitializer() && !isa<UndefValue>(Val: GV->getInitializer())) {
372 OutContext.reportError(L: {},
373 Msg: Twine(GV->getName()) +
374 ": unsupported initializer for address space");
375 return;
376 }
377
378 const Triple::OSType OS = TM.getTargetTriple().getOS();
379 if (OS == Triple::AMDHSA || OS == Triple::AMDPAL) {
380 if (!AMDGPUTargetMachine::EnableObjectLinking)
381 return;
382 // With object linking, LDS definitions should have been externalized
383 // by earlier passes (e.g. LDS lowering, named barrier lowering).
384 // Only declarations reach here, emitted as SHN_AMDGPU_LDS symbols
385 // so the linker can assign their offsets.
386 assert(GV->isDeclaration() &&
387 "LDS definitions should have been externalized when object "
388 "linking is enabled");
389 }
390
391 MCSymbol *GVSym = getSymbol(GV);
392
393 GVSym->redefineIfPossible();
394 if (GVSym->isDefined() || GVSym->isVariable())
395 report_fatal_error(reason: "symbol '" + Twine(GVSym->getName()) +
396 "' is already defined");
397
398 const DataLayout &DL = GV->getDataLayout();
399 uint64_t Size = GV->getGlobalSize(DL);
400 Align Alignment = GV->getAlign().value_or(u: Align(4));
401
402 emitVisibility(Sym: GVSym, Visibility: GV->getVisibility(), IsDefinition: !GV->isDeclaration());
403 emitLinkage(GV, GVSym);
404 auto *TS = getTargetStreamer();
405 TS->emitAMDGPULDS(Symbol: GVSym, Size, Alignment);
406 return;
407 }
408
409 AsmPrinter::emitGlobalVariable(GV);
410}
411
412bool AMDGPUAsmPrinter::doInitialization(Module &M) {
413 const llvm::Triple &TT = M.getTargetTriple();
414 CodeObjectVersion = AMDGPU::getAMDHSACodeObjectVersion(M);
415
416 if (TT.getOS() == Triple::AMDHSA) {
417 switch (CodeObjectVersion) {
418 case AMDGPU::AMDHSA_COV4:
419 HSAMetadataStream = std::make_unique<HSAMD::MetadataStreamerMsgPackV4>();
420 break;
421 case AMDGPU::AMDHSA_COV5:
422 HSAMetadataStream = std::make_unique<HSAMD::MetadataStreamerMsgPackV5>();
423 break;
424 case AMDGPU::AMDHSA_COV6:
425 HSAMetadataStream = std::make_unique<HSAMD::MetadataStreamerMsgPackV6>();
426 break;
427 default:
428 reportFatalUsageError(reason: "unsupported code object version");
429 }
430
431 addAsmPrinterHandler(Handler: std::make_unique<AMDGPUAsmPrinterHandler>(args: this));
432 }
433
434 return AsmPrinter::doInitialization(M);
435}
436
437/// Mimics GCNSubtarget::computeOccupancy for MCExpr.
438///
439/// Remove dependency on GCNSubtarget and depend only only the necessary values
440/// for said occupancy computation. Should match computeOccupancy implementation
441/// without passing \p STM on.
442const AMDGPUMCExpr *createOccupancy(unsigned InitOcc, const MCExpr *NumSGPRs,
443 const MCExpr *NumVGPRs,
444 unsigned DynamicVGPRBlockSize,
445 const GCNSubtarget &STM, MCContext &Ctx) {
446 unsigned MaxWaves = IsaInfo::getMaxWavesPerEU(STI: STM);
447 unsigned Granule = IsaInfo::getVGPRAllocGranule(STI: STM, DynamicVGPRBlockSize);
448 unsigned TargetTotalNumVGPRs = IsaInfo::getTotalNumVGPRs(STI: STM);
449
450 // Bake the per-function SGPR budget into the operands so the late-evaluated
451 // MCExpr stays arithmetic. The trap reservation in particular is implicit on
452 // amdhsa and lives on STM, not on the assembler's MCSubtargetInfo.
453 unsigned SGPRTotal = IsaInfo::getTotalNumSGPRs(STI: STM);
454 unsigned SGPRGranule = IsaInfo::getSGPRAllocGranule(STI: STM);
455 unsigned SGPRTrapReserve = STM.hasTrapHandler() ? IsaInfo::TRAP_NUM_SGPRS : 0;
456
457 auto CreateExpr = [&Ctx](unsigned Value) {
458 return MCConstantExpr::create(Value, Ctx);
459 };
460
461 // Zero SGPR count when SGPRs don't limit occupancy, so the MCExpr skips the
462 // SGPR term without having to test the generation itself.
463 const MCExpr *SGPRArg =
464 IsaInfo::isSGPROccupancyLimited(STI: STM) ? NumSGPRs : CreateExpr(0);
465
466 return AMDGPUMCExpr::create(Kind: AMDGPUMCExpr::AGVK_Occupancy,
467 Args: {CreateExpr(MaxWaves), CreateExpr(Granule),
468 CreateExpr(TargetTotalNumVGPRs),
469 CreateExpr(InitOcc), CreateExpr(SGPRTotal),
470 CreateExpr(SGPRGranule),
471 CreateExpr(SGPRTrapReserve), SGPRArg, NumVGPRs},
472 Ctx);
473}
474
475void AMDGPUAsmPrinter::validateMCResourceInfo(Function &F) {
476 if (F.isDeclaration() || !AMDGPU::isModuleEntryFunctionCC(CC: F.getCallingConv()))
477 return;
478
479 using RIK = MCResourceInfo::ResourceInfoKind;
480 const GCNSubtarget &STM = TM.getSubtarget<GCNSubtarget>(F);
481 MCSymbol *FnSym = TM.getSymbol(GV: &F);
482
483 auto TryGetMCExprValue = [](const MCExpr *Value, uint64_t &Res) -> bool {
484 int64_t Val;
485 if (Value->evaluateAsAbsolute(Res&: Val)) {
486 Res = Val;
487 return true;
488 }
489 return false;
490 };
491
492 const uint64_t MaxScratchPerWorkitem =
493 STM.getMaxWaveScratchSize() / STM.getWavefrontSize();
494 MCSymbol *ScratchSizeSymbol =
495 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_PrivateSegSize, OutContext);
496 uint64_t ScratchSize;
497 if (ScratchSizeSymbol->isVariable() &&
498 TryGetMCExprValue(ScratchSizeSymbol->getVariableValue(), ScratchSize) &&
499 ScratchSize > MaxScratchPerWorkitem) {
500 DiagnosticInfoStackSize DiagStackSize(F, ScratchSize, MaxScratchPerWorkitem,
501 DS_Error);
502 F.getContext().diagnose(DI: DiagStackSize);
503 }
504
505 // Validate addressable scalar registers (i.e., prior to added implicit
506 // SGPRs).
507 MCSymbol *NumSGPRSymbol =
508 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_NumSGPR, OutContext);
509 if (STM.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
510 !STM.hasSGPRInitBug()) {
511 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
512 uint64_t NumSgpr;
513 if (NumSGPRSymbol->isVariable() &&
514 TryGetMCExprValue(NumSGPRSymbol->getVariableValue(), NumSgpr) &&
515 NumSgpr > MaxAddressableNumSGPRs) {
516 F.getContext().diagnose(DI: DiagnosticInfoResourceLimit(
517 F, "addressable scalar registers", NumSgpr, MaxAddressableNumSGPRs,
518 DS_Error, DK_ResourceLimit));
519 return;
520 }
521 }
522
523 MCSymbol *VCCUsedSymbol =
524 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_UsesVCC, OutContext);
525 MCSymbol *FlatUsedSymbol =
526 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_UsesFlatScratch, OutContext);
527 uint64_t VCCUsed, FlatUsed, NumSgpr;
528
529 if (NumSGPRSymbol->isVariable() && VCCUsedSymbol->isVariable() &&
530 FlatUsedSymbol->isVariable() &&
531 TryGetMCExprValue(NumSGPRSymbol->getVariableValue(), NumSgpr) &&
532 TryGetMCExprValue(VCCUsedSymbol->getVariableValue(), VCCUsed) &&
533 TryGetMCExprValue(FlatUsedSymbol->getVariableValue(), FlatUsed)) {
534
535 // Recomputes NumSgprs + implicit SGPRs but all symbols should now be
536 // resolvable.
537 NumSgpr += IsaInfo::getNumExtraSGPRs(
538 STI: STM, VCCUsed, FlatScrUsed: FlatUsed,
539 XNACKUsed: getTargetStreamer()->getTargetID()->isXnackOnOrAny());
540 if (STM.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS ||
541 STM.hasSGPRInitBug()) {
542 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
543 if (NumSgpr > MaxAddressableNumSGPRs) {
544 F.getContext().diagnose(DI: DiagnosticInfoResourceLimit(
545 F, "scalar registers", NumSgpr, MaxAddressableNumSGPRs, DS_Error,
546 DK_ResourceLimit));
547 return;
548 }
549 }
550
551 MCSymbol *NumVgprSymbol =
552 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_NumVGPR, OutContext);
553 MCSymbol *NumAgprSymbol =
554 RI.getSymbol(FuncName: FnSym->getName(), RIK: RIK::RIK_NumAGPR, OutContext);
555 uint64_t NumVgpr, NumAgpr;
556
557 MachineModuleInfo &MMI = *GetMMI();
558 MachineFunction *MF = MMI.getMachineFunction(F);
559 if (MF && NumVgprSymbol->isVariable() && NumAgprSymbol->isVariable() &&
560 TryGetMCExprValue(NumVgprSymbol->getVariableValue(), NumVgpr) &&
561 TryGetMCExprValue(NumAgprSymbol->getVariableValue(), NumAgpr)) {
562 const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
563 unsigned MaxWaves = MFI.getMaxWavesPerEU();
564 uint64_t TotalNumVgpr =
565 getTotalNumVGPRs(has90AInsts: STM.hasGFX90AInsts(), ArgNumAGPR: NumAgpr, ArgNumVGPR: NumVgpr);
566 uint64_t NumVGPRsForWavesPerEU =
567 std::max(l: {TotalNumVgpr, (uint64_t)1,
568 (uint64_t)STM.getMinNumVGPRs(
569 WavesPerEU: MaxWaves, DynamicVGPRBlockSize: MFI.getDynamicVGPRBlockSize())});
570 uint64_t NumSGPRsForWavesPerEU = std::max(
571 l: {NumSgpr, (uint64_t)1, (uint64_t)STM.getMinNumSGPRs(WavesPerEU: MaxWaves)});
572 const MCExpr *OccupancyExpr = createOccupancy(
573 InitOcc: STM.getOccupancyWithWorkGroupSizes(MF: *MF).second,
574 NumSGPRs: MCConstantExpr::create(Value: NumSGPRsForWavesPerEU, Ctx&: OutContext),
575 NumVGPRs: MCConstantExpr::create(Value: NumVGPRsForWavesPerEU, Ctx&: OutContext),
576 DynamicVGPRBlockSize: MFI.getDynamicVGPRBlockSize(), STM, Ctx&: OutContext);
577 uint64_t Occupancy;
578
579 const auto [MinWEU, MaxWEU] = AMDGPU::getIntegerPairAttribute(
580 F, Name: "amdgpu-waves-per-eu", Default: {0, 0}, OnlyFirstRequired: true);
581
582 if (TryGetMCExprValue(OccupancyExpr, Occupancy) && Occupancy < MinWEU) {
583 DiagnosticInfoOptimizationFailure Diag(
584 F, F.getSubprogram(),
585 "failed to meet occupancy target given by 'amdgpu-waves-per-eu' in "
586 "'" +
587 F.getName() + "': desired occupancy was " + Twine(MinWEU) +
588 ", final occupancy is " + Twine(Occupancy));
589 F.getContext().diagnose(DI: Diag);
590 return;
591 }
592 }
593 }
594}
595
596static void appendTypeEncoding(std::string &Enc, Type *Ty, const DataLayout &DL,
597 bool IsReturnType) {
598 if (Ty->isVoidTy()) {
599 Enc += 'v';
600 return;
601 }
602 unsigned Bits = DL.getTypeSizeInBits(Ty);
603 // Zero-sized non-void types (e.g. `{}` or `[0 x i8]`) consume no ABI
604 // registers. For returns, emit the same no-result marker as void so the
605 // parameter encoding still has an explicit return-type prefix.
606 if (Bits == 0) {
607 if (IsReturnType)
608 Enc += 'v';
609 return;
610 }
611 if (Bits <= 32)
612 Enc += 'i';
613 else if (Bits <= 64)
614 Enc += 'l';
615 else
616 Enc.append(n: divideCeil(Numerator: Bits, Denominator: 32), c: 'i');
617}
618
619static std::string computeTypeId(const FunctionType *FTy,
620 const DataLayout &DL) {
621 std::string Enc;
622 appendTypeEncoding(Enc, Ty: FTy->getReturnType(), DL, /*IsReturnType=*/true);
623 for (Type *ParamTy : FTy->params())
624 appendTypeEncoding(Enc, Ty: ParamTy, DL, /*IsReturnType=*/false);
625 return Enc;
626}
627
628void AMDGPUAsmPrinter::collectCallEdge(const MachineInstr &MI) {
629 if (!AMDGPUTargetMachine::EnableObjectLinking)
630 return;
631 const SIInstrInfo *TII = MF->getSubtarget<GCNSubtarget>().getInstrInfo();
632 const MachineOperand *Callee =
633 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::callee);
634 if (!Callee || !Callee->isGlobal())
635 return;
636 DirectCallEdges.insert(
637 X: {getSymbol(GV: &MF->getFunction()), getSymbol(GV: Callee->getGlobal())});
638}
639
640void AMDGPUAsmPrinter::emitAMDGPUInfo(Module &M) {
641 if (!AMDGPUTargetMachine::EnableObjectLinking)
642 return;
643
644 const NamedMDNode *LDSMD = M.getNamedMetadata(Name: "amdgpu.lds.uses");
645 bool HasLDSUses = LDSMD && LDSMD->getNumOperands() > 0;
646
647 const NamedMDNode *BarMD = M.getNamedMetadata(Name: "amdgpu.named_barrier.uses");
648 bool HasNamedBarriers = BarMD && BarMD->getNumOperands() > 0;
649
650 // Collect address-taken functions (with type IDs) and indirect call sites.
651 DenseMap<const Function *, std::string> AddrTakenTypeIds;
652 using IndirectCallInfo = std::pair<const Function *, std::string>;
653 SmallVector<IndirectCallInfo, 8> IndirectCalls;
654
655 for (const Function &F : M) {
656 bool IsKernel = AMDGPU::isKernel(CC: F.getCallingConv());
657
658 if (!IsKernel && F.hasAddressTaken(/*PutOffender=*/nullptr,
659 /*IgnoreCallbackUses=*/false,
660 /*IgnoreAssumeLikeCalls=*/true,
661 /*IgnoreLLVMUsed=*/IngoreLLVMUsed: true)) {
662 AddrTakenTypeIds[&F] =
663 computeTypeId(FTy: F.getFunctionType(), DL: M.getDataLayout());
664 }
665
666 if (F.isDeclaration())
667 continue;
668
669 StringSet<> SeenTypeIds;
670 for (const BasicBlock &BB : F) {
671 for (const Instruction &I : BB) {
672 const auto *CB = dyn_cast<CallBase>(Val: &I);
673 if (!CB || !CB->isIndirectCall())
674 continue;
675 std::string TId =
676 computeTypeId(FTy: CB->getFunctionType(), DL: M.getDataLayout());
677 if (SeenTypeIds.insert(key: TId).second)
678 IndirectCalls.push_back(Elt: {&F, std::move(TId)});
679 }
680 }
681 }
682
683 if (FunctionInfos.empty() && DirectCallEdges.empty() && !HasLDSUses &&
684 !HasNamedBarriers && AddrTakenTypeIds.empty() && IndirectCalls.empty())
685 return;
686
687 AMDGPU::InfoSectionData Data;
688 Data.Funcs = std::move(FunctionInfos);
689
690 for (auto &[F, TypeId] : AddrTakenTypeIds) {
691 MCSymbol *Sym = getSymbol(GV: F);
692 Data.TypeIds.push_back(Elt: {Sym, TypeId});
693 }
694
695 for (auto &[CallerSym, CalleeSym] : DirectCallEdges)
696 Data.Calls.push_back(Elt: {CallerSym, CalleeSym});
697 DirectCallEdges.clear();
698
699 if (HasLDSUses) {
700 for (const MDNode *N : LDSMD->operands()) {
701 auto *Func = mdconst::extract<Function>(MD: N->getOperand(I: 0));
702 auto *LdsVar = mdconst::extract<GlobalVariable>(MD: N->getOperand(I: 1));
703 Data.Uses.push_back(Elt: {getSymbol(GV: Func), getSymbol(GV: LdsVar)});
704 }
705 }
706
707 if (HasNamedBarriers) {
708 for (const MDNode *N : BarMD->operands()) {
709 auto *BarVar = mdconst::extract<GlobalVariable>(MD: N->getOperand(I: 0));
710 MCSymbol *BarSym = getSymbol(GV: BarVar);
711 for (unsigned I = 1, E = N->getNumOperands(); I < E; ++I) {
712 auto *Func = mdconst::extract<Function>(MD: N->getOperand(I));
713 Data.Uses.push_back(Elt: {getSymbol(GV: Func), BarSym});
714 }
715 }
716 }
717
718 for (auto &[Caller, Enc] : IndirectCalls) {
719 MCSymbol *CallerSym = getSymbol(GV: Caller);
720 Data.IndirectCalls.push_back(Elt: {CallerSym, Enc});
721 }
722
723 getTargetStreamer()->emitAMDGPUInfo(Data);
724}
725
726bool AMDGPUAsmPrinter::doFinalization(Module &M) {
727 const Triple &TT = M.getTargetTriple();
728
729 // Pad with s_code_end to help tools and guard against instruction prefetch
730 // causing stale data in caches. Arguably this should be done by the linker,
731 // which is why this isn't done for Mesa.
732 // Don't do it if there is no code.
733 const MCSubtargetInfo &STI = *getGlobalSTI();
734 if ((AMDGPU::isGFX10Plus(STI) || AMDGPU::isGFX90A(STI)) &&
735 (TT.getOS() == Triple::AMDHSA || TT.getOS() == Triple::AMDPAL)) {
736 MCSection *TextSect = getObjFileLowering().getTextSection();
737 if (TextSect->hasInstructions()) {
738 OutStreamer->switchSection(Section: TextSect);
739 getTargetStreamer()->EmitCodeEnd(STI);
740 }
741 }
742
743 // Emit the unified .amdgpu.info section (per-function resources, call graph,
744 // LDS/named-barrier use edges, indirect calls, and address-taken type IDs).
745 emitAMDGPUInfo(M);
746
747 // Assign expressions which can only be resolved when all other functions are
748 // known.
749 RI.finalize(OutContext);
750
751 // Switch section and emit all GPR maximums within the processed module.
752 OutStreamer->pushSection();
753 MCSectionELF *MaxGPRSection =
754 OutContext.getELFSection(Section: ".AMDGPU.gpr_maximums", Type: ELF::SHT_PROGBITS, Flags: 0);
755 OutStreamer->switchSection(Section: MaxGPRSection);
756 getTargetStreamer()->EmitMCResourceMaximums(
757 MaxVGPR: RI.getMaxVGPRSymbol(OutContext), MaxAGPR: RI.getMaxAGPRSymbol(OutContext),
758 MaxSGPR: RI.getMaxSGPRSymbol(OutContext), MaxNamedBarrier: RI.getMaxNamedBarrierSymbol(OutContext));
759 OutStreamer->popSection();
760
761 // In the object-linking pipeline per-function resource MCExprs reference
762 // external callee symbols that cannot be evaluated here, so cross-TU limit
763 // checks would silently no-op for every non-leaf function. Defer resource
764 // sanity checking to the linker, which re-validates against the aggregated
765 // call graph in the combined .amdgpu.info metadata.
766 if (!AMDGPUTargetMachine::EnableObjectLinking) {
767 for (Function &F : M.functions())
768 validateMCResourceInfo(F);
769 }
770
771 RI.reset();
772
773 return AsmPrinter::doFinalization(M);
774}
775
776SmallString<128> AMDGPUAsmPrinter::getMCExprStr(const MCExpr *Value) {
777 SmallString<128> Str;
778 raw_svector_ostream OSS(Str);
779 auto &Streamer = getTargetStreamer()->getStreamer();
780 auto &Context = Streamer.getContext();
781 const MCExpr *New = foldAMDGPUMCExpr(Expr: Value, Ctx&: Context);
782 printAMDGPUMCExpr(Expr: New, OS&: OSS, MAI: &MAI);
783 return Str;
784}
785
786// Print comments that apply to both callable functions and entry points.
787void AMDGPUAsmPrinter::emitCommonFunctionComments(
788 const MCExpr *NumVGPR, const MCExpr *NumAGPR, const MCExpr *TotalNumVGPR,
789 const MCExpr *NumSGPR, const MCExpr *ScratchSize, uint64_t CodeSize,
790 const AMDGPUMachineFunctionInfo *MFI) {
791 OutStreamer->emitRawComment(T: " codeLenInByte = " + Twine(CodeSize), TabPrefix: false);
792 OutStreamer->emitRawComment(T: " TotalNumSgprs: " + getMCExprStr(Value: NumSGPR),
793 TabPrefix: false);
794 OutStreamer->emitRawComment(T: " NumVgprs: " + getMCExprStr(Value: NumVGPR), TabPrefix: false);
795 if (NumAGPR && TotalNumVGPR) {
796 OutStreamer->emitRawComment(T: " NumAgprs: " + getMCExprStr(Value: NumAGPR), TabPrefix: false);
797 OutStreamer->emitRawComment(T: " TotalNumVgprs: " + getMCExprStr(Value: TotalNumVGPR),
798 TabPrefix: false);
799 }
800 OutStreamer->emitRawComment(T: " ScratchSize: " + getMCExprStr(Value: ScratchSize),
801 TabPrefix: false);
802 OutStreamer->emitRawComment(T: " MemoryBound: " + Twine(MFI->isMemoryBound()),
803 TabPrefix: false);
804}
805
806const MCExpr *AMDGPUAsmPrinter::getAmdhsaKernelCodeProperties(
807 const MachineFunction &MF) const {
808 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
809 MCContext &Ctx = MF.getContext();
810 uint16_t KernelCodeProperties = 0;
811 const GCNUserSGPRUsageInfo &UserSGPRInfo = MFI.getUserSGPRInfo();
812
813 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
814 KernelCodeProperties |=
815 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER;
816 }
817 if (UserSGPRInfo.hasDispatchPtr()) {
818 KernelCodeProperties |=
819 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR;
820 }
821 if (UserSGPRInfo.hasQueuePtr()) {
822 KernelCodeProperties |= amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR;
823 }
824 if (UserSGPRInfo.hasKernargSegmentPtr()) {
825 KernelCodeProperties |=
826 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR;
827 }
828 if (UserSGPRInfo.hasDispatchID()) {
829 KernelCodeProperties |=
830 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID;
831 }
832 if (UserSGPRInfo.hasFlatScratchInit()) {
833 KernelCodeProperties |=
834 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT;
835 }
836 if (UserSGPRInfo.hasPrivateSegmentSize()) {
837 KernelCodeProperties |=
838 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE;
839 }
840 if (MF.getSubtarget<GCNSubtarget>().isWave32()) {
841 KernelCodeProperties |=
842 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;
843 }
844
845 // CurrentProgramInfo.DynamicCallStack is a MCExpr and could be
846 // un-evaluatable at this point so it cannot be conditionally checked here.
847 // Instead, we'll directly shift the possibly unknown MCExpr into its place
848 // and bitwise-or it into KernelCodeProperties.
849 const MCExpr *KernelCodePropExpr =
850 MCConstantExpr::create(Value: KernelCodeProperties, Ctx);
851 const MCExpr *OrValue = MCConstantExpr::create(
852 Value: amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK_SHIFT, Ctx);
853 OrValue = MCBinaryExpr::createShl(LHS: CurrentProgramInfo.DynamicCallStack,
854 RHS: OrValue, Ctx);
855 KernelCodePropExpr = MCBinaryExpr::createOr(LHS: KernelCodePropExpr, RHS: OrValue, Ctx);
856
857 return KernelCodePropExpr;
858}
859
860MCKernelDescriptor
861AMDGPUAsmPrinter::getAmdhsaKernelDescriptor(const MachineFunction &MF,
862 const SIProgramInfo &PI) const {
863 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
864 const Function &F = MF.getFunction();
865 const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
866 MCContext &Ctx = MF.getContext();
867
868 MCKernelDescriptor KernelDescriptor;
869
870 KernelDescriptor.group_segment_fixed_size =
871 MCConstantExpr::create(Value: PI.LDSSize, Ctx);
872 KernelDescriptor.private_segment_fixed_size = PI.ScratchSize;
873
874 Align MaxKernArgAlign;
875 KernelDescriptor.kernarg_size = MCConstantExpr::create(
876 Value: STM.getKernArgSegmentSize(F, MaxAlign&: MaxKernArgAlign), Ctx);
877
878 KernelDescriptor.compute_pgm_rsrc1 = PI.getComputePGMRSrc1(ST: STM, Ctx);
879 KernelDescriptor.compute_pgm_rsrc2 = PI.getComputePGMRSrc2(ST: STM, Ctx);
880 KernelDescriptor.kernel_code_properties = getAmdhsaKernelCodeProperties(MF);
881
882 int64_t PGM_Rsrc3 = 1;
883 bool EvaluatableRsrc3 =
884 CurrentProgramInfo.ComputePGMRSrc3->evaluateAsAbsolute(Res&: PGM_Rsrc3);
885 (void)PGM_Rsrc3;
886 (void)EvaluatableRsrc3;
887 assert(STM.getGeneration() >= AMDGPUSubtarget::GFX10 ||
888 STM.hasGFX90AInsts() || STM.hasGFX1250Insts() || !EvaluatableRsrc3 ||
889 static_cast<uint64_t>(PGM_Rsrc3) == 0);
890 KernelDescriptor.compute_pgm_rsrc3 = CurrentProgramInfo.ComputePGMRSrc3;
891
892 KernelDescriptor.kernarg_preload = MCConstantExpr::create(
893 Value: AMDGPU::hasKernargPreload(STI: STM) ? Info->getNumKernargPreloadedSGPRs() : 0,
894 Ctx);
895
896 return KernelDescriptor;
897}
898
899bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
900 // Init target streamer lazily on the first function so that previous passes
901 // can set metadata.
902 if (!IsTargetStreamerInitialized)
903 initTargetStreamer(M&: *MF.getFunction().getParent());
904
905 ResourceUsage = GetResourceUsage(MF);
906 CurrentProgramInfo.reset(MF);
907
908 const AMDGPUMachineFunctionInfo *MFI =
909 MF.getInfo<AMDGPUMachineFunctionInfo>();
910 MCContext &Ctx = MF.getContext();
911
912 // The starting address of all shader programs must be 256 bytes aligned.
913 // Regular functions just need the basic required instruction alignment.
914 MF.ensureAlignment(A: MFI->isEntryFunction() ? Align(256) : Align(4));
915
916 SetupMachineFunction(MF);
917
918 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
919 MCContext &Context = getObjFileLowering().getContext();
920 // FIXME: This should be an explicit check for Mesa.
921 if (!STM.isAmdHsaOS() && !STM.isAmdPalOS()) {
922 MCSectionELF *ConfigSection =
923 Context.getELFSection(Section: ".AMDGPU.config", Type: ELF::SHT_PROGBITS, Flags: 0);
924 OutStreamer->switchSection(Section: ConfigSection);
925 }
926
927 RI.gatherResourceInfo(MF, FRI: *ResourceUsage, OutContext);
928
929 if (AMDGPUTargetMachine::EnableObjectLinking) {
930 const AMDGPUResourceUsageAnalysisWrapperPass::FunctionResourceInfo &RU =
931 *ResourceUsage;
932 FunctionInfos.push_back(
933 Elt: {/*NumSGPR=*/static_cast<uint32_t>(RU.NumExplicitSGPR),
934 /*NumArchVGPR=*/static_cast<uint32_t>(RU.NumVGPR),
935 /*NumAccVGPR=*/static_cast<uint32_t>(RU.NumAGPR),
936 /*PrivateSegmentSize=*/static_cast<uint32_t>(RU.PrivateSegmentSize),
937 /*UsesVCC=*/RU.UsesVCC,
938 /*UsesFlatScratch=*/RU.UsesFlatScratch,
939 /*HasDynStack=*/RU.HasDynamicallySizedStack,
940 /*Sym=*/getSymbol(GV: &MF.getFunction())});
941 }
942
943 if (MFI->isModuleEntryFunction()) {
944 getSIProgramInfo(Out&: CurrentProgramInfo, MF);
945 }
946
947 if (STM.isAmdPalOS()) {
948 if (MFI->isEntryFunction())
949 EmitPALMetadata(MF, KernelInfo: CurrentProgramInfo);
950 else if (MFI->isModuleEntryFunction())
951 emitPALFunctionMetadata(MF);
952 } else if (!STM.isAmdHsaOS()) {
953 EmitProgramInfoSI(MF, KernelInfo: CurrentProgramInfo);
954 }
955
956 DumpCodeInstEmitter = nullptr;
957 if (STM.dumpCode()) {
958 // For -dumpcode, get the assembler out of the streamer. This only works
959 // with -filetype=obj.
960 MCAssembler *Assembler = OutStreamer->getAssemblerPtr();
961 if (Assembler)
962 DumpCodeInstEmitter = Assembler->getEmitterPtr();
963 }
964
965 DisasmLines.clear();
966 HexLines.clear();
967 DisasmLineMaxLen = 0;
968
969 emitFunctionBody();
970
971 emitResourceUsageRemarks(MF, CurrentProgramInfo, isModuleEntryFunction: MFI->isModuleEntryFunction(),
972 hasMAIInsts: STM.hasMAIInsts());
973
974 {
975 using RIK = MCResourceInfo::ResourceInfoKind;
976 getTargetStreamer()->EmitMCResourceInfo(
977 NumVGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumVGPR, OutContext),
978 NumAGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumAGPR, OutContext),
979 NumExplicitSGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumSGPR, OutContext),
980 NumNamedBarrier: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumNamedBarrier,
981 OutContext),
982 PrivateSegmentSize: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_PrivateSegSize,
983 OutContext),
984 UsesVCC: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_UsesVCC, OutContext),
985 UsesFlatScratch: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_UsesFlatScratch,
986 OutContext),
987 HasDynamicallySizedStack: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_HasDynSizedStack,
988 OutContext),
989 HasRecursion: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_HasRecursion,
990 OutContext),
991 HasIndirectCall: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_HasIndirectCall,
992 OutContext));
993 }
994
995 // Emit _dvgpr$ symbol when appropriate.
996 emitDVgprSymbol(MF);
997
998 if (isVerbose()) {
999 MCSectionELF *CommentSection =
1000 Context.getELFSection(Section: ".AMDGPU.csdata", Type: ELF::SHT_PROGBITS, Flags: 0);
1001 OutStreamer->switchSection(Section: CommentSection);
1002
1003 if (!MFI->isEntryFunction()) {
1004 using RIK = MCResourceInfo::ResourceInfoKind;
1005 OutStreamer->emitRawComment(T: " Function info:", TabPrefix: false);
1006
1007 emitCommonFunctionComments(
1008 NumVGPR: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_NumVGPR, OutContext)
1009 ->getVariableValue(),
1010 NumAGPR: STM.hasMAIInsts() ? RI.getSymbol(FuncName: CurrentFnSym->getName(),
1011 RIK: RIK::RIK_NumAGPR, OutContext)
1012 ->getVariableValue()
1013 : nullptr,
1014 TotalNumVGPR: RI.createTotalNumVGPRs(MF, Ctx),
1015 NumSGPR: RI.createTotalNumSGPRs(
1016 MF,
1017 hasXnack: MF.getSubtarget<GCNSubtarget>().getTargetID().isXnackOnOrAny(),
1018 Ctx),
1019 ScratchSize: RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK: RIK::RIK_PrivateSegSize,
1020 OutContext)
1021 ->getVariableValue(),
1022 CodeSize: CurrentProgramInfo.getFunctionCodeSize(MF), MFI);
1023 return false;
1024 }
1025
1026 OutStreamer->emitRawComment(T: " Kernel info:", TabPrefix: false);
1027 emitCommonFunctionComments(
1028 NumVGPR: CurrentProgramInfo.NumArchVGPR,
1029 NumAGPR: STM.hasMAIInsts() ? CurrentProgramInfo.NumAccVGPR : nullptr,
1030 TotalNumVGPR: CurrentProgramInfo.NumVGPR, NumSGPR: CurrentProgramInfo.NumSGPR,
1031 ScratchSize: CurrentProgramInfo.ScratchSize,
1032 CodeSize: CurrentProgramInfo.getFunctionCodeSize(MF), MFI);
1033
1034 OutStreamer->emitRawComment(
1035 T: " FloatMode: " + Twine(CurrentProgramInfo.FloatMode), TabPrefix: false);
1036 OutStreamer->emitRawComment(
1037 T: " IeeeMode: " + Twine(CurrentProgramInfo.IEEEMode), TabPrefix: false);
1038 OutStreamer->emitRawComment(
1039 T: " LDSByteSize: " + Twine(CurrentProgramInfo.LDSSize) +
1040 " bytes/workgroup (compile time only)",
1041 TabPrefix: false);
1042
1043 OutStreamer->emitRawComment(
1044 T: " SGPRBlocks: " + getMCExprStr(Value: CurrentProgramInfo.SGPRBlocks), TabPrefix: false);
1045
1046 OutStreamer->emitRawComment(
1047 T: " VGPRBlocks: " + getMCExprStr(Value: CurrentProgramInfo.VGPRBlocks), TabPrefix: false);
1048
1049 OutStreamer->emitRawComment(
1050 T: " NumSGPRsForWavesPerEU: " +
1051 getMCExprStr(Value: CurrentProgramInfo.NumSGPRsForWavesPerEU),
1052 TabPrefix: false);
1053 OutStreamer->emitRawComment(
1054 T: " NumVGPRsForWavesPerEU: " +
1055 getMCExprStr(Value: CurrentProgramInfo.NumVGPRsForWavesPerEU),
1056 TabPrefix: false);
1057
1058 if (STM.hasGFX90AInsts()) {
1059 const MCExpr *AdjustedAccum = MCBinaryExpr::createAdd(
1060 LHS: CurrentProgramInfo.AccumOffset, RHS: MCConstantExpr::create(Value: 1, Ctx), Ctx);
1061 AdjustedAccum = MCBinaryExpr::createMul(
1062 LHS: AdjustedAccum, RHS: MCConstantExpr::create(Value: 4, Ctx), Ctx);
1063 OutStreamer->emitRawComment(
1064 T: " AccumOffset: " + getMCExprStr(Value: AdjustedAccum), TabPrefix: false);
1065 }
1066
1067 if (STM.hasGFX1250Insts())
1068 OutStreamer->emitRawComment(
1069 T: " NamedBarCnt: " + getMCExprStr(Value: CurrentProgramInfo.NamedBarCnt),
1070 TabPrefix: false);
1071
1072 OutStreamer->emitRawComment(
1073 T: " Occupancy: " + getMCExprStr(Value: CurrentProgramInfo.Occupancy), TabPrefix: false);
1074
1075 OutStreamer->emitRawComment(
1076 T: " WaveLimiterHint : " + Twine(MFI->needsWaveLimiter()), TabPrefix: false);
1077
1078 OutStreamer->emitRawComment(
1079 T: " COMPUTE_PGM_RSRC2:SCRATCH_EN: " +
1080 getMCExprStr(Value: CurrentProgramInfo.ScratchEnable),
1081 TabPrefix: false);
1082 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:USER_SGPR: " +
1083 Twine(CurrentProgramInfo.UserSGPR),
1084 TabPrefix: false);
1085 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TRAP_HANDLER: " +
1086 Twine(CurrentProgramInfo.TrapHandlerEnable),
1087 TabPrefix: false);
1088 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TGID_X_EN: " +
1089 Twine(CurrentProgramInfo.TGIdXEnable),
1090 TabPrefix: false);
1091 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TGID_Y_EN: " +
1092 Twine(CurrentProgramInfo.TGIdYEnable),
1093 TabPrefix: false);
1094 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TGID_Z_EN: " +
1095 Twine(CurrentProgramInfo.TGIdZEnable),
1096 TabPrefix: false);
1097 OutStreamer->emitRawComment(T: " COMPUTE_PGM_RSRC2:TIDIG_COMP_CNT: " +
1098 Twine(CurrentProgramInfo.TIdIGCompCount),
1099 TabPrefix: false);
1100
1101 [[maybe_unused]] int64_t PGMRSrc3;
1102 assert(STM.getGeneration() >= AMDGPUSubtarget::GFX10 ||
1103 STM.hasGFX90AInsts() || STM.hasGFX1250Insts() ||
1104 (CurrentProgramInfo.ComputePGMRSrc3->evaluateAsAbsolute(PGMRSrc3) &&
1105 static_cast<uint64_t>(PGMRSrc3) == 0));
1106 if (STM.hasGFX90AInsts()) {
1107 OutStreamer->emitRawComment(
1108 T: " COMPUTE_PGM_RSRC3_GFX90A:ACCUM_OFFSET: " +
1109 getMCExprStr(Value: MCKernelDescriptor::bits_get(
1110 Src: CurrentProgramInfo.ComputePGMRSrc3,
1111 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
1112 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, Ctx)),
1113 TabPrefix: false);
1114 OutStreamer->emitRawComment(
1115 T: " COMPUTE_PGM_RSRC3_GFX90A:TG_SPLIT: " +
1116 getMCExprStr(Value: MCKernelDescriptor::bits_get(
1117 Src: CurrentProgramInfo.ComputePGMRSrc3,
1118 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT,
1119 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, Ctx)),
1120 TabPrefix: false);
1121 }
1122 }
1123
1124 if (DumpCodeInstEmitter) {
1125
1126 OutStreamer->switchSection(
1127 Section: Context.getELFSection(Section: ".AMDGPU.disasm", Type: ELF::SHT_PROGBITS, Flags: 0));
1128
1129 for (size_t i = 0; i < DisasmLines.size(); ++i) {
1130 std::string Comment = "\n";
1131 if (!HexLines[i].empty()) {
1132 Comment = std::string(DisasmLineMaxLen - DisasmLines[i].size(), ' ');
1133 Comment += " ; " + HexLines[i] + "\n";
1134 }
1135
1136 OutStreamer->emitBytes(Data: StringRef(DisasmLines[i]));
1137 OutStreamer->emitBytes(Data: StringRef(Comment));
1138 }
1139 }
1140
1141 return false;
1142}
1143
1144// When appropriate, add a _dvgpr$ symbol, with the value of the function
1145// symbol, plus an offset encoding one less than the number of VGPR blocks used
1146// by the function in bits 5..3 of the symbol value. A "VGPR block" can be
1147// either 16 VGPRs (for a max of 128), or 32 VGPRs (for a max of 256). This is
1148// used by a front-end to have functions that are chained rather than called,
1149// and a dispatcher that dynamically resizes the VGPR count before dispatching
1150// to a function.
1151void AMDGPUAsmPrinter::emitDVgprSymbol(MachineFunction &MF) {
1152 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
1153 if (MFI.isDynamicVGPREnabled() &&
1154 MF.getFunction().getCallingConv() == CallingConv::AMDGPU_CS_Chain) {
1155 MCContext &Ctx = MF.getContext();
1156 unsigned BlockSize = MFI.getDynamicVGPRBlockSize();
1157
1158 const MCExpr *EncodedBlocks;
1159 MCValue NumVGPRs;
1160 if (CurrentProgramInfo.NumVGPRsForWavesPerEU->evaluateAsRelocatable(
1161 Res&: NumVGPRs, Asm: nullptr) &&
1162 NumVGPRs.isAbsolute()) {
1163
1164 // Calculate number of VGPR blocks.
1165 // Treat 0 VGPRs as 1 VGPR to avoid underflowing.
1166 unsigned NumBlocks =
1167 divideCeil(Numerator: std::max(a: unsigned(NumVGPRs.getConstant()), b: 1U), Denominator: BlockSize);
1168
1169 if (NumBlocks > AMDGPU::IsaInfo::MaxDynamicVGPRBlocks) {
1170 OutContext.reportError(
1171 L: {}, Msg: "DVGPR block count " + Twine(NumBlocks) +
1172 " exceeds maximum of " +
1173 Twine(AMDGPU::IsaInfo::MaxDynamicVGPRBlocks) +
1174 " for __dvgpr$ symbol for '" +
1175 Twine(CurrentFnSym->getName()) + "'");
1176 return;
1177 }
1178 unsigned EncodedNumBlocks = (NumBlocks - 1) << 3;
1179 EncodedBlocks = MCConstantExpr::create(Value: EncodedNumBlocks, Ctx);
1180 } else {
1181 // Value not yet available so build a symbolic MCExpr:
1182 // ((alignTo(max(NumVGPRs, 1), BlockSize) / BlockSize - 1) << 3
1183 const MCExpr *One = MCConstantExpr::create(Value: 1, Ctx);
1184 const MCExpr *BlockSizeConst = MCConstantExpr::create(Value: BlockSize, Ctx);
1185 const MCExpr *MaxVGPRs = AMDGPUMCExpr::createMax(
1186 Args: {CurrentProgramInfo.NumVGPRsForWavesPerEU, One}, Ctx);
1187 const MCExpr *NumBlocks = MCBinaryExpr::createDiv(
1188 LHS: AMDGPUMCExpr::createAlignTo(Value: MaxVGPRs, Align: BlockSizeConst, Ctx),
1189 RHS: BlockSizeConst, Ctx);
1190 EncodedBlocks =
1191 MCBinaryExpr::createShl(LHS: MCBinaryExpr::createSub(LHS: NumBlocks, RHS: One, Ctx),
1192 RHS: MCConstantExpr::create(Value: 3, Ctx), Ctx);
1193 }
1194
1195 // Add to function symbol to create _dvgpr$ symbol.
1196 const MCExpr *DVgprFuncVal = MCBinaryExpr::createAdd(
1197 LHS: MCSymbolRefExpr::create(Symbol: CurrentFnSym, Ctx), RHS: EncodedBlocks, Ctx);
1198 MCSymbol *DVgprFuncSym =
1199 Ctx.getOrCreateSymbol(Name: Twine("_dvgpr$") + CurrentFnSym->getName());
1200 OutStreamer->emitAssignment(Symbol: DVgprFuncSym, Value: DVgprFuncVal);
1201 emitVisibility(Sym: DVgprFuncSym, Visibility: MF.getFunction().getVisibility());
1202 emitLinkage(GV: &MF.getFunction(), GVSym: DVgprFuncSym);
1203 }
1204}
1205
1206// TODO: Fold this into emitFunctionBodyStart.
1207void AMDGPUAsmPrinter::initializeTargetID(const Module &M) {
1208 // In the beginning all features are either 'Any' or 'NotSupported',
1209 // depending on global target features. This will cover empty modules.
1210 getTargetStreamer()->initializeTargetID(STI: *getGlobalSTI(),
1211 FeatureString: getGlobalSTI()->getFeatureString());
1212
1213 // If module is empty, we are done.
1214 if (M.empty())
1215 return;
1216
1217 // If module is not empty, need to find first 'Off' or 'On' feature
1218 // setting per feature from functions in module.
1219 for (auto &F : M) {
1220 auto &TSTargetID = getTargetStreamer()->getTargetID();
1221 if ((!TSTargetID->isXnackSupported() || TSTargetID->isXnackOnOrOff()) &&
1222 (!TSTargetID->isSramEccSupported() || TSTargetID->isSramEccOnOrOff()))
1223 break;
1224
1225 const GCNSubtarget &STM = TM.getSubtarget<GCNSubtarget>(F);
1226 const AMDGPU::TargetID &STMTargetID = STM.getTargetID();
1227 if (TSTargetID->isXnackSupported())
1228 if (TSTargetID->getXnackSetting() == AMDGPU::TargetIDSetting::Any)
1229 TSTargetID->setXnackSetting(STMTargetID.getXnackSetting());
1230 if (TSTargetID->isSramEccSupported())
1231 if (TSTargetID->getSramEccSetting() == AMDGPU::TargetIDSetting::Any)
1232 TSTargetID->setSramEccSetting(STMTargetID.getSramEccSetting());
1233 }
1234}
1235
1236// AccumOffset computed for the MCExpr equivalent of:
1237// alignTo(std::max(1, NumVGPR), 4) / 4 - 1;
1238static const MCExpr *computeAccumOffset(const MCExpr *NumVGPR, MCContext &Ctx) {
1239 const MCExpr *ConstFour = MCConstantExpr::create(Value: 4, Ctx);
1240 const MCExpr *ConstOne = MCConstantExpr::create(Value: 1, Ctx);
1241
1242 // Can't be lower than 1 for subsequent alignTo.
1243 const MCExpr *MaximumTaken =
1244 AMDGPUMCExpr::createMax(Args: {ConstOne, NumVGPR}, Ctx);
1245
1246 // Practically, it's computing divideCeil(MaximumTaken, 4).
1247 const MCExpr *DivCeil = MCBinaryExpr::createDiv(
1248 LHS: AMDGPUMCExpr::createAlignTo(Value: MaximumTaken, Align: ConstFour, Ctx), RHS: ConstFour,
1249 Ctx);
1250
1251 return MCBinaryExpr::createSub(LHS: DivCeil, RHS: ConstOne, Ctx);
1252}
1253
1254void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
1255 const MachineFunction &MF) {
1256 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1257 MCContext &Ctx = MF.getContext();
1258
1259 auto CreateExpr = [&Ctx](int64_t Value) {
1260 return MCConstantExpr::create(Value, Ctx);
1261 };
1262
1263 auto TryGetMCExprValue = [](const MCExpr *Value, uint64_t &Res) -> bool {
1264 int64_t Val;
1265 if (Value->evaluateAsAbsolute(Res&: Val)) {
1266 Res = Val;
1267 return true;
1268 }
1269 return false;
1270 };
1271
1272 auto GetSymRefExpr =
1273 [&](MCResourceInfo::ResourceInfoKind RIK) -> const MCExpr * {
1274 MCSymbol *Sym = RI.getSymbol(FuncName: CurrentFnSym->getName(), RIK, OutContext);
1275 return MCSymbolRefExpr::create(Symbol: Sym, Ctx);
1276 };
1277
1278 using RIK = MCResourceInfo::ResourceInfoKind;
1279 ProgInfo.NumArchVGPR = GetSymRefExpr(RIK::RIK_NumVGPR);
1280 ProgInfo.NumAccVGPR = GetSymRefExpr(RIK::RIK_NumAGPR);
1281 ProgInfo.NumVGPR = AMDGPUMCExpr::createTotalNumVGPR(
1282 NumAGPR: ProgInfo.NumAccVGPR, NumVGPR: ProgInfo.NumArchVGPR, Ctx);
1283
1284 ProgInfo.AccumOffset = computeAccumOffset(NumVGPR: ProgInfo.NumArchVGPR, Ctx);
1285 ProgInfo.TgSplit =
1286 STM.hasTgSplitSupport() && AMDGPU::isTgSplitEnabled(F: MF.getFunction());
1287 ProgInfo.NumSGPR = GetSymRefExpr(RIK::RIK_NumSGPR);
1288 ProgInfo.ScratchSize = GetSymRefExpr(RIK::RIK_PrivateSegSize);
1289 ProgInfo.VCCUsed = GetSymRefExpr(RIK::RIK_UsesVCC);
1290 ProgInfo.FlatUsed = GetSymRefExpr(RIK::RIK_UsesFlatScratch);
1291 ProgInfo.DynamicCallStack =
1292 MCBinaryExpr::createOr(LHS: GetSymRefExpr(RIK::RIK_HasDynSizedStack),
1293 RHS: GetSymRefExpr(RIK::RIK_HasRecursion), Ctx);
1294
1295 const MCExpr *BarBlkConst = MCConstantExpr::create(Value: 4, Ctx);
1296 const MCExpr *AlignToBlk = AMDGPUMCExpr::createAlignTo(
1297 Value: GetSymRefExpr(RIK::RIK_NumNamedBarrier), Align: BarBlkConst, Ctx);
1298 ProgInfo.NamedBarCnt = MCBinaryExpr::createDiv(LHS: AlignToBlk, RHS: BarBlkConst, Ctx);
1299
1300 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1301
1302 // The calculations related to SGPR/VGPR blocks are
1303 // duplicated in part in AMDGPUAsmParser::calculateGPRBlocks, and could be
1304 // unified.
1305 const MCExpr *ExtraSGPRs = AMDGPUMCExpr::createExtraSGPRs(
1306 VCCUsed: ProgInfo.VCCUsed, FlatScrUsed: ProgInfo.FlatUsed,
1307 XNACKUsed: getTargetStreamer()->getTargetID()->isXnackOnOrAny(), Ctx);
1308
1309 // Check the addressable register limit before we add ExtraSGPRs.
1310 if (STM.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
1311 !STM.hasSGPRInitBug()) {
1312 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
1313 uint64_t NumSgpr;
1314 if (TryGetMCExprValue(ProgInfo.NumSGPR, NumSgpr) &&
1315 NumSgpr > MaxAddressableNumSGPRs) {
1316 // This can happen due to a compiler bug or when using inline asm.
1317 LLVMContext &Ctx = MF.getFunction().getContext();
1318 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1319 MF.getFunction(), "addressable scalar registers", NumSgpr,
1320 MaxAddressableNumSGPRs, DS_Error, DK_ResourceLimit));
1321 ProgInfo.NumSGPR = CreateExpr(MaxAddressableNumSGPRs - 1);
1322 }
1323 }
1324
1325 // Account for extra SGPRs and VGPRs reserved for debugger use.
1326 ProgInfo.NumSGPR = MCBinaryExpr::createAdd(LHS: ProgInfo.NumSGPR, RHS: ExtraSGPRs, Ctx);
1327
1328 const Function &F = MF.getFunction();
1329
1330 // Ensure there are enough SGPRs and VGPRs for wave dispatch, where wave
1331 // dispatch registers as function args.
1332 unsigned WaveDispatchNumSGPR = MFI->getNumWaveDispatchSGPRs(),
1333 WaveDispatchNumVGPR = MFI->getNumWaveDispatchVGPRs();
1334
1335 if (WaveDispatchNumSGPR) {
1336 ProgInfo.NumSGPR = AMDGPUMCExpr::createMax(
1337 Args: {ProgInfo.NumSGPR,
1338 MCBinaryExpr::createAdd(LHS: CreateExpr(WaveDispatchNumSGPR), RHS: ExtraSGPRs,
1339 Ctx)},
1340 Ctx);
1341 }
1342
1343 if (WaveDispatchNumVGPR) {
1344 ProgInfo.NumArchVGPR = AMDGPUMCExpr::createMax(
1345 Args: {ProgInfo.NumVGPR, CreateExpr(WaveDispatchNumVGPR)}, Ctx);
1346
1347 ProgInfo.NumVGPR = AMDGPUMCExpr::createTotalNumVGPR(
1348 NumAGPR: ProgInfo.NumAccVGPR, NumVGPR: ProgInfo.NumArchVGPR, Ctx);
1349 }
1350
1351 // Adjust number of registers used to meet default/requested minimum/maximum
1352 // number of waves per execution unit request.
1353 unsigned MaxWaves = MFI->getMaxWavesPerEU();
1354 ProgInfo.NumSGPRsForWavesPerEU =
1355 AMDGPUMCExpr::createMax(Args: {ProgInfo.NumSGPR, CreateExpr(1ul),
1356 CreateExpr(STM.getMinNumSGPRs(WavesPerEU: MaxWaves))},
1357 Ctx);
1358 ProgInfo.NumVGPRsForWavesPerEU =
1359 AMDGPUMCExpr::createMax(Args: {ProgInfo.NumVGPR, CreateExpr(1ul),
1360 CreateExpr(STM.getMinNumVGPRs(
1361 WavesPerEU: MaxWaves, DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize()))},
1362 Ctx);
1363
1364 if (STM.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS ||
1365 STM.hasSGPRInitBug()) {
1366 unsigned MaxAddressableNumSGPRs = STM.getAddressableNumSGPRs();
1367 uint64_t NumSgpr;
1368 if (TryGetMCExprValue(ProgInfo.NumSGPR, NumSgpr) &&
1369 NumSgpr > MaxAddressableNumSGPRs) {
1370 // This can happen due to a compiler bug or when using inline asm to use
1371 // the registers which are usually reserved for vcc etc.
1372 LLVMContext &Ctx = MF.getFunction().getContext();
1373 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1374 MF.getFunction(), "scalar registers", NumSgpr, MaxAddressableNumSGPRs,
1375 DS_Error, DK_ResourceLimit));
1376 ProgInfo.NumSGPR = CreateExpr(MaxAddressableNumSGPRs);
1377 ProgInfo.NumSGPRsForWavesPerEU = CreateExpr(MaxAddressableNumSGPRs);
1378 }
1379 }
1380
1381 if (STM.hasSGPRInitBug()) {
1382 ProgInfo.NumSGPR =
1383 CreateExpr(AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG);
1384 ProgInfo.NumSGPRsForWavesPerEU =
1385 CreateExpr(AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG);
1386 }
1387
1388 if (MFI->getNumUserSGPRs() > STM.getMaxNumUserSGPRs()) {
1389 LLVMContext &Ctx = MF.getFunction().getContext();
1390 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1391 MF.getFunction(), "user SGPRs", MFI->getNumUserSGPRs(),
1392 STM.getMaxNumUserSGPRs(), DS_Error));
1393 }
1394
1395 if (MFI->getLDSSize() > STM.getAddressableLocalMemorySize()) {
1396 LLVMContext &Ctx = MF.getFunction().getContext();
1397 Ctx.diagnose(DI: DiagnosticInfoResourceLimit(
1398 MF.getFunction(), "local memory", MFI->getLDSSize(),
1399 STM.getAddressableLocalMemorySize(), DS_Error));
1400 }
1401 // The MCExpr equivalent of getNumSGPRBlocks/getNumVGPRBlocks:
1402 // (alignTo(max(1u, NumGPR), GPREncodingGranule) / GPREncodingGranule) - 1
1403 auto GetNumGPRBlocks = [&CreateExpr, &Ctx](const MCExpr *NumGPR,
1404 unsigned Granule) {
1405 const MCExpr *OneConst = CreateExpr(1ul);
1406 const MCExpr *GranuleConst = CreateExpr(Granule);
1407 const MCExpr *MaxNumGPR = AMDGPUMCExpr::createMax(Args: {NumGPR, OneConst}, Ctx);
1408 const MCExpr *AlignToGPR =
1409 AMDGPUMCExpr::createAlignTo(Value: MaxNumGPR, Align: GranuleConst, Ctx);
1410 const MCExpr *DivGPR =
1411 MCBinaryExpr::createDiv(LHS: AlignToGPR, RHS: GranuleConst, Ctx);
1412 const MCExpr *SubGPR = MCBinaryExpr::createSub(LHS: DivGPR, RHS: OneConst, Ctx);
1413 return SubGPR;
1414 };
1415 // GFX10+ will always allocate 128 SGPRs and this field must be 0
1416 if (STM.getGeneration() >= AMDGPUSubtarget::GFX10) {
1417 ProgInfo.SGPRBlocks = CreateExpr(0ul);
1418 } else {
1419 ProgInfo.SGPRBlocks = GetNumGPRBlocks(ProgInfo.NumSGPRsForWavesPerEU,
1420 IsaInfo::getSGPREncodingGranule(STI: STM));
1421 }
1422 ProgInfo.VGPRBlocks = GetNumGPRBlocks(ProgInfo.NumVGPRsForWavesPerEU,
1423 IsaInfo::getVGPREncodingGranule(STI: STM));
1424
1425 const SIModeRegisterDefaults Mode = MFI->getMode();
1426
1427 // Set the value to initialize FP_ROUND and FP_DENORM parts of the mode
1428 // register.
1429 ProgInfo.FloatMode = getFPMode(Mode);
1430
1431 ProgInfo.IEEEMode = Mode.IEEE;
1432
1433 // Make clamp modifier on NaN input returns 0.
1434 ProgInfo.DX10Clamp = Mode.DX10Clamp;
1435
1436 unsigned LDSAlignShift = 8;
1437 switch (getLdsDwGranularity(ST: STM)) {
1438 case 512:
1439 case 320:
1440 LDSAlignShift = 11;
1441 break;
1442 case 128:
1443 LDSAlignShift = 9;
1444 break;
1445 case 64:
1446 LDSAlignShift = 8;
1447 break;
1448 default:
1449 llvm_unreachable("invald LDS block size");
1450 }
1451
1452 ProgInfo.SGPRSpill = MFI->getNumSpilledSGPRs();
1453 ProgInfo.VGPRSpill = MFI->getNumSpilledVGPRs();
1454
1455 ProgInfo.LDSSize = MFI->getLDSSize();
1456 ProgInfo.LDSBlocks =
1457 alignTo(Value: ProgInfo.LDSSize, Align: 1ULL << LDSAlignShift) >> LDSAlignShift;
1458
1459 // The MCExpr equivalent of divideCeil.
1460 auto DivideCeil = [&Ctx](const MCExpr *Numerator, const MCExpr *Denominator) {
1461 const MCExpr *Ceil =
1462 AMDGPUMCExpr::createAlignTo(Value: Numerator, Align: Denominator, Ctx);
1463 return MCBinaryExpr::createDiv(LHS: Ceil, RHS: Denominator, Ctx);
1464 };
1465
1466 // Scratch is allocated in 64-dword or 256-dword blocks.
1467 unsigned ScratchAlignShift =
1468 STM.getGeneration() >= AMDGPUSubtarget::GFX11 ? 8 : 10;
1469 // We need to program the hardware with the amount of scratch memory that
1470 // is used by the entire wave. ProgInfo.ScratchSize is the amount of
1471 // scratch memory used per thread.
1472 ProgInfo.ScratchBlocks = DivideCeil(
1473 MCBinaryExpr::createMul(LHS: ProgInfo.ScratchSize,
1474 RHS: CreateExpr(STM.getWavefrontSize()), Ctx),
1475 CreateExpr(1ULL << ScratchAlignShift));
1476
1477 if (STM.supportsWGP()) {
1478 ProgInfo.WgpMode = STM.isCuModeEnabled() ? 0 : 1;
1479 }
1480
1481 if (getIsaVersion(GPU: getGlobalSTI()->getCPU()).Major >= 10) {
1482 ProgInfo.MemOrdered = 1;
1483 ProgInfo.FwdProgress = !F.hasFnAttribute(Kind: "amdgpu-no-fwd-progress");
1484 }
1485
1486 // 0 = X, 1 = XY, 2 = XYZ
1487 unsigned TIDIGCompCnt = 0;
1488 if (MFI->hasWorkItemIDZ())
1489 TIDIGCompCnt = 2;
1490 else if (MFI->hasWorkItemIDY())
1491 TIDIGCompCnt = 1;
1492
1493 // The private segment wave byte offset is the last of the system SGPRs. We
1494 // initially assumed it was allocated, and may have used it. It shouldn't harm
1495 // anything to disable it if we know the stack isn't used here. We may still
1496 // have emitted code reading it to initialize scratch, but if that's unused
1497 // reading garbage should be OK.
1498 ProgInfo.ScratchEnable = MCBinaryExpr::createLOr(
1499 LHS: MCBinaryExpr::createGT(LHS: ProgInfo.ScratchBlocks,
1500 RHS: MCConstantExpr::create(Value: 0, Ctx), Ctx),
1501 RHS: ProgInfo.DynamicCallStack, Ctx);
1502
1503 ProgInfo.UserSGPR = MFI->getNumUserSGPRs();
1504 // For AMDHSA, TRAP_HANDLER must be zero, as it is populated by the CP.
1505 ProgInfo.TrapHandlerEnable = STM.isAmdHsaOS() ? 0 : STM.hasTrapHandler();
1506 ProgInfo.TGIdXEnable = MFI->hasWorkGroupIDX();
1507 ProgInfo.TGIdYEnable = MFI->hasWorkGroupIDY();
1508 ProgInfo.TGIdZEnable = MFI->hasWorkGroupIDZ();
1509 ProgInfo.TGSizeEnable = MFI->hasWorkGroupInfo();
1510 ProgInfo.TIdIGCompCount = TIDIGCompCnt;
1511 ProgInfo.EXCPEnMSB = 0;
1512 // For AMDHSA, LDS_SIZE must be zero, as it is populated by the CP.
1513 ProgInfo.LdsSize = STM.isAmdHsaOS() ? 0 : ProgInfo.LDSBlocks;
1514 ProgInfo.EXCPEnable = 0;
1515
1516 if (STM.hasGFX90AInsts()) {
1517 ProgInfo.ComputePGMRSrc3 =
1518 setBits(Dst: ProgInfo.ComputePGMRSrc3, Value: ProgInfo.AccumOffset,
1519 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
1520 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT, Ctx);
1521 ProgInfo.ComputePGMRSrc3 =
1522 setBits(Dst: ProgInfo.ComputePGMRSrc3, Value: CreateExpr(ProgInfo.TgSplit),
1523 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT,
1524 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT, Ctx);
1525 }
1526
1527 if (STM.hasGFX1250Insts())
1528 ProgInfo.ComputePGMRSrc3 =
1529 setBits(Dst: ProgInfo.ComputePGMRSrc3, Value: ProgInfo.NamedBarCnt,
1530 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
1531 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT, Ctx);
1532
1533 ProgInfo.Occupancy = createOccupancy(
1534 InitOcc: STM.computeOccupancy(F, LDSSize: ProgInfo.LDSSize).second,
1535 NumSGPRs: ProgInfo.NumSGPRsForWavesPerEU, NumVGPRs: ProgInfo.NumVGPRsForWavesPerEU,
1536 DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize(), STM, Ctx);
1537
1538 const auto [MinWEU, MaxWEU] =
1539 AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-waves-per-eu", Default: {0, 0}, OnlyFirstRequired: true);
1540 uint64_t Occupancy;
1541 if (TryGetMCExprValue(ProgInfo.Occupancy, Occupancy) && Occupancy < MinWEU) {
1542 DiagnosticInfoOptimizationFailure Diag(
1543 F, F.getSubprogram(),
1544 "failed to meet occupancy target given by 'amdgpu-waves-per-eu' in "
1545 "'" +
1546 F.getName() + "': desired occupancy was " + Twine(MinWEU) +
1547 ", final occupancy is " + Twine(Occupancy));
1548 F.getContext().diagnose(DI: Diag);
1549 }
1550}
1551
1552static unsigned getRsrcReg(CallingConv::ID CallConv) {
1553 switch (CallConv) {
1554 default:
1555 [[fallthrough]];
1556 case CallingConv::AMDGPU_CS:
1557 return R_00B848_COMPUTE_PGM_RSRC1;
1558 case CallingConv::AMDGPU_LS:
1559 return R_00B528_SPI_SHADER_PGM_RSRC1_LS;
1560 case CallingConv::AMDGPU_HS:
1561 return R_00B428_SPI_SHADER_PGM_RSRC1_HS;
1562 case CallingConv::AMDGPU_ES:
1563 return R_00B328_SPI_SHADER_PGM_RSRC1_ES;
1564 case CallingConv::AMDGPU_GS:
1565 return R_00B228_SPI_SHADER_PGM_RSRC1_GS;
1566 case CallingConv::AMDGPU_VS:
1567 return R_00B128_SPI_SHADER_PGM_RSRC1_VS;
1568 case CallingConv::AMDGPU_PS:
1569 return R_00B028_SPI_SHADER_PGM_RSRC1_PS;
1570 }
1571}
1572
1573void AMDGPUAsmPrinter::EmitProgramInfoSI(
1574 const MachineFunction &MF, const SIProgramInfo &CurrentProgramInfo) {
1575 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1576 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1577 unsigned RsrcReg = getRsrcReg(CallConv: MF.getFunction().getCallingConv());
1578 MCContext &Ctx = MF.getContext();
1579
1580 // (((Value) & Mask) << Shift)
1581 auto SetBits = [&Ctx](const MCExpr *Value, uint32_t Mask, uint32_t Shift) {
1582 const MCExpr *msk = MCConstantExpr::create(Value: Mask, Ctx);
1583 const MCExpr *shft = MCConstantExpr::create(Value: Shift, Ctx);
1584 return MCBinaryExpr::createShl(LHS: MCBinaryExpr::createAnd(LHS: Value, RHS: msk, Ctx),
1585 RHS: shft, Ctx);
1586 };
1587
1588 auto EmitResolvedOrExpr = [this](const MCExpr *Value, unsigned Size) {
1589 int64_t Val;
1590 if (Value->evaluateAsAbsolute(Res&: Val))
1591 OutStreamer->emitIntValue(Value: static_cast<uint64_t>(Val), Size);
1592 else
1593 OutStreamer->emitValue(Value, Size);
1594 };
1595
1596 if (AMDGPU::isCompute(CC: MF.getFunction().getCallingConv())) {
1597 OutStreamer->emitInt32(R_00B848_COMPUTE_PGM_RSRC1);
1598
1599 EmitResolvedOrExpr(CurrentProgramInfo.getComputePGMRSrc1(ST: STM, Ctx),
1600 /*Size=*/4);
1601
1602 OutStreamer->emitInt32(R_00B84C_COMPUTE_PGM_RSRC2);
1603 EmitResolvedOrExpr(CurrentProgramInfo.getComputePGMRSrc2(ST: STM, Ctx),
1604 /*Size=*/4);
1605
1606 OutStreamer->emitInt32(R_00B860_COMPUTE_TMPRING_SIZE);
1607
1608 // Sets bits according to S_0286E8_WAVESIZE_* mask and shift values for the
1609 // appropriate generation.
1610 if (STM.getGeneration() >= AMDGPUSubtarget::GFX12) {
1611 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1612 /*Mask=*/0x3FFFF, /*Shift=*/12),
1613 /*Size=*/4);
1614 } else if (STM.getGeneration() == AMDGPUSubtarget::GFX11) {
1615 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1616 /*Mask=*/0x7FFF, /*Shift=*/12),
1617 /*Size=*/4);
1618 } else {
1619 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1620 /*Mask=*/0x1FFF, /*Shift=*/12),
1621 /*Size=*/4);
1622 }
1623
1624 // TODO: Should probably note flat usage somewhere. SC emits a "FlatPtr32 =
1625 // 0" comment but I don't see a corresponding field in the register spec.
1626 } else {
1627 OutStreamer->emitInt32(Value: RsrcReg);
1628
1629 const MCExpr *GPRBlocks = MCBinaryExpr::createOr(
1630 LHS: SetBits(CurrentProgramInfo.VGPRBlocks, /*Mask=*/0x3F, /*Shift=*/0),
1631 RHS: SetBits(CurrentProgramInfo.SGPRBlocks, /*Mask=*/0x0F, /*Shift=*/6),
1632 Ctx&: MF.getContext());
1633 EmitResolvedOrExpr(GPRBlocks, /*Size=*/4);
1634 OutStreamer->emitInt32(R_0286E8_SPI_TMPRING_SIZE);
1635
1636 // Sets bits according to S_0286E8_WAVESIZE_* mask and shift values for the
1637 // appropriate generation.
1638 if (STM.getGeneration() >= AMDGPUSubtarget::GFX12) {
1639 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1640 /*Mask=*/0x3FFFF, /*Shift=*/12),
1641 /*Size=*/4);
1642 } else if (STM.getGeneration() == AMDGPUSubtarget::GFX11) {
1643 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1644 /*Mask=*/0x7FFF, /*Shift=*/12),
1645 /*Size=*/4);
1646 } else {
1647 EmitResolvedOrExpr(SetBits(CurrentProgramInfo.ScratchBlocks,
1648 /*Mask=*/0x1FFF, /*Shift=*/12),
1649 /*Size=*/4);
1650 }
1651 }
1652
1653 if (MF.getFunction().getCallingConv() == CallingConv::AMDGPU_PS) {
1654 OutStreamer->emitInt32(R_00B02C_SPI_SHADER_PGM_RSRC2_PS);
1655 unsigned ExtraLDSSize = STM.getGeneration() >= AMDGPUSubtarget::GFX11
1656 ? divideCeil(Numerator: CurrentProgramInfo.LDSBlocks, Denominator: 2)
1657 : CurrentProgramInfo.LDSBlocks;
1658 OutStreamer->emitInt32(S_00B02C_EXTRA_LDS_SIZE(ExtraLDSSize));
1659 OutStreamer->emitInt32(R_0286CC_SPI_PS_INPUT_ENA);
1660 OutStreamer->emitInt32(Value: MFI->getPSInputEnable());
1661 OutStreamer->emitInt32(R_0286D0_SPI_PS_INPUT_ADDR);
1662 OutStreamer->emitInt32(Value: MFI->getPSInputAddr());
1663 }
1664
1665 OutStreamer->emitInt32(R_SPILLED_SGPRS);
1666 OutStreamer->emitInt32(Value: MFI->getNumSpilledSGPRs());
1667 OutStreamer->emitInt32(R_SPILLED_VGPRS);
1668 OutStreamer->emitInt32(Value: MFI->getNumSpilledVGPRs());
1669}
1670
1671// Helper function to add common PAL Metadata 3.0+
1672static void EmitPALMetadataCommon(AMDGPUPALMetadata *MD,
1673 const SIProgramInfo &CurrentProgramInfo,
1674 CallingConv::ID CC, const GCNSubtarget &ST,
1675 unsigned DynamicVGPRBlockSize) {
1676 if (ST.hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode))
1677 MD->setHwStage(CC, field: ".ieee_mode", Val: (bool)CurrentProgramInfo.IEEEMode);
1678
1679 MD->setHwStage(CC, field: ".wgp_mode", Val: (bool)CurrentProgramInfo.WgpMode);
1680 MD->setHwStage(CC, field: ".mem_ordered", Val: (bool)CurrentProgramInfo.MemOrdered);
1681 MD->setHwStage(CC, field: ".forward_progress", Val: (bool)CurrentProgramInfo.FwdProgress);
1682
1683 if (AMDGPU::isCompute(CC)) {
1684 MD->setHwStage(CC, field: ".trap_present",
1685 Val: (bool)CurrentProgramInfo.TrapHandlerEnable);
1686 MD->setHwStage(CC, field: ".excp_en", Val: CurrentProgramInfo.EXCPEnable);
1687
1688 if (DynamicVGPRBlockSize != 0)
1689 MD->setComputeRegisters(field: ".dynamic_vgpr_en", Val: true);
1690 }
1691
1692 MD->updateHwStageMaximum(
1693 CC, field: ".lds_size",
1694 Val: (unsigned)(CurrentProgramInfo.LdsSize * getLdsDwGranularity(ST) *
1695 sizeof(uint32_t)));
1696}
1697
1698// This is the equivalent of EmitProgramInfoSI above, but for when the OS type
1699// is AMDPAL. It stores each compute/SPI register setting and other PAL
1700// metadata items into the PALMD::Metadata, combining with any provided by the
1701// frontend as LLVM metadata. Once all functions are written, the PAL metadata
1702// is then written as a single block in the .note section.
1703void AMDGPUAsmPrinter::EmitPALMetadata(
1704 const MachineFunction &MF, const SIProgramInfo &CurrentProgramInfo) {
1705 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1706 auto CC = MF.getFunction().getCallingConv();
1707 auto *MD = getTargetStreamer()->getPALMetadata();
1708 auto &Ctx = MF.getContext();
1709
1710 MD->setEntryPoint(CC, Name: MF.getFunction().getName());
1711 MD->setNumUsedVgprs(CC, Val: CurrentProgramInfo.NumVGPRsForWavesPerEU, Ctx);
1712
1713 // For targets that support dynamic VGPRs, set the number of saved dynamic
1714 // VGPRs (if any) in the PAL metadata.
1715 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1716 if (MFI->isDynamicVGPREnabled() &&
1717 MFI->getScratchReservedForDynamicVGPRs() > 0)
1718 MD->setHwStage(CC, field: ".dynamic_vgpr_saved_count",
1719 Val: MFI->getScratchReservedForDynamicVGPRs() / 4);
1720
1721 // Only set AGPRs for supported devices
1722 if (STM.hasMAIInsts()) {
1723 MD->setNumUsedAgprs(CC, Val: CurrentProgramInfo.NumAccVGPR);
1724 }
1725
1726 MD->setNumUsedSgprs(CC, Val: CurrentProgramInfo.NumSGPRsForWavesPerEU, Ctx);
1727 if (MD->getPALMajorVersion() < 3) {
1728 MD->setRsrc1(CC, Val: CurrentProgramInfo.getPGMRSrc1(CC, ST: STM, Ctx), Ctx);
1729 if (AMDGPU::isCompute(CC)) {
1730 MD->setRsrc2(CC, Val: CurrentProgramInfo.getComputePGMRSrc2(ST: STM, Ctx), Ctx);
1731 } else {
1732 const MCExpr *HasScratchBlocks =
1733 MCBinaryExpr::createGT(LHS: CurrentProgramInfo.ScratchBlocks,
1734 RHS: MCConstantExpr::create(Value: 0, Ctx), Ctx);
1735 auto [Shift, Mask] = getShiftMask(C_00B84C_SCRATCH_EN);
1736 MD->setRsrc2(CC, Val: maskShiftSet(Val: HasScratchBlocks, Mask, Shift, Ctx), Ctx);
1737 }
1738 } else {
1739 MD->setHwStage(CC, field: ".debug_mode", Val: (bool)CurrentProgramInfo.DebugMode);
1740 MD->setHwStage(CC, field: ".scratch_en", Type: msgpack::Type::Boolean,
1741 Val: CurrentProgramInfo.ScratchEnable);
1742 EmitPALMetadataCommon(MD, CurrentProgramInfo, CC, ST: STM,
1743 DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize());
1744 }
1745
1746 // ScratchSize is in bytes, 16 aligned.
1747 MD->setScratchSize(
1748 CC,
1749 Val: AMDGPUMCExpr::createAlignTo(Value: CurrentProgramInfo.ScratchSize,
1750 Align: MCConstantExpr::create(Value: 16, Ctx), Ctx),
1751 Ctx);
1752
1753 if (MF.getFunction().getCallingConv() == CallingConv::AMDGPU_PS) {
1754 unsigned ExtraLDSSize = STM.getGeneration() >= AMDGPUSubtarget::GFX11
1755 ? divideCeil(Numerator: CurrentProgramInfo.LDSBlocks, Denominator: 2)
1756 : CurrentProgramInfo.LDSBlocks;
1757 if (MD->getPALMajorVersion() < 3) {
1758 MD->setRsrc2(
1759 CC,
1760 Val: MCConstantExpr::create(S_00B02C_EXTRA_LDS_SIZE(ExtraLDSSize), Ctx),
1761 Ctx);
1762 MD->setSpiPsInputEna(MFI->getPSInputEnable());
1763 MD->setSpiPsInputAddr(MFI->getPSInputAddr());
1764 } else {
1765 // Graphics registers
1766 const unsigned ExtraLdsDwGranularity =
1767 STM.getGeneration() >= AMDGPUSubtarget::GFX11 ? 256 : 128;
1768 MD->setGraphicsRegisters(
1769 field: ".ps_extra_lds_size",
1770 Val: (unsigned)(ExtraLDSSize * ExtraLdsDwGranularity * sizeof(uint32_t)));
1771
1772 // Set PsInputEna and PsInputAddr .spi_ps_input_ena and .spi_ps_input_addr
1773 static StringLiteral const PsInputFields[] = {
1774 ".persp_sample_ena", ".persp_center_ena",
1775 ".persp_centroid_ena", ".persp_pull_model_ena",
1776 ".linear_sample_ena", ".linear_center_ena",
1777 ".linear_centroid_ena", ".line_stipple_tex_ena",
1778 ".pos_x_float_ena", ".pos_y_float_ena",
1779 ".pos_z_float_ena", ".pos_w_float_ena",
1780 ".front_face_ena", ".ancillary_ena",
1781 ".sample_coverage_ena", ".pos_fixed_pt_ena"};
1782 unsigned PSInputEna = MFI->getPSInputEnable();
1783 unsigned PSInputAddr = MFI->getPSInputAddr();
1784 for (auto [Idx, Field] : enumerate(First: PsInputFields)) {
1785 MD->setGraphicsRegisters(field1: ".spi_ps_input_ena", field2: Field,
1786 Val: (bool)((PSInputEna >> Idx) & 1));
1787 MD->setGraphicsRegisters(field1: ".spi_ps_input_addr", field2: Field,
1788 Val: (bool)((PSInputAddr >> Idx) & 1));
1789 }
1790 }
1791 }
1792
1793 // For version 3 and above the wave front size is already set in the metadata
1794 if (MD->getPALMajorVersion() < 3 && STM.isWave32())
1795 MD->setWave32(MF.getFunction().getCallingConv());
1796}
1797
1798void AMDGPUAsmPrinter::emitPALFunctionMetadata(const MachineFunction &MF) {
1799 auto *MD = getTargetStreamer()->getPALMetadata();
1800 const MachineFrameInfo &MFI = MF.getFrameInfo();
1801 StringRef FnName = MF.getFunction().getName();
1802 MD->setFunctionScratchSize(FnName, Val: MFI.getStackSize());
1803 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1804 MCContext &Ctx = MF.getContext();
1805
1806 if (MD->getPALMajorVersion() < 3) {
1807 // Set compute registers
1808 MD->setRsrc1(
1809 CC: CallingConv::AMDGPU_CS,
1810 Val: CurrentProgramInfo.getPGMRSrc1(CC: CallingConv::AMDGPU_CS, ST, Ctx), Ctx);
1811 MD->setRsrc2(CC: CallingConv::AMDGPU_CS,
1812 Val: CurrentProgramInfo.getComputePGMRSrc2(ST, Ctx), Ctx);
1813 } else {
1814 EmitPALMetadataCommon(
1815 MD, CurrentProgramInfo, CC: CallingConv::AMDGPU_CS, ST,
1816 DynamicVGPRBlockSize: MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize());
1817 }
1818
1819 // Set optional info
1820 MD->setFunctionLdsSize(FnName, Val: CurrentProgramInfo.LDSSize);
1821 MD->setFunctionNumUsedVgprs(FnName, Val: CurrentProgramInfo.NumVGPRsForWavesPerEU);
1822 MD->setFunctionNumUsedSgprs(FnName, Val: CurrentProgramInfo.NumSGPRsForWavesPerEU);
1823}
1824
1825// This is supposed to be log2(Size)
1826static amd_element_byte_size_t getElementByteSizeValue(unsigned Size) {
1827 switch (Size) {
1828 case 4:
1829 return AMD_ELEMENT_4_BYTES;
1830 case 8:
1831 return AMD_ELEMENT_8_BYTES;
1832 case 16:
1833 return AMD_ELEMENT_16_BYTES;
1834 default:
1835 llvm_unreachable("invalid private_element_size");
1836 }
1837}
1838
1839void AMDGPUAsmPrinter::getAmdKernelCode(AMDGPUMCKernelCodeT &Out,
1840 const SIProgramInfo &CurrentProgramInfo,
1841 const MachineFunction &MF) const {
1842 const Function &F = MF.getFunction();
1843 assert(F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
1844 F.getCallingConv() == CallingConv::SPIR_KERNEL);
1845
1846 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1847 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
1848 MCContext &Ctx = MF.getContext();
1849
1850 Out.initDefault(STI: STM, Ctx, /*InitMCExpr=*/false);
1851
1852 Out.compute_pgm_resource1_registers =
1853 CurrentProgramInfo.getComputePGMRSrc1(ST: STM, Ctx);
1854 Out.compute_pgm_resource2_registers =
1855 CurrentProgramInfo.getComputePGMRSrc2(ST: STM, Ctx);
1856 Out.code_properties |= AMD_CODE_PROPERTY_IS_PTR64;
1857
1858 Out.is_dynamic_callstack = CurrentProgramInfo.DynamicCallStack;
1859
1860 AMD_HSA_BITS_SET(Out.code_properties, AMD_CODE_PROPERTY_PRIVATE_ELEMENT_SIZE,
1861 getElementByteSizeValue(STM.getMaxPrivateElementSize(true)));
1862
1863 const GCNUserSGPRUsageInfo &UserSGPRInfo = MFI->getUserSGPRInfo();
1864 if (UserSGPRInfo.hasPrivateSegmentBuffer()) {
1865 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER;
1866 }
1867
1868 if (UserSGPRInfo.hasDispatchPtr())
1869 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR;
1870
1871 if (UserSGPRInfo.hasQueuePtr())
1872 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR;
1873
1874 if (UserSGPRInfo.hasKernargSegmentPtr())
1875 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR;
1876
1877 if (UserSGPRInfo.hasDispatchID())
1878 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID;
1879
1880 if (UserSGPRInfo.hasFlatScratchInit())
1881 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT;
1882
1883 if (UserSGPRInfo.hasPrivateSegmentSize())
1884 Out.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE;
1885
1886 if (STM.isXNACKEnabled())
1887 Out.code_properties |= AMD_CODE_PROPERTY_IS_XNACK_SUPPORTED;
1888
1889 Align MaxKernArgAlign;
1890 Out.kernarg_segment_byte_size = STM.getKernArgSegmentSize(F, MaxAlign&: MaxKernArgAlign);
1891 Out.wavefront_sgpr_count = CurrentProgramInfo.NumSGPR;
1892 Out.workitem_vgpr_count = CurrentProgramInfo.NumVGPR;
1893 Out.workitem_private_segment_byte_size = CurrentProgramInfo.ScratchSize;
1894 Out.workgroup_group_segment_byte_size = CurrentProgramInfo.LDSSize;
1895
1896 // kernarg_segment_alignment is specified as log of the alignment.
1897 // The minimum alignment is 16.
1898 // FIXME: The metadata treats the minimum as 4?
1899 Out.kernarg_segment_alignment = Log2(A: std::max(a: Align(16), b: MaxKernArgAlign));
1900}
1901
1902bool AMDGPUAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
1903 const char *ExtraCode, raw_ostream &O) {
1904 // First try the generic code, which knows about modifiers like 'c' and 'n'.
1905 if (!AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, OS&: O))
1906 return false;
1907
1908 if (ExtraCode && ExtraCode[0]) {
1909 if (ExtraCode[1] != 0)
1910 return true; // Unknown modifier.
1911
1912 switch (ExtraCode[0]) {
1913 case 'r':
1914 break;
1915 default:
1916 return true;
1917 }
1918 }
1919
1920 // TODO: Should be able to support other operand types like globals.
1921 const MachineOperand &MO = MI->getOperand(i: OpNo);
1922 if (MO.isReg()) {
1923 AMDGPUInstPrinter::printRegOperand(Reg: MO.getReg(), O,
1924 MRI: *MF->getSubtarget().getRegisterInfo());
1925 return false;
1926 }
1927 if (MO.isImm()) {
1928 int64_t Val = MO.getImm();
1929 if (AMDGPU::isInlinableIntLiteral(Literal: Val)) {
1930 O << Val;
1931 } else if (isUInt<16>(x: Val)) {
1932 O << format(Fmt: "0x%" PRIx16, Vals: static_cast<uint16_t>(Val));
1933 } else if (isUInt<32>(x: Val)) {
1934 O << format(Fmt: "0x%" PRIx32, Vals: static_cast<uint32_t>(Val));
1935 } else {
1936 O << format(Fmt: "0x%" PRIx64, Vals: static_cast<uint64_t>(Val));
1937 }
1938 return false;
1939 }
1940 return true;
1941}
1942
1943void AMDGPUAsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const {
1944 AU.addRequired<AMDGPUResourceUsageAnalysisWrapperPass>();
1945 AU.addPreserved<AMDGPUResourceUsageAnalysisWrapperPass>();
1946 AU.addRequired<MachineModuleInfoWrapperPass>();
1947 AU.addPreserved<MachineModuleInfoWrapperPass>();
1948 AsmPrinter::getAnalysisUsage(AU);
1949}
1950
1951void AMDGPUAsmPrinter::emitResourceUsageRemarks(
1952 const MachineFunction &MF, const SIProgramInfo &CurrentProgramInfo,
1953 bool isModuleEntryFunction, bool hasMAIInsts) {
1954 if (!ORE)
1955 return;
1956
1957 const char *Name = "kernel-resource-usage";
1958 const char *Indent = " ";
1959
1960 // If the remark is not specifically enabled, do not output to yaml
1961 LLVMContext &Ctx = MF.getFunction().getContext();
1962 if (!Ctx.getDiagHandlerPtr()->isAnalysisRemarkEnabled(PassName: Name))
1963 return;
1964
1965 // Currently non-kernel functions have no resources to emit.
1966 if (!isEntryFunctionCC(CC: MF.getFunction().getCallingConv()))
1967 return;
1968
1969 auto EmitResourceUsageRemark = [&](StringRef RemarkName,
1970 StringRef RemarkLabel, auto Argument) {
1971 // Add an indent for every line besides the line with the kernel name. This
1972 // makes it easier to tell which resource usage go with which kernel since
1973 // the kernel name will always be displayed first.
1974 std::string LabelStr = RemarkLabel.str() + ": ";
1975 if (RemarkName != "FunctionName")
1976 LabelStr = Indent + LabelStr;
1977
1978 ORE->emit([&]() {
1979 return MachineOptimizationRemarkAnalysis(Name, RemarkName,
1980 MF.getFunction().getSubprogram(),
1981 &MF.front())
1982 << LabelStr << ore::NV(RemarkName, Argument);
1983 });
1984 };
1985
1986 // FIXME: Formatting here is pretty nasty because clang does not accept
1987 // newlines from diagnostics. This forces us to emit multiple diagnostic
1988 // remarks to simulate newlines. If and when clang does accept newlines, this
1989 // formatting should be aggregated into one remark with newlines to avoid
1990 // printing multiple diagnostic location and diag opts.
1991 EmitResourceUsageRemark("FunctionName", "Function Name",
1992 MF.getFunction().getName());
1993 EmitResourceUsageRemark("NumSGPR", "TotalSGPRs",
1994 getMCExprStr(Value: CurrentProgramInfo.NumSGPR));
1995 EmitResourceUsageRemark("NumVGPR", "VGPRs",
1996 getMCExprStr(Value: CurrentProgramInfo.NumArchVGPR));
1997 if (hasMAIInsts) {
1998 EmitResourceUsageRemark("NumAGPR", "AGPRs",
1999 getMCExprStr(Value: CurrentProgramInfo.NumAccVGPR));
2000 }
2001 EmitResourceUsageRemark("ScratchSize", "ScratchSize [bytes/lane]",
2002 getMCExprStr(Value: CurrentProgramInfo.ScratchSize));
2003 int64_t DynStack;
2004 bool DynStackEvaluatable =
2005 CurrentProgramInfo.DynamicCallStack->evaluateAsAbsolute(Res&: DynStack);
2006 StringRef DynamicStackStr =
2007 DynStackEvaluatable && DynStack ? "True" : "False";
2008 EmitResourceUsageRemark("DynamicStack", "Dynamic Stack", DynamicStackStr);
2009 EmitResourceUsageRemark("Occupancy", "Occupancy [waves/SIMD]",
2010 getMCExprStr(Value: CurrentProgramInfo.Occupancy));
2011 EmitResourceUsageRemark("SGPRSpill", "SGPRs Spill",
2012 CurrentProgramInfo.SGPRSpill);
2013 EmitResourceUsageRemark("VGPRSpill", "VGPRs Spill",
2014 CurrentProgramInfo.VGPRSpill);
2015 if (isModuleEntryFunction)
2016 EmitResourceUsageRemark("BytesLDS", "LDS Size [bytes/block]",
2017 CurrentProgramInfo.LDSSize);
2018}
2019
2020PreservedAnalyses AMDGPUAsmPrinterBeginPass::run(Module &M,
2021 ModuleAnalysisManager &MAM) {
2022
2023 AMDGPUAsmPrinter &AsmPrinter = static_cast<AMDGPUAsmPrinter &>(
2024 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
2025 setupModuleAsmPrinter(M, MAM, AsmPrinter);
2026 AsmPrinter.doInitialization(M);
2027 return PreservedAnalyses::all();
2028}
2029
2030PreservedAnalyses
2031AMDGPUAsmPrinterPass::run(MachineFunction &MF,
2032 MachineFunctionAnalysisManager &MFAM) {
2033 AMDGPUAsmPrinter &AsmPrinter = static_cast<AMDGPUAsmPrinter &>(
2034 MFAM.getResult<ModuleAnalysisManagerMachineFunctionProxy>(IR&: MF)
2035 .getCachedResult<AsmPrinterAnalysis>(IR&: *MF.getFunction().getParent())
2036 ->getPrinter());
2037 setupMachineFunctionAsmPrinter(MFAM, MF, AsmPrinter);
2038 AsmPrinter.GetResourceUsage = [&MFAM](MachineFunction &MF)
2039 -> const AMDGPUResourceUsageAnalysisImpl::SIFunctionResourceInfo * {
2040 return &MFAM.getResult<AMDGPUResourceUsageAnalysis>(IR&: MF);
2041 };
2042 AsmPrinter.runOnMachineFunction(MF);
2043 return PreservedAnalyses::all();
2044}
2045
2046PreservedAnalyses AMDGPUAsmPrinterEndPass::run(Module &M,
2047 ModuleAnalysisManager &MAM) {
2048 AMDGPUAsmPrinter &AsmPrinter = static_cast<AMDGPUAsmPrinter &>(
2049 MAM.getResult<AsmPrinterAnalysis>(IR&: M).getPrinter());
2050 setupModuleAsmPrinter(M, MAM, AsmPrinter);
2051 AsmPrinter.doFinalization(M);
2052 return PreservedAnalyses::all();
2053}
2054
2055char AMDGPUAsmPrinter::ID = 0;
2056
2057INITIALIZE_PASS(AMDGPUAsmPrinter, "amdgpu-asm-printer",
2058 "AMDGPU Assembly Printer", false, false)
2059