1//===-- AMDGPUTargetStreamer.cpp - Mips Target Streamer Methods -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides AMDGPU specific target streamer methods.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AMDGPUTargetStreamer.h"
14#include "AMDGPUMCExpr.h"
15#include "AMDGPUMCKernelDescriptor.h"
16#include "AMDGPUMCTargetDesc.h"
17#include "AMDGPUPTNote.h"
18#include "Utils/AMDGPUBaseInfo.h"
19#include "Utils/AMDKernelCodeTUtils.h"
20#include "llvm/BinaryFormat/AMDGPUMetadataVerifier.h"
21#include "llvm/BinaryFormat/ELF.h"
22#include "llvm/MC/MCAsmInfo.h"
23#include "llvm/MC/MCAssembler.h"
24#include "llvm/MC/MCContext.h"
25#include "llvm/MC/MCELFObjectWriter.h"
26#include "llvm/MC/MCELFStreamer.h"
27#include "llvm/MC/MCSubtargetInfo.h"
28#include "llvm/MC/StringTableBuilder.h"
29#include "llvm/Support/AMDGPUMetadata.h"
30#include "llvm/Support/AMDGPUObjLinkingInfo.h"
31#include "llvm/Support/AMDHSAKernelDescriptor.h"
32#include "llvm/Support/CommandLine.h"
33#include "llvm/Support/FormattedStream.h"
34#include "llvm/TargetParser/AMDGPUTargetParser.h"
35
36using namespace llvm;
37using namespace llvm::AMDGPU;
38
39//===----------------------------------------------------------------------===//
40// AMDGPUTargetStreamer
41//===----------------------------------------------------------------------===//
42
43static cl::opt<unsigned>
44 ForceGenericVersion("amdgpu-force-generic-version",
45 cl::desc("Force a specific generic_v<N> flag to be "
46 "added. For testing purposes only."),
47 cl::ReallyHidden, cl::init(Val: 0));
48
49void AMDGPUTargetStreamer::initializeTargetID(const MCSubtargetInfo &STI,
50 bool ApplyFeatureString) {
51 assert(TargetID == std::nullopt && "TargetID can only be initialized once");
52 // Apply xnack/sramecc from subtarget features only in MC contexts
53 // (assembler), not in codegen where they come from module flags
54 TargetID = AMDGPU::createAMDGPUTargetID(
55 STI, FeatureString: ApplyFeatureString ? STI.getFeatureString() : "");
56}
57
58bool AMDGPUTargetStreamer::EmitHSAMetadataV3(StringRef HSAMetadataString) {
59 msgpack::Document HSAMetadataDoc;
60 if (!HSAMetadataDoc.fromYAML(S: HSAMetadataString))
61 return false;
62 return EmitHSAMetadata(HSAMetadata&: HSAMetadataDoc, Strict: false);
63}
64
65unsigned AMDGPUTargetStreamer::getElfMach(StringRef GPU) {
66 AMDGPU::GPUKind AK = parseArchAMDGCN(CPU: GPU);
67 if (AK == AMDGPU::GPUKind::GK_NONE)
68 AK = parseArchR600(CPU: GPU);
69
70 // clang-format off
71 switch (AK) {
72 case GK_R600: return ELF::EF_AMDGPU_MACH_R600_R600;
73 case GK_R630: return ELF::EF_AMDGPU_MACH_R600_R630;
74 case GK_RS880: return ELF::EF_AMDGPU_MACH_R600_RS880;
75 case GK_RV670: return ELF::EF_AMDGPU_MACH_R600_RV670;
76 case GK_RV710: return ELF::EF_AMDGPU_MACH_R600_RV710;
77 case GK_RV730: return ELF::EF_AMDGPU_MACH_R600_RV730;
78 case GK_RV770: return ELF::EF_AMDGPU_MACH_R600_RV770;
79 case GK_CEDAR: return ELF::EF_AMDGPU_MACH_R600_CEDAR;
80 case GK_CYPRESS: return ELF::EF_AMDGPU_MACH_R600_CYPRESS;
81 case GK_JUNIPER: return ELF::EF_AMDGPU_MACH_R600_JUNIPER;
82 case GK_REDWOOD: return ELF::EF_AMDGPU_MACH_R600_REDWOOD;
83 case GK_SUMO: return ELF::EF_AMDGPU_MACH_R600_SUMO;
84 case GK_BARTS: return ELF::EF_AMDGPU_MACH_R600_BARTS;
85 case GK_CAICOS: return ELF::EF_AMDGPU_MACH_R600_CAICOS;
86 case GK_CAYMAN: return ELF::EF_AMDGPU_MACH_R600_CAYMAN;
87 case GK_TURKS: return ELF::EF_AMDGPU_MACH_R600_TURKS;
88 case GK_GFX600: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX600;
89 case GK_GFX601: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX601;
90 case GK_GFX602: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX602;
91 case GK_GFX700: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX700;
92 case GK_GFX701: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX701;
93 case GK_GFX702: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX702;
94 case GK_GFX703: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX703;
95 case GK_GFX704: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX704;
96 case GK_GFX705: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX705;
97 case GK_GFX801: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX801;
98 case GK_GFX802: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX802;
99 case GK_GFX803: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX803;
100 case GK_GFX805: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX805;
101 case GK_GFX810: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX810;
102 case GK_GFX900: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX900;
103 case GK_GFX902: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX902;
104 case GK_GFX904: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX904;
105 case GK_GFX906: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX906;
106 case GK_GFX908: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX908;
107 case GK_GFX909: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX909;
108 case GK_GFX90A: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX90A;
109 case GK_GFX90C: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX90C;
110 case GK_GFX942: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX942;
111 case GK_GFX950: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX950;
112 case GK_GFX1010: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1010;
113 case GK_GFX1011: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1011;
114 case GK_GFX1012: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1012;
115 case GK_GFX1013: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1013;
116 case GK_GFX1030: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1030;
117 case GK_GFX1031: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1031;
118 case GK_GFX1032: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1032;
119 case GK_GFX1033: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1033;
120 case GK_GFX1034: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1034;
121 case GK_GFX1035: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1035;
122 case GK_GFX1036: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1036;
123 case GK_GFX1100: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1100;
124 case GK_GFX1101: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1101;
125 case GK_GFX1102: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1102;
126 case GK_GFX1103: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1103;
127 case GK_GFX1150: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1150;
128 case GK_GFX1151: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1151;
129 case GK_GFX1152: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1152;
130 case GK_GFX1153: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1153;
131 case GK_GFX1154: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1154;
132 case GK_GFX1170: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1170;
133 case GK_GFX1171: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1171;
134 case GK_GFX1172: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1172;
135 case GK_GFX1200: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1200;
136 case GK_GFX1201: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1201;
137 case GK_GFX1250_STRICT: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1250_STRICT;
138 case GK_GFX1250: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1250;
139 case GK_GFX1251: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1251;
140 case GK_GFX1310: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1310;
141 case GK_GFX9_GENERIC: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC;
142 case GK_GFX9_4_GENERIC: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX9_4_GENERIC;
143 case GK_GFX10_1_GENERIC: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX10_1_GENERIC;
144 case GK_GFX10_3_GENERIC: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX10_3_GENERIC;
145 case GK_GFX11_GENERIC: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX11_GENERIC;
146 case GK_GFX11_7_GENERIC: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX11_7_GENERIC;
147 case GK_GFX12_GENERIC: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX12_GENERIC;
148 case GK_GFX12_5_GENERIC: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX12_5_GENERIC;
149 case GK_GFX13_GENERIC: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX13_GENERIC;
150 case GK_GENERIC:
151 case GK_GENERIC_HSA:
152 case GK_NONE: return ELF::EF_AMDGPU_MACH_NONE;
153 }
154 // clang-format on
155
156 llvm_unreachable("unknown GPU");
157}
158
159//===----------------------------------------------------------------------===//
160// AMDGPUTargetAsmStreamer
161//===----------------------------------------------------------------------===//
162
163AMDGPUTargetAsmStreamer::AMDGPUTargetAsmStreamer(MCStreamer &S,
164 formatted_raw_ostream &OS)
165 : AMDGPUTargetStreamer(S), OS(OS) {}
166
167// A hook for emitting stuff at the end.
168// We use it for emitting the accumulated PAL metadata as directives.
169// The PAL metadata is reset after it is emitted.
170void AMDGPUTargetAsmStreamer::finish() {
171 std::string S;
172 getPALMetadata()->toString(S);
173 OS << S;
174
175 // Reset the pal metadata so its data will not affect a compilation that
176 // reuses this object.
177 getPALMetadata()->reset();
178}
179
180void AMDGPUTargetAsmStreamer::EmitDirectiveAMDGCNTarget() {
181 OS << "\t.amdgcn_target \"" << *getTargetID() << "\"\n";
182}
183
184void AMDGPUTargetAsmStreamer::EmitDirectiveAMDHSACodeObjectVersion(
185 unsigned COV) {
186 AMDGPUTargetStreamer::EmitDirectiveAMDHSACodeObjectVersion(COV);
187 OS << "\t.amdhsa_code_object_version " << COV << '\n';
188}
189
190void AMDGPUTargetAsmStreamer::EmitAMDKernelCodeT(AMDGPUMCKernelCodeT &Header) {
191 auto FoldAndPrint = [&](const MCExpr *Expr, raw_ostream &OS,
192 const MCAsmInfo *MAI) {
193 printAMDGPUMCExpr(Expr: foldAMDGPUMCExpr(Expr, Ctx&: getContext()), OS, MAI);
194 };
195
196 OS << "\t.amd_kernel_code_t\n";
197 Header.EmitKernelCodeT(OS, Ctx&: getContext(), Helper: FoldAndPrint);
198 OS << "\t.end_amd_kernel_code_t\n";
199}
200
201void AMDGPUTargetAsmStreamer::EmitAMDGPUSymbolType(StringRef SymbolName,
202 unsigned Type) {
203 switch (Type) {
204 default:
205 llvm_unreachable("Invalid AMDGPU symbol type");
206 case ELF::STT_AMDGPU_HSA_KERNEL:
207 OS << "\t.amdgpu_hsa_kernel " << SymbolName << '\n';
208 break;
209 }
210}
211
212void AMDGPUTargetAsmStreamer::emitAMDGPULDS(MCSymbol *Symbol, unsigned Size,
213 Align Alignment) {
214 OS << "\t.amdgpu_lds " << Symbol->getName() << ", " << Size << ", "
215 << Alignment.value() << '\n';
216}
217
218void AMDGPUTargetAsmStreamer::EmitMCResourceInfo(
219 const MCSymbol *NumVGPR, const MCSymbol *NumAGPR,
220 const MCSymbol *NumExplicitSGPR, const MCSymbol *NumNamedBarrier,
221 const MCSymbol *PrivateSegmentSize, const MCSymbol *UsesVCC,
222 const MCSymbol *UsesFlatScratch, const MCSymbol *HasDynamicallySizedStack,
223 const MCSymbol *HasRecursion, const MCSymbol *HasIndirectCall) {
224#define PRINT_RES_INFO(ARG) \
225 OS << "\t.set "; \
226 ARG->print(OS, &getContext().getAsmInfo()); \
227 OS << ", "; \
228 getContext().getAsmInfo().printExpr(OS, *ARG->getVariableValue()); \
229 Streamer.addBlankLine();
230
231 PRINT_RES_INFO(NumVGPR);
232 PRINT_RES_INFO(NumAGPR);
233 PRINT_RES_INFO(NumExplicitSGPR);
234 PRINT_RES_INFO(NumNamedBarrier);
235 PRINT_RES_INFO(PrivateSegmentSize);
236 PRINT_RES_INFO(UsesVCC);
237 PRINT_RES_INFO(UsesFlatScratch);
238 PRINT_RES_INFO(HasDynamicallySizedStack);
239 PRINT_RES_INFO(HasRecursion);
240 PRINT_RES_INFO(HasIndirectCall);
241#undef PRINT_RES_INFO
242}
243
244void AMDGPUTargetAsmStreamer::EmitMCResourceMaximums(
245 const MCSymbol *MaxVGPR, const MCSymbol *MaxAGPR, const MCSymbol *MaxSGPR,
246 const MCSymbol *MaxNamedBarrier) {
247#define PRINT_RES_INFO(ARG) \
248 OS << "\t.set "; \
249 ARG->print(OS, &getContext().getAsmInfo()); \
250 OS << ", "; \
251 getContext().getAsmInfo().printExpr(OS, *ARG->getVariableValue()); \
252 Streamer.addBlankLine();
253
254 PRINT_RES_INFO(MaxVGPR);
255 PRINT_RES_INFO(MaxAGPR);
256 PRINT_RES_INFO(MaxSGPR);
257 PRINT_RES_INFO(MaxNamedBarrier);
258#undef PRINT_RES_INFO
259}
260
261bool AMDGPUTargetAsmStreamer::EmitISAVersion() {
262 OS << "\t.amd_amdgpu_isa \"" << getTargetID() << "\"\n";
263 return true;
264}
265
266bool AMDGPUTargetAsmStreamer::EmitHSAMetadata(msgpack::Document &HSAMetadataDoc,
267 bool Strict) {
268 HSAMD::V3::MetadataVerifier Verifier(Strict);
269 if (!Verifier.verify(HSAMetadataRoot&: HSAMetadataDoc.getRoot()))
270 return false;
271
272 std::string HSAMetadataString;
273 raw_string_ostream StrOS(HSAMetadataString);
274 HSAMetadataDoc.toYAML(OS&: StrOS);
275
276 OS << '\t' << HSAMD::V3::AssemblerDirectiveBegin << '\n';
277 OS << StrOS.str() << '\n';
278 OS << '\t' << HSAMD::V3::AssemblerDirectiveEnd << '\n';
279 return true;
280}
281
282bool AMDGPUTargetAsmStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) {
283 const uint32_t Encoded_s_code_end = 0xbf9f0000;
284 const uint32_t Encoded_s_nop = 0xbf800000;
285 uint32_t Encoded_pad = Encoded_s_code_end;
286
287 // Instruction cache line size in bytes.
288 const unsigned Log2CacheLineSize = AMDGPU::isGFX11Plus(STI) ? 7 : 6;
289 const unsigned CacheLineSize = 1u << Log2CacheLineSize;
290
291 // Extra padding amount in bytes to support prefetch mode 3.
292 unsigned FillSize = 3 * CacheLineSize;
293
294 if (AMDGPU::isGFX90A(STI)) {
295 Encoded_pad = Encoded_s_nop;
296 FillSize = 16 * CacheLineSize;
297 }
298
299 OS << "\t.p2alignl " << Log2CacheLineSize << ", " << Encoded_pad << '\n';
300 OS << "\t.fill " << (FillSize / 4) << ", 4, " << Encoded_pad << '\n';
301 return true;
302}
303
304void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(
305 const MCSubtargetInfo &STI, StringRef KernelName,
306 const MCKernelDescriptor &KD, const MCExpr *NextVGPR,
307 const MCExpr *NextSGPR, const MCExpr *ReserveVCC,
308 const MCExpr *ReserveFlatScr) {
309 IsaVersion IVersion = getIsaVersion(GPU: STI.getCPU());
310 const MCAsmInfo &MAI = getContext().getAsmInfo();
311
312 OS << "\t.amdhsa_kernel " << KernelName << '\n';
313
314 auto PrintField = [&](const MCExpr *Expr, uint32_t Shift, uint32_t Mask,
315 StringRef Directive) {
316 OS << "\t\t" << Directive << ' ';
317 const MCExpr *ShiftedAndMaskedExpr =
318 MCKernelDescriptor::bits_get(Src: Expr, Shift, Mask, Ctx&: getContext());
319 const MCExpr *New = foldAMDGPUMCExpr(Expr: ShiftedAndMaskedExpr, Ctx&: getContext());
320 printAMDGPUMCExpr(Expr: New, OS, MAI: &MAI);
321 OS << '\n';
322 };
323
324 auto EmitMCExpr = [&](const MCExpr *Value) {
325 const MCExpr *NewExpr = foldAMDGPUMCExpr(Expr: Value, Ctx&: getContext());
326 printAMDGPUMCExpr(Expr: NewExpr, OS, MAI: &MAI);
327 };
328
329 OS << "\t\t.amdhsa_group_segment_fixed_size ";
330 EmitMCExpr(KD.group_segment_fixed_size);
331 OS << '\n';
332
333 OS << "\t\t.amdhsa_private_segment_fixed_size ";
334 EmitMCExpr(KD.private_segment_fixed_size);
335 OS << '\n';
336
337 OS << "\t\t.amdhsa_kernarg_size ";
338 EmitMCExpr(KD.kernarg_size);
339 OS << '\n';
340
341 if (isGFX1250Plus(STI)) {
342 PrintField(KD.compute_pgm_rsrc2,
343 amdhsa::COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT_SHIFT,
344 amdhsa::COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT,
345 ".amdhsa_user_sgpr_count");
346 } else {
347 PrintField(KD.compute_pgm_rsrc2,
348 amdhsa::COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT_SHIFT,
349 amdhsa::COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT,
350 ".amdhsa_user_sgpr_count");
351 }
352
353 if (!hasArchitectedFlatScratch(STI))
354 PrintField(
355 KD.kernel_code_properties,
356 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER_SHIFT,
357 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
358 ".amdhsa_user_sgpr_private_segment_buffer");
359 PrintField(KD.kernel_code_properties,
360 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR_SHIFT,
361 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR,
362 ".amdhsa_user_sgpr_dispatch_ptr");
363 PrintField(KD.kernel_code_properties,
364 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR_SHIFT,
365 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR,
366 ".amdhsa_user_sgpr_queue_ptr");
367 PrintField(KD.kernel_code_properties,
368 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR_SHIFT,
369 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
370 ".amdhsa_user_sgpr_kernarg_segment_ptr");
371 PrintField(KD.kernel_code_properties,
372 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID_SHIFT,
373 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID,
374 ".amdhsa_user_sgpr_dispatch_id");
375 if (!hasArchitectedFlatScratch(STI))
376 PrintField(KD.kernel_code_properties,
377 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT_SHIFT,
378 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT,
379 ".amdhsa_user_sgpr_flat_scratch_init");
380 if (hasKernargPreload(STI)) {
381 PrintField(KD.kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_LENGTH_SHIFT,
382 amdhsa::KERNARG_PRELOAD_SPEC_LENGTH,
383 ".amdhsa_user_sgpr_kernarg_preload_length");
384 PrintField(KD.kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_OFFSET_SHIFT,
385 amdhsa::KERNARG_PRELOAD_SPEC_OFFSET,
386 ".amdhsa_user_sgpr_kernarg_preload_offset");
387 }
388 PrintField(
389 KD.kernel_code_properties,
390 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE_SHIFT,
391 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
392 ".amdhsa_user_sgpr_private_segment_size");
393 if (IVersion.Major >= 10)
394 PrintField(KD.kernel_code_properties,
395 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32_SHIFT,
396 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32,
397 ".amdhsa_wavefront_size32");
398 if (CodeObjectVersion >= AMDGPU::AMDHSA_COV5)
399 PrintField(KD.kernel_code_properties,
400 amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK_SHIFT,
401 amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK,
402 ".amdhsa_uses_dynamic_stack");
403 PrintField(KD.compute_pgm_rsrc2,
404 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT_SHIFT,
405 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT,
406 (hasArchitectedFlatScratch(STI)
407 ? ".amdhsa_enable_private_segment"
408 : ".amdhsa_system_sgpr_private_segment_wavefront_offset"));
409 PrintField(KD.compute_pgm_rsrc2,
410 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X_SHIFT,
411 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X,
412 ".amdhsa_system_sgpr_workgroup_id_x");
413 PrintField(KD.compute_pgm_rsrc2,
414 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y_SHIFT,
415 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y,
416 ".amdhsa_system_sgpr_workgroup_id_y");
417 PrintField(KD.compute_pgm_rsrc2,
418 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z_SHIFT,
419 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z,
420 ".amdhsa_system_sgpr_workgroup_id_z");
421 PrintField(KD.compute_pgm_rsrc2,
422 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO_SHIFT,
423 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO,
424 ".amdhsa_system_sgpr_workgroup_info");
425 PrintField(KD.compute_pgm_rsrc2,
426 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID_SHIFT,
427 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID,
428 ".amdhsa_system_vgpr_workitem_id");
429
430 // These directives are required.
431 OS << "\t\t.amdhsa_next_free_vgpr ";
432 EmitMCExpr(NextVGPR);
433 OS << '\n';
434
435 OS << "\t\t.amdhsa_next_free_sgpr ";
436 EmitMCExpr(NextSGPR);
437 OS << '\n';
438
439 if (AMDGPU::isGFX90A(STI)) {
440 // MCExpr equivalent of taking the (accum_offset + 1) * 4.
441 const MCExpr *accum_bits = MCKernelDescriptor::bits_get(
442 Src: KD.compute_pgm_rsrc3,
443 Shift: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
444 Mask: amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, Ctx&: getContext());
445 accum_bits = MCBinaryExpr::createAdd(
446 LHS: accum_bits, RHS: MCConstantExpr::create(Value: 1, Ctx&: getContext()), Ctx&: getContext());
447 accum_bits = MCBinaryExpr::createMul(
448 LHS: accum_bits, RHS: MCConstantExpr::create(Value: 4, Ctx&: getContext()), Ctx&: getContext());
449 OS << "\t\t.amdhsa_accum_offset ";
450 const MCExpr *New = foldAMDGPUMCExpr(Expr: accum_bits, Ctx&: getContext());
451 printAMDGPUMCExpr(Expr: New, OS, MAI: &MAI);
452 OS << '\n';
453 }
454
455 if (isGFX1250Plus(STI))
456 PrintField(KD.compute_pgm_rsrc3,
457 amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT,
458 amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
459 ".amdhsa_named_barrier_count");
460
461 OS << "\t\t.amdhsa_reserve_vcc ";
462 EmitMCExpr(ReserveVCC);
463 OS << '\n';
464
465 if (IVersion.Major >= 7 && !hasArchitectedFlatScratch(STI)) {
466 OS << "\t\t.amdhsa_reserve_flat_scratch ";
467 EmitMCExpr(ReserveFlatScr);
468 OS << '\n';
469 }
470
471 switch (CodeObjectVersion) {
472 default:
473 break;
474 case AMDGPU::AMDHSA_COV4:
475 case AMDGPU::AMDHSA_COV5:
476 if (STI.hasFeature(Feature: AMDGPU::FeatureSupportsXNACK)) {
477 bool XnackOn = getTargetID()->isXnackOnOrAny();
478 OS << "\t\t.amdhsa_reserve_xnack_mask " << XnackOn << '\n';
479 }
480 break;
481 }
482
483 PrintField(KD.compute_pgm_rsrc1,
484 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32_SHIFT,
485 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32,
486 ".amdhsa_float_round_mode_32");
487 PrintField(KD.compute_pgm_rsrc1,
488 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64_SHIFT,
489 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64,
490 ".amdhsa_float_round_mode_16_64");
491 PrintField(KD.compute_pgm_rsrc1,
492 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32_SHIFT,
493 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32,
494 ".amdhsa_float_denorm_mode_32");
495 PrintField(KD.compute_pgm_rsrc1,
496 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64_SHIFT,
497 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64,
498 ".amdhsa_float_denorm_mode_16_64");
499 if (STI.hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode)) {
500 PrintField(KD.compute_pgm_rsrc1,
501 amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP_SHIFT,
502 amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP,
503 ".amdhsa_dx10_clamp");
504 PrintField(KD.compute_pgm_rsrc1,
505 amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE_SHIFT,
506 amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE,
507 ".amdhsa_ieee_mode");
508 }
509 if (IVersion.Major >= 9) {
510 PrintField(KD.compute_pgm_rsrc1,
511 amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL_SHIFT,
512 amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL,
513 ".amdhsa_fp16_overflow");
514 }
515 if (AMDGPU::isGFX90A(STI))
516 PrintField(KD.compute_pgm_rsrc3,
517 amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT,
518 amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, ".amdhsa_tg_split");
519 if (AMDGPU::supportsWGP(STI))
520 PrintField(KD.compute_pgm_rsrc1,
521 amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE_SHIFT,
522 amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE,
523 ".amdhsa_workgroup_processor_mode");
524 if (IVersion.Major >= 10) {
525 PrintField(KD.compute_pgm_rsrc1,
526 amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED_SHIFT,
527 amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED,
528 ".amdhsa_memory_ordered");
529 PrintField(KD.compute_pgm_rsrc1,
530 amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS_SHIFT,
531 amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS,
532 ".amdhsa_forward_progress");
533 }
534 if (IVersion.Major >= 10 && IVersion.Major < 12) {
535 PrintField(KD.compute_pgm_rsrc3,
536 amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT_SHIFT,
537 amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT,
538 ".amdhsa_shared_vgpr_count");
539 }
540 if (IVersion.Major == 11) {
541 PrintField(KD.compute_pgm_rsrc3,
542 amdhsa::COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE_SHIFT,
543 amdhsa::COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE,
544 ".amdhsa_inst_pref_size");
545 }
546 if (IVersion.Major >= 12) {
547 PrintField(KD.compute_pgm_rsrc3,
548 amdhsa::COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE_SHIFT,
549 amdhsa::COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE,
550 ".amdhsa_inst_pref_size");
551 PrintField(KD.compute_pgm_rsrc1,
552 amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN_SHIFT,
553 amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN,
554 ".amdhsa_round_robin_scheduling");
555 }
556 PrintField(
557 KD.compute_pgm_rsrc2,
558 amdhsa::
559 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION_SHIFT,
560 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION,
561 ".amdhsa_exception_fp_ieee_invalid_op");
562 PrintField(
563 KD.compute_pgm_rsrc2,
564 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE_SHIFT,
565 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
566 ".amdhsa_exception_fp_denorm_src");
567 PrintField(
568 KD.compute_pgm_rsrc2,
569 amdhsa::
570 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO_SHIFT,
571 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO,
572 ".amdhsa_exception_fp_ieee_div_zero");
573 PrintField(
574 KD.compute_pgm_rsrc2,
575 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW_SHIFT,
576 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
577 ".amdhsa_exception_fp_ieee_overflow");
578 PrintField(
579 KD.compute_pgm_rsrc2,
580 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW_SHIFT,
581 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
582 ".amdhsa_exception_fp_ieee_underflow");
583 PrintField(
584 KD.compute_pgm_rsrc2,
585 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT_SHIFT,
586 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
587 ".amdhsa_exception_fp_ieee_inexact");
588 PrintField(
589 KD.compute_pgm_rsrc2,
590 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO_SHIFT,
591 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
592 ".amdhsa_exception_int_div_zero");
593
594 OS << "\t.end_amdhsa_kernel\n";
595}
596
597namespace {
598/// Callback type invoked by \c forEachInfoScope for each function scope in
599/// the canonical iteration order. The scope is emitted exactly once per
600/// unique \p Sym regardless of how many flat entries reference it.
601using InfoScopeEmitter = function_ref<void(
602 MCSymbol *Sym, const AMDGPU::FuncInfo *Info, ArrayRef<MCSymbol *> Uses,
603 ArrayRef<MCSymbol *> Calls, ArrayRef<StringRef> IndirectCallTypeIds,
604 ArrayRef<StringRef> TypeIds)>;
605
606/// Group the flat edge lists in \p Data by source function symbol and drive
607/// per-scope emission. A scope is opened for every function with attached
608/// info and for every function that appears only as an edge source; each
609/// scope is emitted exactly once. Both the asm and ELF streamers share this
610/// iteration logic and only differ in the per-scope emission callback.
611static void forEachInfoScope(const AMDGPU::InfoSectionData &Data,
612 InfoScopeEmitter Emit) {
613 DenseMap<MCSymbol *, SmallVector<MCSymbol *, 2>> FuncUses;
614 DenseMap<MCSymbol *, SmallVector<MCSymbol *, 4>> FuncCalls;
615 DenseMap<MCSymbol *, SmallVector<StringRef, 2>> FuncIndirectCalls;
616 DenseMap<MCSymbol *, SmallVector<StringRef, 1>> FuncTypeIds;
617 for (const auto &[Func, Res] : Data.Uses)
618 FuncUses[Func].push_back(Elt: Res);
619 for (const auto &[Src, Dst] : Data.Calls)
620 FuncCalls[Src].push_back(Elt: Dst);
621 for (const auto &[Func, TypeId] : Data.IndirectCalls)
622 FuncIndirectCalls[Func].push_back(Elt: TypeId);
623 for (const auto &[Sym, TypeId] : Data.TypeIds)
624 FuncTypeIds[Sym].push_back(Elt: TypeId);
625
626 DenseSet<MCSymbol *> Emitted;
627 auto EmitIfNew = [&](MCSymbol *Sym, const AMDGPU::FuncInfo *Info) {
628 if (!Emitted.insert(V: Sym).second)
629 return;
630 ArrayRef<MCSymbol *> Uses, Calls;
631 ArrayRef<StringRef> IndirectCallTypeIds, TypeIds;
632 if (auto It = FuncUses.find(Val: Sym); It != FuncUses.end())
633 Uses = It->second;
634 if (auto It = FuncCalls.find(Val: Sym); It != FuncCalls.end())
635 Calls = It->second;
636 if (auto It = FuncIndirectCalls.find(Val: Sym); It != FuncIndirectCalls.end())
637 IndirectCallTypeIds = It->second;
638 if (auto It = FuncTypeIds.find(Val: Sym); It != FuncTypeIds.end())
639 TypeIds = It->second;
640 Emit(Sym, Info, Uses, Calls, IndirectCallTypeIds, TypeIds);
641 };
642
643 for (const AMDGPU::FuncInfo &Func : Data.Funcs)
644 EmitIfNew(Func.Sym, &Func);
645 // Emit scopes for functions that only appear as edge sources (e.g. typeid
646 // tags on address-taken declarations, or callers of external functions).
647 for (const auto &[Sym, TypeId] : Data.TypeIds)
648 EmitIfNew(Sym, nullptr);
649 for (const auto &[Sym, Res] : Data.Uses)
650 EmitIfNew(Sym, nullptr);
651 for (const auto &[Sym, Dst] : Data.Calls)
652 EmitIfNew(Sym, nullptr);
653 for (const auto &[Sym, TypeId] : Data.IndirectCalls)
654 EmitIfNew(Sym, nullptr);
655}
656} // namespace
657
658void AMDGPUTargetAsmStreamer::emitAMDGPUInfo(
659 const AMDGPU::InfoSectionData &Data) {
660 forEachInfoScope(Data, Emit: [&](MCSymbol *Sym, const AMDGPU::FuncInfo *Info,
661 ArrayRef<MCSymbol *> Uses,
662 ArrayRef<MCSymbol *> Calls,
663 ArrayRef<StringRef> IndirectCallTypeIds,
664 ArrayRef<StringRef> TypeIds) {
665 OS << "\t.amdgpu_info " << Sym->getName() << '\n';
666 if (Info) {
667 AMDGPU::FuncInfoFlags Flags{};
668 if (Info->UsesVCC)
669 Flags |= AMDGPU::FuncInfoFlags::FUNC_USES_VCC;
670 if (Info->UsesFlatScratch)
671 Flags |= AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH;
672 if (Info->HasDynStack)
673 Flags |= AMDGPU::FuncInfoFlags::FUNC_HAS_DYN_STACK;
674 OS << "\t\t.amdgpu_flags " << llvm::to_underlying(E: Flags) << '\n';
675 OS << "\t\t.amdgpu_num_sgpr " << Info->NumSGPR << '\n';
676 OS << "\t\t.amdgpu_num_vgpr " << Info->NumArchVGPR << '\n';
677 if (Info->NumAccVGPR)
678 OS << "\t\t.amdgpu_num_agpr " << Info->NumAccVGPR << '\n';
679 OS << "\t\t.amdgpu_private_segment_size " << Info->PrivateSegmentSize
680 << '\n';
681 }
682 for (MCSymbol *Res : Uses)
683 OS << "\t\t.amdgpu_use " << Res->getName() << '\n';
684 for (MCSymbol *Dst : Calls)
685 OS << "\t\t.amdgpu_call " << Dst->getName() << '\n';
686 for (StringRef TypeId : IndirectCallTypeIds)
687 OS << "\t\t.amdgpu_indirect_call \"" << TypeId << "\"\n";
688 for (StringRef TypeId : TypeIds)
689 OS << "\t\t.amdgpu_typeid \"" << TypeId << "\"\n";
690 OS << "\t.end_amdgpu_info\n\n";
691 });
692}
693
694//===----------------------------------------------------------------------===//
695// AMDGPUTargetELFStreamer
696//===----------------------------------------------------------------------===//
697
698AMDGPUTargetELFStreamer::AMDGPUTargetELFStreamer(MCStreamer &S,
699 const MCSubtargetInfo &STI)
700 : AMDGPUTargetStreamer(S), STI(STI), Streamer(S) {}
701
702MCELFStreamer &AMDGPUTargetELFStreamer::getStreamer() {
703 return static_cast<MCELFStreamer &>(Streamer);
704}
705
706// A hook for emitting stuff at the end.
707// We use it for emitting the accumulated PAL metadata as a .note record.
708// The PAL metadata is reset after it is emitted.
709void AMDGPUTargetELFStreamer::finish() {
710 ELFObjectWriter &W = getStreamer().getWriter();
711 W.setELFHeaderEFlags(getEFlags());
712 W.setOverrideABIVersion(
713 getELFABIVersion(OS: STI.getTargetTriple(), CodeObjectVersion));
714
715 std::string Blob;
716 const char *Vendor = getPALMetadata()->getVendor();
717 unsigned Type = getPALMetadata()->getType();
718 getPALMetadata()->toBlob(Type, S&: Blob);
719 if (Blob.empty())
720 return;
721 EmitNote(Name: Vendor, DescSize: MCConstantExpr::create(Value: Blob.size(), Ctx&: getContext()), NoteType: Type,
722 EmitDesc: [&](MCELFStreamer &OS) { OS.emitBytes(Data: Blob); });
723
724 // Reset the pal metadata so its data will not affect a compilation that
725 // reuses this object.
726 getPALMetadata()->reset();
727}
728
729void AMDGPUTargetELFStreamer::EmitNote(
730 StringRef Name, const MCExpr *DescSZ, unsigned NoteType,
731 function_ref<void(MCELFStreamer &)> EmitDesc) {
732 auto &S = getStreamer();
733 auto &Context = S.getContext();
734
735 auto NameSZ = Name.size() + 1;
736
737 unsigned NoteFlags = 0;
738 // TODO Apparently, this is currently needed for OpenCL as mentioned in
739 // https://reviews.llvm.org/D74995
740 if (isHsaAbi(STI))
741 NoteFlags = ELF::SHF_ALLOC;
742
743 S.pushSection();
744 S.switchSection(
745 Section: Context.getELFSection(Section: ElfNote::SectionName, Type: ELF::SHT_NOTE, Flags: NoteFlags));
746 S.emitInt32(Value: NameSZ); // namesz
747 S.emitValue(Value: DescSZ, Size: 4); // descz
748 S.emitInt32(Value: NoteType); // type
749 S.emitBytes(Data: Name); // name
750 S.emitInt8(Value: 0); // null terminator
751 S.emitValueToAlignment(Alignment: Align(4), Fill: 0, FillLen: 1, MaxBytesToEmit: 0); // padding 0
752 EmitDesc(S); // desc
753 S.emitValueToAlignment(Alignment: Align(4), Fill: 0, FillLen: 1, MaxBytesToEmit: 0); // padding 0
754 S.popSection();
755}
756
757unsigned AMDGPUTargetELFStreamer::getEFlags() {
758 switch (STI.getTargetTriple().getArch()) {
759 default:
760 llvm_unreachable("Unsupported Arch");
761 case Triple::r600:
762 return getEFlagsR600();
763 case Triple::amdgpu:
764 return getEFlagsAMDGCN();
765 }
766}
767
768unsigned AMDGPUTargetELFStreamer::getEFlagsR600() {
769 assert(STI.getTargetTriple().getArch() == Triple::r600);
770
771 return getElfMach(GPU: STI.getCPU());
772}
773
774unsigned AMDGPUTargetELFStreamer::getEFlagsAMDGCN() {
775 assert(STI.getTargetTriple().isAMDGCN());
776
777 switch (STI.getTargetTriple().getOS()) {
778 default:
779 // TODO: Why are some tests have "mingw" listed as OS?
780 // llvm_unreachable("Unsupported OS");
781 case Triple::UnknownOS:
782 return getEFlagsUnknownOS();
783 case Triple::AMDHSA:
784 return getEFlagsAMDHSA();
785 case Triple::AMDPAL:
786 return getEFlagsAMDPAL();
787 case Triple::Mesa3D:
788 return getEFlagsMesa3D();
789 }
790}
791
792unsigned AMDGPUTargetELFStreamer::getEFlagsUnknownOS() {
793 // TODO: Why are some tests have "mingw" listed as OS?
794 // assert(STI.getTargetTriple().getOS() == Triple::UnknownOS);
795
796 return getEFlagsV3();
797}
798
799unsigned AMDGPUTargetELFStreamer::getEFlagsAMDHSA() {
800 assert(isHsaAbi(STI));
801
802 if (CodeObjectVersion >= 6)
803 return getEFlagsV6();
804 return getEFlagsV4();
805}
806
807unsigned AMDGPUTargetELFStreamer::getEFlagsAMDPAL() {
808 assert(STI.getTargetTriple().getOS() == Triple::AMDPAL);
809
810 return getEFlagsV3();
811}
812
813unsigned AMDGPUTargetELFStreamer::getEFlagsMesa3D() {
814 assert(STI.getTargetTriple().getOS() == Triple::Mesa3D);
815
816 return getEFlagsV3();
817}
818
819unsigned AMDGPUTargetELFStreamer::getEFlagsV3() {
820 unsigned EFlagsV3 = 0;
821
822 // mach.
823 EFlagsV3 |= getElfMach(GPU: STI.getCPU());
824
825 // xnack.
826 if (getTargetID()->isXnackOnOrAny())
827 EFlagsV3 |= ELF::EF_AMDGPU_FEATURE_XNACK_V3;
828 // sramecc.
829 if (getTargetID()->isSramEccOnOrAny())
830 EFlagsV3 |= ELF::EF_AMDGPU_FEATURE_SRAMECC_V3;
831
832 return EFlagsV3;
833}
834
835unsigned AMDGPUTargetELFStreamer::getEFlagsV4() {
836 unsigned EFlagsV4 = 0;
837
838 // mach.
839 EFlagsV4 |= getElfMach(GPU: STI.getCPU());
840
841 // xnack.
842 // Hardwired-on XNACK is implied by the processor, not an ELF mode selection.
843 AMDGPU::TargetIDSetting XnackSetting =
844 STI.hasFeature(Feature: AMDGPU::FeatureXNACKOnOffModes)
845 ? getTargetID()->getXnackSetting()
846 : AMDGPU::TargetIDSetting::Unsupported;
847 switch (XnackSetting) {
848 case AMDGPU::TargetIDSetting::Unsupported:
849 break;
850 case AMDGPU::TargetIDSetting::Any:
851 EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_XNACK_ANY_V4;
852 break;
853 case AMDGPU::TargetIDSetting::Off:
854 EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_XNACK_OFF_V4;
855 break;
856 case AMDGPU::TargetIDSetting::On:
857 EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_XNACK_ON_V4;
858 break;
859 }
860 // sramecc.
861 // Hardwired-on SRAMECC is implied by the processor, not an ELF mode.
862 AMDGPU::TargetIDSetting SramEccSetting =
863 STI.hasFeature(Feature: AMDGPU::FeatureSRAMECCOnOffModes)
864 ? getTargetID()->getSramEccSetting()
865 : AMDGPU::TargetIDSetting::Unsupported;
866 switch (SramEccSetting) {
867 case AMDGPU::TargetIDSetting::Unsupported:
868 break;
869 case AMDGPU::TargetIDSetting::Any:
870 EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_SRAMECC_ANY_V4;
871 break;
872 case AMDGPU::TargetIDSetting::Off:
873 EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_SRAMECC_OFF_V4;
874 break;
875 case AMDGPU::TargetIDSetting::On:
876 EFlagsV4 |= ELF::EF_AMDGPU_FEATURE_SRAMECC_ON_V4;
877 break;
878 }
879
880 return EFlagsV4;
881}
882
883unsigned AMDGPUTargetELFStreamer::getEFlagsV6() {
884 unsigned Flags = getEFlagsV4();
885
886 unsigned Version = ForceGenericVersion;
887 if (!Version) {
888 switch (parseArchAMDGCN(CPU: STI.getCPU())) {
889 case AMDGPU::GK_GFX9_GENERIC:
890 Version = GenericVersion::GFX9;
891 break;
892 case AMDGPU::GK_GFX9_4_GENERIC:
893 Version = GenericVersion::GFX9_4;
894 break;
895 case AMDGPU::GK_GFX10_1_GENERIC:
896 Version = GenericVersion::GFX10_1;
897 break;
898 case AMDGPU::GK_GFX10_3_GENERIC:
899 Version = GenericVersion::GFX10_3;
900 break;
901 case AMDGPU::GK_GFX11_GENERIC:
902 Version = GenericVersion::GFX11;
903 break;
904 case AMDGPU::GK_GFX11_7_GENERIC:
905 Version = GenericVersion::GFX11_7;
906 break;
907 case AMDGPU::GK_GFX12_GENERIC:
908 Version = GenericVersion::GFX12;
909 break;
910 case AMDGPU::GK_GFX12_5_GENERIC:
911 Version = GenericVersion::GFX12_5;
912 break;
913 case AMDGPU::GK_GFX13_GENERIC:
914 Version = GenericVersion::GFX13;
915 break;
916 default:
917 break;
918 }
919 }
920
921 // Versions start at 1.
922 if (Version) {
923 if (Version > ELF::EF_AMDGPU_GENERIC_VERSION_MAX)
924 report_fatal_error(reason: "Cannot encode generic code object version " +
925 Twine(Version) +
926 " - no ELF flag can represent this version!");
927 Flags |= (Version << ELF::EF_AMDGPU_GENERIC_VERSION_OFFSET);
928 }
929
930 return Flags;
931}
932
933void AMDGPUTargetELFStreamer::EmitDirectiveAMDGCNTarget() {}
934
935void AMDGPUTargetELFStreamer::EmitAMDKernelCodeT(AMDGPUMCKernelCodeT &Header) {
936 MCStreamer &OS = getStreamer();
937 OS.pushSection();
938 Header.EmitKernelCodeT(OS, Ctx&: getContext());
939 OS.popSection();
940}
941
942void AMDGPUTargetELFStreamer::EmitAMDGPUSymbolType(StringRef SymbolName,
943 unsigned Type) {
944 auto *Symbol = static_cast<MCSymbolELF *>(
945 getStreamer().getContext().getOrCreateSymbol(Name: SymbolName));
946 Symbol->setType(Type);
947}
948
949void AMDGPUTargetELFStreamer::emitAMDGPULDS(MCSymbol *Symbol, unsigned Size,
950 Align Alignment) {
951 auto *SymbolELF = static_cast<MCSymbolELF *>(Symbol);
952 SymbolELF->setType(ELF::STT_OBJECT);
953
954 if (!SymbolELF->isBindingSet())
955 SymbolELF->setBinding(ELF::STB_GLOBAL);
956
957 if (SymbolELF->declareCommon(Size, Alignment)) {
958 report_fatal_error(reason: "Symbol: " + Symbol->getName() +
959 " redeclared as different type");
960 }
961
962 SymbolELF->setIndex(ELF::SHN_AMDGPU_LDS);
963 SymbolELF->setSize(MCConstantExpr::create(Value: Size, Ctx&: getContext()));
964}
965
966bool AMDGPUTargetELFStreamer::EmitISAVersion() {
967 // Create two labels to mark the beginning and end of the desc field
968 // and a MCExpr to calculate the size of the desc field.
969 auto &Context = getContext();
970 auto *DescBegin = Context.createTempSymbol();
971 auto *DescEnd = Context.createTempSymbol();
972 auto *DescSZ = MCBinaryExpr::createSub(
973 LHS: MCSymbolRefExpr::create(Symbol: DescEnd, Ctx&: Context),
974 RHS: MCSymbolRefExpr::create(Symbol: DescBegin, Ctx&: Context), Ctx&: Context);
975
976 EmitNote(Name: ElfNote::NoteNameV2, DescSZ, NoteType: ELF::NT_AMD_HSA_ISA_NAME,
977 EmitDesc: [&](MCELFStreamer &OS) {
978 OS.emitLabel(Symbol: DescBegin);
979
980 SmallString<32> Str;
981 raw_svector_ostream StrOS(Str);
982 StrOS << *getTargetID();
983
984 OS.emitBytes(Data: StrOS.str());
985 OS.emitLabel(Symbol: DescEnd);
986 });
987 return true;
988}
989
990bool AMDGPUTargetELFStreamer::EmitHSAMetadata(msgpack::Document &HSAMetadataDoc,
991 bool Strict) {
992 HSAMD::V3::MetadataVerifier Verifier(Strict);
993 if (!Verifier.verify(HSAMetadataRoot&: HSAMetadataDoc.getRoot()))
994 return false;
995
996 std::string HSAMetadataString;
997 HSAMetadataDoc.writeToBlob(Blob&: HSAMetadataString);
998
999 // Create two labels to mark the beginning and end of the desc field
1000 // and a MCExpr to calculate the size of the desc field.
1001 auto &Context = getContext();
1002 auto *DescBegin = Context.createTempSymbol();
1003 auto *DescEnd = Context.createTempSymbol();
1004 auto *DescSZ = MCBinaryExpr::createSub(
1005 LHS: MCSymbolRefExpr::create(Symbol: DescEnd, Ctx&: Context),
1006 RHS: MCSymbolRefExpr::create(Symbol: DescBegin, Ctx&: Context), Ctx&: Context);
1007
1008 EmitNote(Name: ElfNote::NoteNameV3, DescSZ, NoteType: ELF::NT_AMDGPU_METADATA,
1009 EmitDesc: [&](MCELFStreamer &OS) {
1010 OS.emitLabel(Symbol: DescBegin);
1011 OS.emitBytes(Data: HSAMetadataString);
1012 OS.emitLabel(Symbol: DescEnd);
1013 });
1014 return true;
1015}
1016
1017bool AMDGPUTargetELFStreamer::EmitCodeEnd(const MCSubtargetInfo &STI) {
1018 const uint32_t Encoded_s_code_end = 0xbf9f0000;
1019 const uint32_t Encoded_s_nop = 0xbf800000;
1020 uint32_t Encoded_pad = Encoded_s_code_end;
1021
1022 // Instruction cache line size in bytes.
1023 const unsigned Log2CacheLineSize = AMDGPU::isGFX11Plus(STI) ? 7 : 6;
1024 const unsigned CacheLineSize = 1u << Log2CacheLineSize;
1025
1026 // Extra padding amount in bytes to support prefetch mode 3.
1027 unsigned FillSize = 3 * CacheLineSize;
1028
1029 if (AMDGPU::isGFX90A(STI)) {
1030 Encoded_pad = Encoded_s_nop;
1031 FillSize = 16 * CacheLineSize;
1032 }
1033
1034 MCStreamer &OS = getStreamer();
1035 OS.pushSection();
1036 OS.emitValueToAlignment(Alignment: Align(CacheLineSize), Fill: Encoded_pad, FillLen: 4);
1037 for (unsigned I = 0; I < FillSize; I += 4)
1038 OS.emitInt32(Value: Encoded_pad);
1039 OS.popSection();
1040 return true;
1041}
1042
1043void AMDGPUTargetELFStreamer::EmitAmdhsaKernelDescriptor(
1044 const MCSubtargetInfo &STI, StringRef KernelName,
1045 const MCKernelDescriptor &KernelDescriptor, const MCExpr *NextVGPR,
1046 const MCExpr *NextSGPR, const MCExpr *ReserveVCC,
1047 const MCExpr *ReserveFlatScr) {
1048 auto &Streamer = getStreamer();
1049 auto &Context = Streamer.getContext();
1050
1051 auto *KernelCodeSymbol =
1052 static_cast<MCSymbolELF *>(Context.getOrCreateSymbol(Name: Twine(KernelName)));
1053 auto *KernelDescriptorSymbol = static_cast<MCSymbolELF *>(
1054 Context.getOrCreateSymbol(Name: Twine(KernelName) + Twine(".kd")));
1055
1056 // Copy kernel descriptor symbol's binding, other and visibility from the
1057 // kernel code symbol.
1058 KernelDescriptorSymbol->setBinding(KernelCodeSymbol->getBinding());
1059 KernelDescriptorSymbol->setOther(KernelCodeSymbol->getOther());
1060 KernelDescriptorSymbol->setVisibility(KernelCodeSymbol->getVisibility());
1061 // Kernel descriptor symbol's type and size are fixed.
1062 KernelDescriptorSymbol->setType(ELF::STT_OBJECT);
1063 KernelDescriptorSymbol->setSize(
1064 MCConstantExpr::create(Value: sizeof(amdhsa::kernel_descriptor_t), Ctx&: Context));
1065
1066 // The visibility of the kernel code symbol must be protected or less to allow
1067 // static relocations from the kernel descriptor to be used.
1068 if (KernelCodeSymbol->getVisibility() == ELF::STV_DEFAULT)
1069 KernelCodeSymbol->setVisibility(ELF::STV_PROTECTED);
1070
1071 Streamer.emitLabel(Symbol: KernelDescriptorSymbol);
1072 Streamer.emitValue(
1073 Value: KernelDescriptor.group_segment_fixed_size,
1074 Size: sizeof(amdhsa::kernel_descriptor_t::group_segment_fixed_size));
1075 Streamer.emitValue(
1076 Value: KernelDescriptor.private_segment_fixed_size,
1077 Size: sizeof(amdhsa::kernel_descriptor_t::private_segment_fixed_size));
1078 Streamer.emitValue(Value: KernelDescriptor.kernarg_size,
1079 Size: sizeof(amdhsa::kernel_descriptor_t::kernarg_size));
1080
1081 for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved0); ++i)
1082 Streamer.emitInt8(Value: 0u);
1083
1084 // FIXME: Remove the use of VK_AMDGPU_REL64 in the expression below. The
1085 // expression being created is:
1086 // (start of kernel code) - (start of kernel descriptor)
1087 // It implies R_AMDGPU_REL64, but ends up being R_AMDGPU_ABS64.
1088 Streamer.emitValue(
1089 Value: MCBinaryExpr::createSub(
1090 LHS: MCSymbolRefExpr::create(Symbol: KernelCodeSymbol, specifier: AMDGPUMCExpr::S_REL64,
1091 Ctx&: Context),
1092 RHS: MCSymbolRefExpr::create(Symbol: KernelDescriptorSymbol, Ctx&: Context), Ctx&: Context),
1093 Size: sizeof(amdhsa::kernel_descriptor_t::kernel_code_entry_byte_offset));
1094 for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved1); ++i)
1095 Streamer.emitInt8(Value: 0u);
1096 Streamer.emitValue(Value: KernelDescriptor.compute_pgm_rsrc3,
1097 Size: sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc3));
1098 Streamer.emitValue(Value: KernelDescriptor.compute_pgm_rsrc1,
1099 Size: sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc1));
1100 Streamer.emitValue(Value: KernelDescriptor.compute_pgm_rsrc2,
1101 Size: sizeof(amdhsa::kernel_descriptor_t::compute_pgm_rsrc2));
1102 Streamer.emitValue(
1103 Value: KernelDescriptor.kernel_code_properties,
1104 Size: sizeof(amdhsa::kernel_descriptor_t::kernel_code_properties));
1105 Streamer.emitValue(Value: KernelDescriptor.kernarg_preload,
1106 Size: sizeof(amdhsa::kernel_descriptor_t::kernarg_preload));
1107 for (uint32_t i = 0; i < sizeof(amdhsa::kernel_descriptor_t::reserved3); ++i)
1108 Streamer.emitInt8(Value: 0u);
1109}
1110
1111void AMDGPUTargetELFStreamer::emitAMDGPUInfo(
1112 const AMDGPU::InfoSectionData &Data) {
1113 MCELFStreamer &S = getStreamer();
1114 MCContext &Context = S.getContext();
1115
1116 StringTableBuilder StrTab(StringTableBuilder::ELF);
1117 auto getOrAddString = [&](StringRef Str) -> uint32_t {
1118 if (Str.empty())
1119 return UINT32_MAX;
1120 return StrTab.add(S: Str);
1121 };
1122
1123 auto EmitU32Entry = [&](AMDGPU::InfoKind Kind, uint32_t Val) {
1124 S.emitInt8(Value: static_cast<uint8_t>(Kind));
1125 S.emitInt8(Value: 4);
1126 S.emitInt32(Value: Val);
1127 };
1128 auto EmitSymEntry = [&](AMDGPU::InfoKind Kind, MCSymbol *Sym) {
1129 S.emitInt8(Value: static_cast<uint8_t>(Kind));
1130 S.emitInt8(Value: 8);
1131 S.emitValue(Value: MCSymbolRefExpr::create(Symbol: Sym, Ctx&: Context), Size: 8);
1132 };
1133
1134 S.pushSection();
1135 MCSectionELF *InfoSec = Context.getELFSection(
1136 Section: ".amdgpu.info", Type: ELF::SHT_PROGBITS, Flags: ELF::SHF_EXCLUDE);
1137 S.switchSection(Section: InfoSec);
1138
1139 forEachInfoScope(Data, Emit: [&](MCSymbol *Sym, const AMDGPU::FuncInfo *Info,
1140 ArrayRef<MCSymbol *> Uses,
1141 ArrayRef<MCSymbol *> Calls,
1142 ArrayRef<StringRef> IndirectCallTypeIds,
1143 ArrayRef<StringRef> TypeIds) {
1144 EmitSymEntry(AMDGPU::InfoKind::INFO_FUNC, Sym);
1145
1146 if (Info) {
1147 AMDGPU::FuncInfoFlags Flags{};
1148 if (Info->UsesVCC)
1149 Flags |= AMDGPU::FuncInfoFlags::FUNC_USES_VCC;
1150 if (Info->UsesFlatScratch)
1151 Flags |= AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH;
1152 if (Info->HasDynStack)
1153 Flags |= AMDGPU::FuncInfoFlags::FUNC_HAS_DYN_STACK;
1154 EmitU32Entry(AMDGPU::InfoKind::INFO_FLAGS, llvm::to_underlying(E: Flags));
1155 EmitU32Entry(AMDGPU::InfoKind::INFO_NUM_SGPR, Info->NumSGPR);
1156 EmitU32Entry(AMDGPU::InfoKind::INFO_NUM_VGPR, Info->NumArchVGPR);
1157 // INFO_NUM_AGPR is only emitted when the function actually uses AGPRs,
1158 // since AGPRs are not available on all architectures.
1159 if (Info->NumAccVGPR)
1160 EmitU32Entry(AMDGPU::InfoKind::INFO_NUM_AGPR, Info->NumAccVGPR);
1161 EmitU32Entry(AMDGPU::InfoKind::INFO_PRIVATE_SEGMENT_SIZE,
1162 Info->PrivateSegmentSize);
1163 }
1164
1165 for (MCSymbol *Res : Uses)
1166 EmitSymEntry(AMDGPU::InfoKind::INFO_USE, Res);
1167 for (MCSymbol *Dst : Calls)
1168 EmitSymEntry(AMDGPU::InfoKind::INFO_CALL, Dst);
1169 for (StringRef TypeId : IndirectCallTypeIds) {
1170 EmitU32Entry(AMDGPU::InfoKind::INFO_INDIRECT_CALL,
1171 getOrAddString(TypeId));
1172 }
1173 for (StringRef TypeId : TypeIds)
1174 EmitU32Entry(AMDGPU::InfoKind::INFO_TYPEID, getOrAddString(TypeId));
1175 });
1176
1177 if (!StrTab.empty()) {
1178 StrTab.finalizeInOrder();
1179 MCSectionELF *Sec = Context.getELFSection(Section: ".amdgpu.strtab", Type: ELF::SHT_STRTAB,
1180 Flags: ELF::SHF_EXCLUDE);
1181 S.switchSection(Section: Sec);
1182 SmallString<128> Buf;
1183 raw_svector_ostream OS(Buf);
1184 StrTab.write(OS);
1185 S.emitBytes(Data: Buf);
1186 }
1187
1188 S.popSection();
1189}
1190