1//===--- AMDGPUHSAMetadataStreamer.cpp --------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// AMDGPU HSA Metadata Streamer.
11///
12//
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPUHSAMetadataStreamer.h"
16#include "AMDGPU.h"
17#include "GCNSubtarget.h"
18#include "MCTargetDesc/AMDGPUTargetStreamer.h"
19#include "SIMachineFunctionInfo.h"
20#include "SIProgramInfo.h"
21#include "llvm/IR/Module.h"
22#include "llvm/MC/MCContext.h"
23#include "llvm/MC/MCExpr.h"
24#include "llvm/Target/TargetLoweringObjectFile.h"
25
26using namespace llvm;
27
28static std::pair<Type *, Align> getArgumentTypeAlign(const Argument &Arg,
29 const DataLayout &DL) {
30 Type *Ty = Arg.getType();
31 MaybeAlign ArgAlign;
32 if (Arg.hasByRefAttr()) {
33 Ty = Arg.getParamByRefType();
34 ArgAlign = Arg.getParamAlign();
35 }
36
37 if (!ArgAlign)
38 ArgAlign = DL.getABITypeAlign(Ty);
39
40 return std::pair(Ty, *ArgAlign);
41}
42
43/// Find the mangled symbol name for the runtime handle for \p EnqueuedBlock
44static std::string getEnqueuedBlockSymbolName(const AMDGPUTargetMachine &TM,
45 const Function &EnqueuedBlock) {
46 const MDNode *Associated =
47 EnqueuedBlock.getMetadata(KindID: LLVMContext::MD_associated);
48 if (!Associated)
49 return "";
50
51 auto *VM = cast<ValueAsMetadata>(Val: Associated->getOperand(I: 0));
52 auto *RuntimeHandle =
53 dyn_cast<GlobalVariable>(Val: VM->getValue()->stripPointerCasts());
54 if (!RuntimeHandle ||
55 RuntimeHandle->getSection() != ".amdgpu.kernel.runtime.handle")
56 return "";
57
58 SmallString<128> Name;
59 TM.getNameWithPrefix(Name, GV: RuntimeHandle,
60 Mang&: TM.getObjFileLowering()->getMangler());
61 return Name.str().str();
62}
63
64namespace llvm {
65
66namespace AMDGPU::HSAMD {
67
68//===----------------------------------------------------------------------===//
69// HSAMetadataStreamerV4
70//===----------------------------------------------------------------------===//
71
72void MetadataStreamerMsgPackV4::dump(StringRef HSAMetadataString) const {
73 errs() << "AMDGPU HSA Metadata:\n" << HSAMetadataString << '\n';
74}
75
76void MetadataStreamerMsgPackV4::verify(StringRef HSAMetadataString) const {
77 errs() << "AMDGPU HSA Metadata Parser Test: ";
78
79 msgpack::Document FromHSAMetadataString;
80
81 if (!FromHSAMetadataString.fromYAML(S: HSAMetadataString)) {
82 errs() << "FAIL\n";
83 return;
84 }
85
86 std::string ToHSAMetadataString;
87 raw_string_ostream StrOS(ToHSAMetadataString);
88 FromHSAMetadataString.toYAML(OS&: StrOS);
89
90 errs() << (HSAMetadataString == StrOS.str() ? "PASS" : "FAIL") << '\n';
91 if (HSAMetadataString != ToHSAMetadataString) {
92 errs() << "Original input: " << HSAMetadataString << '\n'
93 << "Produced output: " << StrOS.str() << '\n';
94 }
95}
96
97std::optional<StringRef>
98MetadataStreamerMsgPackV4::getAccessQualifier(StringRef AccQual) const {
99 return StringSwitch<std::optional<StringRef>>(AccQual)
100 .Case(S: "read_only", Value: StringRef("read_only"))
101 .Case(S: "write_only", Value: StringRef("write_only"))
102 .Case(S: "read_write", Value: StringRef("read_write"))
103 .Default(Value: std::nullopt);
104}
105
106std::optional<StringRef> MetadataStreamerMsgPackV4::getAddressSpaceQualifier(
107 unsigned AddressSpace) const {
108 switch (AddressSpace) {
109 case AMDGPUAS::PRIVATE_ADDRESS:
110 return StringRef("private");
111 case AMDGPUAS::GLOBAL_ADDRESS:
112 return StringRef("global");
113 case AMDGPUAS::CONSTANT_ADDRESS:
114 return StringRef("constant");
115 case AMDGPUAS::LOCAL_ADDRESS:
116 return StringRef("local");
117 case AMDGPUAS::FLAT_ADDRESS:
118 return StringRef("generic");
119 case AMDGPUAS::REGION_ADDRESS:
120 return StringRef("region");
121 default:
122 return std::nullopt;
123 }
124}
125
126StringRef
127MetadataStreamerMsgPackV4::getValueKind(Type *Ty, StringRef TypeQual,
128 StringRef BaseTypeName) const {
129 if (TypeQual.contains(Other: "pipe"))
130 return "pipe";
131
132 return StringSwitch<StringRef>(BaseTypeName)
133 .Case(S: "image1d_t", Value: "image")
134 .Case(S: "image1d_array_t", Value: "image")
135 .Case(S: "image1d_buffer_t", Value: "image")
136 .Case(S: "image2d_t", Value: "image")
137 .Case(S: "image2d_array_t", Value: "image")
138 .Case(S: "image2d_array_depth_t", Value: "image")
139 .Case(S: "image2d_array_msaa_t", Value: "image")
140 .Case(S: "image2d_array_msaa_depth_t", Value: "image")
141 .Case(S: "image2d_depth_t", Value: "image")
142 .Case(S: "image2d_msaa_t", Value: "image")
143 .Case(S: "image2d_msaa_depth_t", Value: "image")
144 .Case(S: "image3d_t", Value: "image")
145 .Case(S: "sampler_t", Value: "sampler")
146 .Case(S: "queue_t", Value: "queue")
147 .Default(Value: isa<PointerType>(Val: Ty)
148 ? (Ty->getPointerAddressSpace() == AMDGPUAS::LOCAL_ADDRESS
149 ? "dynamic_shared_pointer"
150 : "global_buffer")
151 : "by_value");
152}
153
154std::string MetadataStreamerMsgPackV4::getTypeName(Type *Ty,
155 bool Signed) const {
156 switch (Ty->getTypeID()) {
157 case Type::IntegerTyID: {
158 if (!Signed)
159 return (Twine('u') + getTypeName(Ty, Signed: true)).str();
160
161 auto BitWidth = Ty->getIntegerBitWidth();
162 switch (BitWidth) {
163 case 8:
164 return "char";
165 case 16:
166 return "short";
167 case 32:
168 return "int";
169 case 64:
170 return "long";
171 default:
172 return (Twine('i') + Twine(BitWidth)).str();
173 }
174 }
175 case Type::HalfTyID:
176 return "half";
177 case Type::FloatTyID:
178 return "float";
179 case Type::DoubleTyID:
180 return "double";
181 case Type::FixedVectorTyID: {
182 auto *VecTy = cast<FixedVectorType>(Val: Ty);
183 auto *ElTy = VecTy->getElementType();
184 auto NumElements = VecTy->getNumElements();
185 return (Twine(getTypeName(Ty: ElTy, Signed)) + Twine(NumElements)).str();
186 }
187 default:
188 return "unknown";
189 }
190}
191
192msgpack::ArrayDocNode
193MetadataStreamerMsgPackV4::getWorkGroupDimensions(MDNode *Node) const {
194 auto Dims = HSAMetadataDoc->getArrayNode();
195 if (Node->getNumOperands() != 3)
196 return Dims;
197
198 for (auto &Op : Node->operands())
199 Dims.push_back(N: Dims.getDocument()->getNode(
200 V: mdconst::extract<ConstantInt>(MD: Op)->getZExtValue()));
201 return Dims;
202}
203
204void MetadataStreamerMsgPackV4::emitVersion() {
205 auto Version = HSAMetadataDoc->getArrayNode();
206 Version.push_back(N: Version.getDocument()->getNode(V: VersionMajorV4));
207 Version.push_back(N: Version.getDocument()->getNode(V: VersionMinorV4));
208 getRootMetadata(Key: "amdhsa.version") = Version;
209}
210
211void MetadataStreamerMsgPackV4::emitTargetID(const TargetID &TargetID) {
212 getRootMetadata(Key: "amdhsa.target") =
213 HSAMetadataDoc->getNode(V: TargetID.toString(), /*Copy=*/true);
214}
215
216void MetadataStreamerMsgPackV4::emitPrintf(const Module &Mod) {
217 auto *Node = Mod.getNamedMetadata(Name: "llvm.printf.fmts");
218 if (!Node)
219 return;
220
221 auto Printf = HSAMetadataDoc->getArrayNode();
222 for (auto *Op : Node->operands())
223 if (Op->getNumOperands())
224 Printf.push_back(N: Printf.getDocument()->getNode(
225 V: cast<MDString>(Val: Op->getOperand(I: 0))->getString(), /*Copy=*/true));
226 getRootMetadata(Key: "amdhsa.printf") = Printf;
227}
228
229void MetadataStreamerMsgPackV4::emitKernelLanguage(const Function &Func,
230 msgpack::MapDocNode Kern) {
231 // TODO: What about other languages?
232 auto *Node = Func.getParent()->getNamedMetadata(Name: "opencl.ocl.version");
233 if (!Node || !Node->getNumOperands())
234 return;
235 auto *Op0 = Node->getOperand(i: 0);
236 if (Op0->getNumOperands() <= 1)
237 return;
238
239 Kern[".language"] = Kern.getDocument()->getNode(V: "OpenCL C");
240 auto LanguageVersion = Kern.getDocument()->getArrayNode();
241 LanguageVersion.push_back(N: Kern.getDocument()->getNode(
242 V: mdconst::extract<ConstantInt>(MD: Op0->getOperand(I: 0))->getZExtValue()));
243 LanguageVersion.push_back(N: Kern.getDocument()->getNode(
244 V: mdconst::extract<ConstantInt>(MD: Op0->getOperand(I: 1))->getZExtValue()));
245 Kern[".language_version"] = LanguageVersion;
246}
247
248void MetadataStreamerMsgPackV4::emitKernelAttrs(const AMDGPUTargetMachine &TM,
249 const MachineFunction &MF,
250 msgpack::MapDocNode Kern) {
251 const Function &Func = MF.getFunction();
252 if (auto *Node = Func.getMetadata(Kind: "reqd_work_group_size"))
253 Kern[".reqd_workgroup_size"] = getWorkGroupDimensions(Node);
254 if (auto *Node = Func.getMetadata(Kind: "work_group_size_hint"))
255 Kern[".workgroup_size_hint"] = getWorkGroupDimensions(Node);
256 if (auto *Node = Func.getMetadata(Kind: "vec_type_hint")) {
257 Kern[".vec_type_hint"] = Kern.getDocument()->getNode(
258 V: getTypeName(
259 Ty: cast<ValueAsMetadata>(Val: Node->getOperand(I: 0))->getType(),
260 Signed: mdconst::extract<ConstantInt>(MD: Node->getOperand(I: 1))->getZExtValue()),
261 /*Copy=*/true);
262 }
263
264 std::string HandleName = getEnqueuedBlockSymbolName(TM, EnqueuedBlock: Func);
265 if (!HandleName.empty()) {
266 Kern[".device_enqueue_symbol"] =
267 Kern.getDocument()->getNode(V: std::move(HandleName), /*Copy=*/true);
268 }
269
270 if (Func.hasFnAttribute(Kind: "device-init"))
271 Kern[".kind"] = Kern.getDocument()->getNode(V: "init");
272 else if (Func.hasFnAttribute(Kind: "device-fini"))
273 Kern[".kind"] = Kern.getDocument()->getNode(V: "fini");
274}
275
276void MetadataStreamerMsgPackV4::emitKernelArgs(const MachineFunction &MF,
277 msgpack::MapDocNode Kern) {
278 auto &Func = MF.getFunction();
279 uint64_t Offset = 0;
280 auto Args = HSAMetadataDoc->getArrayNode();
281 for (auto &Arg : Func.args()) {
282 if (Arg.hasAttribute(Kind: "amdgpu-hidden-argument"))
283 continue;
284
285 emitKernelArg(Arg, Offset, Args);
286 }
287
288 emitHiddenKernelArgs(MF, Offset, Args);
289
290 Kern[".args"] = Args;
291}
292
293void MetadataStreamerMsgPackV4::emitKernelArg(const Argument &Arg,
294 uint64_t &Offset,
295 msgpack::ArrayDocNode Args) {
296 const auto *Func = Arg.getParent();
297 auto ArgNo = Arg.getArgNo();
298 const MDNode *Node;
299
300 StringRef Name;
301 Node = Func->getMetadata(Kind: "kernel_arg_name");
302 if (Node && ArgNo < Node->getNumOperands())
303 Name = cast<MDString>(Val: Node->getOperand(I: ArgNo))->getString();
304 else if (Arg.hasName())
305 Name = Arg.getName();
306
307 StringRef TypeName;
308 Node = Func->getMetadata(Kind: "kernel_arg_type");
309 if (Node && ArgNo < Node->getNumOperands())
310 TypeName = cast<MDString>(Val: Node->getOperand(I: ArgNo))->getString();
311
312 StringRef BaseTypeName;
313 Node = Func->getMetadata(Kind: "kernel_arg_base_type");
314 if (Node && ArgNo < Node->getNumOperands())
315 BaseTypeName = cast<MDString>(Val: Node->getOperand(I: ArgNo))->getString();
316
317 StringRef ActAccQual;
318 // Do we really need NoAlias check here?
319 if (Arg.getType()->isPointerTy() && Arg.hasNoAliasAttr()) {
320 if (Arg.onlyReadsMemory())
321 ActAccQual = "read_only";
322 else if (Arg.hasAttribute(Kind: Attribute::WriteOnly))
323 ActAccQual = "write_only";
324 }
325
326 StringRef AccQual;
327 Node = Func->getMetadata(Kind: "kernel_arg_access_qual");
328 if (Node && ArgNo < Node->getNumOperands())
329 AccQual = cast<MDString>(Val: Node->getOperand(I: ArgNo))->getString();
330
331 StringRef TypeQual;
332 Node = Func->getMetadata(Kind: "kernel_arg_type_qual");
333 if (Node && ArgNo < Node->getNumOperands())
334 TypeQual = cast<MDString>(Val: Node->getOperand(I: ArgNo))->getString();
335
336 const DataLayout &DL = Func->getDataLayout();
337
338 MaybeAlign PointeeAlign;
339 Type *Ty = Arg.hasByRefAttr() ? Arg.getParamByRefType() : Arg.getType();
340
341 // FIXME: Need to distinguish in memory alignment from pointer alignment.
342 if (auto *PtrTy = dyn_cast<PointerType>(Val: Ty)) {
343 if (PtrTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS)
344 PointeeAlign = Arg.getParamAlign().valueOrOne();
345 }
346
347 // There's no distinction between byval aggregates and raw aggregates.
348 Type *ArgTy;
349 Align ArgAlign;
350 std::tie(args&: ArgTy, args&: ArgAlign) = getArgumentTypeAlign(Arg, DL);
351
352 emitKernelArg(DL, Ty: ArgTy, Alignment: ArgAlign,
353 ValueKind: getValueKind(Ty: ArgTy, TypeQual, BaseTypeName), Offset, Args,
354 PointeeAlign, Name, TypeName, BaseTypeName, ActAccQual,
355 AccQual, TypeQual);
356}
357
358void MetadataStreamerMsgPackV4::emitKernelArg(
359 const DataLayout &DL, Type *Ty, Align Alignment, StringRef ValueKind,
360 uint64_t &Offset, msgpack::ArrayDocNode Args, MaybeAlign PointeeAlign,
361 StringRef Name, StringRef TypeName, StringRef BaseTypeName,
362 StringRef ActAccQual, StringRef AccQual, StringRef TypeQual) {
363 auto Arg = Args.getDocument()->getMapNode();
364
365 if (!Name.empty())
366 Arg[".name"] = Arg.getDocument()->getNode(V: Name, /*Copy=*/true);
367 if (!TypeName.empty())
368 Arg[".type_name"] = Arg.getDocument()->getNode(V: TypeName, /*Copy=*/true);
369 auto Size = DL.getTypeAllocSize(Ty);
370 Arg[".size"] = Arg.getDocument()->getNode(V: Size);
371 Offset = alignTo(Size: Offset, A: Alignment);
372 Arg[".offset"] = Arg.getDocument()->getNode(V: Offset);
373 Offset += Size;
374 Arg[".value_kind"] = Arg.getDocument()->getNode(V: ValueKind, /*Copy=*/true);
375 if (PointeeAlign)
376 Arg[".pointee_align"] = Arg.getDocument()->getNode(V: PointeeAlign->value());
377
378 if (auto *PtrTy = dyn_cast<PointerType>(Val: Ty))
379 if (auto Qualifier = getAddressSpaceQualifier(AddressSpace: PtrTy->getAddressSpace()))
380 // Limiting address space to emit only for a certain ValueKind.
381 if (ValueKind == "global_buffer" || ValueKind == "dynamic_shared_pointer")
382 Arg[".address_space"] = Arg.getDocument()->getNode(V: *Qualifier,
383 /*Copy=*/true);
384
385 if (auto AQ = getAccessQualifier(AccQual))
386 Arg[".access"] = Arg.getDocument()->getNode(V: *AQ, /*Copy=*/true);
387
388 if (auto AAQ = getAccessQualifier(AccQual: ActAccQual))
389 Arg[".actual_access"] = Arg.getDocument()->getNode(V: *AAQ, /*Copy=*/true);
390
391 SmallVector<StringRef, 1> SplitTypeQuals;
392 TypeQual.split(A&: SplitTypeQuals, Separator: " ", MaxSplit: -1, KeepEmpty: false);
393 for (StringRef Key : SplitTypeQuals) {
394 if (Key == "const")
395 Arg[".is_const"] = Arg.getDocument()->getNode(V: true);
396 else if (Key == "restrict")
397 Arg[".is_restrict"] = Arg.getDocument()->getNode(V: true);
398 else if (Key == "volatile")
399 Arg[".is_volatile"] = Arg.getDocument()->getNode(V: true);
400 else if (Key == "pipe")
401 Arg[".is_pipe"] = Arg.getDocument()->getNode(V: true);
402 }
403
404 Args.push_back(N: Arg);
405}
406
407void MetadataStreamerMsgPackV4::emitHiddenKernelArgs(
408 const MachineFunction &MF, uint64_t &Offset, msgpack::ArrayDocNode Args) {
409 auto &Func = MF.getFunction();
410 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
411
412 unsigned HiddenArgNumBytes = ST.getImplicitArgNumBytes(F: Func);
413 if (!HiddenArgNumBytes)
414 return;
415
416 const Module *M = Func.getParent();
417 auto &DL = M->getDataLayout();
418 auto *Int64Ty = Type::getInt64Ty(C&: Func.getContext());
419
420 Offset = alignTo(Size: Offset, A: ST.getAlignmentForImplicitArgPtr());
421
422 if (HiddenArgNumBytes >= 8)
423 emitKernelArg(DL, Ty: Int64Ty, Alignment: Align(8), ValueKind: "hidden_global_offset_x", Offset,
424 Args);
425 if (HiddenArgNumBytes >= 16)
426 emitKernelArg(DL, Ty: Int64Ty, Alignment: Align(8), ValueKind: "hidden_global_offset_y", Offset,
427 Args);
428 if (HiddenArgNumBytes >= 24)
429 emitKernelArg(DL, Ty: Int64Ty, Alignment: Align(8), ValueKind: "hidden_global_offset_z", Offset,
430 Args);
431
432 auto *Int8PtrTy =
433 PointerType::get(C&: Func.getContext(), AddressSpace: AMDGPUAS::GLOBAL_ADDRESS);
434
435 if (HiddenArgNumBytes >= 32) {
436 // We forbid the use of features requiring hostcall when compiling OpenCL
437 // before code object V5, which makes the mutual exclusion between the
438 // "printf buffer" and "hostcall buffer" here sound.
439 if (M->getNamedMetadata(Name: "llvm.printf.fmts"))
440 emitKernelArg(DL, Ty: Int8PtrTy, Alignment: Align(8), ValueKind: "hidden_printf_buffer", Offset,
441 Args);
442 else if (!Func.hasFnAttribute(Kind: "amdgpu-no-hostcall-ptr"))
443 emitKernelArg(DL, Ty: Int8PtrTy, Alignment: Align(8), ValueKind: "hidden_hostcall_buffer", Offset,
444 Args);
445 else
446 emitKernelArg(DL, Ty: Int8PtrTy, Alignment: Align(8), ValueKind: "hidden_none", Offset, Args);
447 }
448
449 // Emit "default queue" and "completion action" arguments if enqueue kernel is
450 // used, otherwise emit dummy "none" arguments.
451 if (HiddenArgNumBytes >= 40) {
452 if (!Func.hasFnAttribute(Kind: "amdgpu-no-default-queue")) {
453 emitKernelArg(DL, Ty: Int8PtrTy, Alignment: Align(8), ValueKind: "hidden_default_queue", Offset,
454 Args);
455 } else {
456 emitKernelArg(DL, Ty: Int8PtrTy, Alignment: Align(8), ValueKind: "hidden_none", Offset, Args);
457 }
458 }
459
460 if (HiddenArgNumBytes >= 48) {
461 if (!Func.hasFnAttribute(Kind: "amdgpu-no-completion-action")) {
462 emitKernelArg(DL, Ty: Int8PtrTy, Alignment: Align(8), ValueKind: "hidden_completion_action", Offset,
463 Args);
464 } else {
465 emitKernelArg(DL, Ty: Int8PtrTy, Alignment: Align(8), ValueKind: "hidden_none", Offset, Args);
466 }
467 }
468
469 // Emit the pointer argument for multi-grid object.
470 if (HiddenArgNumBytes >= 56) {
471 if (!Func.hasFnAttribute(Kind: "amdgpu-no-multigrid-sync-arg")) {
472 emitKernelArg(DL, Ty: Int8PtrTy, Alignment: Align(8), ValueKind: "hidden_multigrid_sync_arg", Offset,
473 Args);
474 } else {
475 emitKernelArg(DL, Ty: Int8PtrTy, Alignment: Align(8), ValueKind: "hidden_none", Offset, Args);
476 }
477 }
478}
479
480msgpack::MapDocNode
481MetadataStreamerMsgPackV4::getHSAKernelProps(const MachineFunction &MF,
482 const SIProgramInfo &ProgramInfo,
483 unsigned CodeObjectVersion) const {
484 const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
485 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
486 const Function &F = MF.getFunction();
487
488 auto Kern = HSAMetadataDoc->getMapNode();
489
490 Align MaxKernArgAlign;
491 Kern[".kernarg_segment_size"] = Kern.getDocument()->getNode(
492 V: STM.getKernArgSegmentSize(F, MaxAlign&: MaxKernArgAlign));
493 Kern[".group_segment_fixed_size"] =
494 Kern.getDocument()->getNode(V: ProgramInfo.LDSSize);
495 DelayedExprs->assignDocNode(DN&: Kern[".private_segment_fixed_size"],
496 Type: msgpack::Type::UInt, ExprValue: ProgramInfo.ScratchSize);
497 if (CodeObjectVersion >= AMDGPU::AMDHSA_COV5) {
498 DelayedExprs->assignDocNode(DN&: Kern[".uses_dynamic_stack"],
499 Type: msgpack::Type::Boolean,
500 ExprValue: ProgramInfo.DynamicCallStack);
501 }
502
503 if (CodeObjectVersion >= AMDGPU::AMDHSA_COV5 && STM.hasSupportsWGP())
504 Kern[".workgroup_processor_mode"] =
505 Kern.getDocument()->getNode(V: ProgramInfo.WgpMode);
506
507 // FIXME: The metadata treats the minimum as 16?
508 Kern[".kernarg_segment_align"] =
509 Kern.getDocument()->getNode(V: std::max(a: Align(4), b: MaxKernArgAlign).value());
510 Kern[".wavefront_size"] =
511 Kern.getDocument()->getNode(V: STM.getWavefrontSize());
512 DelayedExprs->assignDocNode(DN&: Kern[".sgpr_count"], Type: msgpack::Type::UInt,
513 ExprValue: ProgramInfo.NumSGPR);
514 DelayedExprs->assignDocNode(DN&: Kern[".vgpr_count"], Type: msgpack::Type::UInt,
515 ExprValue: ProgramInfo.NumVGPR);
516
517 // Only add AGPR count to metadata for supported devices
518 if (STM.hasMAIInsts()) {
519 DelayedExprs->assignDocNode(DN&: Kern[".agpr_count"], Type: msgpack::Type::UInt,
520 ExprValue: ProgramInfo.NumAccVGPR);
521 }
522
523 Kern[".max_flat_workgroup_size"] =
524 Kern.getDocument()->getNode(V: MFI.getMaxFlatWorkGroupSize());
525
526 uint32_t NumWGY = MFI.getMaxNumWorkGroupsY();
527 uint32_t NumWGZ = MFI.getMaxNumWorkGroupsZ();
528 uint32_t NumWGX = MFI.getMaxNumWorkGroupsX();
529
530 // TODO: Should consider 0 invalid and reject in IR verifier.
531 if (NumWGX != std::numeric_limits<uint32_t>::max() && NumWGX != 0)
532 Kern[".max_num_workgroups_x"] = Kern.getDocument()->getNode(V: NumWGX);
533
534 if (NumWGY != std::numeric_limits<uint32_t>::max() && NumWGY != 0)
535 Kern[".max_num_workgroups_y"] = Kern.getDocument()->getNode(V: NumWGY);
536
537 if (NumWGZ != std::numeric_limits<uint32_t>::max() && NumWGZ != 0)
538 Kern[".max_num_workgroups_z"] = Kern.getDocument()->getNode(V: NumWGZ);
539
540 Kern[".sgpr_spill_count"] =
541 Kern.getDocument()->getNode(V: MFI.getNumSpilledSGPRs());
542 Kern[".vgpr_spill_count"] =
543 Kern.getDocument()->getNode(V: MFI.getNumSpilledVGPRs());
544
545 return Kern;
546}
547
548bool MetadataStreamerMsgPackV4::emitTo(AMDGPUTargetStreamer &TargetStreamer) {
549 DelayedExprs->resolveDelayedExpressions();
550 return TargetStreamer.EmitHSAMetadata(HSAMetadata&: *HSAMetadataDoc, Strict: true);
551}
552
553void MetadataStreamerMsgPackV4::begin(const Module &Mod,
554 const TargetID &TargetID) {
555 emitVersion();
556 emitTargetID(TargetID);
557 emitPrintf(Mod);
558 getRootMetadata(Key: "amdhsa.kernels") = HSAMetadataDoc->getArrayNode();
559 DelayedExprs->clear();
560}
561
562void MetadataStreamerMsgPackV4::end() {
563 DelayedExprs->resolveDelayedExpressions();
564 std::string HSAMetadataString;
565 raw_string_ostream StrOS(HSAMetadataString);
566 HSAMetadataDoc->toYAML(OS&: StrOS);
567
568 if (CLOpts.dump_hsa_metadata)
569 dump(HSAMetadataString: StrOS.str());
570 if (CLOpts.verify_hsa_metadata)
571 verify(HSAMetadataString: StrOS.str());
572}
573
574void MetadataStreamerMsgPackV4::emitKernel(const MachineFunction &MF,
575 const SIProgramInfo &ProgramInfo) {
576 auto &Func = MF.getFunction();
577 if (Func.getCallingConv() != CallingConv::AMDGPU_KERNEL &&
578 Func.getCallingConv() != CallingConv::SPIR_KERNEL)
579 return;
580
581 auto CodeObjectVersion =
582 AMDGPU::getAMDHSACodeObjectVersion(M: *Func.getParent());
583 auto Kern = getHSAKernelProps(MF, ProgramInfo, CodeObjectVersion);
584
585 auto Kernels =
586 getRootMetadata(Key: "amdhsa.kernels").getArray(/*Convert=*/true);
587
588 auto &TM = static_cast<const AMDGPUTargetMachine &>(MF.getTarget());
589 {
590 Kern[".name"] = Kern.getDocument()->getNode(V: Func.getName());
591 Kern[".symbol"] = Kern.getDocument()->getNode(
592 V: (Twine(Func.getName()) + Twine(".kd")).str(), /*Copy=*/true);
593 emitKernelLanguage(Func, Kern);
594 emitKernelAttrs(TM, MF, Kern);
595 emitKernelArgs(MF, Kern);
596 }
597
598 Kernels.push_back(N: Kern);
599}
600
601//===----------------------------------------------------------------------===//
602// HSAMetadataStreamerV5
603//===----------------------------------------------------------------------===//
604
605void MetadataStreamerMsgPackV5::emitVersion() {
606 auto Version = HSAMetadataDoc->getArrayNode();
607 Version.push_back(N: Version.getDocument()->getNode(V: VersionMajorV5));
608 Version.push_back(N: Version.getDocument()->getNode(V: VersionMinorV5));
609 getRootMetadata(Key: "amdhsa.version") = Version;
610}
611
612void MetadataStreamerMsgPackV5::emitHiddenKernelArgs(
613 const MachineFunction &MF, uint64_t &Offset, msgpack::ArrayDocNode Args) {
614 auto &Func = MF.getFunction();
615 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
616
617 // No implicit kernel argument is used.
618 if (ST.getImplicitArgNumBytes(F: Func) == 0)
619 return;
620
621 const Module *M = Func.getParent();
622 auto &DL = M->getDataLayout();
623 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
624
625 auto *Int64Ty = Type::getInt64Ty(C&: Func.getContext());
626 auto *Int32Ty = Type::getInt32Ty(C&: Func.getContext());
627 auto *Int16Ty = Type::getInt16Ty(C&: Func.getContext());
628
629 Offset = alignTo(Size: Offset, A: ST.getAlignmentForImplicitArgPtr());
630 emitKernelArg(DL, Ty: Int32Ty, Alignment: Align(4), ValueKind: "hidden_block_count_x", Offset, Args);
631 emitKernelArg(DL, Ty: Int32Ty, Alignment: Align(4), ValueKind: "hidden_block_count_y", Offset, Args);
632 emitKernelArg(DL, Ty: Int32Ty, Alignment: Align(4), ValueKind: "hidden_block_count_z", Offset, Args);
633
634 emitKernelArg(DL, Ty: Int16Ty, Alignment: Align(2), ValueKind: "hidden_group_size_x", Offset, Args);
635 emitKernelArg(DL, Ty: Int16Ty, Alignment: Align(2), ValueKind: "hidden_group_size_y", Offset, Args);
636 emitKernelArg(DL, Ty: Int16Ty, Alignment: Align(2), ValueKind: "hidden_group_size_z", Offset, Args);
637
638 emitKernelArg(DL, Ty: Int16Ty, Alignment: Align(2), ValueKind: "hidden_remainder_x", Offset, Args);
639 emitKernelArg(DL, Ty: Int16Ty, Alignment: Align(2), ValueKind: "hidden_remainder_y", Offset, Args);
640 emitKernelArg(DL, Ty: Int16Ty, Alignment: Align(2), ValueKind: "hidden_remainder_z", Offset, Args);
641
642 // Reserved for hidden_tool_correlation_id.
643 Offset += 8;
644
645 Offset += 8; // Reserved.
646
647 emitKernelArg(DL, Ty: Int64Ty, Alignment: Align(8), ValueKind: "hidden_global_offset_x", Offset, Args);
648 emitKernelArg(DL, Ty: Int64Ty, Alignment: Align(8), ValueKind: "hidden_global_offset_y", Offset, Args);
649 emitKernelArg(DL, Ty: Int64Ty, Alignment: Align(8), ValueKind: "hidden_global_offset_z", Offset, Args);
650
651 emitKernelArg(DL, Ty: Int16Ty, Alignment: Align(2), ValueKind: "hidden_grid_dims", Offset, Args);
652
653 Offset += 6; // Reserved.
654 auto *Int8PtrTy =
655 PointerType::get(C&: Func.getContext(), AddressSpace: AMDGPUAS::GLOBAL_ADDRESS);
656
657 if (M->getNamedMetadata(Name: "llvm.printf.fmts")) {
658 emitKernelArg(DL, Ty: Int8PtrTy, Alignment: Align(8), ValueKind: "hidden_printf_buffer", Offset,
659 Args);
660 } else {
661 Offset += 8; // Skipped.
662 }
663
664 if (!Func.hasFnAttribute(Kind: "amdgpu-no-hostcall-ptr")) {
665 emitKernelArg(DL, Ty: Int8PtrTy, Alignment: Align(8), ValueKind: "hidden_hostcall_buffer", Offset,
666 Args);
667 } else {
668 Offset += 8; // Skipped.
669 }
670
671 if (!Func.hasFnAttribute(Kind: "amdgpu-no-multigrid-sync-arg")) {
672 emitKernelArg(DL, Ty: Int8PtrTy, Alignment: Align(8), ValueKind: "hidden_multigrid_sync_arg", Offset,
673 Args);
674 } else {
675 Offset += 8; // Skipped.
676 }
677
678 if (!Func.hasFnAttribute(Kind: "amdgpu-no-heap-ptr"))
679 emitKernelArg(DL, Ty: Int8PtrTy, Alignment: Align(8), ValueKind: "hidden_heap_v1", Offset, Args);
680 else
681 Offset += 8; // Skipped.
682
683 if (!Func.hasFnAttribute(Kind: "amdgpu-no-default-queue")) {
684 emitKernelArg(DL, Ty: Int8PtrTy, Alignment: Align(8), ValueKind: "hidden_default_queue", Offset,
685 Args);
686 } else {
687 Offset += 8; // Skipped.
688 }
689
690 if (!Func.hasFnAttribute(Kind: "amdgpu-no-completion-action")) {
691 emitKernelArg(DL, Ty: Int8PtrTy, Alignment: Align(8), ValueKind: "hidden_completion_action", Offset,
692 Args);
693 } else {
694 Offset += 8; // Skipped.
695 }
696
697 // Emit argument for hidden dynamic lds size
698 if (MFI.isDynamicLDSUsed()) {
699 emitKernelArg(DL, Ty: Int32Ty, Alignment: Align(4), ValueKind: "hidden_dynamic_lds_size", Offset,
700 Args);
701 } else {
702 Offset += 4; // skipped
703 }
704
705 Offset += 68; // Reserved.
706
707 // hidden_private_base and hidden_shared_base are only when the subtarget has
708 // ApertureRegs.
709 if (!ST.hasApertureRegs()) {
710 emitKernelArg(DL, Ty: Int32Ty, Alignment: Align(4), ValueKind: "hidden_private_base", Offset, Args);
711 emitKernelArg(DL, Ty: Int32Ty, Alignment: Align(4), ValueKind: "hidden_shared_base", Offset, Args);
712 } else {
713 Offset += 8; // Skipped.
714 }
715
716 if (MFI.getUserSGPRInfo().hasQueuePtr())
717 emitKernelArg(DL, Ty: Int8PtrTy, Alignment: Align(8), ValueKind: "hidden_queue_ptr", Offset, Args);
718}
719
720void MetadataStreamerMsgPackV5::emitKernelAttrs(const AMDGPUTargetMachine &TM,
721 const MachineFunction &MF,
722 msgpack::MapDocNode Kern) {
723 MetadataStreamerMsgPackV4::emitKernelAttrs(TM, MF, Kern);
724
725 const Function &Func = MF.getFunction();
726 if (Func.hasFnAttribute(Kind: "uniform-work-group-size"))
727 Kern[".uniform_work_group_size"] = Kern.getDocument()->getNode(V: 1);
728}
729
730//===----------------------------------------------------------------------===//
731// HSAMetadataStreamerV6
732//===----------------------------------------------------------------------===//
733
734void MetadataStreamerMsgPackV6::emitVersion() {
735 auto Version = HSAMetadataDoc->getArrayNode();
736 Version.push_back(N: Version.getDocument()->getNode(V: VersionMajorV6));
737 Version.push_back(N: Version.getDocument()->getNode(V: VersionMinorV6));
738 getRootMetadata(Key: "amdhsa.version") = Version;
739}
740
741void MetadataStreamerMsgPackV6::emitKernelAttrs(const AMDGPUTargetMachine &TM,
742 const MachineFunction &MF,
743 msgpack::MapDocNode Kern) {
744 MetadataStreamerMsgPackV5::emitKernelAttrs(TM, MF, Kern);
745
746 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
747 ClusterDimsAttr Attr = MFI.getClusterDims();
748 if (Attr.isFixedDims()) {
749 msgpack::ArrayDocNode ClusterDimsNode = HSAMetadataDoc->getArrayNode();
750 ClusterDimsNode.push_back(N: HSAMetadataDoc->getNode(V: Attr.getDims()[0]));
751 ClusterDimsNode.push_back(N: HSAMetadataDoc->getNode(V: Attr.getDims()[1]));
752 ClusterDimsNode.push_back(N: HSAMetadataDoc->getNode(V: Attr.getDims()[2]));
753 Kern[".cluster_dims"] = ClusterDimsNode;
754 }
755}
756
757} // end namespace AMDGPU::HSAMD
758} // end namespace llvm
759