1//===- AMDGPUAttributor.cpp -----------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This pass uses Attributor framework to deduce AMDGPU attributes.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AMDGPU.h"
14#include "AMDGPUTargetMachine.h"
15#include "GCNSubtarget.h"
16#include "Utils/AMDGPUBaseInfo.h"
17#include "llvm/IR/IntrinsicsAMDGPU.h"
18#include "llvm/IR/IntrinsicsR600.h"
19#include "llvm/Target/TargetMachine.h"
20#include "llvm/Transforms/IPO/Attributor.h"
21#include <cstdint>
22
23#define DEBUG_TYPE "amdgpu-attributor"
24
25using namespace llvm;
26
27#define AMDGPU_ATTRIBUTE(Name, Str) Name##_POS,
28
29enum ImplicitArgumentPositions {
30#include "AMDGPUAttributes.def"
31 LAST_ARG_POS
32};
33
34#define AMDGPU_ATTRIBUTE(Name, Str) Name = 1 << Name##_POS,
35
36enum ImplicitArgumentMask {
37 UNKNOWN_INTRINSIC = 0,
38#include "AMDGPUAttributes.def"
39 ALL_ARGUMENT_MASK = (1 << LAST_ARG_POS) - 1,
40 NOT_IMPLICIT_INPUT
41};
42
43#define AMDGPU_ATTRIBUTE(Name, Str) {Name, Str},
44static constexpr std::pair<ImplicitArgumentMask, StringLiteral>
45 ImplicitAttrs[] = {
46#include "AMDGPUAttributes.def"
47};
48
49// We do not need to note the x workitem or workgroup id because they are always
50// initialized.
51//
52// TODO: We should not add the attributes if the known compile time workgroup
53// size is 1 for y/z.
54static ImplicitArgumentMask
55intrinsicToAttrMask(Intrinsic::ID ID, bool &NonKernelOnly, bool &NeedsImplicit,
56 bool HasApertureRegs, bool SupportsGetDoorBellID,
57 unsigned CodeObjectVersion) {
58 switch (ID) {
59 case Intrinsic::amdgcn_workitem_id_x:
60 NonKernelOnly = true;
61 return WORKITEM_ID_X;
62 case Intrinsic::amdgcn_workgroup_id_x:
63 NonKernelOnly = true;
64 return WORKGROUP_ID_X;
65 case Intrinsic::amdgcn_workitem_id_y:
66 case Intrinsic::r600_read_tidig_y:
67 return WORKITEM_ID_Y;
68 case Intrinsic::amdgcn_workitem_id_z:
69 case Intrinsic::r600_read_tidig_z:
70 return WORKITEM_ID_Z;
71 case Intrinsic::amdgcn_workgroup_id_y:
72 case Intrinsic::r600_read_tgid_y:
73 return WORKGROUP_ID_Y;
74 case Intrinsic::amdgcn_workgroup_id_z:
75 case Intrinsic::r600_read_tgid_z:
76 return WORKGROUP_ID_Z;
77 case Intrinsic::amdgcn_cluster_id_x:
78 NonKernelOnly = true;
79 return CLUSTER_ID_X;
80 case Intrinsic::amdgcn_cluster_id_y:
81 return CLUSTER_ID_Y;
82 case Intrinsic::amdgcn_cluster_id_z:
83 return CLUSTER_ID_Z;
84 case Intrinsic::amdgcn_lds_kernel_id:
85 return LDS_KERNEL_ID;
86 case Intrinsic::amdgcn_dispatch_ptr:
87 return DISPATCH_PTR;
88 case Intrinsic::amdgcn_dispatch_id:
89 return DISPATCH_ID;
90 case Intrinsic::amdgcn_implicitarg_ptr:
91 return IMPLICIT_ARG_PTR;
92 // Need queue_ptr anyway. But under V5, we also need implicitarg_ptr to access
93 // queue_ptr.
94 case Intrinsic::amdgcn_queue_ptr:
95 NeedsImplicit = (CodeObjectVersion >= AMDGPU::AMDHSA_COV5);
96 return QUEUE_PTR;
97 case Intrinsic::amdgcn_is_shared:
98 case Intrinsic::amdgcn_is_private:
99 if (HasApertureRegs)
100 return NOT_IMPLICIT_INPUT;
101 // Under V5, we need implicitarg_ptr + offsets to access private_base or
102 // shared_base. For pre-V5, however, need to access them through queue_ptr +
103 // offsets.
104 return CodeObjectVersion >= AMDGPU::AMDHSA_COV5 ? IMPLICIT_ARG_PTR
105 : QUEUE_PTR;
106 case Intrinsic::amdgcn_wwm:
107 case Intrinsic::amdgcn_strict_wwm:
108 return WHOLE_WAVE_MODE;
109 case Intrinsic::trap:
110 case Intrinsic::debugtrap:
111 case Intrinsic::ubsantrap:
112 if (SupportsGetDoorBellID) // GetDoorbellID support implemented since V4.
113 return CodeObjectVersion >= AMDGPU::AMDHSA_COV4 ? NOT_IMPLICIT_INPUT
114 : QUEUE_PTR;
115 NeedsImplicit = (CodeObjectVersion >= AMDGPU::AMDHSA_COV5);
116 return QUEUE_PTR;
117 default:
118 return UNKNOWN_INTRINSIC;
119 }
120}
121
122static bool castRequiresQueuePtr(unsigned SrcAS) {
123 return SrcAS == AMDGPUAS::LOCAL_ADDRESS || SrcAS == AMDGPUAS::PRIVATE_ADDRESS;
124}
125
126static bool isDSAddress(const Constant *C) {
127 const GlobalValue *GV = dyn_cast<GlobalValue>(Val: C);
128 if (!GV)
129 return false;
130 unsigned AS = GV->getAddressSpace();
131 return AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS;
132}
133
134/// Returns true if sanitizer attributes are present on a function.
135static bool hasSanitizerAttributes(const Function &F) {
136 return F.hasFnAttribute(Kind: Attribute::SanitizeAddress) ||
137 F.hasFnAttribute(Kind: Attribute::SanitizeThread) ||
138 F.hasFnAttribute(Kind: Attribute::SanitizeConcurrency) ||
139 F.hasFnAttribute(Kind: Attribute::SanitizeMemory) ||
140 F.hasFnAttribute(Kind: Attribute::SanitizeHWAddress) ||
141 F.hasFnAttribute(Kind: Attribute::SanitizeMemTag);
142}
143
144namespace {
145class AMDGPUInformationCache : public InformationCache {
146public:
147 AMDGPUInformationCache(const Module &M, AnalysisGetter &AG,
148 BumpPtrAllocator &Allocator,
149 SetVector<Function *> *CGSCC, TargetMachine &TM)
150 : InformationCache(M, AG, Allocator, CGSCC), TM(TM),
151 SubArch(M.getTargetTriple().getSubArch()),
152 Features(
153 AMDGPU::getFeatureBitset(AK: AMDGPU::getGPUKindFromSubArch(SubArch))),
154 CodeObjectVersion(AMDGPU::getAMDHSACodeObjectVersion(M)) {
155 LLVMContext &Ctx = M.getContext();
156 for (auto [Mask, Name] : ImplicitAttrs)
157 ImplicitAttrList.push_back(Elt: {Mask, Attribute::get(Context&: Ctx, Kind: Name)});
158 // Sorted by name, the order of attributes in an attribute set.
159 llvm::sort(C&: ImplicitAttrList, Comp: [](const auto &A, const auto &B) {
160 return A.second.getKindAsString() < B.second.getKindAsString();
161 });
162
163 unsigned COV = CodeObjectVersion;
164 ImplicitArgRanges = {
165 {MULTIGRID_SYNC_ARG,
166 {AMDGPU::getMultigridSyncArgImplicitArgPosition(COV), 8}},
167 {HOSTCALL_PTR, {AMDGPU::getHostcallImplicitArgPosition(COV), 8}},
168 {DEFAULT_QUEUE, {AMDGPU::getDefaultQueueImplicitArgPosition(COV), 8}},
169 {COMPLETION_ACTION,
170 {AMDGPU::getCompletionActionImplicitArgPosition(COV), 8}}};
171 if (COV >= 5) {
172 ImplicitArgRanges.push_back(
173 Elt: {HEAP_PTR, {AMDGPU::ImplicitArg::HEAP_PTR_OFFSET, 8}});
174 ImplicitArgRanges.push_back(
175 Elt: {QUEUE_PTR, {AMDGPU::ImplicitArg::QUEUE_PTR_OFFSET, 8}});
176 }
177
178 int64_t Begin = INT64_MAX, End = 0;
179 for (auto [Mask, Range] : ImplicitArgRanges) {
180 ImplicitArgsMask |= Mask;
181 Begin = std::min(a: Begin, b: Range.Offset);
182 End = std::max(a: End, b: Range.Offset + Range.Size);
183 }
184
185 ImplicitArgsRange = AA::RangeTy(Begin, End - Begin);
186 }
187
188 TargetMachine &TM;
189
190 /// The attributes for the implicit arguments that are not needed, sorted by
191 /// name.
192 ArrayRef<std::pair<ImplicitArgumentMask, Attribute>>
193 getImplicitAttrList() const {
194 return ImplicitAttrList;
195 }
196
197 enum ConstantStatus : uint8_t {
198 NONE = 0,
199 DS_GLOBAL = 1 << 0,
200 ADDR_SPACE_CAST_PRIVATE_TO_FLAT = 1 << 1,
201 ADDR_SPACE_CAST_LOCAL_TO_FLAT = 1 << 2,
202 ADDR_SPACE_CAST_BOTH_TO_FLAT =
203 ADDR_SPACE_CAST_PRIVATE_TO_FLAT | ADDR_SPACE_CAST_LOCAL_TO_FLAT,
204 CS_WORST = DS_GLOBAL | ADDR_SPACE_CAST_BOTH_TO_FLAT,
205 };
206
207 std::optional<std::pair<unsigned, unsigned>>
208 getFlatWorkGroupSizeAttr(const Function &F) const {
209 auto R = AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-flat-work-group-size");
210 if (!R)
211 return std::nullopt;
212 return std::make_pair(x&: R->first, y&: *(R->second));
213 }
214
215 std::pair<unsigned, unsigned>
216 getDefaultFlatWorkGroupSize(const Function &F) const {
217 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F);
218 return ST.getDefaultFlatWorkGroupSize(CC: F.getCallingConv());
219 }
220
221 std::pair<unsigned, unsigned> getMaximumFlatWorkGroupRange() const {
222 return {AMDGPU::getMinFlatWorkGroupSize(),
223 AMDGPU::getMaxFlatWorkGroupSize()};
224 }
225
226 /// Get code object version.
227 unsigned getCodeObjectVersion() const { return CodeObjectVersion; }
228
229 /// The implicit kernel arguments at a fixed offset from implicitarg_ptr,
230 /// with their ranges.
231 ArrayRef<std::pair<ImplicitArgumentMask, AA::RangeTy>>
232 getImplicitArgRanges() const {
233 return ImplicitArgRanges;
234 }
235
236 /// The mask of the implicit arguments in getImplicitArgRanges().
237 uint32_t getImplicitArgsMask() const { return ImplicitArgsMask; }
238
239 /// The range covering all the implicit arguments in getImplicitArgRanges().
240 AA::RangeTy getImplicitArgsRange() const { return ImplicitArgsRange; }
241
242 /// Get the features of the module target.
243 const AMDGPU::AMDGPUFeatureBitset &getFeatures() const { return Features; }
244
245 std::optional<std::pair<unsigned, unsigned>>
246 getWavesPerEUAttr(const Function &F) {
247 auto Val = AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-waves-per-eu",
248 /*OnlyFirstRequired=*/true);
249 if (!Val)
250 return std::nullopt;
251 if (!Val->second)
252 Val->second = AMDGPU::getMaxWavesPerEU(SubArch);
253 return std::make_pair(x&: Val->first, y&: *(Val->second));
254 }
255
256 unsigned getMaxWavesPerEU() const {
257 return AMDGPU::getMaxWavesPerEU(SubArch);
258 }
259
260 unsigned getMaxAddrSpace() const override {
261 return AMDGPUAS::MAX_AMDGPU_ADDRESS;
262 }
263
264private:
265 /// Check if the ConstantExpr \p CE uses an addrspacecast from private or
266 /// local to flat. These casts may require the queue pointer.
267 static uint8_t visitConstExpr(const ConstantExpr *CE) {
268 uint8_t Status = NONE;
269
270 if (CE->getOpcode() == Instruction::AddrSpaceCast) {
271 unsigned SrcAS = CE->getOperand(i_nocapture: 0)->getType()->getPointerAddressSpace();
272 if (SrcAS == AMDGPUAS::PRIVATE_ADDRESS)
273 Status |= ADDR_SPACE_CAST_PRIVATE_TO_FLAT;
274 else if (SrcAS == AMDGPUAS::LOCAL_ADDRESS)
275 Status |= ADDR_SPACE_CAST_LOCAL_TO_FLAT;
276 }
277
278 return Status;
279 }
280
281 /// Get the constant access bitmap for \p C.
282 uint8_t getConstantAccess(const Constant *C) {
283 if (isa<ConstantData>(Val: C))
284 return NONE;
285
286 const auto &It = ConstantStatus.find(Val: C);
287 if (It != ConstantStatus.end())
288 return It->second.value();
289
290 SmallPtrSet<const Constant *, 8> Visited;
291 SmallVector<const Constant *> Worklist;
292 Worklist.push_back(Elt: C);
293 Visited.insert(Ptr: C);
294
295 uint8_t Result = 0;
296 while (Result != CS_WORST && !Worklist.empty()) {
297 const Constant *CurC = Worklist.pop_back_val();
298
299 std::optional<uint8_t> &CurCResultOrNone = ConstantStatus[CurC];
300 if (CurCResultOrNone) {
301 Result |= CurCResultOrNone.value();
302 continue;
303 }
304 uint8_t CurCResult = 0;
305
306 if (isDSAddress(C: CurC))
307 CurCResult |= DS_GLOBAL;
308
309 if (const auto *CE = dyn_cast<ConstantExpr>(Val: CurC))
310 CurCResult |= visitConstExpr(CE);
311
312 for (const Use &U : CurC->operands()) {
313 if (const auto *OpC = dyn_cast<Constant>(Val: U)) {
314 if (Visited.insert(Ptr: OpC).second)
315 Worklist.push_back(Elt: OpC);
316 }
317 }
318
319 CurCResultOrNone = CurCResult;
320 Result |= CurCResult;
321 }
322
323 ConstantStatus[C] = Result;
324 return Result;
325 }
326
327public:
328 /// Returns true if \p Fn needs the queue pointer because of \p C.
329 bool needsQueuePtr(const Constant *C, Function &Fn) {
330 bool IsNonEntryFunc = !AMDGPU::isEntryFunctionCC(CC: Fn.getCallingConv());
331 bool HasAperture = Features.test(I: AMDGPU::FEAT_APERTURE_REGS);
332
333 // No need to explore the constants.
334 if (!IsNonEntryFunc && HasAperture)
335 return false;
336
337 uint8_t Access = getConstantAccess(C);
338
339 // We need to trap on DS globals in non-entry functions.
340 if (IsNonEntryFunc && (Access & DS_GLOBAL))
341 return true;
342
343 return !HasAperture && (Access & ADDR_SPACE_CAST_BOTH_TO_FLAT);
344 }
345
346 bool checkConstForAddrSpaceCastFromPrivate(const Constant *C) {
347 uint8_t Access = getConstantAccess(C);
348 return Access & ADDR_SPACE_CAST_PRIVATE_TO_FLAT;
349 }
350
351private:
352 /// Used to determine if the Constant needs the queue pointer.
353 DenseMap<const Constant *, std::optional<uint8_t>> ConstantStatus;
354 SmallVector<std::pair<ImplicitArgumentMask, Attribute>, LAST_ARG_POS>
355 ImplicitAttrList;
356 const Triple::SubArchType SubArch;
357 const AMDGPU::AMDGPUFeatureBitset Features;
358 const unsigned CodeObjectVersion;
359 SmallVector<std::pair<ImplicitArgumentMask, AA::RangeTy>, 6>
360 ImplicitArgRanges;
361 uint32_t ImplicitArgsMask = 0;
362 AA::RangeTy ImplicitArgsRange;
363};
364
365struct AAAMDAttributes
366 : public StateWrapper<BitIntegerState<uint32_t, ALL_ARGUMENT_MASK, 0>,
367 AbstractAttribute> {
368 using Base = StateWrapper<BitIntegerState<uint32_t, ALL_ARGUMENT_MASK, 0>,
369 AbstractAttribute>;
370
371 AAAMDAttributes(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
372
373 /// Create an abstract attribute view for the position \p IRP.
374 static AAAMDAttributes &createForPosition(const IRPosition &IRP,
375 Attributor &A);
376
377 StringRef getName() const override { return "AAAMDAttributes"; }
378
379 const char *getIdAddr() const override { return &ID; }
380
381 /// This function should return true if the type of the \p AA is
382 /// AAAMDAttributes.
383 static bool classof(const AbstractAttribute *AA) {
384 return (AA->getIdAddr() == &ID);
385 }
386
387 /// Unique ID (due to the unique address)
388 static const char ID;
389};
390const char AAAMDAttributes::ID = 0;
391
392struct AAUniformWorkGroupSize
393 : public StateWrapper<BooleanState, AbstractAttribute> {
394 using Base = StateWrapper<BooleanState, AbstractAttribute>;
395 AAUniformWorkGroupSize(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
396
397 /// Create an abstract attribute view for the position \p IRP.
398 static AAUniformWorkGroupSize &createForPosition(const IRPosition &IRP,
399 Attributor &A);
400
401 StringRef getName() const override { return "AAUniformWorkGroupSize"; }
402
403 const char *getIdAddr() const override { return &ID; }
404
405 /// This function should return true if the type of the \p AA is
406 /// AAAMDAttributes.
407 static bool classof(const AbstractAttribute *AA) {
408 return (AA->getIdAddr() == &ID);
409 }
410
411 /// Unique ID (due to the unique address)
412 static const char ID;
413};
414const char AAUniformWorkGroupSize::ID = 0;
415
416struct AAUniformWorkGroupSizeFunction : public AAUniformWorkGroupSize {
417 AAUniformWorkGroupSizeFunction(const IRPosition &IRP, Attributor &A)
418 : AAUniformWorkGroupSize(IRP, A) {}
419
420 void initialize(Attributor &A) override {
421 Function *F = getAssociatedFunction();
422 CallingConv::ID CC = F->getCallingConv();
423
424 if (CC != CallingConv::AMDGPU_KERNEL)
425 return;
426
427 bool InitialValue = F->hasFnAttribute(Kind: "uniform-work-group-size");
428
429 if (InitialValue)
430 indicateOptimisticFixpoint();
431 else
432 indicatePessimisticFixpoint();
433 }
434
435 ChangeStatus updateImpl(Attributor &A) override {
436 ChangeStatus Change = ChangeStatus::UNCHANGED;
437
438 auto CheckCallSite = [&](AbstractCallSite CS) {
439 Function *Caller = CS.getInstruction()->getFunction();
440 LLVM_DEBUG(dbgs() << "[AAUniformWorkGroupSize] Call " << Caller->getName()
441 << "->" << getAssociatedFunction()->getName() << "\n");
442
443 const auto *CallerInfo = A.getAAFor<AAUniformWorkGroupSize>(
444 QueryingAA: *this, IRP: IRPosition::function(F: *Caller), DepClass: DepClassTy::REQUIRED);
445 if (!CallerInfo || !CallerInfo->isValidState())
446 return false;
447
448 Change = Change | clampStateAndIndicateChange(S&: this->getState(),
449 R: CallerInfo->getState());
450
451 return true;
452 };
453
454 bool AllCallSitesKnown = true;
455 if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this, RequireAllCallSites: true, UsedAssumedInformation&: AllCallSitesKnown))
456 return indicatePessimisticFixpoint();
457
458 return Change;
459 }
460
461 ChangeStatus manifest(Attributor &A) override {
462 if (!getAssumed())
463 return ChangeStatus::UNCHANGED;
464
465 LLVMContext &Ctx = getAssociatedFunction()->getContext();
466 return A.manifestAttrs(IRP: getIRPosition(),
467 DeducedAttrs: {Attribute::get(Context&: Ctx, Kind: "uniform-work-group-size")},
468 /*ForceReplace=*/true);
469 }
470
471 bool isValidState() const override {
472 // This state is always valid, even when the state is false.
473 return true;
474 }
475
476 const std::string getAsStr(Attributor *) const override {
477 return "AMDWorkGroupSize[" + std::to_string(val: getAssumed()) + "]";
478 }
479
480 void trackStatistics() const override {}
481};
482
483AAUniformWorkGroupSize &
484AAUniformWorkGroupSize::createForPosition(const IRPosition &IRP,
485 Attributor &A) {
486 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
487 return *new (A.Allocator) AAUniformWorkGroupSizeFunction(IRP, A);
488 llvm_unreachable(
489 "AAUniformWorkGroupSize is only valid for function position");
490}
491
492struct AAAMDAttributesFunction : public AAAMDAttributes {
493 AAAMDAttributesFunction(const IRPosition &IRP, Attributor &A)
494 : AAAMDAttributes(IRP, A) {}
495
496 void initialize(Attributor &A) override {
497 Function *F = getAssociatedFunction();
498
499 // If the function requires the implicit arg pointer due to sanitizers,
500 // assume it's needed even if explicitly marked as not requiring it.
501 // Flat scratch initialization is needed because `asan_malloc_impl`
502 // calls introduced later in pipeline will have flat scratch accesses.
503 // FIXME: FLAT_SCRATCH_INIT will not be required here if device-libs
504 // implementation for `asan_malloc_impl` is updated.
505 const bool HasSanitizerAttrs = hasSanitizerAttributes(F: *F);
506 if (HasSanitizerAttrs) {
507 removeAssumedBits(BitsEncoding: IMPLICIT_ARG_PTR);
508 removeAssumedBits(BitsEncoding: HOSTCALL_PTR);
509 removeAssumedBits(BitsEncoding: FLAT_SCRATCH_INIT);
510 }
511
512 for (auto Attr : ImplicitAttrs) {
513 if (HasSanitizerAttrs &&
514 (Attr.first == IMPLICIT_ARG_PTR || Attr.first == HOSTCALL_PTR ||
515 Attr.first == FLAT_SCRATCH_INIT))
516 continue;
517
518 if (F->hasFnAttribute(Kind: Attr.second))
519 addKnownBits(Bits: Attr.first);
520 }
521
522 if (F->isDeclaration())
523 return;
524
525 // Ignore functions with graphics calling conventions, these are currently
526 // not allowed to have kernel arguments.
527 if (AMDGPU::isGraphics(CC: F->getCallingConv())) {
528 indicatePessimisticFixpoint();
529 return;
530 }
531 }
532
533 ChangeStatus updateImpl(Attributor &A) override {
534 Function *F = getAssociatedFunction();
535 // The current assumed state used to determine a change.
536 auto OrigAssumed = getAssumed();
537
538 // Check for Intrinsics and propagate attributes.
539 const AACallEdges *AAEdges = A.getAAFor<AACallEdges>(
540 QueryingAA: *this, IRP: this->getIRPosition(), DepClass: DepClassTy::REQUIRED);
541 if (!AAEdges || !AAEdges->isValidState() ||
542 AAEdges->hasNonAsmUnknownCallee())
543 return indicatePessimisticFixpoint();
544
545 bool IsNonEntryFunc = !AMDGPU::isEntryFunctionCC(CC: F->getCallingConv());
546
547 bool NeedsImplicit = false;
548 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
549 const AMDGPU::AMDGPUFeatureBitset &Features = InfoCache.getFeatures();
550 bool HasApertureRegs = Features.test(I: AMDGPU::FEAT_APERTURE_REGS);
551 bool SupportsGetDoorbellID = Features.test(I: AMDGPU::FEAT_GET_DOORBELL_ID);
552 unsigned COV = InfoCache.getCodeObjectVersion();
553
554 for (Function *Callee : AAEdges->getOptimisticEdges()) {
555 Intrinsic::ID IID = Callee->getIntrinsicID();
556 if (IID == Intrinsic::not_intrinsic) {
557 const AAAMDAttributes *AAAMD = A.getAAFor<AAAMDAttributes>(
558 QueryingAA: *this, IRP: IRPosition::function(F: *Callee), DepClass: DepClassTy::REQUIRED);
559 if (!AAAMD || !AAAMD->isValidState())
560 return indicatePessimisticFixpoint();
561 *this &= *AAAMD;
562 continue;
563 }
564
565 bool NonKernelOnly = false;
566 ImplicitArgumentMask AttrMask =
567 intrinsicToAttrMask(ID: IID, NonKernelOnly, NeedsImplicit,
568 HasApertureRegs, SupportsGetDoorBellID: SupportsGetDoorbellID, CodeObjectVersion: COV);
569
570 if (AttrMask == UNKNOWN_INTRINSIC) {
571 // Assume not-nocallback intrinsics may invoke a function which accesses
572 // implicit arguments.
573 //
574 // FIXME: This isn't really the correct check. We want to ensure it
575 // isn't calling any function that may use implicit arguments regardless
576 // of whether it's internal to the module or not.
577 //
578 // TODO: Ignoring callsite attributes.
579 if (!Callee->hasFnAttribute(Kind: Attribute::NoCallback))
580 return indicatePessimisticFixpoint();
581 continue;
582 }
583
584 if (AttrMask != NOT_IMPLICIT_INPUT) {
585 if ((IsNonEntryFunc || !NonKernelOnly))
586 removeAssumedBits(BitsEncoding: AttrMask);
587 }
588 }
589
590 // Need implicitarg_ptr to acess queue_ptr, private_base, and shared_base.
591 if (NeedsImplicit)
592 removeAssumedBits(BitsEncoding: IMPLICIT_ARG_PTR);
593
594 if (isAssumed(BitsEncoding: QUEUE_PTR) && checkForQueuePtr(A)) {
595 // Under V5, we need implicitarg_ptr + offsets to access private_base or
596 // shared_base. We do not actually need queue_ptr.
597 if (COV >= 5)
598 removeAssumedBits(BitsEncoding: IMPLICIT_ARG_PTR);
599 else
600 removeAssumedBits(BitsEncoding: QUEUE_PTR);
601 }
602
603 uint32_t Retrieved = getRetrievedImplicitArgs(A);
604
605 if (Retrieved & MULTIGRID_SYNC_ARG) {
606 assert(!isAssumed(IMPLICIT_ARG_PTR) &&
607 "multigrid_sync_arg needs implicitarg_ptr");
608 removeAssumedBits(BitsEncoding: MULTIGRID_SYNC_ARG);
609 }
610
611 if (Retrieved & HOSTCALL_PTR) {
612 assert(!isAssumed(IMPLICIT_ARG_PTR) && "hostcall needs implicitarg_ptr");
613 removeAssumedBits(BitsEncoding: HOSTCALL_PTR);
614 }
615
616 if (Retrieved & HEAP_PTR) {
617 assert(!isAssumed(IMPLICIT_ARG_PTR) && "heap_ptr needs implicitarg_ptr");
618 removeAssumedBits(BitsEncoding: HEAP_PTR);
619 }
620
621 if (isAssumed(BitsEncoding: QUEUE_PTR) && (Retrieved & QUEUE_PTR)) {
622 assert(!isAssumed(IMPLICIT_ARG_PTR) && "queue_ptr needs implicitarg_ptr");
623 removeAssumedBits(BitsEncoding: QUEUE_PTR);
624 }
625
626 if (isAssumed(BitsEncoding: LDS_KERNEL_ID) && (Retrieved & LDS_KERNEL_ID)) {
627 removeAssumedBits(BitsEncoding: LDS_KERNEL_ID);
628 }
629
630 if (isAssumed(BitsEncoding: DEFAULT_QUEUE) && (Retrieved & DEFAULT_QUEUE))
631 removeAssumedBits(BitsEncoding: DEFAULT_QUEUE);
632
633 if (isAssumed(BitsEncoding: COMPLETION_ACTION) && (Retrieved & COMPLETION_ACTION))
634 removeAssumedBits(BitsEncoding: COMPLETION_ACTION);
635
636 if (isAssumed(BitsEncoding: FLAT_SCRATCH_INIT) && needFlatScratchInit(A))
637 removeAssumedBits(BitsEncoding: FLAT_SCRATCH_INIT);
638
639 return getAssumed() != OrigAssumed ? ChangeStatus::CHANGED
640 : ChangeStatus::UNCHANGED;
641 }
642
643 ChangeStatus manifest(Attributor &A) override {
644 SmallVector<Attribute, LAST_ARG_POS> AttrList;
645 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
646 for (auto [Mask, Attr] : InfoCache.getImplicitAttrList()) {
647 if (isKnown(BitsEncoding: Mask))
648 AttrList.push_back(Elt: Attr);
649 }
650
651 return A.manifestAttrs(IRP: getIRPosition(), DeducedAttrs: AttrList,
652 /* ForceReplace */ true);
653 }
654
655 const std::string getAsStr(Attributor *) const override {
656 std::string Str;
657 raw_string_ostream OS(Str);
658 OS << "AMDInfo[";
659 for (auto Attr : ImplicitAttrs)
660 if (isAssumed(BitsEncoding: Attr.first))
661 OS << ' ' << Attr.second;
662 OS << " ]";
663 return OS.str();
664 }
665
666 void trackStatistics() const override {}
667
668private:
669 bool checkForQueuePtr(Attributor &A) {
670 Function *F = getAssociatedFunction();
671 bool IsNonEntryFunc = !AMDGPU::isEntryFunctionCC(CC: F->getCallingConv());
672
673 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
674
675 bool NeedsQueuePtr = false;
676
677 auto CheckAddrSpaceCasts = [&](Instruction &I) {
678 unsigned SrcAS = static_cast<AddrSpaceCastInst &>(I).getSrcAddressSpace();
679 if (castRequiresQueuePtr(SrcAS)) {
680 NeedsQueuePtr = true;
681 return false;
682 }
683 return true;
684 };
685
686 bool HasApertureRegs =
687 InfoCache.getFeatures().test(I: AMDGPU::FEAT_APERTURE_REGS);
688
689 // `checkForAllInstructions` is much more cheaper than going through all
690 // instructions, try it first.
691
692 // The queue pointer is not needed if aperture regs is present.
693 if (!HasApertureRegs) {
694 bool UsedAssumedInformation = false;
695 A.checkForAllInstructions(Pred: CheckAddrSpaceCasts, QueryingAA: *this,
696 Opcodes: {Instruction::AddrSpaceCast},
697 UsedAssumedInformation);
698 }
699
700 // If we found that we need the queue pointer, nothing else to do.
701 if (NeedsQueuePtr)
702 return true;
703
704 if (!IsNonEntryFunc && HasApertureRegs)
705 return false;
706
707 for (BasicBlock &BB : *F) {
708 for (Instruction &I : BB) {
709 for (const Use &U : I.operands()) {
710 if (const auto *C = dyn_cast<Constant>(Val: U)) {
711 if (InfoCache.needsQueuePtr(C, Fn&: *F))
712 return true;
713 }
714 }
715 }
716 }
717
718 return false;
719 }
720
721 /// Returns the implicit kernel arguments that may be read through the
722 /// pointer returned by the implicitarg_ptr call \p Call.
723 uint32_t getImplicitArgsReadThrough(Attributor &A, const CallBase &Call) {
724 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
725 uint32_t ArgsMask = InfoCache.getImplicitArgsMask();
726 const auto *PointerInfoAA = A.getAAFor<AAPointerInfo>(
727 QueryingAA: *this, IRP: IRPosition::callsite_returned(CB: Call), DepClass: DepClassTy::REQUIRED);
728 if (!PointerInfoAA || !PointerInfoAA->getState().isValidState())
729 return ArgsMask;
730
731 // An implicit argument is not read if every access that may read any byte
732 // of it is droppable. Check the accesses once for all the implicit
733 // arguments, and stop once all of them may be read.
734 uint32_t Read = 0;
735 auto CheckAccess = [&](const AAPointerInfo::Access &Acc, bool) {
736 if (Acc.getRemoteInst()->isDroppable())
737 return true;
738 for (const AA::RangeTy &AccRange : Acc.getRanges()) {
739 for (auto [Mask, Range] : InfoCache.getImplicitArgRanges()) {
740 if (AccRange.mayOverlap(Range))
741 Read |= Mask;
742 }
743 }
744
745 return Read != ArgsMask;
746 };
747
748 if (!PointerInfoAA->forallInterferingAccesses(
749 Range: InfoCache.getImplicitArgsRange(), CB: CheckAccess))
750 return ArgsMask;
751 return Read;
752 }
753
754 /// Returns the implicit kernel arguments that may be retrieved by the
755 /// function.
756 uint32_t getRetrievedImplicitArgs(Attributor &A) {
757 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
758 uint32_t All = InfoCache.getImplicitArgsMask() | LDS_KERNEL_ID;
759 uint32_t Retrieved = 0;
760 auto CheckCall = [&](Instruction &I) {
761 auto &Call = cast<CallBase>(Val&: I);
762 switch (Call.getIntrinsicID()) {
763 case Intrinsic::amdgcn_implicitarg_ptr:
764 Retrieved |= getImplicitArgsReadThrough(A, Call);
765 break;
766 case Intrinsic::amdgcn_lds_kernel_id:
767 Retrieved |= LDS_KERNEL_ID;
768 break;
769 default:
770 // Other calls do not retrieve anything.
771 return true;
772 }
773
774 // Stop once everything may be retrieved.
775 return Retrieved != All;
776 };
777
778 bool UsedAssumedInformation = false;
779 if (!A.checkForAllCallLikeInstructions(Pred: CheckCall, QueryingAA: *this,
780 UsedAssumedInformation))
781 return All;
782 return Retrieved;
783 }
784
785 // Returns true if FlatScratchInit is needed, i.e., no-flat-scratch-init is
786 // not to be set.
787 bool needFlatScratchInit(Attributor &A) {
788 assert(isAssumed(FLAT_SCRATCH_INIT)); // only called if the bit is still set
789
790 // Check all AddrSpaceCast instructions. FlatScratchInit is needed if
791 // there is a cast from PRIVATE_ADDRESS.
792 auto AddrSpaceCastNotFromPrivate = [](Instruction &I) {
793 return cast<AddrSpaceCastInst>(Val&: I).getSrcAddressSpace() !=
794 AMDGPUAS::PRIVATE_ADDRESS;
795 };
796
797 bool UsedAssumedInformation = false;
798 if (!A.checkForAllInstructions(Pred: AddrSpaceCastNotFromPrivate, QueryingAA: *this,
799 Opcodes: {Instruction::AddrSpaceCast},
800 UsedAssumedInformation))
801 return true;
802
803 // Check for addrSpaceCast from PRIVATE_ADDRESS in constant expressions
804 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
805
806 Function *F = getAssociatedFunction();
807 for (Instruction &I : instructions(F)) {
808 for (const Use &U : I.operands()) {
809 if (const auto *C = dyn_cast<Constant>(Val: U)) {
810 if (InfoCache.checkConstForAddrSpaceCastFromPrivate(C))
811 return true;
812 }
813 }
814 }
815
816 return false;
817 }
818};
819
820AAAMDAttributes &AAAMDAttributes::createForPosition(const IRPosition &IRP,
821 Attributor &A) {
822 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
823 return *new (A.Allocator) AAAMDAttributesFunction(IRP, A);
824 llvm_unreachable("AAAMDAttributes is only valid for function position");
825}
826
827/// Base class to derive different size ranges.
828struct AAAMDSizeRangeAttribute
829 : public StateWrapper<IntegerRangeState, AbstractAttribute, uint32_t> {
830 using Base = StateWrapper<IntegerRangeState, AbstractAttribute, uint32_t>;
831
832 StringRef AttrName;
833
834 AAAMDSizeRangeAttribute(const IRPosition &IRP, Attributor &A,
835 StringRef AttrName)
836 : Base(IRP, 32), AttrName(AttrName) {}
837
838 void trackStatistics() const override {}
839
840 template <class AttributeImpl> ChangeStatus updateImplImpl(Attributor &A) {
841 ChangeStatus Change = ChangeStatus::UNCHANGED;
842
843 auto CheckCallSite = [&](AbstractCallSite CS) {
844 Function *Caller = CS.getInstruction()->getFunction();
845 LLVM_DEBUG(dbgs() << '[' << getName() << "] Call " << Caller->getName()
846 << "->" << getAssociatedFunction()->getName() << '\n');
847
848 const auto *CallerInfo = A.getAAFor<AttributeImpl>(
849 *this, IRPosition::function(F: *Caller), DepClassTy::REQUIRED);
850 if (!CallerInfo || !CallerInfo->isValidState())
851 return false;
852
853 Change |=
854 clampStateAndIndicateChange(this->getState(), CallerInfo->getState());
855
856 return true;
857 };
858
859 bool AllCallSitesKnown = true;
860 if (!A.checkForAllCallSites(CheckCallSite, *this,
861 /*RequireAllCallSites=*/true,
862 AllCallSitesKnown))
863 return indicatePessimisticFixpoint();
864
865 return Change;
866 }
867
868 /// Clamp the assumed range to the default value ([Min, Max]) and emit the
869 /// attribute if it is not same as default.
870 ChangeStatus
871 emitAttributeIfNotDefaultAfterClamp(Attributor &A,
872 std::pair<unsigned, unsigned> Default) {
873 auto [Min, Max] = Default;
874 unsigned Lower = getAssumed().getLower().getZExtValue();
875 unsigned Upper = getAssumed().getUpper().getZExtValue();
876
877 // Clamp the range to the default value.
878 if (Lower < Min)
879 Lower = Min;
880 if (Upper > Max + 1)
881 Upper = Max + 1;
882
883 // No manifest if the value is invalid or same as default after clamp.
884 if ((Lower == Min && Upper == Max + 1) || (Upper < Lower))
885 return ChangeStatus::UNCHANGED;
886
887 Function *F = getAssociatedFunction();
888 LLVMContext &Ctx = F->getContext();
889 SmallString<10> Buffer;
890 raw_svector_ostream OS(Buffer);
891 OS << Lower << ',' << Upper - 1;
892 return A.manifestAttrs(IRP: getIRPosition(),
893 DeducedAttrs: {Attribute::get(Context&: Ctx, Kind: AttrName, Val: OS.str())},
894 /*ForceReplace=*/true);
895 }
896
897 const std::string getAsStr(Attributor *) const override {
898 std::string Str;
899 raw_string_ostream OS(Str);
900 OS << getName() << '[';
901 OS << getAssumed().getLower() << ',' << getAssumed().getUpper() - 1;
902 OS << ']';
903 return OS.str();
904 }
905};
906
907/// Propagate amdgpu-flat-work-group-size attribute.
908struct AAAMDFlatWorkGroupSize : public AAAMDSizeRangeAttribute {
909 AAAMDFlatWorkGroupSize(const IRPosition &IRP, Attributor &A)
910 : AAAMDSizeRangeAttribute(IRP, A, "amdgpu-flat-work-group-size") {}
911
912 void initialize(Attributor &A) override {
913 Function *F = getAssociatedFunction();
914 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
915
916 bool HasAttr = false;
917 auto Range = InfoCache.getDefaultFlatWorkGroupSize(F: *F);
918 auto MaxRange = InfoCache.getMaximumFlatWorkGroupRange();
919
920 if (auto Attr = InfoCache.getFlatWorkGroupSizeAttr(F: *F)) {
921 // We only consider an attribute that is not max range because the front
922 // end always emits the attribute, unfortunately, and sometimes it emits
923 // the max range.
924 if (*Attr != MaxRange) {
925 Range = *Attr;
926 HasAttr = true;
927 }
928 }
929
930 // We don't want to directly clamp the state if it's the max range because
931 // that is basically the worst state.
932 if (Range == MaxRange)
933 return;
934
935 auto [Min, Max] = Range;
936 ConstantRange CR(APInt(32, Min), APInt(32, Max + 1));
937 IntegerRangeState IRS(CR);
938 clampStateAndIndicateChange(S&: this->getState(), R: IRS);
939
940 if (HasAttr || AMDGPU::isEntryFunctionCC(CC: F->getCallingConv()))
941 indicateOptimisticFixpoint();
942 }
943
944 ChangeStatus updateImpl(Attributor &A) override {
945 return updateImplImpl<AAAMDFlatWorkGroupSize>(A);
946 }
947
948 /// Create an abstract attribute view for the position \p IRP.
949 static AAAMDFlatWorkGroupSize &createForPosition(const IRPosition &IRP,
950 Attributor &A);
951
952 ChangeStatus manifest(Attributor &A) override {
953 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
954 return emitAttributeIfNotDefaultAfterClamp(
955 A, Default: InfoCache.getMaximumFlatWorkGroupRange());
956 }
957
958 StringRef getName() const override { return "AAAMDFlatWorkGroupSize"; }
959
960 const char *getIdAddr() const override { return &ID; }
961
962 /// This function should return true if the type of the \p AA is
963 /// AAAMDFlatWorkGroupSize
964 static bool classof(const AbstractAttribute *AA) {
965 return (AA->getIdAddr() == &ID);
966 }
967
968 /// Unique ID (due to the unique address)
969 static const char ID;
970};
971
972const char AAAMDFlatWorkGroupSize::ID = 0;
973
974AAAMDFlatWorkGroupSize &
975AAAMDFlatWorkGroupSize::createForPosition(const IRPosition &IRP,
976 Attributor &A) {
977 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
978 return *new (A.Allocator) AAAMDFlatWorkGroupSize(IRP, A);
979 llvm_unreachable(
980 "AAAMDFlatWorkGroupSize is only valid for function position");
981}
982
983struct TupleDecIntegerRangeState : public AbstractState {
984 DecIntegerState<uint32_t> X, Y, Z;
985
986 bool isValidState() const override {
987 return X.isValidState() && Y.isValidState() && Z.isValidState();
988 }
989
990 bool isAtFixpoint() const override {
991 return X.isAtFixpoint() && Y.isAtFixpoint() && Z.isAtFixpoint();
992 }
993
994 ChangeStatus indicateOptimisticFixpoint() override {
995 return X.indicateOptimisticFixpoint() | Y.indicateOptimisticFixpoint() |
996 Z.indicateOptimisticFixpoint();
997 }
998
999 ChangeStatus indicatePessimisticFixpoint() override {
1000 return X.indicatePessimisticFixpoint() | Y.indicatePessimisticFixpoint() |
1001 Z.indicatePessimisticFixpoint();
1002 }
1003
1004 TupleDecIntegerRangeState operator^=(const TupleDecIntegerRangeState &Other) {
1005 X ^= Other.X;
1006 Y ^= Other.Y;
1007 Z ^= Other.Z;
1008 return *this;
1009 }
1010
1011 bool operator==(const TupleDecIntegerRangeState &Other) const {
1012 return X == Other.X && Y == Other.Y && Z == Other.Z;
1013 }
1014
1015 TupleDecIntegerRangeState &getAssumed() { return *this; }
1016 const TupleDecIntegerRangeState &getAssumed() const { return *this; }
1017};
1018
1019using AAAMDMaxNumWorkgroupsState =
1020 StateWrapper<TupleDecIntegerRangeState, AbstractAttribute, uint32_t>;
1021
1022/// Propagate amdgpu-max-num-workgroups attribute.
1023struct AAAMDMaxNumWorkgroups
1024 : public StateWrapper<TupleDecIntegerRangeState, AbstractAttribute> {
1025 using Base = StateWrapper<TupleDecIntegerRangeState, AbstractAttribute>;
1026
1027 AAAMDMaxNumWorkgroups(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
1028
1029 void initialize(Attributor &A) override {
1030 Function *F = getAssociatedFunction();
1031
1032 SmallVector<unsigned> MaxNumWorkgroups = AMDGPU::getMaxNumWorkGroups(F: *F);
1033
1034 X.takeKnownMinimum(Value: MaxNumWorkgroups[0]);
1035 Y.takeKnownMinimum(Value: MaxNumWorkgroups[1]);
1036 Z.takeKnownMinimum(Value: MaxNumWorkgroups[2]);
1037
1038 if (AMDGPU::isEntryFunctionCC(CC: F->getCallingConv()))
1039 indicatePessimisticFixpoint();
1040 }
1041
1042 ChangeStatus updateImpl(Attributor &A) override {
1043 ChangeStatus Change = ChangeStatus::UNCHANGED;
1044
1045 auto CheckCallSite = [&](AbstractCallSite CS) {
1046 Function *Caller = CS.getInstruction()->getFunction();
1047 LLVM_DEBUG(dbgs() << "[AAAMDMaxNumWorkgroups] Call " << Caller->getName()
1048 << "->" << getAssociatedFunction()->getName() << '\n');
1049
1050 const auto *CallerInfo = A.getAAFor<AAAMDMaxNumWorkgroups>(
1051 QueryingAA: *this, IRP: IRPosition::function(F: *Caller), DepClass: DepClassTy::REQUIRED);
1052 if (!CallerInfo || !CallerInfo->isValidState())
1053 return false;
1054
1055 Change |=
1056 clampStateAndIndicateChange(S&: this->getState(), R: CallerInfo->getState());
1057 return true;
1058 };
1059
1060 bool AllCallSitesKnown = true;
1061 if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this,
1062 /*RequireAllCallSites=*/true,
1063 UsedAssumedInformation&: AllCallSitesKnown))
1064 return indicatePessimisticFixpoint();
1065
1066 return Change;
1067 }
1068
1069 /// Create an abstract attribute view for the position \p IRP.
1070 static AAAMDMaxNumWorkgroups &createForPosition(const IRPosition &IRP,
1071 Attributor &A);
1072
1073 ChangeStatus manifest(Attributor &A) override {
1074 Function *F = getAssociatedFunction();
1075 LLVMContext &Ctx = F->getContext();
1076 SmallString<32> Buffer;
1077 raw_svector_ostream OS(Buffer);
1078 OS << X.getAssumed() << ',' << Y.getAssumed() << ',' << Z.getAssumed();
1079
1080 // TODO: Should annotate loads of the group size for this to do anything
1081 // useful.
1082 return A.manifestAttrs(
1083 IRP: getIRPosition(),
1084 DeducedAttrs: {Attribute::get(Context&: Ctx, Kind: "amdgpu-max-num-workgroups", Val: OS.str())},
1085 /* ForceReplace= */ true);
1086 }
1087
1088 StringRef getName() const override { return "AAAMDMaxNumWorkgroups"; }
1089
1090 const std::string getAsStr(Attributor *) const override {
1091 std::string Buffer = "AAAMDMaxNumWorkgroupsState[";
1092 raw_string_ostream OS(Buffer);
1093 OS << X.getAssumed() << ',' << Y.getAssumed() << ',' << Z.getAssumed()
1094 << ']';
1095 return OS.str();
1096 }
1097
1098 const char *getIdAddr() const override { return &ID; }
1099
1100 /// This function should return true if the type of the \p AA is
1101 /// AAAMDMaxNumWorkgroups
1102 static bool classof(const AbstractAttribute *AA) {
1103 return (AA->getIdAddr() == &ID);
1104 }
1105
1106 void trackStatistics() const override {}
1107
1108 /// Unique ID (due to the unique address)
1109 static const char ID;
1110};
1111
1112const char AAAMDMaxNumWorkgroups::ID = 0;
1113
1114AAAMDMaxNumWorkgroups &
1115AAAMDMaxNumWorkgroups::createForPosition(const IRPosition &IRP, Attributor &A) {
1116 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
1117 return *new (A.Allocator) AAAMDMaxNumWorkgroups(IRP, A);
1118 llvm_unreachable("AAAMDMaxNumWorkgroups is only valid for function position");
1119}
1120
1121/// Propagate amdgpu-waves-per-eu attribute.
1122struct AAAMDWavesPerEU : public AAAMDSizeRangeAttribute {
1123 AAAMDWavesPerEU(const IRPosition &IRP, Attributor &A)
1124 : AAAMDSizeRangeAttribute(IRP, A, "amdgpu-waves-per-eu") {}
1125
1126 void initialize(Attributor &A) override {
1127 Function *F = getAssociatedFunction();
1128 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
1129
1130 // If the attribute exists, we will honor it if it is not the default.
1131 if (auto Attr = InfoCache.getWavesPerEUAttr(F: *F)) {
1132 std::pair<unsigned, unsigned> MaxWavesPerEURange{
1133 1U, InfoCache.getMaxWavesPerEU()};
1134 if (*Attr != MaxWavesPerEURange) {
1135 auto [Min, Max] = *Attr;
1136 ConstantRange Range(APInt(32, Min), APInt(32, Max + 1));
1137 IntegerRangeState RangeState(Range);
1138 this->getState() = RangeState;
1139 indicateOptimisticFixpoint();
1140 return;
1141 }
1142 }
1143
1144 if (AMDGPU::isEntryFunctionCC(CC: F->getCallingConv()))
1145 indicatePessimisticFixpoint();
1146 }
1147
1148 ChangeStatus updateImpl(Attributor &A) override {
1149 ChangeStatus Change = ChangeStatus::UNCHANGED;
1150
1151 auto CheckCallSite = [&](AbstractCallSite CS) {
1152 Function *Caller = CS.getInstruction()->getFunction();
1153 Function *Func = getAssociatedFunction();
1154 LLVM_DEBUG(dbgs() << '[' << getName() << "] Call " << Caller->getName()
1155 << "->" << Func->getName() << '\n');
1156 (void)Func;
1157
1158 const auto *CallerAA = A.getAAFor<AAAMDWavesPerEU>(
1159 QueryingAA: *this, IRP: IRPosition::function(F: *Caller), DepClass: DepClassTy::REQUIRED);
1160 if (!CallerAA || !CallerAA->isValidState())
1161 return false;
1162
1163 ConstantRange Assumed = getAssumed();
1164 unsigned Min = std::max(a: Assumed.getLower().getZExtValue(),
1165 b: CallerAA->getAssumed().getLower().getZExtValue());
1166 unsigned Max = std::max(a: Assumed.getUpper().getZExtValue(),
1167 b: CallerAA->getAssumed().getUpper().getZExtValue());
1168 ConstantRange Range(APInt(32, Min), APInt(32, Max));
1169 IntegerRangeState RangeState(Range);
1170 getState() = RangeState;
1171 Change |= getState() == Assumed ? ChangeStatus::UNCHANGED
1172 : ChangeStatus::CHANGED;
1173
1174 return true;
1175 };
1176
1177 bool AllCallSitesKnown = true;
1178 if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this, RequireAllCallSites: true, UsedAssumedInformation&: AllCallSitesKnown))
1179 return indicatePessimisticFixpoint();
1180
1181 return Change;
1182 }
1183
1184 /// Create an abstract attribute view for the position \p IRP.
1185 static AAAMDWavesPerEU &createForPosition(const IRPosition &IRP,
1186 Attributor &A);
1187
1188 ChangeStatus manifest(Attributor &A) override {
1189 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
1190 return emitAttributeIfNotDefaultAfterClamp(
1191 A, Default: {1U, InfoCache.getMaxWavesPerEU()});
1192 }
1193
1194 StringRef getName() const override { return "AAAMDWavesPerEU"; }
1195
1196 const char *getIdAddr() const override { return &ID; }
1197
1198 /// This function should return true if the type of the \p AA is
1199 /// AAAMDWavesPerEU
1200 static bool classof(const AbstractAttribute *AA) {
1201 return (AA->getIdAddr() == &ID);
1202 }
1203
1204 /// Unique ID (due to the unique address)
1205 static const char ID;
1206};
1207
1208const char AAAMDWavesPerEU::ID = 0;
1209
1210AAAMDWavesPerEU &AAAMDWavesPerEU::createForPosition(const IRPosition &IRP,
1211 Attributor &A) {
1212 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
1213 return *new (A.Allocator) AAAMDWavesPerEU(IRP, A);
1214 llvm_unreachable("AAAMDWavesPerEU is only valid for function position");
1215}
1216
1217/// Compute the minimum number of AGPRs required to allocate the inline asm.
1218static unsigned inlineAsmGetNumRequiredAGPRs(const InlineAsm *IA,
1219 const CallBase &Call) {
1220 unsigned ArgNo = 0;
1221 unsigned ResNo = 0;
1222 unsigned AGPRDefCount = 0;
1223 unsigned AGPRUseCount = 0;
1224 unsigned MaxPhysReg = 0;
1225 const DataLayout &DL = Call.getFunction()->getDataLayout();
1226
1227 // TODO: Overestimates due to not accounting for tied operands
1228 for (const InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
1229 Type *Ty = nullptr;
1230 switch (CI.Type) {
1231 case InlineAsm::isOutput: {
1232 Ty = Call.getType();
1233 if (auto *STy = dyn_cast<StructType>(Val: Ty))
1234 Ty = STy->getElementType(N: ResNo);
1235 ++ResNo;
1236 break;
1237 }
1238 case InlineAsm::isInput: {
1239 Ty = Call.getArgOperand(i: ArgNo++)->getType();
1240 break;
1241 }
1242 case InlineAsm::isLabel:
1243 continue;
1244 case InlineAsm::isClobber:
1245 // Parse the physical register reference.
1246 break;
1247 }
1248
1249 for (StringRef Code : CI.Codes) {
1250 unsigned RegCount = 0;
1251 if (Code.starts_with(Prefix: "a")) {
1252 // Virtual register, compute number of registers based on the type.
1253 //
1254 // We ought to be going through TargetLowering to get the number of
1255 // registers, but we should avoid the dependence on CodeGen here.
1256 RegCount = divideCeil(Numerator: DL.getTypeSizeInBits(Ty), Denominator: 32);
1257 } else {
1258 // Physical register reference
1259 auto [Kind, RegIdx, NumRegs] = AMDGPU::parseAsmConstraintPhysReg(Constraint: Code);
1260 if (Kind == 'a') {
1261 RegCount = NumRegs;
1262 MaxPhysReg = std::max(a: MaxPhysReg, b: std::min(a: RegIdx + NumRegs, b: 256u));
1263 }
1264
1265 continue;
1266 }
1267
1268 if (CI.Type == InlineAsm::isOutput) {
1269 // Apply tuple alignment requirement
1270 //
1271 // TODO: This is more conservative than necessary.
1272 AGPRDefCount = alignTo(Value: AGPRDefCount, Align: RegCount);
1273
1274 AGPRDefCount += RegCount;
1275 if (CI.isEarlyClobber) {
1276 AGPRUseCount = alignTo(Value: AGPRUseCount, Align: RegCount);
1277 AGPRUseCount += RegCount;
1278 }
1279 } else {
1280 AGPRUseCount = alignTo(Value: AGPRUseCount, Align: RegCount);
1281 AGPRUseCount += RegCount;
1282 }
1283 }
1284 }
1285
1286 unsigned MaxVirtReg = std::max(a: AGPRUseCount, b: AGPRDefCount);
1287
1288 // TODO: This is overly conservative. If there are any physical registers,
1289 // allocate any virtual registers after them so we don't have to solve optimal
1290 // packing.
1291 return std::min(a: MaxVirtReg + MaxPhysReg, b: 256u);
1292}
1293
1294/// Call \p Pred on each function that \p CB may call. Return false if a callee
1295/// is unknown, or if \p Pred returns false for a callee.
1296static bool
1297checkForAllPossibleCallees(Attributor &A, const AbstractAttribute &QueryingAA,
1298 const CallBase &CB,
1299 function_ref<bool(const Function &)> Pred) {
1300 // A direct call without callbacks only calls its callee, so it does not need
1301 // a call site AACallEdges.
1302 auto *Callee = dyn_cast<Function>(Val: CB.getCalledOperand());
1303 if (Callee && !Callee->hasMetadata(KindID: LLVMContext::MD_callback))
1304 return Pred(*Callee);
1305
1306 const AACallEdges *CBEdges = A.getAAFor<AACallEdges>(
1307 QueryingAA, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::REQUIRED);
1308 if (!CBEdges || CBEdges->hasUnknownCallee())
1309 return false;
1310 return all_of(Range: CBEdges->getOptimisticEdges(),
1311 P: [&](const Function *Fn) { return Pred(*Fn); });
1312}
1313
1314struct AAAMDGPUMinAGPRAlloc
1315 : public StateWrapper<DecIntegerState<>, AbstractAttribute> {
1316 using Base = StateWrapper<DecIntegerState<>, AbstractAttribute>;
1317 AAAMDGPUMinAGPRAlloc(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
1318
1319 static AAAMDGPUMinAGPRAlloc &createForPosition(const IRPosition &IRP,
1320 Attributor &A) {
1321 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
1322 return *new (A.Allocator) AAAMDGPUMinAGPRAlloc(IRP, A);
1323 llvm_unreachable(
1324 "AAAMDGPUMinAGPRAlloc is only valid for function position");
1325 }
1326
1327 void initialize(Attributor &A) override {
1328 Function *F = getAssociatedFunction();
1329 auto [MinNumAGPR, MaxNumAGPR] =
1330 AMDGPU::getIntegerPairAttribute(F: *F, Name: "amdgpu-agpr-alloc", Default: {~0u, ~0u},
1331 /*OnlyFirstRequired=*/true);
1332 if (MinNumAGPR == 0) {
1333 indicateOptimisticFixpoint();
1334 return;
1335 }
1336
1337 if (hasSanitizerAttributes(F: *F))
1338 indicatePessimisticFixpoint();
1339 }
1340
1341 const std::string getAsStr(Attributor *A) const override {
1342 std::string Str = "amdgpu-agpr-alloc=";
1343 raw_string_ostream OS(Str);
1344 OS << getAssumed();
1345 return OS.str();
1346 }
1347
1348 void trackStatistics() const override {}
1349
1350 ChangeStatus updateImpl(Attributor &A) override {
1351 DecIntegerState<> Maximum;
1352
1353 // Check for cases which require allocation of AGPRs. The only cases where
1354 // AGPRs are required are if there are direct references to AGPRs, so inline
1355 // assembly and special intrinsics.
1356 auto CheckForMinAGPRAllocs = [&](Instruction &I) {
1357 const auto &CB = cast<CallBase>(Val&: I);
1358 const Value *CalleeOp = CB.getCalledOperand();
1359
1360 if (const InlineAsm *IA = dyn_cast<InlineAsm>(Val: CalleeOp)) {
1361 // Technically, the inline asm could be invoking a call to an unknown
1362 // external function that requires AGPRs, but ignore that.
1363 unsigned NumRegs = inlineAsmGetNumRequiredAGPRs(IA, Call: CB);
1364 Maximum.takeAssumedMaximum(Value: NumRegs);
1365 return true;
1366 }
1367 switch (CB.getIntrinsicID()) {
1368 case Intrinsic::not_intrinsic:
1369 break;
1370 case Intrinsic::write_register:
1371 case Intrinsic::read_register:
1372 case Intrinsic::read_volatile_register: {
1373 const MDString *RegName = cast<MDString>(
1374 Val: cast<MDNode>(
1375 Val: cast<MetadataAsValue>(Val: CB.getArgOperand(i: 0))->getMetadata())
1376 ->getOperand(I: 0));
1377 auto [Kind, RegIdx, NumRegs] =
1378 AMDGPU::parseAsmPhysRegName(TupleString: RegName->getString());
1379 if (Kind == 'a')
1380 Maximum.takeAssumedMaximum(Value: std::min(a: RegIdx + NumRegs, b: 256u));
1381
1382 return true;
1383 }
1384 // Trap-like intrinsics such as llvm.trap and llvm.debugtrap do not have
1385 // the nocallback attribute, so the AMDGPU attributor can conservatively
1386 // drop all implicitly-known inputs and AGPR allocation information. Make
1387 // sure we still infer that no implicit inputs are required and that the
1388 // AGPR allocation stays at zero. Trap-like intrinsics may invoke a
1389 // function which requires AGPRs, so we need to check if the called
1390 // function has the "trap-func-name" attribute.
1391 case Intrinsic::trap:
1392 case Intrinsic::debugtrap:
1393 case Intrinsic::ubsantrap:
1394 return CB.hasFnAttr(Kind: Attribute::NoCallback) ||
1395 !CB.hasFnAttr(Kind: "trap-func-name");
1396 default:
1397 // Some intrinsics may use AGPRs, but if we have a choice, we are not
1398 // required to use AGPRs.
1399 // Assume !nocallback intrinsics may call a function which requires
1400 // AGPRs.
1401 return CB.hasFnAttr(Kind: Attribute::NoCallback);
1402 }
1403
1404 // TODO: Handle callsite attributes
1405 auto CheckCallee = [&](const Function &Callee) {
1406 const AAAMDGPUMinAGPRAlloc *CalleeInfo =
1407 A.getAAFor<AAAMDGPUMinAGPRAlloc>(
1408 QueryingAA: *this, IRP: IRPosition::function(F: Callee), DepClass: DepClassTy::REQUIRED);
1409 if (!CalleeInfo || !CalleeInfo->isValidState())
1410 return false;
1411 Maximum.takeAssumedMaximum(Value: CalleeInfo->getAssumed());
1412 return true;
1413 };
1414 if (!checkForAllPossibleCallees(A, QueryingAA: *this, CB, Pred: CheckCallee)) {
1415 Maximum.indicatePessimisticFixpoint();
1416 return false;
1417 }
1418
1419 return true;
1420 };
1421
1422 bool UsedAssumedInformation = false;
1423 if (!A.checkForAllCallLikeInstructions(Pred: CheckForMinAGPRAllocs, QueryingAA: *this,
1424 UsedAssumedInformation))
1425 return indicatePessimisticFixpoint();
1426
1427 return clampStateAndIndicateChange(S&: getState(), R: Maximum);
1428 }
1429
1430 ChangeStatus manifest(Attributor &A) override {
1431 LLVMContext &Ctx = getAssociatedFunction()->getContext();
1432 SmallString<4> Buffer;
1433 raw_svector_ostream OS(Buffer);
1434 OS << getAssumed();
1435
1436 return A.manifestAttrs(
1437 IRP: getIRPosition(), DeducedAttrs: {Attribute::get(Context&: Ctx, Kind: "amdgpu-agpr-alloc", Val: OS.str())});
1438 }
1439
1440 StringRef getName() const override { return "AAAMDGPUMinAGPRAlloc"; }
1441 const char *getIdAddr() const override { return &ID; }
1442
1443 /// This function should return true if the type of the \p AA is
1444 /// AAAMDGPUMinAGPRAllocs
1445 static bool classof(const AbstractAttribute *AA) {
1446 return (AA->getIdAddr() == &ID);
1447 }
1448
1449 static const char ID;
1450};
1451
1452const char AAAMDGPUMinAGPRAlloc::ID = 0;
1453
1454/// Deduce the function attribute "amdgpu-no-lds-dma"
1455struct AAAMDGPUNoLDSDMA : public StateWrapper<BooleanState, AbstractAttribute> {
1456 using Base = StateWrapper<BooleanState, AbstractAttribute>;
1457 AAAMDGPUNoLDSDMA(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
1458
1459 /// Create an abstract attribute view for the position \p IRP.
1460 static AAAMDGPUNoLDSDMA &createForPosition(const IRPosition &IRP,
1461 Attributor &A) {
1462 assert(IRP.getPositionKind() == IRPosition::IRP_FUNCTION &&
1463 "AAAMDGPUNoLDSDMA is only valid for function position");
1464 return *new (A.Allocator) AAAMDGPUNoLDSDMA(IRP, A);
1465 }
1466
1467 void initialize(Attributor &A) override {
1468 const Function *F = getAssociatedFunction();
1469 if (F->hasFnAttribute(Kind: "amdgpu-no-lds-dma")) {
1470 indicateOptimisticFixpoint();
1471 return;
1472 }
1473
1474 if (F->isDeclaration())
1475 indicatePessimisticFixpoint();
1476 }
1477
1478 ChangeStatus updateImpl(Attributor &A) override {
1479 auto CheckNoLDSDMA = [&](Instruction &I) {
1480 const auto &CB = cast<CallBase>(Val&: I);
1481
1482 // Inline assembly may hold any instruction, including an LDSDMA
1483 // operation.
1484 if (isa<InlineAsm>(Val: CB.getCalledOperand()))
1485 return false;
1486
1487 Intrinsic::ID IID = CB.getIntrinsicID();
1488 if (IID != Intrinsic::not_intrinsic) {
1489 // Assume !nocallback intrinsics may call a function which issues an
1490 // LDSDMA operation.
1491 return !AMDGPU::isLDSDMAIntrinsic(IntrID: IID) &&
1492 CB.hasFnAttr(Kind: Attribute::NoCallback);
1493 }
1494
1495 return checkForAllPossibleCallees(
1496 A, QueryingAA: *this, CB, Pred: [&](const Function &Callee) {
1497 const AAAMDGPUNoLDSDMA *CalleeInfo = A.getAAFor<AAAMDGPUNoLDSDMA>(
1498 QueryingAA: *this, IRP: IRPosition::function(F: Callee), DepClass: DepClassTy::REQUIRED);
1499 return CalleeInfo && CalleeInfo->getAssumed();
1500 });
1501 };
1502
1503 bool UsedAssumedInformation = false;
1504 if (!A.checkForAllCallLikeInstructions(Pred: CheckNoLDSDMA, QueryingAA: *this,
1505 UsedAssumedInformation))
1506 return indicatePessimisticFixpoint();
1507
1508 return ChangeStatus::UNCHANGED;
1509 }
1510
1511 ChangeStatus manifest(Attributor &A) override {
1512 if (!getAssumed())
1513 return ChangeStatus::UNCHANGED;
1514
1515 LLVMContext &Ctx = getAssociatedFunction()->getContext();
1516 return A.manifestAttrs(IRP: getIRPosition(),
1517 DeducedAttrs: {Attribute::get(Context&: Ctx, Kind: "amdgpu-no-lds-dma")});
1518 }
1519
1520 bool isValidState() const override { return true; }
1521
1522 const std::string getAsStr(Attributor *A) const override {
1523 return "AMDGPUNoLDSDMA[" + std::to_string(val: getAssumed()) + "]";
1524 }
1525
1526 void trackStatistics() const override {}
1527
1528 StringRef getName() const override { return "AAAMDGPUNoLDSDMA"; }
1529 const char *getIdAddr() const override { return &ID; }
1530
1531 /// This function should return true if the type of the \p AA is
1532 /// AAAMDGPUNoLDSDMA
1533 static bool classof(const AbstractAttribute *AA) {
1534 return (AA->getIdAddr() == &ID);
1535 }
1536
1537 static const char ID;
1538};
1539
1540const char AAAMDGPUNoLDSDMA::ID = 0;
1541
1542/// An abstract attribute to propagate the function attribute
1543/// "amdgpu-cluster-dims" from kernel entry functions to device functions.
1544struct AAAMDGPUClusterDims
1545 : public StateWrapper<BooleanState, AbstractAttribute> {
1546 using Base = StateWrapper<BooleanState, AbstractAttribute>;
1547 AAAMDGPUClusterDims(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
1548
1549 /// Create an abstract attribute view for the position \p IRP.
1550 static AAAMDGPUClusterDims &createForPosition(const IRPosition &IRP,
1551 Attributor &A);
1552
1553 StringRef getName() const override { return "AAAMDGPUClusterDims"; }
1554
1555 const char *getIdAddr() const override { return &ID; }
1556
1557 /// This function should return true if the type of the \p AA is
1558 /// AAAMDGPUClusterDims.
1559 static bool classof(const AbstractAttribute *AA) {
1560 return AA->getIdAddr() == &ID;
1561 }
1562
1563 virtual const AMDGPU::ClusterDimsAttr &getClusterDims() const = 0;
1564
1565 /// Unique ID (due to the unique address)
1566 static const char ID;
1567};
1568
1569const char AAAMDGPUClusterDims::ID = 0;
1570
1571struct AAAMDGPUClusterDimsFunction : public AAAMDGPUClusterDims {
1572 AAAMDGPUClusterDimsFunction(const IRPosition &IRP, Attributor &A)
1573 : AAAMDGPUClusterDims(IRP, A) {}
1574
1575 void initialize(Attributor &A) override {
1576 Function *F = getAssociatedFunction();
1577 assert(F && "empty associated function");
1578
1579 Attr = AMDGPU::ClusterDimsAttr::get(F: *F);
1580
1581 // No matter what a kernel function has, it is final.
1582 if (AMDGPU::isEntryFunctionCC(CC: F->getCallingConv())) {
1583 if (Attr.isUnknown())
1584 indicatePessimisticFixpoint();
1585 else
1586 indicateOptimisticFixpoint();
1587 }
1588 }
1589
1590 const std::string getAsStr(Attributor *A) const override {
1591 if (!getAssumed() || Attr.isUnknown())
1592 return "unknown";
1593 if (Attr.isNoCluster())
1594 return "no";
1595 if (Attr.isVariableDims())
1596 return "variable";
1597 return Attr.to_string();
1598 }
1599
1600 void trackStatistics() const override {}
1601
1602 ChangeStatus updateImpl(Attributor &A) override {
1603 auto OldState = Attr;
1604
1605 auto CheckCallSite = [&](AbstractCallSite CS) {
1606 const auto *CallerAA = A.getAAFor<AAAMDGPUClusterDims>(
1607 QueryingAA: *this, IRP: IRPosition::function(F: *CS.getInstruction()->getFunction()),
1608 DepClass: DepClassTy::REQUIRED);
1609 if (!CallerAA || !CallerAA->isValidState())
1610 return false;
1611
1612 return merge(Other: CallerAA->getClusterDims());
1613 };
1614
1615 bool UsedAssumedInformation = false;
1616 if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this,
1617 /*RequireAllCallSites=*/true,
1618 UsedAssumedInformation))
1619 return indicatePessimisticFixpoint();
1620
1621 return OldState == Attr ? ChangeStatus::UNCHANGED : ChangeStatus::CHANGED;
1622 }
1623
1624 ChangeStatus manifest(Attributor &A) override {
1625 if (Attr.isUnknown())
1626 return ChangeStatus::UNCHANGED;
1627 return A.manifestAttrs(
1628 IRP: getIRPosition(),
1629 DeducedAttrs: {Attribute::get(Context&: getAssociatedFunction()->getContext(), Kind: AttrName,
1630 Val: Attr.to_string())},
1631 /*ForceReplace=*/true);
1632 }
1633
1634 const AMDGPU::ClusterDimsAttr &getClusterDims() const override {
1635 return Attr;
1636 }
1637
1638private:
1639 bool merge(const AMDGPU::ClusterDimsAttr &Other) {
1640 // Case 1: Both of them are unknown yet, we do nothing and continue wait for
1641 // propagation.
1642 if (Attr.isUnknown() && Other.isUnknown())
1643 return true;
1644
1645 // Case 2: The other is determined, but we are unknown yet, we simply take
1646 // the other's value.
1647 if (Attr.isUnknown()) {
1648 Attr = Other;
1649 return true;
1650 }
1651
1652 // Case 3: We are determined but the other is unknown yet, we simply keep
1653 // everything unchanged.
1654 if (Other.isUnknown())
1655 return true;
1656
1657 // After this point, both are determined.
1658
1659 // Case 4: If they are same, we do nothing.
1660 if (Attr == Other)
1661 return true;
1662
1663 // Now they are not same.
1664
1665 // Case 5: If either of us uses cluster (but not both; otherwise case 4
1666 // would hold), then it is unknown whether cluster will be used, and the
1667 // state is final, unlike case 1.
1668 if (Attr.isNoCluster() || Other.isNoCluster()) {
1669 Attr.setUnknown();
1670 return false;
1671 }
1672
1673 // Case 6: Both of us use cluster, but the dims are different, so the result
1674 // is, cluster is used, but we just don't have a fixed dims.
1675 Attr.setVariableDims();
1676 return true;
1677 }
1678
1679 AMDGPU::ClusterDimsAttr Attr;
1680
1681 static constexpr char AttrName[] = "amdgpu-cluster-dims";
1682};
1683
1684AAAMDGPUClusterDims &
1685AAAMDGPUClusterDims::createForPosition(const IRPosition &IRP, Attributor &A) {
1686 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
1687 return *new (A.Allocator) AAAMDGPUClusterDimsFunction(IRP, A);
1688 llvm_unreachable("AAAMDGPUClusterDims is only valid for function position");
1689}
1690
1691static bool runImpl(SetVector<Function *> &Functions, bool IsModulePass,
1692 bool DeleteFns, Module &M, AnalysisGetter &AG,
1693 TargetMachine &TM, AMDGPUAttributorOptions Options,
1694 ThinOrFullLTOPhase LTOPhase) {
1695
1696 CallGraphUpdater CGUpdater;
1697 BumpPtrAllocator Allocator;
1698 AMDGPUInformationCache InfoCache(M, AG, Allocator, nullptr, TM);
1699 static constexpr AAKindSet Allowed =
1700 AAKindSet::get<AAAMDAttributes, AAUniformWorkGroupSize, AAPotentialValues,
1701 AAAMDFlatWorkGroupSize, AAAMDMaxNumWorkgroups,
1702 AAAMDWavesPerEU, AAAMDGPUMinAGPRAlloc, AACallEdges,
1703 AAPointerInfo, AAPotentialConstantValues,
1704 AAUnderlyingObjects, AANoAliasAddrSpace, AAAddressSpace,
1705 AAIndirectCallInfo, AAAMDGPUClusterDims, AAAMDGPUNoLDSDMA,
1706 AAAlign>();
1707
1708 AttributorConfig AC(CGUpdater);
1709 AC.IsClosedWorldModule = Options.IsClosedWorld;
1710 AC.Allowed = Allowed;
1711 AC.IsModulePass = IsModulePass;
1712 AC.DeleteFns = DeleteFns;
1713 AC.DefaultInitializeLiveInternals = false;
1714 // AAIsDead is not allowed, so nothing is ever assumed dead.
1715 AC.UseLiveness = false;
1716 unsigned IndirectCallSpecializationThreshold =
1717 static_cast<const AMDGPUTargetMachine &>(TM)
1718 .getCLOpts()
1719 .indirect_call_specialization_threshold;
1720 AC.IndirectCalleeSpecializationCallback =
1721 [IndirectCallSpecializationThreshold](
1722 Attributor &A, const AbstractAttribute &AA, CallBase &CB,
1723 Function &Callee, unsigned NumAssumedCallees) {
1724 return !AMDGPU::isEntryFunctionCC(CC: Callee.getCallingConv()) &&
1725 (NumAssumedCallees <= IndirectCallSpecializationThreshold);
1726 };
1727 AC.IPOAmendableCB = [](const Function &F) {
1728 return F.getCallingConv() == CallingConv::AMDGPU_KERNEL;
1729 };
1730
1731 Attributor A(Functions, InfoCache, AC);
1732
1733 LLVM_DEBUG({
1734 StringRef LTOPhaseStr = to_string(LTOPhase);
1735 dbgs() << "[AMDGPUAttributor] Running at phase " << LTOPhaseStr << '\n'
1736 << "[AMDGPUAttributor] Module " << M.getName() << " is "
1737 << (AC.IsClosedWorldModule ? "" : "not ")
1738 << "assumed to be a closed world.\n";
1739 });
1740
1741 for (auto *F : Functions) {
1742 A.getOrCreateAAFor<AAAMDAttributes>(IRP: IRPosition::function(F: *F));
1743 A.getOrCreateAAFor<AAUniformWorkGroupSize>(IRP: IRPosition::function(F: *F));
1744 A.getOrCreateAAFor<AAAMDMaxNumWorkgroups>(IRP: IRPosition::function(F: *F));
1745 A.getOrCreateAAFor<AAAMDGPUNoLDSDMA>(IRP: IRPosition::function(F: *F));
1746 CallingConv::ID CC = F->getCallingConv();
1747 if (!AMDGPU::isEntryFunctionCC(CC)) {
1748 A.getOrCreateAAFor<AAAMDFlatWorkGroupSize>(IRP: IRPosition::function(F: *F));
1749 A.getOrCreateAAFor<AAAMDWavesPerEU>(IRP: IRPosition::function(F: *F));
1750 }
1751
1752 const AMDGPU::AMDGPUFeatureBitset &Features = InfoCache.getFeatures();
1753 if (!F->isDeclaration() && Features.test(I: AMDGPU::FEAT_CLUSTERS))
1754 A.getOrCreateAAFor<AAAMDGPUClusterDims>(IRP: IRPosition::function(F: *F));
1755
1756 if (Features.test(I: AMDGPU::FEAT_AGPR_ALLOC))
1757 A.getOrCreateAAFor<AAAMDGPUMinAGPRAlloc>(IRP: IRPosition::function(F: *F));
1758
1759 for (auto &I : instructions(F)) {
1760 Value *Ptr = nullptr;
1761 if (auto *LI = dyn_cast<LoadInst>(Val: &I))
1762 Ptr = LI->getPointerOperand();
1763 else if (auto *SI = dyn_cast<StoreInst>(Val: &I))
1764 Ptr = SI->getPointerOperand();
1765 else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: &I))
1766 Ptr = RMW->getPointerOperand();
1767 else if (auto *CmpX = dyn_cast<AtomicCmpXchgInst>(Val: &I))
1768 Ptr = CmpX->getPointerOperand();
1769
1770 if (Ptr) {
1771 // Only flat pointers can be refined to a specific address space.
1772 if (Ptr->getType()->getPointerAddressSpace() ==
1773 AMDGPUAS::FLAT_ADDRESS) {
1774 A.getOrCreateAAFor<AAAddressSpace>(IRP: IRPosition::value(V: *Ptr));
1775 A.getOrCreateAAFor<AANoAliasAddrSpace>(IRP: IRPosition::value(V: *Ptr));
1776 }
1777 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: Ptr)) {
1778 if (II->getIntrinsicID() == Intrinsic::amdgcn_make_buffer_rsrc)
1779 A.getOrCreateAAFor<AAAlign>(IRP: IRPosition::value(V: *Ptr));
1780 }
1781 }
1782 }
1783 }
1784
1785 return A.run() == ChangeStatus::CHANGED;
1786}
1787} // namespace
1788
1789PreservedAnalyses llvm::AMDGPUAttributorPass::run(Module &M,
1790 ModuleAnalysisManager &AM) {
1791
1792 FunctionAnalysisManager &FAM =
1793 AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
1794 AnalysisGetter AG(FAM);
1795
1796 SetVector<Function *> Functions;
1797 for (Function &F : M) {
1798 if (!F.isDeclaration())
1799 Functions.insert(X: &F);
1800 }
1801
1802 // TODO: Probably preserves CFG
1803 return runImpl(Functions, /*IsModulePass=*/true, /*DeleteFns=*/true, M, AG,
1804 TM, Options, LTOPhase)
1805 ? PreservedAnalyses::none()
1806 : PreservedAnalyses::all();
1807}
1808
1809PreservedAnalyses llvm::AMDGPUAttributorCGSCCPass::run(LazyCallGraph::SCC &C,
1810 CGSCCAnalysisManager &AM,
1811 LazyCallGraph &CG,
1812 CGSCCUpdateResult &UR) {
1813
1814 FunctionAnalysisManager &FAM =
1815 AM.getResult<FunctionAnalysisManagerCGSCCProxy>(IR&: C, ExtraArgs&: CG).getManager();
1816 AnalysisGetter AG(FAM);
1817
1818 SetVector<Function *> Functions;
1819 for (LazyCallGraph::Node &N : C) {
1820 Function *F = &N.getFunction();
1821 if (!F->isIntrinsic())
1822 Functions.insert(X: F);
1823 }
1824
1825 AMDGPUAttributorOptions Options;
1826 Module *M = C.begin()->getFunction().getParent();
1827 // In the CGSCC pipeline, avoid untracked call graph modifications by
1828 // disabling function deletion, mirroring the generic AttributorCGSCCPass.
1829 return runImpl(Functions, /*IsModulePass=*/false, /*DeleteFns=*/false, M&: *M, AG,
1830 TM, Options, LTOPhase: ThinOrFullLTOPhase::None)
1831 ? PreservedAnalyses::none()
1832 : PreservedAnalyses::all();
1833}
1834