1//===- AMDGPUAttributor.cpp -----------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This pass uses Attributor framework to deduce AMDGPU attributes.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AMDGPU.h"
14#include "AMDGPUTargetMachine.h"
15#include "GCNSubtarget.h"
16#include "Utils/AMDGPUBaseInfo.h"
17#include "llvm/IR/IntrinsicsAMDGPU.h"
18#include "llvm/IR/IntrinsicsR600.h"
19#include "llvm/Target/TargetMachine.h"
20#include "llvm/Transforms/IPO/Attributor.h"
21#include <cstdint>
22
23#define DEBUG_TYPE "amdgpu-attributor"
24
25using namespace llvm;
26
27static cl::opt<unsigned> IndirectCallSpecializationThreshold(
28 "amdgpu-indirect-call-specialization-threshold",
29 cl::desc(
30 "A threshold controls whether an indirect call will be specialized"),
31 cl::init(Val: 3));
32
33#define AMDGPU_ATTRIBUTE(Name, Str) Name##_POS,
34
35enum ImplicitArgumentPositions {
36#include "AMDGPUAttributes.def"
37 LAST_ARG_POS
38};
39
40#define AMDGPU_ATTRIBUTE(Name, Str) Name = 1 << Name##_POS,
41
42enum ImplicitArgumentMask {
43 UNKNOWN_INTRINSIC = 0,
44#include "AMDGPUAttributes.def"
45 ALL_ARGUMENT_MASK = (1 << LAST_ARG_POS) - 1,
46 NOT_IMPLICIT_INPUT
47};
48
49#define AMDGPU_ATTRIBUTE(Name, Str) {Name, Str},
50static constexpr std::pair<ImplicitArgumentMask, StringLiteral>
51 ImplicitAttrs[] = {
52#include "AMDGPUAttributes.def"
53};
54
55// We do not need to note the x workitem or workgroup id because they are always
56// initialized.
57//
58// TODO: We should not add the attributes if the known compile time workgroup
59// size is 1 for y/z.
60static ImplicitArgumentMask
61intrinsicToAttrMask(Intrinsic::ID ID, bool &NonKernelOnly, bool &NeedsImplicit,
62 bool HasApertureRegs, bool SupportsGetDoorBellID,
63 unsigned CodeObjectVersion) {
64 switch (ID) {
65 case Intrinsic::amdgcn_workitem_id_x:
66 NonKernelOnly = true;
67 return WORKITEM_ID_X;
68 case Intrinsic::amdgcn_workgroup_id_x:
69 NonKernelOnly = true;
70 return WORKGROUP_ID_X;
71 case Intrinsic::amdgcn_workitem_id_y:
72 case Intrinsic::r600_read_tidig_y:
73 return WORKITEM_ID_Y;
74 case Intrinsic::amdgcn_workitem_id_z:
75 case Intrinsic::r600_read_tidig_z:
76 return WORKITEM_ID_Z;
77 case Intrinsic::amdgcn_workgroup_id_y:
78 case Intrinsic::r600_read_tgid_y:
79 return WORKGROUP_ID_Y;
80 case Intrinsic::amdgcn_workgroup_id_z:
81 case Intrinsic::r600_read_tgid_z:
82 return WORKGROUP_ID_Z;
83 case Intrinsic::amdgcn_cluster_id_x:
84 NonKernelOnly = true;
85 return CLUSTER_ID_X;
86 case Intrinsic::amdgcn_cluster_id_y:
87 return CLUSTER_ID_Y;
88 case Intrinsic::amdgcn_cluster_id_z:
89 return CLUSTER_ID_Z;
90 case Intrinsic::amdgcn_lds_kernel_id:
91 return LDS_KERNEL_ID;
92 case Intrinsic::amdgcn_dispatch_ptr:
93 return DISPATCH_PTR;
94 case Intrinsic::amdgcn_dispatch_id:
95 return DISPATCH_ID;
96 case Intrinsic::amdgcn_implicitarg_ptr:
97 return IMPLICIT_ARG_PTR;
98 // Need queue_ptr anyway. But under V5, we also need implicitarg_ptr to access
99 // queue_ptr.
100 case Intrinsic::amdgcn_queue_ptr:
101 NeedsImplicit = (CodeObjectVersion >= AMDGPU::AMDHSA_COV5);
102 return QUEUE_PTR;
103 case Intrinsic::amdgcn_is_shared:
104 case Intrinsic::amdgcn_is_private:
105 if (HasApertureRegs)
106 return NOT_IMPLICIT_INPUT;
107 // Under V5, we need implicitarg_ptr + offsets to access private_base or
108 // shared_base. For pre-V5, however, need to access them through queue_ptr +
109 // offsets.
110 return CodeObjectVersion >= AMDGPU::AMDHSA_COV5 ? IMPLICIT_ARG_PTR
111 : QUEUE_PTR;
112 case Intrinsic::amdgcn_wwm:
113 case Intrinsic::amdgcn_strict_wwm:
114 return WHOLE_WAVE_MODE;
115 case Intrinsic::trap:
116 case Intrinsic::debugtrap:
117 case Intrinsic::ubsantrap:
118 if (SupportsGetDoorBellID) // GetDoorbellID support implemented since V4.
119 return CodeObjectVersion >= AMDGPU::AMDHSA_COV4 ? NOT_IMPLICIT_INPUT
120 : QUEUE_PTR;
121 NeedsImplicit = (CodeObjectVersion >= AMDGPU::AMDHSA_COV5);
122 return QUEUE_PTR;
123 default:
124 return UNKNOWN_INTRINSIC;
125 }
126}
127
128static bool castRequiresQueuePtr(unsigned SrcAS) {
129 return SrcAS == AMDGPUAS::LOCAL_ADDRESS || SrcAS == AMDGPUAS::PRIVATE_ADDRESS;
130}
131
132static bool isDSAddress(const Constant *C) {
133 const GlobalValue *GV = dyn_cast<GlobalValue>(Val: C);
134 if (!GV)
135 return false;
136 unsigned AS = GV->getAddressSpace();
137 return AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS;
138}
139
140/// Returns true if sanitizer attributes are present on a function.
141static bool hasSanitizerAttributes(const Function &F) {
142 return F.hasFnAttribute(Kind: Attribute::SanitizeAddress) ||
143 F.hasFnAttribute(Kind: Attribute::SanitizeThread) ||
144 F.hasFnAttribute(Kind: Attribute::SanitizeMemory) ||
145 F.hasFnAttribute(Kind: Attribute::SanitizeHWAddress) ||
146 F.hasFnAttribute(Kind: Attribute::SanitizeMemTag);
147}
148
149namespace {
150class AMDGPUInformationCache : public InformationCache {
151public:
152 AMDGPUInformationCache(const Module &M, AnalysisGetter &AG,
153 BumpPtrAllocator &Allocator,
154 SetVector<Function *> *CGSCC, TargetMachine &TM)
155 : InformationCache(M, AG, Allocator, CGSCC), TM(TM),
156 SubArch(M.getTargetTriple().getSubArch()),
157 Features(
158 AMDGPU::getFeatureBitset(AK: AMDGPU::getGPUKindFromSubArch(SubArch))),
159 CodeObjectVersion(AMDGPU::getAMDHSACodeObjectVersion(M)) {
160 LLVMContext &Ctx = M.getContext();
161 for (auto [Mask, Name] : ImplicitAttrs)
162 ImplicitAttrList.push_back(Elt: {Mask, Attribute::get(Context&: Ctx, Kind: Name)});
163 // Sorted by name, the order of attributes in an attribute set.
164 llvm::sort(C&: ImplicitAttrList, Comp: [](const auto &A, const auto &B) {
165 return A.second.getKindAsString() < B.second.getKindAsString();
166 });
167
168 unsigned COV = CodeObjectVersion;
169 ImplicitArgRanges = {
170 {MULTIGRID_SYNC_ARG,
171 {AMDGPU::getMultigridSyncArgImplicitArgPosition(COV), 8}},
172 {HOSTCALL_PTR, {AMDGPU::getHostcallImplicitArgPosition(COV), 8}},
173 {DEFAULT_QUEUE, {AMDGPU::getDefaultQueueImplicitArgPosition(COV), 8}},
174 {COMPLETION_ACTION,
175 {AMDGPU::getCompletionActionImplicitArgPosition(COV), 8}}};
176 if (COV >= 5) {
177 ImplicitArgRanges.push_back(
178 Elt: {HEAP_PTR, {AMDGPU::ImplicitArg::HEAP_PTR_OFFSET, 8}});
179 ImplicitArgRanges.push_back(
180 Elt: {QUEUE_PTR, {AMDGPU::ImplicitArg::QUEUE_PTR_OFFSET, 8}});
181 }
182
183 int64_t Begin = INT64_MAX, End = 0;
184 for (auto [Mask, Range] : ImplicitArgRanges) {
185 ImplicitArgsMask |= Mask;
186 Begin = std::min(a: Begin, b: Range.Offset);
187 End = std::max(a: End, b: Range.Offset + Range.Size);
188 }
189
190 ImplicitArgsRange = AA::RangeTy(Begin, End - Begin);
191 }
192
193 TargetMachine &TM;
194
195 /// The attributes for the implicit arguments that are not needed, sorted by
196 /// name.
197 ArrayRef<std::pair<ImplicitArgumentMask, Attribute>>
198 getImplicitAttrList() const {
199 return ImplicitAttrList;
200 }
201
202 enum ConstantStatus : uint8_t {
203 NONE = 0,
204 DS_GLOBAL = 1 << 0,
205 ADDR_SPACE_CAST_PRIVATE_TO_FLAT = 1 << 1,
206 ADDR_SPACE_CAST_LOCAL_TO_FLAT = 1 << 2,
207 ADDR_SPACE_CAST_BOTH_TO_FLAT =
208 ADDR_SPACE_CAST_PRIVATE_TO_FLAT | ADDR_SPACE_CAST_LOCAL_TO_FLAT,
209 CS_WORST = DS_GLOBAL | ADDR_SPACE_CAST_BOTH_TO_FLAT,
210 };
211
212 std::optional<std::pair<unsigned, unsigned>>
213 getFlatWorkGroupSizeAttr(const Function &F) const {
214 auto R = AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-flat-work-group-size");
215 if (!R)
216 return std::nullopt;
217 return std::make_pair(x&: R->first, y&: *(R->second));
218 }
219
220 std::pair<unsigned, unsigned>
221 getDefaultFlatWorkGroupSize(const Function &F) const {
222 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F);
223 return ST.getDefaultFlatWorkGroupSize(CC: F.getCallingConv());
224 }
225
226 std::pair<unsigned, unsigned> getMaximumFlatWorkGroupRange() const {
227 return {AMDGPU::getMinFlatWorkGroupSize(),
228 AMDGPU::getMaxFlatWorkGroupSize()};
229 }
230
231 /// Get code object version.
232 unsigned getCodeObjectVersion() const { return CodeObjectVersion; }
233
234 /// The implicit kernel arguments at a fixed offset from implicitarg_ptr,
235 /// with their ranges.
236 ArrayRef<std::pair<ImplicitArgumentMask, AA::RangeTy>>
237 getImplicitArgRanges() const {
238 return ImplicitArgRanges;
239 }
240
241 /// The mask of the implicit arguments in getImplicitArgRanges().
242 uint32_t getImplicitArgsMask() const { return ImplicitArgsMask; }
243
244 /// The range covering all the implicit arguments in getImplicitArgRanges().
245 AA::RangeTy getImplicitArgsRange() const { return ImplicitArgsRange; }
246
247 /// Get the features of the module target.
248 const AMDGPU::AMDGPUFeatureBitset &getFeatures() const { return Features; }
249
250 std::optional<std::pair<unsigned, unsigned>>
251 getWavesPerEUAttr(const Function &F) {
252 auto Val = AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-waves-per-eu",
253 /*OnlyFirstRequired=*/true);
254 if (!Val)
255 return std::nullopt;
256 if (!Val->second)
257 Val->second = AMDGPU::getMaxWavesPerEU(SubArch);
258 return std::make_pair(x&: Val->first, y&: *(Val->second));
259 }
260
261 unsigned getMaxWavesPerEU() const {
262 return AMDGPU::getMaxWavesPerEU(SubArch);
263 }
264
265 unsigned getMaxAddrSpace() const override {
266 return AMDGPUAS::MAX_AMDGPU_ADDRESS;
267 }
268
269private:
270 /// Check if the ConstantExpr \p CE uses an addrspacecast from private or
271 /// local to flat. These casts may require the queue pointer.
272 static uint8_t visitConstExpr(const ConstantExpr *CE) {
273 uint8_t Status = NONE;
274
275 if (CE->getOpcode() == Instruction::AddrSpaceCast) {
276 unsigned SrcAS = CE->getOperand(i_nocapture: 0)->getType()->getPointerAddressSpace();
277 if (SrcAS == AMDGPUAS::PRIVATE_ADDRESS)
278 Status |= ADDR_SPACE_CAST_PRIVATE_TO_FLAT;
279 else if (SrcAS == AMDGPUAS::LOCAL_ADDRESS)
280 Status |= ADDR_SPACE_CAST_LOCAL_TO_FLAT;
281 }
282
283 return Status;
284 }
285
286 /// Get the constant access bitmap for \p C.
287 uint8_t getConstantAccess(const Constant *C) {
288 if (isa<ConstantData>(Val: C))
289 return NONE;
290
291 const auto &It = ConstantStatus.find(Val: C);
292 if (It != ConstantStatus.end())
293 return It->second.value();
294
295 SmallPtrSet<const Constant *, 8> Visited;
296 SmallVector<const Constant *> Worklist;
297 Worklist.push_back(Elt: C);
298 Visited.insert(Ptr: C);
299
300 uint8_t Result = 0;
301 while (Result != CS_WORST && !Worklist.empty()) {
302 const Constant *CurC = Worklist.pop_back_val();
303
304 std::optional<uint8_t> &CurCResultOrNone = ConstantStatus[CurC];
305 if (CurCResultOrNone) {
306 Result |= CurCResultOrNone.value();
307 continue;
308 }
309 uint8_t CurCResult = 0;
310
311 if (isDSAddress(C: CurC))
312 CurCResult |= DS_GLOBAL;
313
314 if (const auto *CE = dyn_cast<ConstantExpr>(Val: CurC))
315 CurCResult |= visitConstExpr(CE);
316
317 for (const Use &U : CurC->operands()) {
318 if (const auto *OpC = dyn_cast<Constant>(Val: U)) {
319 if (Visited.insert(Ptr: OpC).second)
320 Worklist.push_back(Elt: OpC);
321 }
322 }
323
324 CurCResultOrNone = CurCResult;
325 Result |= CurCResult;
326 }
327
328 ConstantStatus[C] = Result;
329 return Result;
330 }
331
332public:
333 /// Returns true if \p Fn needs the queue pointer because of \p C.
334 bool needsQueuePtr(const Constant *C, Function &Fn) {
335 bool IsNonEntryFunc = !AMDGPU::isEntryFunctionCC(CC: Fn.getCallingConv());
336 bool HasAperture = Features.test(I: AMDGPU::FEAT_APERTURE_REGS);
337
338 // No need to explore the constants.
339 if (!IsNonEntryFunc && HasAperture)
340 return false;
341
342 uint8_t Access = getConstantAccess(C);
343
344 // We need to trap on DS globals in non-entry functions.
345 if (IsNonEntryFunc && (Access & DS_GLOBAL))
346 return true;
347
348 return !HasAperture && (Access & ADDR_SPACE_CAST_BOTH_TO_FLAT);
349 }
350
351 bool checkConstForAddrSpaceCastFromPrivate(const Constant *C) {
352 uint8_t Access = getConstantAccess(C);
353 return Access & ADDR_SPACE_CAST_PRIVATE_TO_FLAT;
354 }
355
356private:
357 /// Used to determine if the Constant needs the queue pointer.
358 DenseMap<const Constant *, std::optional<uint8_t>> ConstantStatus;
359 SmallVector<std::pair<ImplicitArgumentMask, Attribute>, LAST_ARG_POS>
360 ImplicitAttrList;
361 const Triple::SubArchType SubArch;
362 const AMDGPU::AMDGPUFeatureBitset Features;
363 const unsigned CodeObjectVersion;
364 SmallVector<std::pair<ImplicitArgumentMask, AA::RangeTy>, 6>
365 ImplicitArgRanges;
366 uint32_t ImplicitArgsMask = 0;
367 AA::RangeTy ImplicitArgsRange;
368};
369
370struct AAAMDAttributes
371 : public StateWrapper<BitIntegerState<uint32_t, ALL_ARGUMENT_MASK, 0>,
372 AbstractAttribute> {
373 using Base = StateWrapper<BitIntegerState<uint32_t, ALL_ARGUMENT_MASK, 0>,
374 AbstractAttribute>;
375
376 AAAMDAttributes(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
377
378 /// Create an abstract attribute view for the position \p IRP.
379 static AAAMDAttributes &createForPosition(const IRPosition &IRP,
380 Attributor &A);
381
382 /// See AbstractAttribute::getName().
383 StringRef getName() const override { return "AAAMDAttributes"; }
384
385 /// See AbstractAttribute::getIdAddr().
386 const char *getIdAddr() const override { return &ID; }
387
388 /// This function should return true if the type of the \p AA is
389 /// AAAMDAttributes.
390 static bool classof(const AbstractAttribute *AA) {
391 return (AA->getIdAddr() == &ID);
392 }
393
394 /// Unique ID (due to the unique address)
395 static const char ID;
396};
397const char AAAMDAttributes::ID = 0;
398
399struct AAUniformWorkGroupSize
400 : public StateWrapper<BooleanState, AbstractAttribute> {
401 using Base = StateWrapper<BooleanState, AbstractAttribute>;
402 AAUniformWorkGroupSize(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
403
404 /// Create an abstract attribute view for the position \p IRP.
405 static AAUniformWorkGroupSize &createForPosition(const IRPosition &IRP,
406 Attributor &A);
407
408 /// See AbstractAttribute::getName().
409 StringRef getName() const override { return "AAUniformWorkGroupSize"; }
410
411 /// See AbstractAttribute::getIdAddr().
412 const char *getIdAddr() const override { return &ID; }
413
414 /// This function should return true if the type of the \p AA is
415 /// AAAMDAttributes.
416 static bool classof(const AbstractAttribute *AA) {
417 return (AA->getIdAddr() == &ID);
418 }
419
420 /// Unique ID (due to the unique address)
421 static const char ID;
422};
423const char AAUniformWorkGroupSize::ID = 0;
424
425struct AAUniformWorkGroupSizeFunction : public AAUniformWorkGroupSize {
426 AAUniformWorkGroupSizeFunction(const IRPosition &IRP, Attributor &A)
427 : AAUniformWorkGroupSize(IRP, A) {}
428
429 void initialize(Attributor &A) override {
430 Function *F = getAssociatedFunction();
431 CallingConv::ID CC = F->getCallingConv();
432
433 if (CC != CallingConv::AMDGPU_KERNEL)
434 return;
435
436 bool InitialValue = F->hasFnAttribute(Kind: "uniform-work-group-size");
437
438 if (InitialValue)
439 indicateOptimisticFixpoint();
440 else
441 indicatePessimisticFixpoint();
442 }
443
444 ChangeStatus updateImpl(Attributor &A) override {
445 ChangeStatus Change = ChangeStatus::UNCHANGED;
446
447 auto CheckCallSite = [&](AbstractCallSite CS) {
448 Function *Caller = CS.getInstruction()->getFunction();
449 LLVM_DEBUG(dbgs() << "[AAUniformWorkGroupSize] Call " << Caller->getName()
450 << "->" << getAssociatedFunction()->getName() << "\n");
451
452 const auto *CallerInfo = A.getAAFor<AAUniformWorkGroupSize>(
453 QueryingAA: *this, IRP: IRPosition::function(F: *Caller), DepClass: DepClassTy::REQUIRED);
454 if (!CallerInfo || !CallerInfo->isValidState())
455 return false;
456
457 Change = Change | clampStateAndIndicateChange(S&: this->getState(),
458 R: CallerInfo->getState());
459
460 return true;
461 };
462
463 bool AllCallSitesKnown = true;
464 if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this, RequireAllCallSites: true, UsedAssumedInformation&: AllCallSitesKnown))
465 return indicatePessimisticFixpoint();
466
467 return Change;
468 }
469
470 ChangeStatus manifest(Attributor &A) override {
471 if (!getAssumed())
472 return ChangeStatus::UNCHANGED;
473
474 LLVMContext &Ctx = getAssociatedFunction()->getContext();
475 return A.manifestAttrs(IRP: getIRPosition(),
476 DeducedAttrs: {Attribute::get(Context&: Ctx, Kind: "uniform-work-group-size")},
477 /*ForceReplace=*/true);
478 }
479
480 bool isValidState() const override {
481 // This state is always valid, even when the state is false.
482 return true;
483 }
484
485 const std::string getAsStr(Attributor *) const override {
486 return "AMDWorkGroupSize[" + std::to_string(val: getAssumed()) + "]";
487 }
488
489 /// See AbstractAttribute::trackStatistics()
490 void trackStatistics() const override {}
491};
492
493AAUniformWorkGroupSize &
494AAUniformWorkGroupSize::createForPosition(const IRPosition &IRP,
495 Attributor &A) {
496 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
497 return *new (A.Allocator) AAUniformWorkGroupSizeFunction(IRP, A);
498 llvm_unreachable(
499 "AAUniformWorkGroupSize is only valid for function position");
500}
501
502struct AAAMDAttributesFunction : public AAAMDAttributes {
503 AAAMDAttributesFunction(const IRPosition &IRP, Attributor &A)
504 : AAAMDAttributes(IRP, A) {}
505
506 void initialize(Attributor &A) override {
507 Function *F = getAssociatedFunction();
508
509 // If the function requires the implicit arg pointer due to sanitizers,
510 // assume it's needed even if explicitly marked as not requiring it.
511 // Flat scratch initialization is needed because `asan_malloc_impl`
512 // calls introduced later in pipeline will have flat scratch accesses.
513 // FIXME: FLAT_SCRATCH_INIT will not be required here if device-libs
514 // implementation for `asan_malloc_impl` is updated.
515 const bool HasSanitizerAttrs = hasSanitizerAttributes(F: *F);
516 if (HasSanitizerAttrs) {
517 removeAssumedBits(BitsEncoding: IMPLICIT_ARG_PTR);
518 removeAssumedBits(BitsEncoding: HOSTCALL_PTR);
519 removeAssumedBits(BitsEncoding: FLAT_SCRATCH_INIT);
520 }
521
522 for (auto Attr : ImplicitAttrs) {
523 if (HasSanitizerAttrs &&
524 (Attr.first == IMPLICIT_ARG_PTR || Attr.first == HOSTCALL_PTR ||
525 Attr.first == FLAT_SCRATCH_INIT))
526 continue;
527
528 if (F->hasFnAttribute(Kind: Attr.second))
529 addKnownBits(Bits: Attr.first);
530 }
531
532 if (F->isDeclaration())
533 return;
534
535 // Ignore functions with graphics calling conventions, these are currently
536 // not allowed to have kernel arguments.
537 if (AMDGPU::isGraphics(CC: F->getCallingConv())) {
538 indicatePessimisticFixpoint();
539 return;
540 }
541 }
542
543 ChangeStatus updateImpl(Attributor &A) override {
544 Function *F = getAssociatedFunction();
545 // The current assumed state used to determine a change.
546 auto OrigAssumed = getAssumed();
547
548 // Check for Intrinsics and propagate attributes.
549 const AACallEdges *AAEdges = A.getAAFor<AACallEdges>(
550 QueryingAA: *this, IRP: this->getIRPosition(), DepClass: DepClassTy::REQUIRED);
551 if (!AAEdges || !AAEdges->isValidState() ||
552 AAEdges->hasNonAsmUnknownCallee())
553 return indicatePessimisticFixpoint();
554
555 bool IsNonEntryFunc = !AMDGPU::isEntryFunctionCC(CC: F->getCallingConv());
556
557 bool NeedsImplicit = false;
558 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
559 const AMDGPU::AMDGPUFeatureBitset &Features = InfoCache.getFeatures();
560 bool HasApertureRegs = Features.test(I: AMDGPU::FEAT_APERTURE_REGS);
561 bool SupportsGetDoorbellID = Features.test(I: AMDGPU::FEAT_GET_DOORBELL_ID);
562 unsigned COV = InfoCache.getCodeObjectVersion();
563
564 for (Function *Callee : AAEdges->getOptimisticEdges()) {
565 Intrinsic::ID IID = Callee->getIntrinsicID();
566 if (IID == Intrinsic::not_intrinsic) {
567 const AAAMDAttributes *AAAMD = A.getAAFor<AAAMDAttributes>(
568 QueryingAA: *this, IRP: IRPosition::function(F: *Callee), DepClass: DepClassTy::REQUIRED);
569 if (!AAAMD || !AAAMD->isValidState())
570 return indicatePessimisticFixpoint();
571 *this &= *AAAMD;
572 continue;
573 }
574
575 bool NonKernelOnly = false;
576 ImplicitArgumentMask AttrMask =
577 intrinsicToAttrMask(ID: IID, NonKernelOnly, NeedsImplicit,
578 HasApertureRegs, SupportsGetDoorBellID: SupportsGetDoorbellID, CodeObjectVersion: COV);
579
580 if (AttrMask == UNKNOWN_INTRINSIC) {
581 // Assume not-nocallback intrinsics may invoke a function which accesses
582 // implicit arguments.
583 //
584 // FIXME: This isn't really the correct check. We want to ensure it
585 // isn't calling any function that may use implicit arguments regardless
586 // of whether it's internal to the module or not.
587 //
588 // TODO: Ignoring callsite attributes.
589 if (!Callee->hasFnAttribute(Kind: Attribute::NoCallback))
590 return indicatePessimisticFixpoint();
591 continue;
592 }
593
594 if (AttrMask != NOT_IMPLICIT_INPUT) {
595 if ((IsNonEntryFunc || !NonKernelOnly))
596 removeAssumedBits(BitsEncoding: AttrMask);
597 }
598 }
599
600 // Need implicitarg_ptr to acess queue_ptr, private_base, and shared_base.
601 if (NeedsImplicit)
602 removeAssumedBits(BitsEncoding: IMPLICIT_ARG_PTR);
603
604 if (isAssumed(BitsEncoding: QUEUE_PTR) && checkForQueuePtr(A)) {
605 // Under V5, we need implicitarg_ptr + offsets to access private_base or
606 // shared_base. We do not actually need queue_ptr.
607 if (COV >= 5)
608 removeAssumedBits(BitsEncoding: IMPLICIT_ARG_PTR);
609 else
610 removeAssumedBits(BitsEncoding: QUEUE_PTR);
611 }
612
613 uint32_t Retrieved = getRetrievedImplicitArgs(A);
614
615 if (Retrieved & MULTIGRID_SYNC_ARG) {
616 assert(!isAssumed(IMPLICIT_ARG_PTR) &&
617 "multigrid_sync_arg needs implicitarg_ptr");
618 removeAssumedBits(BitsEncoding: MULTIGRID_SYNC_ARG);
619 }
620
621 if (Retrieved & HOSTCALL_PTR) {
622 assert(!isAssumed(IMPLICIT_ARG_PTR) && "hostcall needs implicitarg_ptr");
623 removeAssumedBits(BitsEncoding: HOSTCALL_PTR);
624 }
625
626 if (Retrieved & HEAP_PTR) {
627 assert(!isAssumed(IMPLICIT_ARG_PTR) && "heap_ptr needs implicitarg_ptr");
628 removeAssumedBits(BitsEncoding: HEAP_PTR);
629 }
630
631 if (isAssumed(BitsEncoding: QUEUE_PTR) && (Retrieved & QUEUE_PTR)) {
632 assert(!isAssumed(IMPLICIT_ARG_PTR) && "queue_ptr needs implicitarg_ptr");
633 removeAssumedBits(BitsEncoding: QUEUE_PTR);
634 }
635
636 if (isAssumed(BitsEncoding: LDS_KERNEL_ID) && (Retrieved & LDS_KERNEL_ID)) {
637 removeAssumedBits(BitsEncoding: LDS_KERNEL_ID);
638 }
639
640 if (isAssumed(BitsEncoding: DEFAULT_QUEUE) && (Retrieved & DEFAULT_QUEUE))
641 removeAssumedBits(BitsEncoding: DEFAULT_QUEUE);
642
643 if (isAssumed(BitsEncoding: COMPLETION_ACTION) && (Retrieved & COMPLETION_ACTION))
644 removeAssumedBits(BitsEncoding: COMPLETION_ACTION);
645
646 if (isAssumed(BitsEncoding: FLAT_SCRATCH_INIT) && needFlatScratchInit(A))
647 removeAssumedBits(BitsEncoding: FLAT_SCRATCH_INIT);
648
649 return getAssumed() != OrigAssumed ? ChangeStatus::CHANGED
650 : ChangeStatus::UNCHANGED;
651 }
652
653 ChangeStatus manifest(Attributor &A) override {
654 SmallVector<Attribute, LAST_ARG_POS> AttrList;
655 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
656 for (auto [Mask, Attr] : InfoCache.getImplicitAttrList()) {
657 if (isKnown(BitsEncoding: Mask))
658 AttrList.push_back(Elt: Attr);
659 }
660
661 return A.manifestAttrs(IRP: getIRPosition(), DeducedAttrs: AttrList,
662 /* ForceReplace */ true);
663 }
664
665 const std::string getAsStr(Attributor *) const override {
666 std::string Str;
667 raw_string_ostream OS(Str);
668 OS << "AMDInfo[";
669 for (auto Attr : ImplicitAttrs)
670 if (isAssumed(BitsEncoding: Attr.first))
671 OS << ' ' << Attr.second;
672 OS << " ]";
673 return OS.str();
674 }
675
676 /// See AbstractAttribute::trackStatistics()
677 void trackStatistics() const override {}
678
679private:
680 bool checkForQueuePtr(Attributor &A) {
681 Function *F = getAssociatedFunction();
682 bool IsNonEntryFunc = !AMDGPU::isEntryFunctionCC(CC: F->getCallingConv());
683
684 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
685
686 bool NeedsQueuePtr = false;
687
688 auto CheckAddrSpaceCasts = [&](Instruction &I) {
689 unsigned SrcAS = static_cast<AddrSpaceCastInst &>(I).getSrcAddressSpace();
690 if (castRequiresQueuePtr(SrcAS)) {
691 NeedsQueuePtr = true;
692 return false;
693 }
694 return true;
695 };
696
697 bool HasApertureRegs =
698 InfoCache.getFeatures().test(I: AMDGPU::FEAT_APERTURE_REGS);
699
700 // `checkForAllInstructions` is much more cheaper than going through all
701 // instructions, try it first.
702
703 // The queue pointer is not needed if aperture regs is present.
704 if (!HasApertureRegs) {
705 bool UsedAssumedInformation = false;
706 A.checkForAllInstructions(Pred: CheckAddrSpaceCasts, QueryingAA: *this,
707 Opcodes: {Instruction::AddrSpaceCast},
708 UsedAssumedInformation);
709 }
710
711 // If we found that we need the queue pointer, nothing else to do.
712 if (NeedsQueuePtr)
713 return true;
714
715 if (!IsNonEntryFunc && HasApertureRegs)
716 return false;
717
718 for (BasicBlock &BB : *F) {
719 for (Instruction &I : BB) {
720 for (const Use &U : I.operands()) {
721 if (const auto *C = dyn_cast<Constant>(Val: U)) {
722 if (InfoCache.needsQueuePtr(C, Fn&: *F))
723 return true;
724 }
725 }
726 }
727 }
728
729 return false;
730 }
731
732 /// Returns the implicit kernel arguments that may be read through the
733 /// pointer returned by the implicitarg_ptr call \p Call.
734 uint32_t getImplicitArgsReadThrough(Attributor &A, const CallBase &Call) {
735 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
736 uint32_t ArgsMask = InfoCache.getImplicitArgsMask();
737 const auto *PointerInfoAA = A.getAAFor<AAPointerInfo>(
738 QueryingAA: *this, IRP: IRPosition::callsite_returned(CB: Call), DepClass: DepClassTy::REQUIRED);
739 if (!PointerInfoAA || !PointerInfoAA->getState().isValidState())
740 return ArgsMask;
741
742 // An implicit argument is not read if every access that may read any byte
743 // of it is droppable. Check the accesses once for all the implicit
744 // arguments, and stop once all of them may be read.
745 uint32_t Read = 0;
746 auto CheckAccess = [&](const AAPointerInfo::Access &Acc, bool) {
747 if (Acc.getRemoteInst()->isDroppable())
748 return true;
749 for (const AA::RangeTy &AccRange : Acc.getRanges()) {
750 for (auto [Mask, Range] : InfoCache.getImplicitArgRanges()) {
751 if (AccRange.mayOverlap(Range))
752 Read |= Mask;
753 }
754 }
755
756 return Read != ArgsMask;
757 };
758
759 if (!PointerInfoAA->forallInterferingAccesses(
760 Range: InfoCache.getImplicitArgsRange(), CB: CheckAccess))
761 return ArgsMask;
762 return Read;
763 }
764
765 /// Returns the implicit kernel arguments that may be retrieved by the
766 /// function.
767 uint32_t getRetrievedImplicitArgs(Attributor &A) {
768 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
769 uint32_t All = InfoCache.getImplicitArgsMask() | LDS_KERNEL_ID;
770 uint32_t Retrieved = 0;
771 auto CheckCall = [&](Instruction &I) {
772 auto &Call = cast<CallBase>(Val&: I);
773 switch (Call.getIntrinsicID()) {
774 case Intrinsic::amdgcn_implicitarg_ptr:
775 Retrieved |= getImplicitArgsReadThrough(A, Call);
776 break;
777 case Intrinsic::amdgcn_lds_kernel_id:
778 Retrieved |= LDS_KERNEL_ID;
779 break;
780 default:
781 // Other calls do not retrieve anything.
782 return true;
783 }
784
785 // Stop once everything may be retrieved.
786 return Retrieved != All;
787 };
788
789 bool UsedAssumedInformation = false;
790 if (!A.checkForAllCallLikeInstructions(Pred: CheckCall, QueryingAA: *this,
791 UsedAssumedInformation))
792 return All;
793 return Retrieved;
794 }
795
796 // Returns true if FlatScratchInit is needed, i.e., no-flat-scratch-init is
797 // not to be set.
798 bool needFlatScratchInit(Attributor &A) {
799 assert(isAssumed(FLAT_SCRATCH_INIT)); // only called if the bit is still set
800
801 // Check all AddrSpaceCast instructions. FlatScratchInit is needed if
802 // there is a cast from PRIVATE_ADDRESS.
803 auto AddrSpaceCastNotFromPrivate = [](Instruction &I) {
804 return cast<AddrSpaceCastInst>(Val&: I).getSrcAddressSpace() !=
805 AMDGPUAS::PRIVATE_ADDRESS;
806 };
807
808 bool UsedAssumedInformation = false;
809 if (!A.checkForAllInstructions(Pred: AddrSpaceCastNotFromPrivate, QueryingAA: *this,
810 Opcodes: {Instruction::AddrSpaceCast},
811 UsedAssumedInformation))
812 return true;
813
814 // Check for addrSpaceCast from PRIVATE_ADDRESS in constant expressions
815 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
816
817 Function *F = getAssociatedFunction();
818 for (Instruction &I : instructions(F)) {
819 for (const Use &U : I.operands()) {
820 if (const auto *C = dyn_cast<Constant>(Val: U)) {
821 if (InfoCache.checkConstForAddrSpaceCastFromPrivate(C))
822 return true;
823 }
824 }
825 }
826
827 return false;
828 }
829};
830
831AAAMDAttributes &AAAMDAttributes::createForPosition(const IRPosition &IRP,
832 Attributor &A) {
833 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
834 return *new (A.Allocator) AAAMDAttributesFunction(IRP, A);
835 llvm_unreachable("AAAMDAttributes is only valid for function position");
836}
837
838/// Base class to derive different size ranges.
839struct AAAMDSizeRangeAttribute
840 : public StateWrapper<IntegerRangeState, AbstractAttribute, uint32_t> {
841 using Base = StateWrapper<IntegerRangeState, AbstractAttribute, uint32_t>;
842
843 StringRef AttrName;
844
845 AAAMDSizeRangeAttribute(const IRPosition &IRP, Attributor &A,
846 StringRef AttrName)
847 : Base(IRP, 32), AttrName(AttrName) {}
848
849 /// See AbstractAttribute::trackStatistics()
850 void trackStatistics() const override {}
851
852 template <class AttributeImpl> ChangeStatus updateImplImpl(Attributor &A) {
853 ChangeStatus Change = ChangeStatus::UNCHANGED;
854
855 auto CheckCallSite = [&](AbstractCallSite CS) {
856 Function *Caller = CS.getInstruction()->getFunction();
857 LLVM_DEBUG(dbgs() << '[' << getName() << "] Call " << Caller->getName()
858 << "->" << getAssociatedFunction()->getName() << '\n');
859
860 const auto *CallerInfo = A.getAAFor<AttributeImpl>(
861 *this, IRPosition::function(F: *Caller), DepClassTy::REQUIRED);
862 if (!CallerInfo || !CallerInfo->isValidState())
863 return false;
864
865 Change |=
866 clampStateAndIndicateChange(this->getState(), CallerInfo->getState());
867
868 return true;
869 };
870
871 bool AllCallSitesKnown = true;
872 if (!A.checkForAllCallSites(CheckCallSite, *this,
873 /*RequireAllCallSites=*/true,
874 AllCallSitesKnown))
875 return indicatePessimisticFixpoint();
876
877 return Change;
878 }
879
880 /// Clamp the assumed range to the default value ([Min, Max]) and emit the
881 /// attribute if it is not same as default.
882 ChangeStatus
883 emitAttributeIfNotDefaultAfterClamp(Attributor &A,
884 std::pair<unsigned, unsigned> Default) {
885 auto [Min, Max] = Default;
886 unsigned Lower = getAssumed().getLower().getZExtValue();
887 unsigned Upper = getAssumed().getUpper().getZExtValue();
888
889 // Clamp the range to the default value.
890 if (Lower < Min)
891 Lower = Min;
892 if (Upper > Max + 1)
893 Upper = Max + 1;
894
895 // No manifest if the value is invalid or same as default after clamp.
896 if ((Lower == Min && Upper == Max + 1) || (Upper < Lower))
897 return ChangeStatus::UNCHANGED;
898
899 Function *F = getAssociatedFunction();
900 LLVMContext &Ctx = F->getContext();
901 SmallString<10> Buffer;
902 raw_svector_ostream OS(Buffer);
903 OS << Lower << ',' << Upper - 1;
904 return A.manifestAttrs(IRP: getIRPosition(),
905 DeducedAttrs: {Attribute::get(Context&: Ctx, Kind: AttrName, Val: OS.str())},
906 /*ForceReplace=*/true);
907 }
908
909 const std::string getAsStr(Attributor *) const override {
910 std::string Str;
911 raw_string_ostream OS(Str);
912 OS << getName() << '[';
913 OS << getAssumed().getLower() << ',' << getAssumed().getUpper() - 1;
914 OS << ']';
915 return OS.str();
916 }
917};
918
919/// Propagate amdgpu-flat-work-group-size attribute.
920struct AAAMDFlatWorkGroupSize : public AAAMDSizeRangeAttribute {
921 AAAMDFlatWorkGroupSize(const IRPosition &IRP, Attributor &A)
922 : AAAMDSizeRangeAttribute(IRP, A, "amdgpu-flat-work-group-size") {}
923
924 void initialize(Attributor &A) override {
925 Function *F = getAssociatedFunction();
926 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
927
928 bool HasAttr = false;
929 auto Range = InfoCache.getDefaultFlatWorkGroupSize(F: *F);
930 auto MaxRange = InfoCache.getMaximumFlatWorkGroupRange();
931
932 if (auto Attr = InfoCache.getFlatWorkGroupSizeAttr(F: *F)) {
933 // We only consider an attribute that is not max range because the front
934 // end always emits the attribute, unfortunately, and sometimes it emits
935 // the max range.
936 if (*Attr != MaxRange) {
937 Range = *Attr;
938 HasAttr = true;
939 }
940 }
941
942 // We don't want to directly clamp the state if it's the max range because
943 // that is basically the worst state.
944 if (Range == MaxRange)
945 return;
946
947 auto [Min, Max] = Range;
948 ConstantRange CR(APInt(32, Min), APInt(32, Max + 1));
949 IntegerRangeState IRS(CR);
950 clampStateAndIndicateChange(S&: this->getState(), R: IRS);
951
952 if (HasAttr || AMDGPU::isEntryFunctionCC(CC: F->getCallingConv()))
953 indicateOptimisticFixpoint();
954 }
955
956 ChangeStatus updateImpl(Attributor &A) override {
957 return updateImplImpl<AAAMDFlatWorkGroupSize>(A);
958 }
959
960 /// Create an abstract attribute view for the position \p IRP.
961 static AAAMDFlatWorkGroupSize &createForPosition(const IRPosition &IRP,
962 Attributor &A);
963
964 ChangeStatus manifest(Attributor &A) override {
965 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
966 return emitAttributeIfNotDefaultAfterClamp(
967 A, Default: InfoCache.getMaximumFlatWorkGroupRange());
968 }
969
970 /// See AbstractAttribute::getName()
971 StringRef getName() const override { return "AAAMDFlatWorkGroupSize"; }
972
973 /// See AbstractAttribute::getIdAddr()
974 const char *getIdAddr() const override { return &ID; }
975
976 /// This function should return true if the type of the \p AA is
977 /// AAAMDFlatWorkGroupSize
978 static bool classof(const AbstractAttribute *AA) {
979 return (AA->getIdAddr() == &ID);
980 }
981
982 /// Unique ID (due to the unique address)
983 static const char ID;
984};
985
986const char AAAMDFlatWorkGroupSize::ID = 0;
987
988AAAMDFlatWorkGroupSize &
989AAAMDFlatWorkGroupSize::createForPosition(const IRPosition &IRP,
990 Attributor &A) {
991 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
992 return *new (A.Allocator) AAAMDFlatWorkGroupSize(IRP, A);
993 llvm_unreachable(
994 "AAAMDFlatWorkGroupSize is only valid for function position");
995}
996
997struct TupleDecIntegerRangeState : public AbstractState {
998 DecIntegerState<uint32_t> X, Y, Z;
999
1000 bool isValidState() const override {
1001 return X.isValidState() && Y.isValidState() && Z.isValidState();
1002 }
1003
1004 bool isAtFixpoint() const override {
1005 return X.isAtFixpoint() && Y.isAtFixpoint() && Z.isAtFixpoint();
1006 }
1007
1008 ChangeStatus indicateOptimisticFixpoint() override {
1009 return X.indicateOptimisticFixpoint() | Y.indicateOptimisticFixpoint() |
1010 Z.indicateOptimisticFixpoint();
1011 }
1012
1013 ChangeStatus indicatePessimisticFixpoint() override {
1014 return X.indicatePessimisticFixpoint() | Y.indicatePessimisticFixpoint() |
1015 Z.indicatePessimisticFixpoint();
1016 }
1017
1018 TupleDecIntegerRangeState operator^=(const TupleDecIntegerRangeState &Other) {
1019 X ^= Other.X;
1020 Y ^= Other.Y;
1021 Z ^= Other.Z;
1022 return *this;
1023 }
1024
1025 bool operator==(const TupleDecIntegerRangeState &Other) const {
1026 return X == Other.X && Y == Other.Y && Z == Other.Z;
1027 }
1028
1029 TupleDecIntegerRangeState &getAssumed() { return *this; }
1030 const TupleDecIntegerRangeState &getAssumed() const { return *this; }
1031};
1032
1033using AAAMDMaxNumWorkgroupsState =
1034 StateWrapper<TupleDecIntegerRangeState, AbstractAttribute, uint32_t>;
1035
1036/// Propagate amdgpu-max-num-workgroups attribute.
1037struct AAAMDMaxNumWorkgroups
1038 : public StateWrapper<TupleDecIntegerRangeState, AbstractAttribute> {
1039 using Base = StateWrapper<TupleDecIntegerRangeState, AbstractAttribute>;
1040
1041 AAAMDMaxNumWorkgroups(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
1042
1043 void initialize(Attributor &A) override {
1044 Function *F = getAssociatedFunction();
1045
1046 SmallVector<unsigned> MaxNumWorkgroups = AMDGPU::getMaxNumWorkGroups(F: *F);
1047
1048 X.takeKnownMinimum(Value: MaxNumWorkgroups[0]);
1049 Y.takeKnownMinimum(Value: MaxNumWorkgroups[1]);
1050 Z.takeKnownMinimum(Value: MaxNumWorkgroups[2]);
1051
1052 if (AMDGPU::isEntryFunctionCC(CC: F->getCallingConv()))
1053 indicatePessimisticFixpoint();
1054 }
1055
1056 ChangeStatus updateImpl(Attributor &A) override {
1057 ChangeStatus Change = ChangeStatus::UNCHANGED;
1058
1059 auto CheckCallSite = [&](AbstractCallSite CS) {
1060 Function *Caller = CS.getInstruction()->getFunction();
1061 LLVM_DEBUG(dbgs() << "[AAAMDMaxNumWorkgroups] Call " << Caller->getName()
1062 << "->" << getAssociatedFunction()->getName() << '\n');
1063
1064 const auto *CallerInfo = A.getAAFor<AAAMDMaxNumWorkgroups>(
1065 QueryingAA: *this, IRP: IRPosition::function(F: *Caller), DepClass: DepClassTy::REQUIRED);
1066 if (!CallerInfo || !CallerInfo->isValidState())
1067 return false;
1068
1069 Change |=
1070 clampStateAndIndicateChange(S&: this->getState(), R: CallerInfo->getState());
1071 return true;
1072 };
1073
1074 bool AllCallSitesKnown = true;
1075 if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this,
1076 /*RequireAllCallSites=*/true,
1077 UsedAssumedInformation&: AllCallSitesKnown))
1078 return indicatePessimisticFixpoint();
1079
1080 return Change;
1081 }
1082
1083 /// Create an abstract attribute view for the position \p IRP.
1084 static AAAMDMaxNumWorkgroups &createForPosition(const IRPosition &IRP,
1085 Attributor &A);
1086
1087 ChangeStatus manifest(Attributor &A) override {
1088 Function *F = getAssociatedFunction();
1089 LLVMContext &Ctx = F->getContext();
1090 SmallString<32> Buffer;
1091 raw_svector_ostream OS(Buffer);
1092 OS << X.getAssumed() << ',' << Y.getAssumed() << ',' << Z.getAssumed();
1093
1094 // TODO: Should annotate loads of the group size for this to do anything
1095 // useful.
1096 return A.manifestAttrs(
1097 IRP: getIRPosition(),
1098 DeducedAttrs: {Attribute::get(Context&: Ctx, Kind: "amdgpu-max-num-workgroups", Val: OS.str())},
1099 /* ForceReplace= */ true);
1100 }
1101
1102 StringRef getName() const override { return "AAAMDMaxNumWorkgroups"; }
1103
1104 const std::string getAsStr(Attributor *) const override {
1105 std::string Buffer = "AAAMDMaxNumWorkgroupsState[";
1106 raw_string_ostream OS(Buffer);
1107 OS << X.getAssumed() << ',' << Y.getAssumed() << ',' << Z.getAssumed()
1108 << ']';
1109 return OS.str();
1110 }
1111
1112 const char *getIdAddr() const override { return &ID; }
1113
1114 /// This function should return true if the type of the \p AA is
1115 /// AAAMDMaxNumWorkgroups
1116 static bool classof(const AbstractAttribute *AA) {
1117 return (AA->getIdAddr() == &ID);
1118 }
1119
1120 void trackStatistics() const override {}
1121
1122 /// Unique ID (due to the unique address)
1123 static const char ID;
1124};
1125
1126const char AAAMDMaxNumWorkgroups::ID = 0;
1127
1128AAAMDMaxNumWorkgroups &
1129AAAMDMaxNumWorkgroups::createForPosition(const IRPosition &IRP, Attributor &A) {
1130 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
1131 return *new (A.Allocator) AAAMDMaxNumWorkgroups(IRP, A);
1132 llvm_unreachable("AAAMDMaxNumWorkgroups is only valid for function position");
1133}
1134
1135/// Propagate amdgpu-waves-per-eu attribute.
1136struct AAAMDWavesPerEU : public AAAMDSizeRangeAttribute {
1137 AAAMDWavesPerEU(const IRPosition &IRP, Attributor &A)
1138 : AAAMDSizeRangeAttribute(IRP, A, "amdgpu-waves-per-eu") {}
1139
1140 void initialize(Attributor &A) override {
1141 Function *F = getAssociatedFunction();
1142 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
1143
1144 // If the attribute exists, we will honor it if it is not the default.
1145 if (auto Attr = InfoCache.getWavesPerEUAttr(F: *F)) {
1146 std::pair<unsigned, unsigned> MaxWavesPerEURange{
1147 1U, InfoCache.getMaxWavesPerEU()};
1148 if (*Attr != MaxWavesPerEURange) {
1149 auto [Min, Max] = *Attr;
1150 ConstantRange Range(APInt(32, Min), APInt(32, Max + 1));
1151 IntegerRangeState RangeState(Range);
1152 this->getState() = RangeState;
1153 indicateOptimisticFixpoint();
1154 return;
1155 }
1156 }
1157
1158 if (AMDGPU::isEntryFunctionCC(CC: F->getCallingConv()))
1159 indicatePessimisticFixpoint();
1160 }
1161
1162 ChangeStatus updateImpl(Attributor &A) override {
1163 ChangeStatus Change = ChangeStatus::UNCHANGED;
1164
1165 auto CheckCallSite = [&](AbstractCallSite CS) {
1166 Function *Caller = CS.getInstruction()->getFunction();
1167 Function *Func = getAssociatedFunction();
1168 LLVM_DEBUG(dbgs() << '[' << getName() << "] Call " << Caller->getName()
1169 << "->" << Func->getName() << '\n');
1170 (void)Func;
1171
1172 const auto *CallerAA = A.getAAFor<AAAMDWavesPerEU>(
1173 QueryingAA: *this, IRP: IRPosition::function(F: *Caller), DepClass: DepClassTy::REQUIRED);
1174 if (!CallerAA || !CallerAA->isValidState())
1175 return false;
1176
1177 ConstantRange Assumed = getAssumed();
1178 unsigned Min = std::max(a: Assumed.getLower().getZExtValue(),
1179 b: CallerAA->getAssumed().getLower().getZExtValue());
1180 unsigned Max = std::max(a: Assumed.getUpper().getZExtValue(),
1181 b: CallerAA->getAssumed().getUpper().getZExtValue());
1182 ConstantRange Range(APInt(32, Min), APInt(32, Max));
1183 IntegerRangeState RangeState(Range);
1184 getState() = RangeState;
1185 Change |= getState() == Assumed ? ChangeStatus::UNCHANGED
1186 : ChangeStatus::CHANGED;
1187
1188 return true;
1189 };
1190
1191 bool AllCallSitesKnown = true;
1192 if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this, RequireAllCallSites: true, UsedAssumedInformation&: AllCallSitesKnown))
1193 return indicatePessimisticFixpoint();
1194
1195 return Change;
1196 }
1197
1198 /// Create an abstract attribute view for the position \p IRP.
1199 static AAAMDWavesPerEU &createForPosition(const IRPosition &IRP,
1200 Attributor &A);
1201
1202 ChangeStatus manifest(Attributor &A) override {
1203 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
1204 return emitAttributeIfNotDefaultAfterClamp(
1205 A, Default: {1U, InfoCache.getMaxWavesPerEU()});
1206 }
1207
1208 /// See AbstractAttribute::getName()
1209 StringRef getName() const override { return "AAAMDWavesPerEU"; }
1210
1211 /// See AbstractAttribute::getIdAddr()
1212 const char *getIdAddr() const override { return &ID; }
1213
1214 /// This function should return true if the type of the \p AA is
1215 /// AAAMDWavesPerEU
1216 static bool classof(const AbstractAttribute *AA) {
1217 return (AA->getIdAddr() == &ID);
1218 }
1219
1220 /// Unique ID (due to the unique address)
1221 static const char ID;
1222};
1223
1224const char AAAMDWavesPerEU::ID = 0;
1225
1226AAAMDWavesPerEU &AAAMDWavesPerEU::createForPosition(const IRPosition &IRP,
1227 Attributor &A) {
1228 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
1229 return *new (A.Allocator) AAAMDWavesPerEU(IRP, A);
1230 llvm_unreachable("AAAMDWavesPerEU is only valid for function position");
1231}
1232
1233/// Compute the minimum number of AGPRs required to allocate the inline asm.
1234static unsigned inlineAsmGetNumRequiredAGPRs(const InlineAsm *IA,
1235 const CallBase &Call) {
1236 unsigned ArgNo = 0;
1237 unsigned ResNo = 0;
1238 unsigned AGPRDefCount = 0;
1239 unsigned AGPRUseCount = 0;
1240 unsigned MaxPhysReg = 0;
1241 const DataLayout &DL = Call.getFunction()->getDataLayout();
1242
1243 // TODO: Overestimates due to not accounting for tied operands
1244 for (const InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
1245 Type *Ty = nullptr;
1246 switch (CI.Type) {
1247 case InlineAsm::isOutput: {
1248 Ty = Call.getType();
1249 if (auto *STy = dyn_cast<StructType>(Val: Ty))
1250 Ty = STy->getElementType(N: ResNo);
1251 ++ResNo;
1252 break;
1253 }
1254 case InlineAsm::isInput: {
1255 Ty = Call.getArgOperand(i: ArgNo++)->getType();
1256 break;
1257 }
1258 case InlineAsm::isLabel:
1259 continue;
1260 case InlineAsm::isClobber:
1261 // Parse the physical register reference.
1262 break;
1263 }
1264
1265 for (StringRef Code : CI.Codes) {
1266 unsigned RegCount = 0;
1267 if (Code.starts_with(Prefix: "a")) {
1268 // Virtual register, compute number of registers based on the type.
1269 //
1270 // We ought to be going through TargetLowering to get the number of
1271 // registers, but we should avoid the dependence on CodeGen here.
1272 RegCount = divideCeil(Numerator: DL.getTypeSizeInBits(Ty), Denominator: 32);
1273 } else {
1274 // Physical register reference
1275 auto [Kind, RegIdx, NumRegs] = AMDGPU::parseAsmConstraintPhysReg(Constraint: Code);
1276 if (Kind == 'a') {
1277 RegCount = NumRegs;
1278 MaxPhysReg = std::max(a: MaxPhysReg, b: std::min(a: RegIdx + NumRegs, b: 256u));
1279 }
1280
1281 continue;
1282 }
1283
1284 if (CI.Type == InlineAsm::isOutput) {
1285 // Apply tuple alignment requirement
1286 //
1287 // TODO: This is more conservative than necessary.
1288 AGPRDefCount = alignTo(Value: AGPRDefCount, Align: RegCount);
1289
1290 AGPRDefCount += RegCount;
1291 if (CI.isEarlyClobber) {
1292 AGPRUseCount = alignTo(Value: AGPRUseCount, Align: RegCount);
1293 AGPRUseCount += RegCount;
1294 }
1295 } else {
1296 AGPRUseCount = alignTo(Value: AGPRUseCount, Align: RegCount);
1297 AGPRUseCount += RegCount;
1298 }
1299 }
1300 }
1301
1302 unsigned MaxVirtReg = std::max(a: AGPRUseCount, b: AGPRDefCount);
1303
1304 // TODO: This is overly conservative. If there are any physical registers,
1305 // allocate any virtual registers after them so we don't have to solve optimal
1306 // packing.
1307 return std::min(a: MaxVirtReg + MaxPhysReg, b: 256u);
1308}
1309
1310struct AAAMDGPUMinAGPRAlloc
1311 : public StateWrapper<DecIntegerState<>, AbstractAttribute> {
1312 using Base = StateWrapper<DecIntegerState<>, AbstractAttribute>;
1313 AAAMDGPUMinAGPRAlloc(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
1314
1315 static AAAMDGPUMinAGPRAlloc &createForPosition(const IRPosition &IRP,
1316 Attributor &A) {
1317 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
1318 return *new (A.Allocator) AAAMDGPUMinAGPRAlloc(IRP, A);
1319 llvm_unreachable(
1320 "AAAMDGPUMinAGPRAlloc is only valid for function position");
1321 }
1322
1323 void initialize(Attributor &A) override {
1324 Function *F = getAssociatedFunction();
1325 auto [MinNumAGPR, MaxNumAGPR] =
1326 AMDGPU::getIntegerPairAttribute(F: *F, Name: "amdgpu-agpr-alloc", Default: {~0u, ~0u},
1327 /*OnlyFirstRequired=*/true);
1328 if (MinNumAGPR == 0) {
1329 indicateOptimisticFixpoint();
1330 return;
1331 }
1332
1333 if (hasSanitizerAttributes(F: *F))
1334 indicatePessimisticFixpoint();
1335 }
1336
1337 const std::string getAsStr(Attributor *A) const override {
1338 std::string Str = "amdgpu-agpr-alloc=";
1339 raw_string_ostream OS(Str);
1340 OS << getAssumed();
1341 return OS.str();
1342 }
1343
1344 void trackStatistics() const override {}
1345
1346 ChangeStatus updateImpl(Attributor &A) override {
1347 DecIntegerState<> Maximum;
1348
1349 // Check for cases which require allocation of AGPRs. The only cases where
1350 // AGPRs are required are if there are direct references to AGPRs, so inline
1351 // assembly and special intrinsics.
1352 auto CheckForMinAGPRAllocs = [&](Instruction &I) {
1353 const auto &CB = cast<CallBase>(Val&: I);
1354 const Value *CalleeOp = CB.getCalledOperand();
1355
1356 if (const InlineAsm *IA = dyn_cast<InlineAsm>(Val: CalleeOp)) {
1357 // Technically, the inline asm could be invoking a call to an unknown
1358 // external function that requires AGPRs, but ignore that.
1359 unsigned NumRegs = inlineAsmGetNumRequiredAGPRs(IA, Call: CB);
1360 Maximum.takeAssumedMaximum(Value: NumRegs);
1361 return true;
1362 }
1363 switch (CB.getIntrinsicID()) {
1364 case Intrinsic::not_intrinsic:
1365 break;
1366 case Intrinsic::write_register:
1367 case Intrinsic::read_register:
1368 case Intrinsic::read_volatile_register: {
1369 const MDString *RegName = cast<MDString>(
1370 Val: cast<MDNode>(
1371 Val: cast<MetadataAsValue>(Val: CB.getArgOperand(i: 0))->getMetadata())
1372 ->getOperand(I: 0));
1373 auto [Kind, RegIdx, NumRegs] =
1374 AMDGPU::parseAsmPhysRegName(TupleString: RegName->getString());
1375 if (Kind == 'a')
1376 Maximum.takeAssumedMaximum(Value: std::min(a: RegIdx + NumRegs, b: 256u));
1377
1378 return true;
1379 }
1380 // Trap-like intrinsics such as llvm.trap and llvm.debugtrap do not have
1381 // the nocallback attribute, so the AMDGPU attributor can conservatively
1382 // drop all implicitly-known inputs and AGPR allocation information. Make
1383 // sure we still infer that no implicit inputs are required and that the
1384 // AGPR allocation stays at zero. Trap-like intrinsics may invoke a
1385 // function which requires AGPRs, so we need to check if the called
1386 // function has the "trap-func-name" attribute.
1387 case Intrinsic::trap:
1388 case Intrinsic::debugtrap:
1389 case Intrinsic::ubsantrap:
1390 return CB.hasFnAttr(Kind: Attribute::NoCallback) ||
1391 !CB.hasFnAttr(Kind: "trap-func-name");
1392 default:
1393 // Some intrinsics may use AGPRs, but if we have a choice, we are not
1394 // required to use AGPRs.
1395 // Assume !nocallback intrinsics may call a function which requires
1396 // AGPRs.
1397 return CB.hasFnAttr(Kind: Attribute::NoCallback);
1398 }
1399
1400 // TODO: Handle callsite attributes
1401 auto *CBEdges = A.getAAFor<AACallEdges>(
1402 QueryingAA: *this, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::REQUIRED);
1403 if (!CBEdges || CBEdges->hasUnknownCallee()) {
1404 Maximum.indicatePessimisticFixpoint();
1405 return false;
1406 }
1407
1408 for (const Function *PossibleCallee : CBEdges->getOptimisticEdges()) {
1409 const auto *CalleeInfo = A.getAAFor<AAAMDGPUMinAGPRAlloc>(
1410 QueryingAA: *this, IRP: IRPosition::function(F: *PossibleCallee), DepClass: DepClassTy::REQUIRED);
1411 if (!CalleeInfo || !CalleeInfo->isValidState()) {
1412 Maximum.indicatePessimisticFixpoint();
1413 return false;
1414 }
1415
1416 Maximum.takeAssumedMaximum(Value: CalleeInfo->getAssumed());
1417 }
1418
1419 return true;
1420 };
1421
1422 bool UsedAssumedInformation = false;
1423 if (!A.checkForAllCallLikeInstructions(Pred: CheckForMinAGPRAllocs, QueryingAA: *this,
1424 UsedAssumedInformation))
1425 return indicatePessimisticFixpoint();
1426
1427 return clampStateAndIndicateChange(S&: getState(), R: Maximum);
1428 }
1429
1430 ChangeStatus manifest(Attributor &A) override {
1431 LLVMContext &Ctx = getAssociatedFunction()->getContext();
1432 SmallString<4> Buffer;
1433 raw_svector_ostream OS(Buffer);
1434 OS << getAssumed();
1435
1436 return A.manifestAttrs(
1437 IRP: getIRPosition(), DeducedAttrs: {Attribute::get(Context&: Ctx, Kind: "amdgpu-agpr-alloc", Val: OS.str())});
1438 }
1439
1440 StringRef getName() const override { return "AAAMDGPUMinAGPRAlloc"; }
1441 const char *getIdAddr() const override { return &ID; }
1442
1443 /// This function should return true if the type of the \p AA is
1444 /// AAAMDGPUMinAGPRAllocs
1445 static bool classof(const AbstractAttribute *AA) {
1446 return (AA->getIdAddr() == &ID);
1447 }
1448
1449 static const char ID;
1450};
1451
1452const char AAAMDGPUMinAGPRAlloc::ID = 0;
1453
1454/// Deduce the function attribute "amdgpu-no-lds-dma"
1455struct AAAMDGPUNoLDSDMA : public StateWrapper<BooleanState, AbstractAttribute> {
1456 using Base = StateWrapper<BooleanState, AbstractAttribute>;
1457 AAAMDGPUNoLDSDMA(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
1458
1459 /// Create an abstract attribute view for the position \p IRP.
1460 static AAAMDGPUNoLDSDMA &createForPosition(const IRPosition &IRP,
1461 Attributor &A) {
1462 assert(IRP.getPositionKind() == IRPosition::IRP_FUNCTION &&
1463 "AAAMDGPUNoLDSDMA is only valid for function position");
1464 return *new (A.Allocator) AAAMDGPUNoLDSDMA(IRP, A);
1465 }
1466
1467 void initialize(Attributor &A) override {
1468 const Function *F = getAssociatedFunction();
1469 if (F->hasFnAttribute(Kind: "amdgpu-no-lds-dma")) {
1470 indicateOptimisticFixpoint();
1471 return;
1472 }
1473
1474 if (F->isDeclaration())
1475 indicatePessimisticFixpoint();
1476 }
1477
1478 ChangeStatus updateImpl(Attributor &A) override {
1479 auto CheckNoLDSDMA = [&](Instruction &I) {
1480 const auto &CB = cast<CallBase>(Val&: I);
1481
1482 // Inline assembly may hold any instruction, including an LDSDMA
1483 // operation.
1484 if (isa<InlineAsm>(Val: CB.getCalledOperand()))
1485 return false;
1486
1487 Intrinsic::ID IID = CB.getIntrinsicID();
1488 if (IID != Intrinsic::not_intrinsic) {
1489 // Assume !nocallback intrinsics may call a function which issues an
1490 // LDSDMA operation.
1491 return !AMDGPU::isLDSDMAIntrinsic(IntrID: IID) &&
1492 CB.hasFnAttr(Kind: Attribute::NoCallback);
1493 }
1494
1495 const auto *CBEdges = A.getAAFor<AACallEdges>(
1496 QueryingAA: *this, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::REQUIRED);
1497 if (!CBEdges || CBEdges->hasUnknownCallee())
1498 return false;
1499
1500 for (const Function *PossibleCallee : CBEdges->getOptimisticEdges()) {
1501 const auto *CalleeInfo = A.getAAFor<AAAMDGPUNoLDSDMA>(
1502 QueryingAA: *this, IRP: IRPosition::function(F: *PossibleCallee), DepClass: DepClassTy::REQUIRED);
1503 if (!CalleeInfo || !CalleeInfo->getAssumed())
1504 return false;
1505 }
1506
1507 return true;
1508 };
1509
1510 bool UsedAssumedInformation = false;
1511 if (!A.checkForAllCallLikeInstructions(Pred: CheckNoLDSDMA, QueryingAA: *this,
1512 UsedAssumedInformation))
1513 return indicatePessimisticFixpoint();
1514
1515 return ChangeStatus::UNCHANGED;
1516 }
1517
1518 ChangeStatus manifest(Attributor &A) override {
1519 if (!getAssumed())
1520 return ChangeStatus::UNCHANGED;
1521
1522 LLVMContext &Ctx = getAssociatedFunction()->getContext();
1523 return A.manifestAttrs(IRP: getIRPosition(),
1524 DeducedAttrs: {Attribute::get(Context&: Ctx, Kind: "amdgpu-no-lds-dma")});
1525 }
1526
1527 bool isValidState() const override { return true; }
1528
1529 const std::string getAsStr(Attributor *A) const override {
1530 return "AMDGPUNoLDSDMA[" + std::to_string(val: getAssumed()) + "]";
1531 }
1532
1533 void trackStatistics() const override {}
1534
1535 StringRef getName() const override { return "AAAMDGPUNoLDSDMA"; }
1536 const char *getIdAddr() const override { return &ID; }
1537
1538 /// This function should return true if the type of the \p AA is
1539 /// AAAMDGPUNoLDSDMA
1540 static bool classof(const AbstractAttribute *AA) {
1541 return (AA->getIdAddr() == &ID);
1542 }
1543
1544 static const char ID;
1545};
1546
1547const char AAAMDGPUNoLDSDMA::ID = 0;
1548
1549/// An abstract attribute to propagate the function attribute
1550/// "amdgpu-cluster-dims" from kernel entry functions to device functions.
1551struct AAAMDGPUClusterDims
1552 : public StateWrapper<BooleanState, AbstractAttribute> {
1553 using Base = StateWrapper<BooleanState, AbstractAttribute>;
1554 AAAMDGPUClusterDims(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
1555
1556 /// Create an abstract attribute view for the position \p IRP.
1557 static AAAMDGPUClusterDims &createForPosition(const IRPosition &IRP,
1558 Attributor &A);
1559
1560 /// See AbstractAttribute::getName().
1561 StringRef getName() const override { return "AAAMDGPUClusterDims"; }
1562
1563 /// See AbstractAttribute::getIdAddr().
1564 const char *getIdAddr() const override { return &ID; }
1565
1566 /// This function should return true if the type of the \p AA is
1567 /// AAAMDGPUClusterDims.
1568 static bool classof(const AbstractAttribute *AA) {
1569 return AA->getIdAddr() == &ID;
1570 }
1571
1572 virtual const AMDGPU::ClusterDimsAttr &getClusterDims() const = 0;
1573
1574 /// Unique ID (due to the unique address)
1575 static const char ID;
1576};
1577
1578const char AAAMDGPUClusterDims::ID = 0;
1579
1580struct AAAMDGPUClusterDimsFunction : public AAAMDGPUClusterDims {
1581 AAAMDGPUClusterDimsFunction(const IRPosition &IRP, Attributor &A)
1582 : AAAMDGPUClusterDims(IRP, A) {}
1583
1584 void initialize(Attributor &A) override {
1585 Function *F = getAssociatedFunction();
1586 assert(F && "empty associated function");
1587
1588 Attr = AMDGPU::ClusterDimsAttr::get(F: *F);
1589
1590 // No matter what a kernel function has, it is final.
1591 if (AMDGPU::isEntryFunctionCC(CC: F->getCallingConv())) {
1592 if (Attr.isUnknown())
1593 indicatePessimisticFixpoint();
1594 else
1595 indicateOptimisticFixpoint();
1596 }
1597 }
1598
1599 const std::string getAsStr(Attributor *A) const override {
1600 if (!getAssumed() || Attr.isUnknown())
1601 return "unknown";
1602 if (Attr.isNoCluster())
1603 return "no";
1604 if (Attr.isVariableDims())
1605 return "variable";
1606 return Attr.to_string();
1607 }
1608
1609 void trackStatistics() const override {}
1610
1611 ChangeStatus updateImpl(Attributor &A) override {
1612 auto OldState = Attr;
1613
1614 auto CheckCallSite = [&](AbstractCallSite CS) {
1615 const auto *CallerAA = A.getAAFor<AAAMDGPUClusterDims>(
1616 QueryingAA: *this, IRP: IRPosition::function(F: *CS.getInstruction()->getFunction()),
1617 DepClass: DepClassTy::REQUIRED);
1618 if (!CallerAA || !CallerAA->isValidState())
1619 return false;
1620
1621 return merge(Other: CallerAA->getClusterDims());
1622 };
1623
1624 bool UsedAssumedInformation = false;
1625 if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this,
1626 /*RequireAllCallSites=*/true,
1627 UsedAssumedInformation))
1628 return indicatePessimisticFixpoint();
1629
1630 return OldState == Attr ? ChangeStatus::UNCHANGED : ChangeStatus::CHANGED;
1631 }
1632
1633 ChangeStatus manifest(Attributor &A) override {
1634 if (Attr.isUnknown())
1635 return ChangeStatus::UNCHANGED;
1636 return A.manifestAttrs(
1637 IRP: getIRPosition(),
1638 DeducedAttrs: {Attribute::get(Context&: getAssociatedFunction()->getContext(), Kind: AttrName,
1639 Val: Attr.to_string())},
1640 /*ForceReplace=*/true);
1641 }
1642
1643 const AMDGPU::ClusterDimsAttr &getClusterDims() const override {
1644 return Attr;
1645 }
1646
1647private:
1648 bool merge(const AMDGPU::ClusterDimsAttr &Other) {
1649 // Case 1: Both of them are unknown yet, we do nothing and continue wait for
1650 // propagation.
1651 if (Attr.isUnknown() && Other.isUnknown())
1652 return true;
1653
1654 // Case 2: The other is determined, but we are unknown yet, we simply take
1655 // the other's value.
1656 if (Attr.isUnknown()) {
1657 Attr = Other;
1658 return true;
1659 }
1660
1661 // Case 3: We are determined but the other is unknown yet, we simply keep
1662 // everything unchanged.
1663 if (Other.isUnknown())
1664 return true;
1665
1666 // After this point, both are determined.
1667
1668 // Case 4: If they are same, we do nothing.
1669 if (Attr == Other)
1670 return true;
1671
1672 // Now they are not same.
1673
1674 // Case 5: If either of us uses cluster (but not both; otherwise case 4
1675 // would hold), then it is unknown whether cluster will be used, and the
1676 // state is final, unlike case 1.
1677 if (Attr.isNoCluster() || Other.isNoCluster()) {
1678 Attr.setUnknown();
1679 return false;
1680 }
1681
1682 // Case 6: Both of us use cluster, but the dims are different, so the result
1683 // is, cluster is used, but we just don't have a fixed dims.
1684 Attr.setVariableDims();
1685 return true;
1686 }
1687
1688 AMDGPU::ClusterDimsAttr Attr;
1689
1690 static constexpr char AttrName[] = "amdgpu-cluster-dims";
1691};
1692
1693AAAMDGPUClusterDims &
1694AAAMDGPUClusterDims::createForPosition(const IRPosition &IRP, Attributor &A) {
1695 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
1696 return *new (A.Allocator) AAAMDGPUClusterDimsFunction(IRP, A);
1697 llvm_unreachable("AAAMDGPUClusterDims is only valid for function position");
1698}
1699
1700static bool runImpl(SetVector<Function *> &Functions, bool IsModulePass,
1701 bool DeleteFns, Module &M, AnalysisGetter &AG,
1702 TargetMachine &TM, AMDGPUAttributorOptions Options,
1703 ThinOrFullLTOPhase LTOPhase) {
1704
1705 CallGraphUpdater CGUpdater;
1706 BumpPtrAllocator Allocator;
1707 AMDGPUInformationCache InfoCache(M, AG, Allocator, nullptr, TM);
1708 DenseSet<const char *> Allowed(
1709 {&AAAMDAttributes::ID, &AAUniformWorkGroupSize::ID,
1710 &AAPotentialValues::ID, &AAAMDFlatWorkGroupSize::ID,
1711 &AAAMDMaxNumWorkgroups::ID, &AAAMDWavesPerEU::ID,
1712 &AAAMDGPUMinAGPRAlloc::ID, &AACallEdges::ID, &AAPointerInfo::ID,
1713 &AAPotentialConstantValues::ID, &AAUnderlyingObjects::ID,
1714 &AANoAliasAddrSpace::ID, &AAAddressSpace::ID, &AAIndirectCallInfo::ID,
1715 &AAAMDGPUClusterDims::ID, &AAAMDGPUNoLDSDMA::ID, &AAAlign::ID});
1716
1717 AttributorConfig AC(CGUpdater);
1718 AC.IsClosedWorldModule = Options.IsClosedWorld;
1719 AC.Allowed = &Allowed;
1720 AC.IsModulePass = IsModulePass;
1721 AC.DeleteFns = DeleteFns;
1722 AC.DefaultInitializeLiveInternals = false;
1723 // AAIsDead is not allowed, so nothing is ever assumed dead.
1724 AC.UseLiveness = false;
1725 AC.IndirectCalleeSpecializationCallback =
1726 [](Attributor &A, const AbstractAttribute &AA, CallBase &CB,
1727 Function &Callee, unsigned NumAssumedCallees) {
1728 return !AMDGPU::isEntryFunctionCC(CC: Callee.getCallingConv()) &&
1729 (NumAssumedCallees <= IndirectCallSpecializationThreshold);
1730 };
1731 AC.IPOAmendableCB = [](const Function &F) {
1732 return F.getCallingConv() == CallingConv::AMDGPU_KERNEL;
1733 };
1734
1735 Attributor A(Functions, InfoCache, AC);
1736
1737 LLVM_DEBUG({
1738 StringRef LTOPhaseStr = to_string(LTOPhase);
1739 dbgs() << "[AMDGPUAttributor] Running at phase " << LTOPhaseStr << '\n'
1740 << "[AMDGPUAttributor] Module " << M.getName() << " is "
1741 << (AC.IsClosedWorldModule ? "" : "not ")
1742 << "assumed to be a closed world.\n";
1743 });
1744
1745 for (auto *F : Functions) {
1746 A.getOrCreateAAFor<AAAMDAttributes>(IRP: IRPosition::function(F: *F));
1747 A.getOrCreateAAFor<AAUniformWorkGroupSize>(IRP: IRPosition::function(F: *F));
1748 A.getOrCreateAAFor<AAAMDMaxNumWorkgroups>(IRP: IRPosition::function(F: *F));
1749 A.getOrCreateAAFor<AAAMDGPUNoLDSDMA>(IRP: IRPosition::function(F: *F));
1750 CallingConv::ID CC = F->getCallingConv();
1751 if (!AMDGPU::isEntryFunctionCC(CC)) {
1752 A.getOrCreateAAFor<AAAMDFlatWorkGroupSize>(IRP: IRPosition::function(F: *F));
1753 A.getOrCreateAAFor<AAAMDWavesPerEU>(IRP: IRPosition::function(F: *F));
1754 }
1755
1756 const AMDGPU::AMDGPUFeatureBitset &Features = InfoCache.getFeatures();
1757 if (!F->isDeclaration() && Features.test(I: AMDGPU::FEAT_CLUSTERS))
1758 A.getOrCreateAAFor<AAAMDGPUClusterDims>(IRP: IRPosition::function(F: *F));
1759
1760 if (Features.test(I: AMDGPU::FEAT_AGPR_ALLOC))
1761 A.getOrCreateAAFor<AAAMDGPUMinAGPRAlloc>(IRP: IRPosition::function(F: *F));
1762
1763 for (auto &I : instructions(F)) {
1764 Value *Ptr = nullptr;
1765 if (auto *LI = dyn_cast<LoadInst>(Val: &I))
1766 Ptr = LI->getPointerOperand();
1767 else if (auto *SI = dyn_cast<StoreInst>(Val: &I))
1768 Ptr = SI->getPointerOperand();
1769 else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: &I))
1770 Ptr = RMW->getPointerOperand();
1771 else if (auto *CmpX = dyn_cast<AtomicCmpXchgInst>(Val: &I))
1772 Ptr = CmpX->getPointerOperand();
1773
1774 if (Ptr) {
1775 // Only flat pointers can be refined to a specific address space.
1776 if (Ptr->getType()->getPointerAddressSpace() ==
1777 AMDGPUAS::FLAT_ADDRESS) {
1778 A.getOrCreateAAFor<AAAddressSpace>(IRP: IRPosition::value(V: *Ptr));
1779 A.getOrCreateAAFor<AANoAliasAddrSpace>(IRP: IRPosition::value(V: *Ptr));
1780 }
1781 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: Ptr)) {
1782 if (II->getIntrinsicID() == Intrinsic::amdgcn_make_buffer_rsrc)
1783 A.getOrCreateAAFor<AAAlign>(IRP: IRPosition::value(V: *Ptr));
1784 }
1785 }
1786 }
1787 }
1788
1789 return A.run() == ChangeStatus::CHANGED;
1790}
1791} // namespace
1792
1793PreservedAnalyses llvm::AMDGPUAttributorPass::run(Module &M,
1794 ModuleAnalysisManager &AM) {
1795
1796 FunctionAnalysisManager &FAM =
1797 AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
1798 AnalysisGetter AG(FAM);
1799
1800 SetVector<Function *> Functions;
1801 for (Function &F : M) {
1802 if (!F.isDeclaration())
1803 Functions.insert(X: &F);
1804 }
1805
1806 // TODO: Probably preserves CFG
1807 return runImpl(Functions, /*IsModulePass=*/true, /*DeleteFns=*/true, M, AG,
1808 TM, Options, LTOPhase)
1809 ? PreservedAnalyses::none()
1810 : PreservedAnalyses::all();
1811}
1812
1813PreservedAnalyses llvm::AMDGPUAttributorCGSCCPass::run(LazyCallGraph::SCC &C,
1814 CGSCCAnalysisManager &AM,
1815 LazyCallGraph &CG,
1816 CGSCCUpdateResult &UR) {
1817
1818 FunctionAnalysisManager &FAM =
1819 AM.getResult<FunctionAnalysisManagerCGSCCProxy>(IR&: C, ExtraArgs&: CG).getManager();
1820 AnalysisGetter AG(FAM);
1821
1822 SetVector<Function *> Functions;
1823 for (LazyCallGraph::Node &N : C) {
1824 Function *F = &N.getFunction();
1825 if (!F->isIntrinsic())
1826 Functions.insert(X: F);
1827 }
1828
1829 AMDGPUAttributorOptions Options;
1830 Module *M = C.begin()->getFunction().getParent();
1831 // In the CGSCC pipeline, avoid untracked call graph modifications by
1832 // disabling function deletion, mirroring the generic AttributorCGSCCPass.
1833 return runImpl(Functions, /*IsModulePass=*/false, /*DeleteFns=*/false, M&: *M, AG,
1834 TM, Options, LTOPhase: ThinOrFullLTOPhase::None)
1835 ? PreservedAnalyses::none()
1836 : PreservedAnalyses::all();
1837}
1838