1//===- AMDGPUAttributor.cpp -----------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This pass uses Attributor framework to deduce AMDGPU attributes.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AMDGPU.h"
14#include "AMDGPUTargetMachine.h"
15#include "GCNSubtarget.h"
16#include "Utils/AMDGPUBaseInfo.h"
17#include "llvm/IR/IntrinsicsAMDGPU.h"
18#include "llvm/IR/IntrinsicsR600.h"
19#include "llvm/Target/TargetMachine.h"
20#include "llvm/Transforms/IPO/Attributor.h"
21#include <cstdint>
22
23#define DEBUG_TYPE "amdgpu-attributor"
24
25using namespace llvm;
26
27static cl::opt<unsigned> IndirectCallSpecializationThreshold(
28 "amdgpu-indirect-call-specialization-threshold",
29 cl::desc(
30 "A threshold controls whether an indirect call will be specialized"),
31 cl::init(Val: 3));
32
33#define AMDGPU_ATTRIBUTE(Name, Str) Name##_POS,
34
35enum ImplicitArgumentPositions {
36#include "AMDGPUAttributes.def"
37 LAST_ARG_POS
38};
39
40#define AMDGPU_ATTRIBUTE(Name, Str) Name = 1 << Name##_POS,
41
42enum ImplicitArgumentMask {
43 UNKNOWN_INTRINSIC = 0,
44#include "AMDGPUAttributes.def"
45 ALL_ARGUMENT_MASK = (1 << LAST_ARG_POS) - 1,
46 NOT_IMPLICIT_INPUT
47};
48
49#define AMDGPU_ATTRIBUTE(Name, Str) {Name, Str},
50static constexpr std::pair<ImplicitArgumentMask, StringLiteral>
51 ImplicitAttrs[] = {
52#include "AMDGPUAttributes.def"
53};
54
55// We do not need to note the x workitem or workgroup id because they are always
56// initialized.
57//
58// TODO: We should not add the attributes if the known compile time workgroup
59// size is 1 for y/z.
60static ImplicitArgumentMask
61intrinsicToAttrMask(Intrinsic::ID ID, bool &NonKernelOnly, bool &NeedsImplicit,
62 bool HasApertureRegs, bool SupportsGetDoorBellID,
63 unsigned CodeObjectVersion) {
64 switch (ID) {
65 case Intrinsic::amdgcn_workitem_id_x:
66 NonKernelOnly = true;
67 return WORKITEM_ID_X;
68 case Intrinsic::amdgcn_workgroup_id_x:
69 NonKernelOnly = true;
70 return WORKGROUP_ID_X;
71 case Intrinsic::amdgcn_workitem_id_y:
72 case Intrinsic::r600_read_tidig_y:
73 return WORKITEM_ID_Y;
74 case Intrinsic::amdgcn_workitem_id_z:
75 case Intrinsic::r600_read_tidig_z:
76 return WORKITEM_ID_Z;
77 case Intrinsic::amdgcn_workgroup_id_y:
78 case Intrinsic::r600_read_tgid_y:
79 return WORKGROUP_ID_Y;
80 case Intrinsic::amdgcn_workgroup_id_z:
81 case Intrinsic::r600_read_tgid_z:
82 return WORKGROUP_ID_Z;
83 case Intrinsic::amdgcn_cluster_id_x:
84 NonKernelOnly = true;
85 return CLUSTER_ID_X;
86 case Intrinsic::amdgcn_cluster_id_y:
87 return CLUSTER_ID_Y;
88 case Intrinsic::amdgcn_cluster_id_z:
89 return CLUSTER_ID_Z;
90 case Intrinsic::amdgcn_lds_kernel_id:
91 return LDS_KERNEL_ID;
92 case Intrinsic::amdgcn_dispatch_ptr:
93 return DISPATCH_PTR;
94 case Intrinsic::amdgcn_dispatch_id:
95 return DISPATCH_ID;
96 case Intrinsic::amdgcn_implicitarg_ptr:
97 return IMPLICIT_ARG_PTR;
98 // Need queue_ptr anyway. But under V5, we also need implicitarg_ptr to access
99 // queue_ptr.
100 case Intrinsic::amdgcn_queue_ptr:
101 NeedsImplicit = (CodeObjectVersion >= AMDGPU::AMDHSA_COV5);
102 return QUEUE_PTR;
103 case Intrinsic::amdgcn_is_shared:
104 case Intrinsic::amdgcn_is_private:
105 if (HasApertureRegs)
106 return NOT_IMPLICIT_INPUT;
107 // Under V5, we need implicitarg_ptr + offsets to access private_base or
108 // shared_base. For pre-V5, however, need to access them through queue_ptr +
109 // offsets.
110 return CodeObjectVersion >= AMDGPU::AMDHSA_COV5 ? IMPLICIT_ARG_PTR
111 : QUEUE_PTR;
112 case Intrinsic::amdgcn_wwm:
113 case Intrinsic::amdgcn_strict_wwm:
114 return WHOLE_WAVE_MODE;
115 case Intrinsic::trap:
116 case Intrinsic::debugtrap:
117 case Intrinsic::ubsantrap:
118 if (SupportsGetDoorBellID) // GetDoorbellID support implemented since V4.
119 return CodeObjectVersion >= AMDGPU::AMDHSA_COV4 ? NOT_IMPLICIT_INPUT
120 : QUEUE_PTR;
121 NeedsImplicit = (CodeObjectVersion >= AMDGPU::AMDHSA_COV5);
122 return QUEUE_PTR;
123 default:
124 return UNKNOWN_INTRINSIC;
125 }
126}
127
128static bool castRequiresQueuePtr(unsigned SrcAS) {
129 return SrcAS == AMDGPUAS::LOCAL_ADDRESS || SrcAS == AMDGPUAS::PRIVATE_ADDRESS;
130}
131
132static bool isDSAddress(const Constant *C) {
133 const GlobalValue *GV = dyn_cast<GlobalValue>(Val: C);
134 if (!GV)
135 return false;
136 unsigned AS = GV->getAddressSpace();
137 return AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS;
138}
139
140/// Returns true if sanitizer attributes are present on a function.
141static bool hasSanitizerAttributes(const Function &F) {
142 return F.hasFnAttribute(Kind: Attribute::SanitizeAddress) ||
143 F.hasFnAttribute(Kind: Attribute::SanitizeThread) ||
144 F.hasFnAttribute(Kind: Attribute::SanitizeMemory) ||
145 F.hasFnAttribute(Kind: Attribute::SanitizeHWAddress) ||
146 F.hasFnAttribute(Kind: Attribute::SanitizeMemTag);
147}
148
149namespace {
150class AMDGPUInformationCache : public InformationCache {
151public:
152 AMDGPUInformationCache(const Module &M, AnalysisGetter &AG,
153 BumpPtrAllocator &Allocator,
154 SetVector<Function *> *CGSCC, TargetMachine &TM)
155 : InformationCache(M, AG, Allocator, CGSCC), TM(TM),
156 SubArch(M.getTargetTriple().getSubArch()),
157 Features(
158 AMDGPU::getFeatureBitset(AK: AMDGPU::getGPUKindFromSubArch(SubArch))),
159 CodeObjectVersion(AMDGPU::getAMDHSACodeObjectVersion(M)) {}
160
161 TargetMachine &TM;
162
163 enum ConstantStatus : uint8_t {
164 NONE = 0,
165 DS_GLOBAL = 1 << 0,
166 ADDR_SPACE_CAST_PRIVATE_TO_FLAT = 1 << 1,
167 ADDR_SPACE_CAST_LOCAL_TO_FLAT = 1 << 2,
168 ADDR_SPACE_CAST_BOTH_TO_FLAT =
169 ADDR_SPACE_CAST_PRIVATE_TO_FLAT | ADDR_SPACE_CAST_LOCAL_TO_FLAT,
170 CS_WORST = DS_GLOBAL | ADDR_SPACE_CAST_BOTH_TO_FLAT,
171 };
172
173 std::optional<std::pair<unsigned, unsigned>>
174 getFlatWorkGroupSizeAttr(const Function &F) const {
175 auto R = AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-flat-work-group-size");
176 if (!R)
177 return std::nullopt;
178 return std::make_pair(x&: R->first, y&: *(R->second));
179 }
180
181 std::pair<unsigned, unsigned>
182 getDefaultFlatWorkGroupSize(const Function &F) const {
183 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F);
184 return ST.getDefaultFlatWorkGroupSize(CC: F.getCallingConv());
185 }
186
187 std::pair<unsigned, unsigned> getMaximumFlatWorkGroupRange() const {
188 return {AMDGPU::getMinFlatWorkGroupSize(),
189 AMDGPU::getMaxFlatWorkGroupSize()};
190 }
191
192 /// Get code object version.
193 unsigned getCodeObjectVersion() const { return CodeObjectVersion; }
194
195 /// Get the features of the module target.
196 const AMDGPU::AMDGPUFeatureBitset &getFeatures() const { return Features; }
197
198 std::optional<std::pair<unsigned, unsigned>>
199 getWavesPerEUAttr(const Function &F) {
200 auto Val = AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-waves-per-eu",
201 /*OnlyFirstRequired=*/true);
202 if (!Val)
203 return std::nullopt;
204 if (!Val->second)
205 Val->second = AMDGPU::getMaxWavesPerEU(SubArch);
206 return std::make_pair(x&: Val->first, y&: *(Val->second));
207 }
208
209 unsigned getMaxWavesPerEU() const {
210 return AMDGPU::getMaxWavesPerEU(SubArch);
211 }
212
213 unsigned getMaxAddrSpace() const override {
214 return AMDGPUAS::MAX_AMDGPU_ADDRESS;
215 }
216
217private:
218 /// Check if the ConstantExpr \p CE uses an addrspacecast from private or
219 /// local to flat. These casts may require the queue pointer.
220 static uint8_t visitConstExpr(const ConstantExpr *CE) {
221 uint8_t Status = NONE;
222
223 if (CE->getOpcode() == Instruction::AddrSpaceCast) {
224 unsigned SrcAS = CE->getOperand(i_nocapture: 0)->getType()->getPointerAddressSpace();
225 if (SrcAS == AMDGPUAS::PRIVATE_ADDRESS)
226 Status |= ADDR_SPACE_CAST_PRIVATE_TO_FLAT;
227 else if (SrcAS == AMDGPUAS::LOCAL_ADDRESS)
228 Status |= ADDR_SPACE_CAST_LOCAL_TO_FLAT;
229 }
230
231 return Status;
232 }
233
234 /// Get the constant access bitmap for \p C.
235 uint8_t getConstantAccess(const Constant *C) {
236 const auto &It = ConstantStatus.find(Val: C);
237 if (It != ConstantStatus.end())
238 return It->second.value();
239
240 SmallPtrSet<const Constant *, 8> Visited;
241 SmallVector<const Constant *> Worklist;
242 Worklist.push_back(Elt: C);
243 Visited.insert(Ptr: C);
244
245 uint8_t Result = 0;
246 while (Result != CS_WORST && !Worklist.empty()) {
247 const Constant *CurC = Worklist.pop_back_val();
248
249 std::optional<uint8_t> &CurCResultOrNone = ConstantStatus[CurC];
250 if (CurCResultOrNone) {
251 Result |= CurCResultOrNone.value();
252 continue;
253 }
254 uint8_t CurCResult = 0;
255
256 if (isDSAddress(C: CurC))
257 CurCResult |= DS_GLOBAL;
258
259 if (const auto *CE = dyn_cast<ConstantExpr>(Val: CurC))
260 CurCResult |= visitConstExpr(CE);
261
262 for (const Use &U : CurC->operands()) {
263 if (const auto *OpC = dyn_cast<Constant>(Val: U)) {
264 if (Visited.insert(Ptr: OpC).second)
265 Worklist.push_back(Elt: OpC);
266 }
267 }
268
269 CurCResultOrNone = CurCResult;
270 Result |= CurCResult;
271 }
272
273 ConstantStatus[C] = Result;
274 return Result;
275 }
276
277public:
278 /// Returns true if \p Fn needs the queue pointer because of \p C.
279 bool needsQueuePtr(const Constant *C, Function &Fn) {
280 bool IsNonEntryFunc = !AMDGPU::isEntryFunctionCC(CC: Fn.getCallingConv());
281 bool HasAperture = Features.test(I: AMDGPU::FEAT_APERTURE_REGS);
282
283 // No need to explore the constants.
284 if (!IsNonEntryFunc && HasAperture)
285 return false;
286
287 uint8_t Access = getConstantAccess(C);
288
289 // We need to trap on DS globals in non-entry functions.
290 if (IsNonEntryFunc && (Access & DS_GLOBAL))
291 return true;
292
293 return !HasAperture && (Access & ADDR_SPACE_CAST_BOTH_TO_FLAT);
294 }
295
296 bool checkConstForAddrSpaceCastFromPrivate(const Constant *C) {
297 uint8_t Access = getConstantAccess(C);
298 return Access & ADDR_SPACE_CAST_PRIVATE_TO_FLAT;
299 }
300
301private:
302 /// Used to determine if the Constant needs the queue pointer.
303 DenseMap<const Constant *, std::optional<uint8_t>> ConstantStatus;
304 const Triple::SubArchType SubArch;
305 const AMDGPU::AMDGPUFeatureBitset Features;
306 const unsigned CodeObjectVersion;
307};
308
309struct AAAMDAttributes
310 : public StateWrapper<BitIntegerState<uint32_t, ALL_ARGUMENT_MASK, 0>,
311 AbstractAttribute> {
312 using Base = StateWrapper<BitIntegerState<uint32_t, ALL_ARGUMENT_MASK, 0>,
313 AbstractAttribute>;
314
315 AAAMDAttributes(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
316
317 /// Create an abstract attribute view for the position \p IRP.
318 static AAAMDAttributes &createForPosition(const IRPosition &IRP,
319 Attributor &A);
320
321 /// See AbstractAttribute::getName().
322 StringRef getName() const override { return "AAAMDAttributes"; }
323
324 /// See AbstractAttribute::getIdAddr().
325 const char *getIdAddr() const override { return &ID; }
326
327 /// This function should return true if the type of the \p AA is
328 /// AAAMDAttributes.
329 static bool classof(const AbstractAttribute *AA) {
330 return (AA->getIdAddr() == &ID);
331 }
332
333 /// Unique ID (due to the unique address)
334 static const char ID;
335};
336const char AAAMDAttributes::ID = 0;
337
338struct AAUniformWorkGroupSize
339 : public StateWrapper<BooleanState, AbstractAttribute> {
340 using Base = StateWrapper<BooleanState, AbstractAttribute>;
341 AAUniformWorkGroupSize(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
342
343 /// Create an abstract attribute view for the position \p IRP.
344 static AAUniformWorkGroupSize &createForPosition(const IRPosition &IRP,
345 Attributor &A);
346
347 /// See AbstractAttribute::getName().
348 StringRef getName() const override { return "AAUniformWorkGroupSize"; }
349
350 /// See AbstractAttribute::getIdAddr().
351 const char *getIdAddr() const override { return &ID; }
352
353 /// This function should return true if the type of the \p AA is
354 /// AAAMDAttributes.
355 static bool classof(const AbstractAttribute *AA) {
356 return (AA->getIdAddr() == &ID);
357 }
358
359 /// Unique ID (due to the unique address)
360 static const char ID;
361};
362const char AAUniformWorkGroupSize::ID = 0;
363
364struct AAUniformWorkGroupSizeFunction : public AAUniformWorkGroupSize {
365 AAUniformWorkGroupSizeFunction(const IRPosition &IRP, Attributor &A)
366 : AAUniformWorkGroupSize(IRP, A) {}
367
368 void initialize(Attributor &A) override {
369 Function *F = getAssociatedFunction();
370 CallingConv::ID CC = F->getCallingConv();
371
372 if (CC != CallingConv::AMDGPU_KERNEL)
373 return;
374
375 bool InitialValue = F->hasFnAttribute(Kind: "uniform-work-group-size");
376
377 if (InitialValue)
378 indicateOptimisticFixpoint();
379 else
380 indicatePessimisticFixpoint();
381 }
382
383 ChangeStatus updateImpl(Attributor &A) override {
384 ChangeStatus Change = ChangeStatus::UNCHANGED;
385
386 auto CheckCallSite = [&](AbstractCallSite CS) {
387 Function *Caller = CS.getInstruction()->getFunction();
388 LLVM_DEBUG(dbgs() << "[AAUniformWorkGroupSize] Call " << Caller->getName()
389 << "->" << getAssociatedFunction()->getName() << "\n");
390
391 const auto *CallerInfo = A.getAAFor<AAUniformWorkGroupSize>(
392 QueryingAA: *this, IRP: IRPosition::function(F: *Caller), DepClass: DepClassTy::REQUIRED);
393 if (!CallerInfo || !CallerInfo->isValidState())
394 return false;
395
396 Change = Change | clampStateAndIndicateChange(S&: this->getState(),
397 R: CallerInfo->getState());
398
399 return true;
400 };
401
402 bool AllCallSitesKnown = true;
403 if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this, RequireAllCallSites: true, UsedAssumedInformation&: AllCallSitesKnown))
404 return indicatePessimisticFixpoint();
405
406 return Change;
407 }
408
409 ChangeStatus manifest(Attributor &A) override {
410 if (!getAssumed())
411 return ChangeStatus::UNCHANGED;
412
413 LLVMContext &Ctx = getAssociatedFunction()->getContext();
414 return A.manifestAttrs(IRP: getIRPosition(),
415 DeducedAttrs: {Attribute::get(Context&: Ctx, Kind: "uniform-work-group-size")},
416 /*ForceReplace=*/true);
417 }
418
419 bool isValidState() const override {
420 // This state is always valid, even when the state is false.
421 return true;
422 }
423
424 const std::string getAsStr(Attributor *) const override {
425 return "AMDWorkGroupSize[" + std::to_string(val: getAssumed()) + "]";
426 }
427
428 /// See AbstractAttribute::trackStatistics()
429 void trackStatistics() const override {}
430};
431
432AAUniformWorkGroupSize &
433AAUniformWorkGroupSize::createForPosition(const IRPosition &IRP,
434 Attributor &A) {
435 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
436 return *new (A.Allocator) AAUniformWorkGroupSizeFunction(IRP, A);
437 llvm_unreachable(
438 "AAUniformWorkGroupSize is only valid for function position");
439}
440
441struct AAAMDAttributesFunction : public AAAMDAttributes {
442 AAAMDAttributesFunction(const IRPosition &IRP, Attributor &A)
443 : AAAMDAttributes(IRP, A) {}
444
445 void initialize(Attributor &A) override {
446 Function *F = getAssociatedFunction();
447
448 // If the function requires the implicit arg pointer due to sanitizers,
449 // assume it's needed even if explicitly marked as not requiring it.
450 // Flat scratch initialization is needed because `asan_malloc_impl`
451 // calls introduced later in pipeline will have flat scratch accesses.
452 // FIXME: FLAT_SCRATCH_INIT will not be required here if device-libs
453 // implementation for `asan_malloc_impl` is updated.
454 const bool HasSanitizerAttrs = hasSanitizerAttributes(F: *F);
455 if (HasSanitizerAttrs) {
456 removeAssumedBits(BitsEncoding: IMPLICIT_ARG_PTR);
457 removeAssumedBits(BitsEncoding: HOSTCALL_PTR);
458 removeAssumedBits(BitsEncoding: FLAT_SCRATCH_INIT);
459 }
460
461 for (auto Attr : ImplicitAttrs) {
462 if (HasSanitizerAttrs &&
463 (Attr.first == IMPLICIT_ARG_PTR || Attr.first == HOSTCALL_PTR ||
464 Attr.first == FLAT_SCRATCH_INIT))
465 continue;
466
467 if (F->hasFnAttribute(Kind: Attr.second))
468 addKnownBits(Bits: Attr.first);
469 }
470
471 if (F->isDeclaration())
472 return;
473
474 // Ignore functions with graphics calling conventions, these are currently
475 // not allowed to have kernel arguments.
476 if (AMDGPU::isGraphics(CC: F->getCallingConv())) {
477 indicatePessimisticFixpoint();
478 return;
479 }
480 }
481
482 ChangeStatus updateImpl(Attributor &A) override {
483 Function *F = getAssociatedFunction();
484 // The current assumed state used to determine a change.
485 auto OrigAssumed = getAssumed();
486
487 // Check for Intrinsics and propagate attributes.
488 const AACallEdges *AAEdges = A.getAAFor<AACallEdges>(
489 QueryingAA: *this, IRP: this->getIRPosition(), DepClass: DepClassTy::REQUIRED);
490 if (!AAEdges || !AAEdges->isValidState() ||
491 AAEdges->hasNonAsmUnknownCallee())
492 return indicatePessimisticFixpoint();
493
494 bool IsNonEntryFunc = !AMDGPU::isEntryFunctionCC(CC: F->getCallingConv());
495
496 bool NeedsImplicit = false;
497 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
498 const AMDGPU::AMDGPUFeatureBitset &Features = InfoCache.getFeatures();
499 bool HasApertureRegs = Features.test(I: AMDGPU::FEAT_APERTURE_REGS);
500 bool SupportsGetDoorbellID = Features.test(I: AMDGPU::FEAT_GET_DOORBELL_ID);
501 unsigned COV = InfoCache.getCodeObjectVersion();
502
503 for (Function *Callee : AAEdges->getOptimisticEdges()) {
504 Intrinsic::ID IID = Callee->getIntrinsicID();
505 if (IID == Intrinsic::not_intrinsic) {
506 const AAAMDAttributes *AAAMD = A.getAAFor<AAAMDAttributes>(
507 QueryingAA: *this, IRP: IRPosition::function(F: *Callee), DepClass: DepClassTy::REQUIRED);
508 if (!AAAMD || !AAAMD->isValidState())
509 return indicatePessimisticFixpoint();
510 *this &= *AAAMD;
511 continue;
512 }
513
514 bool NonKernelOnly = false;
515 ImplicitArgumentMask AttrMask =
516 intrinsicToAttrMask(ID: IID, NonKernelOnly, NeedsImplicit,
517 HasApertureRegs, SupportsGetDoorBellID: SupportsGetDoorbellID, CodeObjectVersion: COV);
518
519 if (AttrMask == UNKNOWN_INTRINSIC) {
520 // Assume not-nocallback intrinsics may invoke a function which accesses
521 // implicit arguments.
522 //
523 // FIXME: This isn't really the correct check. We want to ensure it
524 // isn't calling any function that may use implicit arguments regardless
525 // of whether it's internal to the module or not.
526 //
527 // TODO: Ignoring callsite attributes.
528 if (!Callee->hasFnAttribute(Kind: Attribute::NoCallback))
529 return indicatePessimisticFixpoint();
530 continue;
531 }
532
533 if (AttrMask != NOT_IMPLICIT_INPUT) {
534 if ((IsNonEntryFunc || !NonKernelOnly))
535 removeAssumedBits(BitsEncoding: AttrMask);
536 }
537 }
538
539 // Need implicitarg_ptr to acess queue_ptr, private_base, and shared_base.
540 if (NeedsImplicit)
541 removeAssumedBits(BitsEncoding: IMPLICIT_ARG_PTR);
542
543 if (isAssumed(BitsEncoding: QUEUE_PTR) && checkForQueuePtr(A)) {
544 // Under V5, we need implicitarg_ptr + offsets to access private_base or
545 // shared_base. We do not actually need queue_ptr.
546 if (COV >= 5)
547 removeAssumedBits(BitsEncoding: IMPLICIT_ARG_PTR);
548 else
549 removeAssumedBits(BitsEncoding: QUEUE_PTR);
550 }
551
552 if (funcRetrievesMultigridSyncArg(A, COV)) {
553 assert(!isAssumed(IMPLICIT_ARG_PTR) &&
554 "multigrid_sync_arg needs implicitarg_ptr");
555 removeAssumedBits(BitsEncoding: MULTIGRID_SYNC_ARG);
556 }
557
558 if (funcRetrievesHostcallPtr(A, COV)) {
559 assert(!isAssumed(IMPLICIT_ARG_PTR) && "hostcall needs implicitarg_ptr");
560 removeAssumedBits(BitsEncoding: HOSTCALL_PTR);
561 }
562
563 if (funcRetrievesHeapPtr(A, COV)) {
564 assert(!isAssumed(IMPLICIT_ARG_PTR) && "heap_ptr needs implicitarg_ptr");
565 removeAssumedBits(BitsEncoding: HEAP_PTR);
566 }
567
568 if (isAssumed(BitsEncoding: QUEUE_PTR) && funcRetrievesQueuePtr(A, COV)) {
569 assert(!isAssumed(IMPLICIT_ARG_PTR) && "queue_ptr needs implicitarg_ptr");
570 removeAssumedBits(BitsEncoding: QUEUE_PTR);
571 }
572
573 if (isAssumed(BitsEncoding: LDS_KERNEL_ID) && funcRetrievesLDSKernelId(A)) {
574 removeAssumedBits(BitsEncoding: LDS_KERNEL_ID);
575 }
576
577 if (isAssumed(BitsEncoding: DEFAULT_QUEUE) && funcRetrievesDefaultQueue(A, COV))
578 removeAssumedBits(BitsEncoding: DEFAULT_QUEUE);
579
580 if (isAssumed(BitsEncoding: COMPLETION_ACTION) && funcRetrievesCompletionAction(A, COV))
581 removeAssumedBits(BitsEncoding: COMPLETION_ACTION);
582
583 if (isAssumed(BitsEncoding: FLAT_SCRATCH_INIT) && needFlatScratchInit(A))
584 removeAssumedBits(BitsEncoding: FLAT_SCRATCH_INIT);
585
586 return getAssumed() != OrigAssumed ? ChangeStatus::CHANGED
587 : ChangeStatus::UNCHANGED;
588 }
589
590 ChangeStatus manifest(Attributor &A) override {
591 SmallVector<Attribute, 8> AttrList;
592 LLVMContext &Ctx = getAssociatedFunction()->getContext();
593
594 for (auto Attr : ImplicitAttrs) {
595 if (isKnown(BitsEncoding: Attr.first))
596 AttrList.push_back(Elt: Attribute::get(Context&: Ctx, Kind: Attr.second));
597 }
598
599 return A.manifestAttrs(IRP: getIRPosition(), DeducedAttrs: AttrList,
600 /* ForceReplace */ true);
601 }
602
603 const std::string getAsStr(Attributor *) const override {
604 std::string Str;
605 raw_string_ostream OS(Str);
606 OS << "AMDInfo[";
607 for (auto Attr : ImplicitAttrs)
608 if (isAssumed(BitsEncoding: Attr.first))
609 OS << ' ' << Attr.second;
610 OS << " ]";
611 return OS.str();
612 }
613
614 /// See AbstractAttribute::trackStatistics()
615 void trackStatistics() const override {}
616
617private:
618 bool checkForQueuePtr(Attributor &A) {
619 Function *F = getAssociatedFunction();
620 bool IsNonEntryFunc = !AMDGPU::isEntryFunctionCC(CC: F->getCallingConv());
621
622 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
623
624 bool NeedsQueuePtr = false;
625
626 auto CheckAddrSpaceCasts = [&](Instruction &I) {
627 unsigned SrcAS = static_cast<AddrSpaceCastInst &>(I).getSrcAddressSpace();
628 if (castRequiresQueuePtr(SrcAS)) {
629 NeedsQueuePtr = true;
630 return false;
631 }
632 return true;
633 };
634
635 bool HasApertureRegs =
636 InfoCache.getFeatures().test(I: AMDGPU::FEAT_APERTURE_REGS);
637
638 // `checkForAllInstructions` is much more cheaper than going through all
639 // instructions, try it first.
640
641 // The queue pointer is not needed if aperture regs is present.
642 if (!HasApertureRegs) {
643 bool UsedAssumedInformation = false;
644 A.checkForAllInstructions(Pred: CheckAddrSpaceCasts, QueryingAA: *this,
645 Opcodes: {Instruction::AddrSpaceCast},
646 UsedAssumedInformation);
647 }
648
649 // If we found that we need the queue pointer, nothing else to do.
650 if (NeedsQueuePtr)
651 return true;
652
653 if (!IsNonEntryFunc && HasApertureRegs)
654 return false;
655
656 for (BasicBlock &BB : *F) {
657 for (Instruction &I : BB) {
658 for (const Use &U : I.operands()) {
659 if (const auto *C = dyn_cast<Constant>(Val: U)) {
660 if (InfoCache.needsQueuePtr(C, Fn&: *F))
661 return true;
662 }
663 }
664 }
665 }
666
667 return false;
668 }
669
670 bool funcRetrievesMultigridSyncArg(Attributor &A, unsigned COV) {
671 auto Pos = llvm::AMDGPU::getMultigridSyncArgImplicitArgPosition(COV);
672 AA::RangeTy Range(Pos, 8);
673 return funcRetrievesImplicitKernelArg(A, Range);
674 }
675
676 bool funcRetrievesHostcallPtr(Attributor &A, unsigned COV) {
677 auto Pos = llvm::AMDGPU::getHostcallImplicitArgPosition(COV);
678 AA::RangeTy Range(Pos, 8);
679 return funcRetrievesImplicitKernelArg(A, Range);
680 }
681
682 bool funcRetrievesDefaultQueue(Attributor &A, unsigned COV) {
683 auto Pos = llvm::AMDGPU::getDefaultQueueImplicitArgPosition(COV);
684 AA::RangeTy Range(Pos, 8);
685 return funcRetrievesImplicitKernelArg(A, Range);
686 }
687
688 bool funcRetrievesCompletionAction(Attributor &A, unsigned COV) {
689 auto Pos = llvm::AMDGPU::getCompletionActionImplicitArgPosition(COV);
690 AA::RangeTy Range(Pos, 8);
691 return funcRetrievesImplicitKernelArg(A, Range);
692 }
693
694 bool funcRetrievesHeapPtr(Attributor &A, unsigned COV) {
695 if (COV < 5)
696 return false;
697 AA::RangeTy Range(AMDGPU::ImplicitArg::HEAP_PTR_OFFSET, 8);
698 return funcRetrievesImplicitKernelArg(A, Range);
699 }
700
701 bool funcRetrievesQueuePtr(Attributor &A, unsigned COV) {
702 if (COV < 5)
703 return false;
704 AA::RangeTy Range(AMDGPU::ImplicitArg::QUEUE_PTR_OFFSET, 8);
705 return funcRetrievesImplicitKernelArg(A, Range);
706 }
707
708 bool funcRetrievesImplicitKernelArg(Attributor &A, AA::RangeTy Range) {
709 // Check if this is a call to the implicitarg_ptr builtin and it
710 // is used to retrieve the hostcall pointer. The implicit arg for
711 // hostcall is not used only if every use of the implicitarg_ptr
712 // is a load that clearly does not retrieve any byte of the
713 // hostcall pointer. We check this by tracing all the uses of the
714 // initial call to the implicitarg_ptr intrinsic.
715 auto DoesNotLeadToKernelArgLoc = [&](Instruction &I) {
716 auto &Call = cast<CallBase>(Val&: I);
717 if (Call.getIntrinsicID() != Intrinsic::amdgcn_implicitarg_ptr)
718 return true;
719
720 const auto *PointerInfoAA = A.getAAFor<AAPointerInfo>(
721 QueryingAA: *this, IRP: IRPosition::callsite_returned(CB: Call), DepClass: DepClassTy::REQUIRED);
722 if (!PointerInfoAA || !PointerInfoAA->getState().isValidState())
723 return false;
724
725 return PointerInfoAA->forallInterferingAccesses(
726 Range, CB: [](const AAPointerInfo::Access &Acc, bool IsExact) {
727 return Acc.getRemoteInst()->isDroppable();
728 });
729 };
730
731 bool UsedAssumedInformation = false;
732 return !A.checkForAllCallLikeInstructions(Pred: DoesNotLeadToKernelArgLoc, QueryingAA: *this,
733 UsedAssumedInformation);
734 }
735
736 bool funcRetrievesLDSKernelId(Attributor &A) {
737 auto DoesNotRetrieve = [&](Instruction &I) {
738 auto &Call = cast<CallBase>(Val&: I);
739 return Call.getIntrinsicID() != Intrinsic::amdgcn_lds_kernel_id;
740 };
741 bool UsedAssumedInformation = false;
742 return !A.checkForAllCallLikeInstructions(Pred: DoesNotRetrieve, QueryingAA: *this,
743 UsedAssumedInformation);
744 }
745
746 // Returns true if FlatScratchInit is needed, i.e., no-flat-scratch-init is
747 // not to be set.
748 bool needFlatScratchInit(Attributor &A) {
749 assert(isAssumed(FLAT_SCRATCH_INIT)); // only called if the bit is still set
750
751 // Check all AddrSpaceCast instructions. FlatScratchInit is needed if
752 // there is a cast from PRIVATE_ADDRESS.
753 auto AddrSpaceCastNotFromPrivate = [](Instruction &I) {
754 return cast<AddrSpaceCastInst>(Val&: I).getSrcAddressSpace() !=
755 AMDGPUAS::PRIVATE_ADDRESS;
756 };
757
758 bool UsedAssumedInformation = false;
759 if (!A.checkForAllInstructions(Pred: AddrSpaceCastNotFromPrivate, QueryingAA: *this,
760 Opcodes: {Instruction::AddrSpaceCast},
761 UsedAssumedInformation))
762 return true;
763
764 // Check for addrSpaceCast from PRIVATE_ADDRESS in constant expressions
765 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
766
767 Function *F = getAssociatedFunction();
768 for (Instruction &I : instructions(F)) {
769 for (const Use &U : I.operands()) {
770 if (const auto *C = dyn_cast<Constant>(Val: U)) {
771 if (InfoCache.checkConstForAddrSpaceCastFromPrivate(C))
772 return true;
773 }
774 }
775 }
776
777 return false;
778 }
779};
780
781AAAMDAttributes &AAAMDAttributes::createForPosition(const IRPosition &IRP,
782 Attributor &A) {
783 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
784 return *new (A.Allocator) AAAMDAttributesFunction(IRP, A);
785 llvm_unreachable("AAAMDAttributes is only valid for function position");
786}
787
788/// Base class to derive different size ranges.
789struct AAAMDSizeRangeAttribute
790 : public StateWrapper<IntegerRangeState, AbstractAttribute, uint32_t> {
791 using Base = StateWrapper<IntegerRangeState, AbstractAttribute, uint32_t>;
792
793 StringRef AttrName;
794
795 AAAMDSizeRangeAttribute(const IRPosition &IRP, Attributor &A,
796 StringRef AttrName)
797 : Base(IRP, 32), AttrName(AttrName) {}
798
799 /// See AbstractAttribute::trackStatistics()
800 void trackStatistics() const override {}
801
802 template <class AttributeImpl> ChangeStatus updateImplImpl(Attributor &A) {
803 ChangeStatus Change = ChangeStatus::UNCHANGED;
804
805 auto CheckCallSite = [&](AbstractCallSite CS) {
806 Function *Caller = CS.getInstruction()->getFunction();
807 LLVM_DEBUG(dbgs() << '[' << getName() << "] Call " << Caller->getName()
808 << "->" << getAssociatedFunction()->getName() << '\n');
809
810 const auto *CallerInfo = A.getAAFor<AttributeImpl>(
811 *this, IRPosition::function(F: *Caller), DepClassTy::REQUIRED);
812 if (!CallerInfo || !CallerInfo->isValidState())
813 return false;
814
815 Change |=
816 clampStateAndIndicateChange(this->getState(), CallerInfo->getState());
817
818 return true;
819 };
820
821 bool AllCallSitesKnown = true;
822 if (!A.checkForAllCallSites(CheckCallSite, *this,
823 /*RequireAllCallSites=*/true,
824 AllCallSitesKnown))
825 return indicatePessimisticFixpoint();
826
827 return Change;
828 }
829
830 /// Clamp the assumed range to the default value ([Min, Max]) and emit the
831 /// attribute if it is not same as default.
832 ChangeStatus
833 emitAttributeIfNotDefaultAfterClamp(Attributor &A,
834 std::pair<unsigned, unsigned> Default) {
835 auto [Min, Max] = Default;
836 unsigned Lower = getAssumed().getLower().getZExtValue();
837 unsigned Upper = getAssumed().getUpper().getZExtValue();
838
839 // Clamp the range to the default value.
840 if (Lower < Min)
841 Lower = Min;
842 if (Upper > Max + 1)
843 Upper = Max + 1;
844
845 // No manifest if the value is invalid or same as default after clamp.
846 if ((Lower == Min && Upper == Max + 1) || (Upper < Lower))
847 return ChangeStatus::UNCHANGED;
848
849 Function *F = getAssociatedFunction();
850 LLVMContext &Ctx = F->getContext();
851 SmallString<10> Buffer;
852 raw_svector_ostream OS(Buffer);
853 OS << Lower << ',' << Upper - 1;
854 return A.manifestAttrs(IRP: getIRPosition(),
855 DeducedAttrs: {Attribute::get(Context&: Ctx, Kind: AttrName, Val: OS.str())},
856 /*ForceReplace=*/true);
857 }
858
859 const std::string getAsStr(Attributor *) const override {
860 std::string Str;
861 raw_string_ostream OS(Str);
862 OS << getName() << '[';
863 OS << getAssumed().getLower() << ',' << getAssumed().getUpper() - 1;
864 OS << ']';
865 return OS.str();
866 }
867};
868
869/// Propagate amdgpu-flat-work-group-size attribute.
870struct AAAMDFlatWorkGroupSize : public AAAMDSizeRangeAttribute {
871 AAAMDFlatWorkGroupSize(const IRPosition &IRP, Attributor &A)
872 : AAAMDSizeRangeAttribute(IRP, A, "amdgpu-flat-work-group-size") {}
873
874 void initialize(Attributor &A) override {
875 Function *F = getAssociatedFunction();
876 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
877
878 bool HasAttr = false;
879 auto Range = InfoCache.getDefaultFlatWorkGroupSize(F: *F);
880 auto MaxRange = InfoCache.getMaximumFlatWorkGroupRange();
881
882 if (auto Attr = InfoCache.getFlatWorkGroupSizeAttr(F: *F)) {
883 // We only consider an attribute that is not max range because the front
884 // end always emits the attribute, unfortunately, and sometimes it emits
885 // the max range.
886 if (*Attr != MaxRange) {
887 Range = *Attr;
888 HasAttr = true;
889 }
890 }
891
892 // We don't want to directly clamp the state if it's the max range because
893 // that is basically the worst state.
894 if (Range == MaxRange)
895 return;
896
897 auto [Min, Max] = Range;
898 ConstantRange CR(APInt(32, Min), APInt(32, Max + 1));
899 IntegerRangeState IRS(CR);
900 clampStateAndIndicateChange(S&: this->getState(), R: IRS);
901
902 if (HasAttr || AMDGPU::isEntryFunctionCC(CC: F->getCallingConv()))
903 indicateOptimisticFixpoint();
904 }
905
906 ChangeStatus updateImpl(Attributor &A) override {
907 return updateImplImpl<AAAMDFlatWorkGroupSize>(A);
908 }
909
910 /// Create an abstract attribute view for the position \p IRP.
911 static AAAMDFlatWorkGroupSize &createForPosition(const IRPosition &IRP,
912 Attributor &A);
913
914 ChangeStatus manifest(Attributor &A) override {
915 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
916 return emitAttributeIfNotDefaultAfterClamp(
917 A, Default: InfoCache.getMaximumFlatWorkGroupRange());
918 }
919
920 /// See AbstractAttribute::getName()
921 StringRef getName() const override { return "AAAMDFlatWorkGroupSize"; }
922
923 /// See AbstractAttribute::getIdAddr()
924 const char *getIdAddr() const override { return &ID; }
925
926 /// This function should return true if the type of the \p AA is
927 /// AAAMDFlatWorkGroupSize
928 static bool classof(const AbstractAttribute *AA) {
929 return (AA->getIdAddr() == &ID);
930 }
931
932 /// Unique ID (due to the unique address)
933 static const char ID;
934};
935
936const char AAAMDFlatWorkGroupSize::ID = 0;
937
938AAAMDFlatWorkGroupSize &
939AAAMDFlatWorkGroupSize::createForPosition(const IRPosition &IRP,
940 Attributor &A) {
941 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
942 return *new (A.Allocator) AAAMDFlatWorkGroupSize(IRP, A);
943 llvm_unreachable(
944 "AAAMDFlatWorkGroupSize is only valid for function position");
945}
946
947struct TupleDecIntegerRangeState : public AbstractState {
948 DecIntegerState<uint32_t> X, Y, Z;
949
950 bool isValidState() const override {
951 return X.isValidState() && Y.isValidState() && Z.isValidState();
952 }
953
954 bool isAtFixpoint() const override {
955 return X.isAtFixpoint() && Y.isAtFixpoint() && Z.isAtFixpoint();
956 }
957
958 ChangeStatus indicateOptimisticFixpoint() override {
959 return X.indicateOptimisticFixpoint() | Y.indicateOptimisticFixpoint() |
960 Z.indicateOptimisticFixpoint();
961 }
962
963 ChangeStatus indicatePessimisticFixpoint() override {
964 return X.indicatePessimisticFixpoint() | Y.indicatePessimisticFixpoint() |
965 Z.indicatePessimisticFixpoint();
966 }
967
968 TupleDecIntegerRangeState operator^=(const TupleDecIntegerRangeState &Other) {
969 X ^= Other.X;
970 Y ^= Other.Y;
971 Z ^= Other.Z;
972 return *this;
973 }
974
975 bool operator==(const TupleDecIntegerRangeState &Other) const {
976 return X == Other.X && Y == Other.Y && Z == Other.Z;
977 }
978
979 TupleDecIntegerRangeState &getAssumed() { return *this; }
980 const TupleDecIntegerRangeState &getAssumed() const { return *this; }
981};
982
983using AAAMDMaxNumWorkgroupsState =
984 StateWrapper<TupleDecIntegerRangeState, AbstractAttribute, uint32_t>;
985
986/// Propagate amdgpu-max-num-workgroups attribute.
987struct AAAMDMaxNumWorkgroups
988 : public StateWrapper<TupleDecIntegerRangeState, AbstractAttribute> {
989 using Base = StateWrapper<TupleDecIntegerRangeState, AbstractAttribute>;
990
991 AAAMDMaxNumWorkgroups(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
992
993 void initialize(Attributor &A) override {
994 Function *F = getAssociatedFunction();
995
996 SmallVector<unsigned> MaxNumWorkgroups = AMDGPU::getMaxNumWorkGroups(F: *F);
997
998 X.takeKnownMinimum(Value: MaxNumWorkgroups[0]);
999 Y.takeKnownMinimum(Value: MaxNumWorkgroups[1]);
1000 Z.takeKnownMinimum(Value: MaxNumWorkgroups[2]);
1001
1002 if (AMDGPU::isEntryFunctionCC(CC: F->getCallingConv()))
1003 indicatePessimisticFixpoint();
1004 }
1005
1006 ChangeStatus updateImpl(Attributor &A) override {
1007 ChangeStatus Change = ChangeStatus::UNCHANGED;
1008
1009 auto CheckCallSite = [&](AbstractCallSite CS) {
1010 Function *Caller = CS.getInstruction()->getFunction();
1011 LLVM_DEBUG(dbgs() << "[AAAMDMaxNumWorkgroups] Call " << Caller->getName()
1012 << "->" << getAssociatedFunction()->getName() << '\n');
1013
1014 const auto *CallerInfo = A.getAAFor<AAAMDMaxNumWorkgroups>(
1015 QueryingAA: *this, IRP: IRPosition::function(F: *Caller), DepClass: DepClassTy::REQUIRED);
1016 if (!CallerInfo || !CallerInfo->isValidState())
1017 return false;
1018
1019 Change |=
1020 clampStateAndIndicateChange(S&: this->getState(), R: CallerInfo->getState());
1021 return true;
1022 };
1023
1024 bool AllCallSitesKnown = true;
1025 if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this,
1026 /*RequireAllCallSites=*/true,
1027 UsedAssumedInformation&: AllCallSitesKnown))
1028 return indicatePessimisticFixpoint();
1029
1030 return Change;
1031 }
1032
1033 /// Create an abstract attribute view for the position \p IRP.
1034 static AAAMDMaxNumWorkgroups &createForPosition(const IRPosition &IRP,
1035 Attributor &A);
1036
1037 ChangeStatus manifest(Attributor &A) override {
1038 Function *F = getAssociatedFunction();
1039 LLVMContext &Ctx = F->getContext();
1040 SmallString<32> Buffer;
1041 raw_svector_ostream OS(Buffer);
1042 OS << X.getAssumed() << ',' << Y.getAssumed() << ',' << Z.getAssumed();
1043
1044 // TODO: Should annotate loads of the group size for this to do anything
1045 // useful.
1046 return A.manifestAttrs(
1047 IRP: getIRPosition(),
1048 DeducedAttrs: {Attribute::get(Context&: Ctx, Kind: "amdgpu-max-num-workgroups", Val: OS.str())},
1049 /* ForceReplace= */ true);
1050 }
1051
1052 StringRef getName() const override { return "AAAMDMaxNumWorkgroups"; }
1053
1054 const std::string getAsStr(Attributor *) const override {
1055 std::string Buffer = "AAAMDMaxNumWorkgroupsState[";
1056 raw_string_ostream OS(Buffer);
1057 OS << X.getAssumed() << ',' << Y.getAssumed() << ',' << Z.getAssumed()
1058 << ']';
1059 return OS.str();
1060 }
1061
1062 const char *getIdAddr() const override { return &ID; }
1063
1064 /// This function should return true if the type of the \p AA is
1065 /// AAAMDMaxNumWorkgroups
1066 static bool classof(const AbstractAttribute *AA) {
1067 return (AA->getIdAddr() == &ID);
1068 }
1069
1070 void trackStatistics() const override {}
1071
1072 /// Unique ID (due to the unique address)
1073 static const char ID;
1074};
1075
1076const char AAAMDMaxNumWorkgroups::ID = 0;
1077
1078AAAMDMaxNumWorkgroups &
1079AAAMDMaxNumWorkgroups::createForPosition(const IRPosition &IRP, Attributor &A) {
1080 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
1081 return *new (A.Allocator) AAAMDMaxNumWorkgroups(IRP, A);
1082 llvm_unreachable("AAAMDMaxNumWorkgroups is only valid for function position");
1083}
1084
1085/// Propagate amdgpu-waves-per-eu attribute.
1086struct AAAMDWavesPerEU : public AAAMDSizeRangeAttribute {
1087 AAAMDWavesPerEU(const IRPosition &IRP, Attributor &A)
1088 : AAAMDSizeRangeAttribute(IRP, A, "amdgpu-waves-per-eu") {}
1089
1090 void initialize(Attributor &A) override {
1091 Function *F = getAssociatedFunction();
1092 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
1093
1094 // If the attribute exists, we will honor it if it is not the default.
1095 if (auto Attr = InfoCache.getWavesPerEUAttr(F: *F)) {
1096 std::pair<unsigned, unsigned> MaxWavesPerEURange{
1097 1U, InfoCache.getMaxWavesPerEU()};
1098 if (*Attr != MaxWavesPerEURange) {
1099 auto [Min, Max] = *Attr;
1100 ConstantRange Range(APInt(32, Min), APInt(32, Max + 1));
1101 IntegerRangeState RangeState(Range);
1102 this->getState() = RangeState;
1103 indicateOptimisticFixpoint();
1104 return;
1105 }
1106 }
1107
1108 if (AMDGPU::isEntryFunctionCC(CC: F->getCallingConv()))
1109 indicatePessimisticFixpoint();
1110 }
1111
1112 ChangeStatus updateImpl(Attributor &A) override {
1113 ChangeStatus Change = ChangeStatus::UNCHANGED;
1114
1115 auto CheckCallSite = [&](AbstractCallSite CS) {
1116 Function *Caller = CS.getInstruction()->getFunction();
1117 Function *Func = getAssociatedFunction();
1118 LLVM_DEBUG(dbgs() << '[' << getName() << "] Call " << Caller->getName()
1119 << "->" << Func->getName() << '\n');
1120 (void)Func;
1121
1122 const auto *CallerAA = A.getAAFor<AAAMDWavesPerEU>(
1123 QueryingAA: *this, IRP: IRPosition::function(F: *Caller), DepClass: DepClassTy::REQUIRED);
1124 if (!CallerAA || !CallerAA->isValidState())
1125 return false;
1126
1127 ConstantRange Assumed = getAssumed();
1128 unsigned Min = std::max(a: Assumed.getLower().getZExtValue(),
1129 b: CallerAA->getAssumed().getLower().getZExtValue());
1130 unsigned Max = std::max(a: Assumed.getUpper().getZExtValue(),
1131 b: CallerAA->getAssumed().getUpper().getZExtValue());
1132 ConstantRange Range(APInt(32, Min), APInt(32, Max));
1133 IntegerRangeState RangeState(Range);
1134 getState() = RangeState;
1135 Change |= getState() == Assumed ? ChangeStatus::UNCHANGED
1136 : ChangeStatus::CHANGED;
1137
1138 return true;
1139 };
1140
1141 bool AllCallSitesKnown = true;
1142 if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this, RequireAllCallSites: true, UsedAssumedInformation&: AllCallSitesKnown))
1143 return indicatePessimisticFixpoint();
1144
1145 return Change;
1146 }
1147
1148 /// Create an abstract attribute view for the position \p IRP.
1149 static AAAMDWavesPerEU &createForPosition(const IRPosition &IRP,
1150 Attributor &A);
1151
1152 ChangeStatus manifest(Attributor &A) override {
1153 auto &InfoCache = static_cast<AMDGPUInformationCache &>(A.getInfoCache());
1154 return emitAttributeIfNotDefaultAfterClamp(
1155 A, Default: {1U, InfoCache.getMaxWavesPerEU()});
1156 }
1157
1158 /// See AbstractAttribute::getName()
1159 StringRef getName() const override { return "AAAMDWavesPerEU"; }
1160
1161 /// See AbstractAttribute::getIdAddr()
1162 const char *getIdAddr() const override { return &ID; }
1163
1164 /// This function should return true if the type of the \p AA is
1165 /// AAAMDWavesPerEU
1166 static bool classof(const AbstractAttribute *AA) {
1167 return (AA->getIdAddr() == &ID);
1168 }
1169
1170 /// Unique ID (due to the unique address)
1171 static const char ID;
1172};
1173
1174const char AAAMDWavesPerEU::ID = 0;
1175
1176AAAMDWavesPerEU &AAAMDWavesPerEU::createForPosition(const IRPosition &IRP,
1177 Attributor &A) {
1178 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
1179 return *new (A.Allocator) AAAMDWavesPerEU(IRP, A);
1180 llvm_unreachable("AAAMDWavesPerEU is only valid for function position");
1181}
1182
1183/// Compute the minimum number of AGPRs required to allocate the inline asm.
1184static unsigned inlineAsmGetNumRequiredAGPRs(const InlineAsm *IA,
1185 const CallBase &Call) {
1186 unsigned ArgNo = 0;
1187 unsigned ResNo = 0;
1188 unsigned AGPRDefCount = 0;
1189 unsigned AGPRUseCount = 0;
1190 unsigned MaxPhysReg = 0;
1191 const DataLayout &DL = Call.getFunction()->getParent()->getDataLayout();
1192
1193 // TODO: Overestimates due to not accounting for tied operands
1194 for (const InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
1195 Type *Ty = nullptr;
1196 switch (CI.Type) {
1197 case InlineAsm::isOutput: {
1198 Ty = Call.getType();
1199 if (auto *STy = dyn_cast<StructType>(Val: Ty))
1200 Ty = STy->getElementType(N: ResNo);
1201 ++ResNo;
1202 break;
1203 }
1204 case InlineAsm::isInput: {
1205 Ty = Call.getArgOperand(i: ArgNo++)->getType();
1206 break;
1207 }
1208 case InlineAsm::isLabel:
1209 continue;
1210 case InlineAsm::isClobber:
1211 // Parse the physical register reference.
1212 break;
1213 }
1214
1215 for (StringRef Code : CI.Codes) {
1216 unsigned RegCount = 0;
1217 if (Code.starts_with(Prefix: "a")) {
1218 // Virtual register, compute number of registers based on the type.
1219 //
1220 // We ought to be going through TargetLowering to get the number of
1221 // registers, but we should avoid the dependence on CodeGen here.
1222 RegCount = divideCeil(Numerator: DL.getTypeSizeInBits(Ty), Denominator: 32);
1223 } else {
1224 // Physical register reference
1225 auto [Kind, RegIdx, NumRegs] = AMDGPU::parseAsmConstraintPhysReg(Constraint: Code);
1226 if (Kind == 'a') {
1227 RegCount = NumRegs;
1228 MaxPhysReg = std::max(a: MaxPhysReg, b: std::min(a: RegIdx + NumRegs, b: 256u));
1229 }
1230
1231 continue;
1232 }
1233
1234 if (CI.Type == InlineAsm::isOutput) {
1235 // Apply tuple alignment requirement
1236 //
1237 // TODO: This is more conservative than necessary.
1238 AGPRDefCount = alignTo(Value: AGPRDefCount, Align: RegCount);
1239
1240 AGPRDefCount += RegCount;
1241 if (CI.isEarlyClobber) {
1242 AGPRUseCount = alignTo(Value: AGPRUseCount, Align: RegCount);
1243 AGPRUseCount += RegCount;
1244 }
1245 } else {
1246 AGPRUseCount = alignTo(Value: AGPRUseCount, Align: RegCount);
1247 AGPRUseCount += RegCount;
1248 }
1249 }
1250 }
1251
1252 unsigned MaxVirtReg = std::max(a: AGPRUseCount, b: AGPRDefCount);
1253
1254 // TODO: This is overly conservative. If there are any physical registers,
1255 // allocate any virtual registers after them so we don't have to solve optimal
1256 // packing.
1257 return std::min(a: MaxVirtReg + MaxPhysReg, b: 256u);
1258}
1259
1260struct AAAMDGPUMinAGPRAlloc
1261 : public StateWrapper<DecIntegerState<>, AbstractAttribute> {
1262 using Base = StateWrapper<DecIntegerState<>, AbstractAttribute>;
1263 AAAMDGPUMinAGPRAlloc(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
1264
1265 static AAAMDGPUMinAGPRAlloc &createForPosition(const IRPosition &IRP,
1266 Attributor &A) {
1267 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
1268 return *new (A.Allocator) AAAMDGPUMinAGPRAlloc(IRP, A);
1269 llvm_unreachable(
1270 "AAAMDGPUMinAGPRAlloc is only valid for function position");
1271 }
1272
1273 void initialize(Attributor &A) override {
1274 Function *F = getAssociatedFunction();
1275 auto [MinNumAGPR, MaxNumAGPR] =
1276 AMDGPU::getIntegerPairAttribute(F: *F, Name: "amdgpu-agpr-alloc", Default: {~0u, ~0u},
1277 /*OnlyFirstRequired=*/true);
1278 if (MinNumAGPR == 0) {
1279 indicateOptimisticFixpoint();
1280 return;
1281 }
1282
1283 if (hasSanitizerAttributes(F: *F))
1284 indicatePessimisticFixpoint();
1285 }
1286
1287 const std::string getAsStr(Attributor *A) const override {
1288 std::string Str = "amdgpu-agpr-alloc=";
1289 raw_string_ostream OS(Str);
1290 OS << getAssumed();
1291 return OS.str();
1292 }
1293
1294 void trackStatistics() const override {}
1295
1296 ChangeStatus updateImpl(Attributor &A) override {
1297 DecIntegerState<> Maximum;
1298
1299 // Check for cases which require allocation of AGPRs. The only cases where
1300 // AGPRs are required are if there are direct references to AGPRs, so inline
1301 // assembly and special intrinsics.
1302 auto CheckForMinAGPRAllocs = [&](Instruction &I) {
1303 const auto &CB = cast<CallBase>(Val&: I);
1304 const Value *CalleeOp = CB.getCalledOperand();
1305
1306 if (const InlineAsm *IA = dyn_cast<InlineAsm>(Val: CalleeOp)) {
1307 // Technically, the inline asm could be invoking a call to an unknown
1308 // external function that requires AGPRs, but ignore that.
1309 unsigned NumRegs = inlineAsmGetNumRequiredAGPRs(IA, Call: CB);
1310 Maximum.takeAssumedMaximum(Value: NumRegs);
1311 return true;
1312 }
1313 switch (CB.getIntrinsicID()) {
1314 case Intrinsic::not_intrinsic:
1315 break;
1316 case Intrinsic::write_register:
1317 case Intrinsic::read_register:
1318 case Intrinsic::read_volatile_register: {
1319 const MDString *RegName = cast<MDString>(
1320 Val: cast<MDNode>(
1321 Val: cast<MetadataAsValue>(Val: CB.getArgOperand(i: 0))->getMetadata())
1322 ->getOperand(I: 0));
1323 auto [Kind, RegIdx, NumRegs] =
1324 AMDGPU::parseAsmPhysRegName(TupleString: RegName->getString());
1325 if (Kind == 'a')
1326 Maximum.takeAssumedMaximum(Value: std::min(a: RegIdx + NumRegs, b: 256u));
1327
1328 return true;
1329 }
1330 // Trap-like intrinsics such as llvm.trap and llvm.debugtrap do not have
1331 // the nocallback attribute, so the AMDGPU attributor can conservatively
1332 // drop all implicitly-known inputs and AGPR allocation information. Make
1333 // sure we still infer that no implicit inputs are required and that the
1334 // AGPR allocation stays at zero. Trap-like intrinsics may invoke a
1335 // function which requires AGPRs, so we need to check if the called
1336 // function has the "trap-func-name" attribute.
1337 case Intrinsic::trap:
1338 case Intrinsic::debugtrap:
1339 case Intrinsic::ubsantrap:
1340 return CB.hasFnAttr(Kind: Attribute::NoCallback) ||
1341 !CB.hasFnAttr(Kind: "trap-func-name");
1342 default:
1343 // Some intrinsics may use AGPRs, but if we have a choice, we are not
1344 // required to use AGPRs.
1345 // Assume !nocallback intrinsics may call a function which requires
1346 // AGPRs.
1347 return CB.hasFnAttr(Kind: Attribute::NoCallback);
1348 }
1349
1350 // TODO: Handle callsite attributes
1351 auto *CBEdges = A.getAAFor<AACallEdges>(
1352 QueryingAA: *this, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::REQUIRED);
1353 if (!CBEdges || CBEdges->hasUnknownCallee()) {
1354 Maximum.indicatePessimisticFixpoint();
1355 return false;
1356 }
1357
1358 for (const Function *PossibleCallee : CBEdges->getOptimisticEdges()) {
1359 const auto *CalleeInfo = A.getAAFor<AAAMDGPUMinAGPRAlloc>(
1360 QueryingAA: *this, IRP: IRPosition::function(F: *PossibleCallee), DepClass: DepClassTy::REQUIRED);
1361 if (!CalleeInfo || !CalleeInfo->isValidState()) {
1362 Maximum.indicatePessimisticFixpoint();
1363 return false;
1364 }
1365
1366 Maximum.takeAssumedMaximum(Value: CalleeInfo->getAssumed());
1367 }
1368
1369 return true;
1370 };
1371
1372 bool UsedAssumedInformation = false;
1373 if (!A.checkForAllCallLikeInstructions(Pred: CheckForMinAGPRAllocs, QueryingAA: *this,
1374 UsedAssumedInformation))
1375 return indicatePessimisticFixpoint();
1376
1377 return clampStateAndIndicateChange(S&: getState(), R: Maximum);
1378 }
1379
1380 ChangeStatus manifest(Attributor &A) override {
1381 LLVMContext &Ctx = getAssociatedFunction()->getContext();
1382 SmallString<4> Buffer;
1383 raw_svector_ostream OS(Buffer);
1384 OS << getAssumed();
1385
1386 return A.manifestAttrs(
1387 IRP: getIRPosition(), DeducedAttrs: {Attribute::get(Context&: Ctx, Kind: "amdgpu-agpr-alloc", Val: OS.str())});
1388 }
1389
1390 StringRef getName() const override { return "AAAMDGPUMinAGPRAlloc"; }
1391 const char *getIdAddr() const override { return &ID; }
1392
1393 /// This function should return true if the type of the \p AA is
1394 /// AAAMDGPUMinAGPRAllocs
1395 static bool classof(const AbstractAttribute *AA) {
1396 return (AA->getIdAddr() == &ID);
1397 }
1398
1399 static const char ID;
1400};
1401
1402const char AAAMDGPUMinAGPRAlloc::ID = 0;
1403
1404/// An abstract attribute to propagate the function attribute
1405/// "amdgpu-cluster-dims" from kernel entry functions to device functions.
1406struct AAAMDGPUClusterDims
1407 : public StateWrapper<BooleanState, AbstractAttribute> {
1408 using Base = StateWrapper<BooleanState, AbstractAttribute>;
1409 AAAMDGPUClusterDims(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
1410
1411 /// Create an abstract attribute view for the position \p IRP.
1412 static AAAMDGPUClusterDims &createForPosition(const IRPosition &IRP,
1413 Attributor &A);
1414
1415 /// See AbstractAttribute::getName().
1416 StringRef getName() const override { return "AAAMDGPUClusterDims"; }
1417
1418 /// See AbstractAttribute::getIdAddr().
1419 const char *getIdAddr() const override { return &ID; }
1420
1421 /// This function should return true if the type of the \p AA is
1422 /// AAAMDGPUClusterDims.
1423 static bool classof(const AbstractAttribute *AA) {
1424 return AA->getIdAddr() == &ID;
1425 }
1426
1427 virtual const AMDGPU::ClusterDimsAttr &getClusterDims() const = 0;
1428
1429 /// Unique ID (due to the unique address)
1430 static const char ID;
1431};
1432
1433const char AAAMDGPUClusterDims::ID = 0;
1434
1435struct AAAMDGPUClusterDimsFunction : public AAAMDGPUClusterDims {
1436 AAAMDGPUClusterDimsFunction(const IRPosition &IRP, Attributor &A)
1437 : AAAMDGPUClusterDims(IRP, A) {}
1438
1439 void initialize(Attributor &A) override {
1440 Function *F = getAssociatedFunction();
1441 assert(F && "empty associated function");
1442
1443 Attr = AMDGPU::ClusterDimsAttr::get(F: *F);
1444
1445 // No matter what a kernel function has, it is final.
1446 if (AMDGPU::isEntryFunctionCC(CC: F->getCallingConv())) {
1447 if (Attr.isUnknown())
1448 indicatePessimisticFixpoint();
1449 else
1450 indicateOptimisticFixpoint();
1451 }
1452 }
1453
1454 const std::string getAsStr(Attributor *A) const override {
1455 if (!getAssumed() || Attr.isUnknown())
1456 return "unknown";
1457 if (Attr.isNoCluster())
1458 return "no";
1459 if (Attr.isVariableDims())
1460 return "variable";
1461 return Attr.to_string();
1462 }
1463
1464 void trackStatistics() const override {}
1465
1466 ChangeStatus updateImpl(Attributor &A) override {
1467 auto OldState = Attr;
1468
1469 auto CheckCallSite = [&](AbstractCallSite CS) {
1470 const auto *CallerAA = A.getAAFor<AAAMDGPUClusterDims>(
1471 QueryingAA: *this, IRP: IRPosition::function(F: *CS.getInstruction()->getFunction()),
1472 DepClass: DepClassTy::REQUIRED);
1473 if (!CallerAA || !CallerAA->isValidState())
1474 return false;
1475
1476 return merge(Other: CallerAA->getClusterDims());
1477 };
1478
1479 bool UsedAssumedInformation = false;
1480 if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this,
1481 /*RequireAllCallSites=*/true,
1482 UsedAssumedInformation))
1483 return indicatePessimisticFixpoint();
1484
1485 return OldState == Attr ? ChangeStatus::UNCHANGED : ChangeStatus::CHANGED;
1486 }
1487
1488 ChangeStatus manifest(Attributor &A) override {
1489 if (Attr.isUnknown())
1490 return ChangeStatus::UNCHANGED;
1491 return A.manifestAttrs(
1492 IRP: getIRPosition(),
1493 DeducedAttrs: {Attribute::get(Context&: getAssociatedFunction()->getContext(), Kind: AttrName,
1494 Val: Attr.to_string())},
1495 /*ForceReplace=*/true);
1496 }
1497
1498 const AMDGPU::ClusterDimsAttr &getClusterDims() const override {
1499 return Attr;
1500 }
1501
1502private:
1503 bool merge(const AMDGPU::ClusterDimsAttr &Other) {
1504 // Case 1: Both of them are unknown yet, we do nothing and continue wait for
1505 // propagation.
1506 if (Attr.isUnknown() && Other.isUnknown())
1507 return true;
1508
1509 // Case 2: The other is determined, but we are unknown yet, we simply take
1510 // the other's value.
1511 if (Attr.isUnknown()) {
1512 Attr = Other;
1513 return true;
1514 }
1515
1516 // Case 3: We are determined but the other is unknown yet, we simply keep
1517 // everything unchanged.
1518 if (Other.isUnknown())
1519 return true;
1520
1521 // After this point, both are determined.
1522
1523 // Case 4: If they are same, we do nothing.
1524 if (Attr == Other)
1525 return true;
1526
1527 // Now they are not same.
1528
1529 // Case 5: If either of us uses cluster (but not both; otherwise case 4
1530 // would hold), then it is unknown whether cluster will be used, and the
1531 // state is final, unlike case 1.
1532 if (Attr.isNoCluster() || Other.isNoCluster()) {
1533 Attr.setUnknown();
1534 return false;
1535 }
1536
1537 // Case 6: Both of us use cluster, but the dims are different, so the result
1538 // is, cluster is used, but we just don't have a fixed dims.
1539 Attr.setVariableDims();
1540 return true;
1541 }
1542
1543 AMDGPU::ClusterDimsAttr Attr;
1544
1545 static constexpr char AttrName[] = "amdgpu-cluster-dims";
1546};
1547
1548AAAMDGPUClusterDims &
1549AAAMDGPUClusterDims::createForPosition(const IRPosition &IRP, Attributor &A) {
1550 if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
1551 return *new (A.Allocator) AAAMDGPUClusterDimsFunction(IRP, A);
1552 llvm_unreachable("AAAMDGPUClusterDims is only valid for function position");
1553}
1554
1555static bool runImpl(SetVector<Function *> &Functions, bool IsModulePass,
1556 bool DeleteFns, Module &M, AnalysisGetter &AG,
1557 TargetMachine &TM, AMDGPUAttributorOptions Options,
1558 ThinOrFullLTOPhase LTOPhase) {
1559
1560 CallGraphUpdater CGUpdater;
1561 BumpPtrAllocator Allocator;
1562 AMDGPUInformationCache InfoCache(M, AG, Allocator, nullptr, TM);
1563 DenseSet<const char *> Allowed(
1564 {&AAAMDAttributes::ID, &AAUniformWorkGroupSize::ID,
1565 &AAPotentialValues::ID, &AAAMDFlatWorkGroupSize::ID,
1566 &AAAMDMaxNumWorkgroups::ID, &AAAMDWavesPerEU::ID,
1567 &AAAMDGPUMinAGPRAlloc::ID, &AACallEdges::ID, &AAPointerInfo::ID,
1568 &AAPotentialConstantValues::ID, &AAUnderlyingObjects::ID,
1569 &AANoAliasAddrSpace::ID, &AAAddressSpace::ID, &AAIndirectCallInfo::ID,
1570 &AAAMDGPUClusterDims::ID, &AAAlign::ID});
1571
1572 AttributorConfig AC(CGUpdater);
1573 AC.IsClosedWorldModule = Options.IsClosedWorld;
1574 AC.Allowed = &Allowed;
1575 AC.IsModulePass = IsModulePass;
1576 AC.DeleteFns = DeleteFns;
1577 AC.DefaultInitializeLiveInternals = false;
1578 AC.IndirectCalleeSpecializationCallback =
1579 [](Attributor &A, const AbstractAttribute &AA, CallBase &CB,
1580 Function &Callee, unsigned NumAssumedCallees) {
1581 return !AMDGPU::isEntryFunctionCC(CC: Callee.getCallingConv()) &&
1582 (NumAssumedCallees <= IndirectCallSpecializationThreshold);
1583 };
1584 AC.IPOAmendableCB = [](const Function &F) {
1585 return F.getCallingConv() == CallingConv::AMDGPU_KERNEL;
1586 };
1587
1588 Attributor A(Functions, InfoCache, AC);
1589
1590 LLVM_DEBUG({
1591 StringRef LTOPhaseStr = to_string(LTOPhase);
1592 dbgs() << "[AMDGPUAttributor] Running at phase " << LTOPhaseStr << '\n'
1593 << "[AMDGPUAttributor] Module " << M.getName() << " is "
1594 << (AC.IsClosedWorldModule ? "" : "not ")
1595 << "assumed to be a closed world.\n";
1596 });
1597
1598 for (auto *F : Functions) {
1599 A.getOrCreateAAFor<AAAMDAttributes>(IRP: IRPosition::function(F: *F));
1600 A.getOrCreateAAFor<AAUniformWorkGroupSize>(IRP: IRPosition::function(F: *F));
1601 A.getOrCreateAAFor<AAAMDMaxNumWorkgroups>(IRP: IRPosition::function(F: *F));
1602 CallingConv::ID CC = F->getCallingConv();
1603 if (!AMDGPU::isEntryFunctionCC(CC)) {
1604 A.getOrCreateAAFor<AAAMDFlatWorkGroupSize>(IRP: IRPosition::function(F: *F));
1605 A.getOrCreateAAFor<AAAMDWavesPerEU>(IRP: IRPosition::function(F: *F));
1606 }
1607
1608 const AMDGPU::AMDGPUFeatureBitset &Features = InfoCache.getFeatures();
1609 if (!F->isDeclaration() && Features.test(I: AMDGPU::FEAT_CLUSTERS))
1610 A.getOrCreateAAFor<AAAMDGPUClusterDims>(IRP: IRPosition::function(F: *F));
1611
1612 if (Features.test(I: AMDGPU::FEAT_AGPR_ALLOC))
1613 A.getOrCreateAAFor<AAAMDGPUMinAGPRAlloc>(IRP: IRPosition::function(F: *F));
1614
1615 for (auto &I : instructions(F)) {
1616 Value *Ptr = nullptr;
1617 if (auto *LI = dyn_cast<LoadInst>(Val: &I))
1618 Ptr = LI->getPointerOperand();
1619 else if (auto *SI = dyn_cast<StoreInst>(Val: &I))
1620 Ptr = SI->getPointerOperand();
1621 else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: &I))
1622 Ptr = RMW->getPointerOperand();
1623 else if (auto *CmpX = dyn_cast<AtomicCmpXchgInst>(Val: &I))
1624 Ptr = CmpX->getPointerOperand();
1625
1626 if (Ptr) {
1627 A.getOrCreateAAFor<AAAddressSpace>(IRP: IRPosition::value(V: *Ptr));
1628 A.getOrCreateAAFor<AANoAliasAddrSpace>(IRP: IRPosition::value(V: *Ptr));
1629 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: Ptr)) {
1630 if (II->getIntrinsicID() == Intrinsic::amdgcn_make_buffer_rsrc)
1631 A.getOrCreateAAFor<AAAlign>(IRP: IRPosition::value(V: *Ptr));
1632 }
1633 }
1634 }
1635 }
1636
1637 return A.run() == ChangeStatus::CHANGED;
1638}
1639} // namespace
1640
1641PreservedAnalyses llvm::AMDGPUAttributorPass::run(Module &M,
1642 ModuleAnalysisManager &AM) {
1643
1644 FunctionAnalysisManager &FAM =
1645 AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
1646 AnalysisGetter AG(FAM);
1647
1648 SetVector<Function *> Functions;
1649 for (Function &F : M) {
1650 if (!F.isDeclaration())
1651 Functions.insert(X: &F);
1652 }
1653
1654 // TODO: Probably preserves CFG
1655 return runImpl(Functions, /*IsModulePass=*/true, /*DeleteFns=*/true, M, AG,
1656 TM, Options, LTOPhase)
1657 ? PreservedAnalyses::none()
1658 : PreservedAnalyses::all();
1659}
1660
1661PreservedAnalyses llvm::AMDGPUAttributorCGSCCPass::run(LazyCallGraph::SCC &C,
1662 CGSCCAnalysisManager &AM,
1663 LazyCallGraph &CG,
1664 CGSCCUpdateResult &UR) {
1665
1666 FunctionAnalysisManager &FAM =
1667 AM.getResult<FunctionAnalysisManagerCGSCCProxy>(IR&: C, ExtraArgs&: CG).getManager();
1668 AnalysisGetter AG(FAM);
1669
1670 SetVector<Function *> Functions;
1671 for (LazyCallGraph::Node &N : C) {
1672 Function *F = &N.getFunction();
1673 if (!F->isIntrinsic())
1674 Functions.insert(X: F);
1675 }
1676
1677 AMDGPUAttributorOptions Options;
1678 Module *M = C.begin()->getFunction().getParent();
1679 // In the CGSCC pipeline, avoid untracked call graph modifications by
1680 // disabling function deletion, mirroring the generic AttributorCGSCCPass.
1681 return runImpl(Functions, /*IsModulePass=*/false, /*DeleteFns=*/false, M&: *M, AG,
1682 TM, Options, LTOPhase: ThinOrFullLTOPhase::None)
1683 ? PreservedAnalyses::none()
1684 : PreservedAnalyses::all();
1685}
1686