1//===- AttributorAttributes.cpp - Attributes for Attributor deduction -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// See the Attributor.h file comment and the class descriptions in that file for
10// more information.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/IPO/Attributor.h"
15
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/ADT/DenseMapInfo.h"
20#include "llvm/ADT/MapVector.h"
21#include "llvm/ADT/SCCIterator.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SetOperations.h"
24#include "llvm/ADT/SetVector.h"
25#include "llvm/ADT/SmallPtrSet.h"
26#include "llvm/ADT/SmallVector.h"
27#include "llvm/ADT/Statistic.h"
28#include "llvm/ADT/StringExtras.h"
29#include "llvm/Analysis/AliasAnalysis.h"
30#include "llvm/Analysis/AssumeBundleQueries.h"
31#include "llvm/Analysis/AssumptionCache.h"
32#include "llvm/Analysis/CaptureTracking.h"
33#include "llvm/Analysis/CycleAnalysis.h"
34#include "llvm/Analysis/InstructionSimplify.h"
35#include "llvm/Analysis/LazyValueInfo.h"
36#include "llvm/Analysis/MemoryBuiltins.h"
37#include "llvm/Analysis/ScalarEvolution.h"
38#include "llvm/Analysis/TargetTransformInfo.h"
39#include "llvm/Analysis/ValueTracking.h"
40#include "llvm/IR/Argument.h"
41#include "llvm/IR/Assumptions.h"
42#include "llvm/IR/Attributes.h"
43#include "llvm/IR/BasicBlock.h"
44#include "llvm/IR/Constant.h"
45#include "llvm/IR/Constants.h"
46#include "llvm/IR/DataLayout.h"
47#include "llvm/IR/DerivedTypes.h"
48#include "llvm/IR/GlobalValue.h"
49#include "llvm/IR/IRBuilder.h"
50#include "llvm/IR/InlineAsm.h"
51#include "llvm/IR/InstrTypes.h"
52#include "llvm/IR/Instruction.h"
53#include "llvm/IR/Instructions.h"
54#include "llvm/IR/IntrinsicInst.h"
55#include "llvm/IR/IntrinsicsAMDGPU.h"
56#include "llvm/IR/IntrinsicsNVPTX.h"
57#include "llvm/IR/LLVMContext.h"
58#include "llvm/IR/MDBuilder.h"
59#include "llvm/IR/NoFolder.h"
60#include "llvm/IR/Value.h"
61#include "llvm/IR/ValueHandle.h"
62#include "llvm/Support/Alignment.h"
63#include "llvm/Support/Casting.h"
64#include "llvm/Support/CommandLine.h"
65#include "llvm/Support/ErrorHandling.h"
66#include "llvm/Support/GraphWriter.h"
67#include "llvm/Support/InterleavedRange.h"
68#include "llvm/Support/KnownFPClass.h"
69#include "llvm/Support/MathExtras.h"
70#include "llvm/Support/TypeSize.h"
71#include "llvm/Support/raw_ostream.h"
72#include "llvm/Transforms/Utils/BasicBlockUtils.h"
73#include "llvm/Transforms/Utils/CallPromotionUtils.h"
74#include "llvm/Transforms/Utils/Local.h"
75#include "llvm/Transforms/Utils/ValueMapper.h"
76#include <cassert>
77#include <numeric>
78#include <optional>
79#include <string>
80
81using namespace llvm;
82
83#define DEBUG_TYPE "attributor"
84
85static cl::opt<bool> ManifestInternal(
86 "attributor-manifest-internal", cl::Hidden,
87 cl::desc("Manifest Attributor internal string attributes."),
88 cl::init(Val: false));
89
90static cl::opt<int> MaxHeapToStackSize("max-heap-to-stack-size", cl::init(Val: 128),
91 cl::Hidden);
92
93template <>
94unsigned llvm::PotentialConstantIntValuesState::MaxPotentialValues = 0;
95
96template <> unsigned llvm::PotentialLLVMValuesState::MaxPotentialValues = -1;
97
98static cl::opt<unsigned, true> MaxPotentialValues(
99 "attributor-max-potential-values", cl::Hidden,
100 cl::desc("Maximum number of potential values to be "
101 "tracked for each position."),
102 cl::location(L&: llvm::PotentialConstantIntValuesState::MaxPotentialValues),
103 cl::init(Val: 7));
104
105static cl::opt<int> MaxPotentialValuesIterations(
106 "attributor-max-potential-values-iterations", cl::Hidden,
107 cl::desc(
108 "Maximum number of iterations we keep dismantling potential values."),
109 cl::init(Val: 64));
110
111STATISTIC(NumAAs, "Number of abstract attributes created");
112STATISTIC(NumIndirectCallsPromoted, "Number of indirect calls promoted");
113
114// Some helper macros to deal with statistics tracking.
115//
116// Usage:
117// For simple IR attribute tracking overload trackStatistics in the abstract
118// attribute and choose the right STATS_DECLTRACK_********* macro,
119// e.g.,:
120// void trackStatistics() const override {
121// STATS_DECLTRACK_ARG_ATTR(returned)
122// }
123// If there is a single "increment" side one can use the macro
124// STATS_DECLTRACK with a custom message. If there are multiple increment
125// sides, STATS_DECL and STATS_TRACK can also be used separately.
126//
127#define BUILD_STAT_MSG_IR_ATTR(TYPE, NAME) \
128 ("Number of " #TYPE " marked '" #NAME "'")
129#define BUILD_STAT_NAME(NAME, TYPE) NumIR##TYPE##_##NAME
130#define STATS_DECL_(NAME, MSG) STATISTIC(NAME, MSG);
131#define STATS_DECL(NAME, TYPE, MSG) \
132 STATS_DECL_(BUILD_STAT_NAME(NAME, TYPE), MSG);
133#define STATS_TRACK(NAME, TYPE) ++(BUILD_STAT_NAME(NAME, TYPE));
134#define STATS_DECLTRACK(NAME, TYPE, MSG) \
135 {STATS_DECL(NAME, TYPE, MSG) STATS_TRACK(NAME, TYPE)}
136#define STATS_DECLTRACK_ARG_ATTR(NAME) \
137 STATS_DECLTRACK(NAME, Arguments, BUILD_STAT_MSG_IR_ATTR(arguments, NAME))
138#define STATS_DECLTRACK_CSARG_ATTR(NAME) \
139 STATS_DECLTRACK(NAME, CSArguments, \
140 BUILD_STAT_MSG_IR_ATTR(call site arguments, NAME))
141#define STATS_DECLTRACK_FN_ATTR(NAME) \
142 STATS_DECLTRACK(NAME, Function, BUILD_STAT_MSG_IR_ATTR(functions, NAME))
143#define STATS_DECLTRACK_CS_ATTR(NAME) \
144 STATS_DECLTRACK(NAME, CS, BUILD_STAT_MSG_IR_ATTR(call site, NAME))
145#define STATS_DECLTRACK_FNRET_ATTR(NAME) \
146 STATS_DECLTRACK(NAME, FunctionReturn, \
147 BUILD_STAT_MSG_IR_ATTR(function returns, NAME))
148#define STATS_DECLTRACK_CSRET_ATTR(NAME) \
149 STATS_DECLTRACK(NAME, CSReturn, \
150 BUILD_STAT_MSG_IR_ATTR(call site returns, NAME))
151#define STATS_DECLTRACK_FLOATING_ATTR(NAME) \
152 STATS_DECLTRACK(NAME, Floating, \
153 ("Number of floating values known to be '" #NAME "'"))
154
155// Specialization of the operator<< for abstract attributes subclasses. This
156// disambiguates situations where multiple operators are applicable.
157namespace llvm {
158#define PIPE_OPERATOR(CLASS) \
159 raw_ostream &operator<<(raw_ostream &OS, const CLASS &AA) { \
160 return OS << static_cast<const AbstractAttribute &>(AA); \
161 }
162
163PIPE_OPERATOR(AAIsDead)
164PIPE_OPERATOR(AANoUnwind)
165PIPE_OPERATOR(AANoSync)
166PIPE_OPERATOR(AANoRecurse)
167PIPE_OPERATOR(AANonConvergent)
168PIPE_OPERATOR(AAWillReturn)
169PIPE_OPERATOR(AANoReturn)
170PIPE_OPERATOR(AANonNull)
171PIPE_OPERATOR(AAMustProgress)
172PIPE_OPERATOR(AANoAlias)
173PIPE_OPERATOR(AADereferenceable)
174PIPE_OPERATOR(AAAlign)
175PIPE_OPERATOR(AAInstanceInfo)
176PIPE_OPERATOR(AANoCapture)
177PIPE_OPERATOR(AAValueSimplify)
178PIPE_OPERATOR(AANoFree)
179PIPE_OPERATOR(AAHeapToStack)
180PIPE_OPERATOR(AAIntraFnReachability)
181PIPE_OPERATOR(AAMemoryBehavior)
182PIPE_OPERATOR(AAMemoryLocation)
183PIPE_OPERATOR(AAValueConstantRange)
184PIPE_OPERATOR(AAPrivatizablePtr)
185PIPE_OPERATOR(AAUndefinedBehavior)
186PIPE_OPERATOR(AAPotentialConstantValues)
187PIPE_OPERATOR(AAPotentialValues)
188PIPE_OPERATOR(AANoUndef)
189PIPE_OPERATOR(AANoFPClass)
190PIPE_OPERATOR(AACallEdges)
191PIPE_OPERATOR(AAInterFnReachability)
192PIPE_OPERATOR(AAPointerInfo)
193PIPE_OPERATOR(AAAssumptionInfo)
194PIPE_OPERATOR(AAUnderlyingObjects)
195PIPE_OPERATOR(AAInvariantLoadPointer)
196PIPE_OPERATOR(AAAddressSpace)
197PIPE_OPERATOR(AANoAliasAddrSpace)
198PIPE_OPERATOR(AAAllocationInfo)
199PIPE_OPERATOR(AAIndirectCallInfo)
200PIPE_OPERATOR(AAGlobalValueInfo)
201PIPE_OPERATOR(AADenormalFPMath)
202
203#undef PIPE_OPERATOR
204
205template <>
206ChangeStatus clampStateAndIndicateChange<DerefState>(DerefState &S,
207 const DerefState &R) {
208 ChangeStatus CS0 =
209 clampStateAndIndicateChange(S&: S.DerefBytesState, R: R.DerefBytesState);
210 ChangeStatus CS1 = clampStateAndIndicateChange(S&: S.GlobalState, R: R.GlobalState);
211 return CS0 | CS1;
212}
213
214} // namespace llvm
215
216static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I,
217 bool HeaderOnly, CycleRef *CPtr = nullptr) {
218 if (!CI)
219 return true;
220 auto *BB = I->getParent();
221 CycleRef C = CI->getCycle(Block: BB);
222 if (!C)
223 return false;
224 if (CPtr)
225 *CPtr = C;
226 return !HeaderOnly || BB == CI->getHeader(C);
227}
228
229/// Checks if a type could have padding bytes.
230static bool isDenselyPacked(Type *Ty, const DataLayout &DL) {
231 // There is no size information, so be conservative.
232 if (!Ty->isSized())
233 return false;
234
235 // If the alloc size is not equal to the storage size, then there are padding
236 // bytes. For x86_fp80 on x86-64, size: 80 alloc size: 128.
237 if (DL.getTypeSizeInBits(Ty) != DL.getTypeAllocSizeInBits(Ty))
238 return false;
239
240 // FIXME: This isn't the right way to check for padding in vectors with
241 // non-byte-size elements.
242 if (VectorType *SeqTy = dyn_cast<VectorType>(Val: Ty))
243 return isDenselyPacked(Ty: SeqTy->getElementType(), DL);
244
245 // For array types, check for padding within members.
246 if (ArrayType *SeqTy = dyn_cast<ArrayType>(Val: Ty))
247 return isDenselyPacked(Ty: SeqTy->getElementType(), DL);
248
249 if (!isa<StructType>(Val: Ty))
250 return true;
251
252 // Check for padding within and between elements of a struct.
253 StructType *StructTy = cast<StructType>(Val: Ty);
254 const StructLayout *Layout = DL.getStructLayout(Ty: StructTy);
255 uint64_t StartPos = 0;
256 for (unsigned I = 0, E = StructTy->getNumElements(); I < E; ++I) {
257 Type *ElTy = StructTy->getElementType(N: I);
258 if (!isDenselyPacked(Ty: ElTy, DL))
259 return false;
260 if (StartPos != Layout->getElementOffsetInBits(Idx: I))
261 return false;
262 StartPos += DL.getTypeAllocSizeInBits(Ty: ElTy);
263 }
264
265 return true;
266}
267
268/// Get pointer operand of memory accessing instruction. If \p I is
269/// not a memory accessing instruction, return nullptr. If \p AllowVolatile,
270/// is set to false and the instruction is volatile, return nullptr.
271static const Value *getPointerOperand(const Instruction *I,
272 bool AllowVolatile) {
273 if (!AllowVolatile && I->isVolatile())
274 return nullptr;
275
276 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
277 return LI->getPointerOperand();
278 }
279
280 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
281 return SI->getPointerOperand();
282 }
283
284 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
285 return CXI->getPointerOperand();
286 }
287
288 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: I)) {
289 return RMWI->getPointerOperand();
290 }
291
292 return nullptr;
293}
294
295/// Helper function to create a pointer based on \p Ptr, and advanced by \p
296/// Offset bytes.
297static Value *constructPointer(Value *Ptr, int64_t Offset,
298 IRBuilder<NoFolder> &IRB) {
299 LLVM_DEBUG(dbgs() << "Construct pointer: " << *Ptr << " + " << Offset
300 << "-bytes\n");
301
302 if (Offset)
303 Ptr = IRB.CreatePtrAdd(Ptr, Offset: IRB.getInt64(C: Offset),
304 Name: Ptr->getName() + ".b" + Twine(Offset));
305 return Ptr;
306}
307
308static const Value *
309stripAndAccumulateOffsets(Attributor &A, const AbstractAttribute &QueryingAA,
310 const Value *Val, const DataLayout &DL, APInt &Offset,
311 bool GetMinOffset, bool AllowNonInbounds,
312 bool UseAssumed = false) {
313
314 auto AttributorAnalysis = [&](Value &V, APInt &ROffset) -> bool {
315 const IRPosition &Pos = IRPosition::value(V);
316 // Only track dependence if we are going to use the assumed info.
317 const AAValueConstantRange *ValueConstantRangeAA =
318 A.getAAFor<AAValueConstantRange>(QueryingAA, IRP: Pos,
319 DepClass: UseAssumed ? DepClassTy::OPTIONAL
320 : DepClassTy::NONE);
321 if (!ValueConstantRangeAA)
322 return false;
323 ConstantRange Range = UseAssumed ? ValueConstantRangeAA->getAssumed()
324 : ValueConstantRangeAA->getKnown();
325 if (Range.isFullSet())
326 return false;
327
328 // We can only use the lower part of the range because the upper part can
329 // be higher than what the value can really be.
330 if (GetMinOffset)
331 ROffset = Range.getSignedMin();
332 else
333 ROffset = Range.getSignedMax();
334 return true;
335 };
336
337 return Val->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds,
338 /* AllowInvariant */ AllowInvariantGroup: true,
339 ExternalAnalysis: AttributorAnalysis);
340}
341
342static const Value *
343getMinimalBaseOfPointer(Attributor &A, const AbstractAttribute &QueryingAA,
344 const Value *Ptr, int64_t &BytesOffset,
345 const DataLayout &DL, bool AllowNonInbounds = false) {
346 APInt OffsetAPInt(DL.getIndexTypeSizeInBits(Ty: Ptr->getType()), 0);
347 const Value *Base =
348 stripAndAccumulateOffsets(A, QueryingAA, Val: Ptr, DL, Offset&: OffsetAPInt,
349 /* GetMinOffset */ true, AllowNonInbounds);
350
351 BytesOffset = OffsetAPInt.getSExtValue();
352 return Base;
353}
354
355/// Clamp the information known for all returned values of a function
356/// (identified by \p QueryingAA) into \p S.
357template <typename AAType, typename StateType = typename AAType::StateType,
358 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
359 bool RecurseForSelectAndPHI = true>
360static void clampReturnedValueStates(
361 Attributor &A, const AAType &QueryingAA, StateType &S,
362 const IRPosition::CallBaseContext *CBContext = nullptr) {
363 LLVM_DEBUG(dbgs() << "[Attributor] Clamp return value states for "
364 << QueryingAA << " into " << S << "\n");
365
366 assert((QueryingAA.getIRPosition().getPositionKind() ==
367 IRPosition::IRP_RETURNED ||
368 QueryingAA.getIRPosition().getPositionKind() ==
369 IRPosition::IRP_CALL_SITE_RETURNED) &&
370 "Can only clamp returned value states for a function returned or call "
371 "site returned position!");
372
373 // Use an optional state as there might not be any return values and we want
374 // to join (IntegerState::operator&) the state of all there are.
375 std::optional<StateType> T;
376
377 // Callback for each possibly returned value.
378 auto CheckReturnValue = [&](Value &RV) -> bool {
379 const IRPosition &RVPos = IRPosition::value(V: RV, CBContext);
380 // If possible, use the hasAssumedIRAttr interface.
381 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
382 bool IsKnown;
383 return AA::hasAssumedIRAttr<IRAttributeKind>(
384 A, &QueryingAA, RVPos, DepClassTy::REQUIRED, IsKnown);
385 }
386
387 const AAType *AA =
388 A.getAAFor<AAType>(QueryingAA, RVPos, DepClassTy::REQUIRED);
389 if (!AA)
390 return false;
391 LLVM_DEBUG(dbgs() << "[Attributor] RV: " << RV
392 << " AA: " << AA->getAsStr(&A) << " @ " << RVPos << "\n");
393 const StateType &AAS = AA->getState();
394 if (!T)
395 T = StateType::getBestState(AAS);
396 *T &= AAS;
397 LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " RV State: " << T
398 << "\n");
399 return T->isValidState();
400 };
401
402 if (!A.checkForAllReturnedValues(Pred: CheckReturnValue, QueryingAA,
403 S: AA::ValueScope::Intraprocedural,
404 RecurseForSelectAndPHI))
405 S.indicatePessimisticFixpoint();
406 else if (T)
407 S ^= *T;
408}
409
410namespace {
411/// Helper class for generic deduction: return value -> returned position.
412template <typename AAType, typename BaseType,
413 typename StateType = typename BaseType::StateType,
414 bool PropagateCallBaseContext = false,
415 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
416 bool RecurseForSelectAndPHI = true>
417struct AAReturnedFromReturnedValues : public BaseType {
418 AAReturnedFromReturnedValues(const IRPosition &IRP, Attributor &A)
419 : BaseType(IRP, A) {}
420
421 /// See AbstractAttribute::updateImpl(...).
422 ChangeStatus updateImpl(Attributor &A) override {
423 StateType S(StateType::getBestState(this->getState()));
424 clampReturnedValueStates<AAType, StateType, IRAttributeKind,
425 RecurseForSelectAndPHI>(
426 A, *this, S,
427 PropagateCallBaseContext ? this->getCallBaseContext() : nullptr);
428 // TODO: If we know we visited all returned values, thus no are assumed
429 // dead, we can take the known information from the state T.
430 return clampStateAndIndicateChange<StateType>(this->getState(), S);
431 }
432};
433
434/// Clamp the information known at all call sites for a given argument
435/// (identified by \p QueryingAA) into \p S.
436template <typename AAType, typename StateType = typename AAType::StateType,
437 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
438static void clampCallSiteArgumentStates(Attributor &A, const AAType &QueryingAA,
439 StateType &S) {
440 LLVM_DEBUG(dbgs() << "[Attributor] Clamp call site argument states for "
441 << QueryingAA << " into " << S << "\n");
442
443 assert(QueryingAA.getIRPosition().getPositionKind() ==
444 IRPosition::IRP_ARGUMENT &&
445 "Can only clamp call site argument states for an argument position!");
446
447 // Use an optional state as there might not be any return values and we want
448 // to join (IntegerState::operator&) the state of all there are.
449 std::optional<StateType> T;
450
451 // The argument number which is also the call site argument number.
452 unsigned ArgNo = QueryingAA.getIRPosition().getCallSiteArgNo();
453
454 auto CallSiteCheck = [&](AbstractCallSite ACS) {
455 const IRPosition &ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
456 // Check if a coresponding argument was found or if it is on not associated
457 // (which can happen for callback calls).
458 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
459 return false;
460
461 // If possible, use the hasAssumedIRAttr interface.
462 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
463 bool IsKnown;
464 return AA::hasAssumedIRAttr<IRAttributeKind>(
465 A, &QueryingAA, ACSArgPos, DepClassTy::REQUIRED, IsKnown);
466 }
467
468 const AAType *AA =
469 A.getAAFor<AAType>(QueryingAA, ACSArgPos, DepClassTy::REQUIRED);
470 if (!AA)
471 return false;
472 LLVM_DEBUG(dbgs() << "[Attributor] ACS: " << *ACS.getInstruction()
473 << " AA: " << AA->getAsStr(&A) << " @" << ACSArgPos
474 << "\n");
475 const StateType &AAS = AA->getState();
476 if (!T)
477 T = StateType::getBestState(AAS);
478 *T &= AAS;
479 LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " CSA State: " << T
480 << "\n");
481 return T->isValidState();
482 };
483
484 bool UsedAssumedInformation = false;
485 if (!A.checkForAllCallSites(CallSiteCheck, QueryingAA, true,
486 UsedAssumedInformation))
487 S.indicatePessimisticFixpoint();
488 else if (T)
489 S ^= *T;
490}
491
492/// This function is the bridge between argument position and the call base
493/// context.
494template <typename AAType, typename BaseType,
495 typename StateType = typename AAType::StateType,
496 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
497bool getArgumentStateFromCallBaseContext(Attributor &A,
498 BaseType &QueryingAttribute,
499 IRPosition &Pos, StateType &State) {
500 assert((Pos.getPositionKind() == IRPosition::IRP_ARGUMENT) &&
501 "Expected an 'argument' position !");
502 const CallBase *CBContext = Pos.getCallBaseContext();
503 if (!CBContext)
504 return false;
505
506 int ArgNo = Pos.getCallSiteArgNo();
507 assert(ArgNo >= 0 && "Invalid Arg No!");
508 const IRPosition CBArgPos = IRPosition::callsite_argument(CB: *CBContext, ArgNo);
509
510 // If possible, use the hasAssumedIRAttr interface.
511 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
512 bool IsKnown;
513 return AA::hasAssumedIRAttr<IRAttributeKind>(
514 A, &QueryingAttribute, CBArgPos, DepClassTy::REQUIRED, IsKnown);
515 }
516
517 const auto *AA =
518 A.getAAFor<AAType>(QueryingAttribute, CBArgPos, DepClassTy::REQUIRED);
519 if (!AA)
520 return false;
521 const StateType &CBArgumentState =
522 static_cast<const StateType &>(AA->getState());
523
524 LLVM_DEBUG(dbgs() << "[Attributor] Briding Call site context to argument"
525 << "Position:" << Pos << "CB Arg state:" << CBArgumentState
526 << "\n");
527
528 // NOTE: If we want to do call site grouping it should happen here.
529 State ^= CBArgumentState;
530 return true;
531}
532
533/// Helper class for generic deduction: call site argument -> argument position.
534template <typename AAType, typename BaseType,
535 typename StateType = typename AAType::StateType,
536 bool BridgeCallBaseContext = false,
537 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
538struct AAArgumentFromCallSiteArguments : public BaseType {
539 AAArgumentFromCallSiteArguments(const IRPosition &IRP, Attributor &A)
540 : BaseType(IRP, A) {}
541
542 /// See AbstractAttribute::updateImpl(...).
543 ChangeStatus updateImpl(Attributor &A) override {
544 StateType S = StateType::getBestState(this->getState());
545
546 if (BridgeCallBaseContext) {
547 bool Success =
548 getArgumentStateFromCallBaseContext<AAType, BaseType, StateType,
549 IRAttributeKind>(
550 A, *this, this->getIRPosition(), S);
551 if (Success)
552 return clampStateAndIndicateChange<StateType>(this->getState(), S);
553 }
554 clampCallSiteArgumentStates<AAType, StateType, IRAttributeKind>(A, *this,
555 S);
556
557 // TODO: If we know we visited all incoming values, thus no are assumed
558 // dead, we can take the known information from the state T.
559 return clampStateAndIndicateChange<StateType>(this->getState(), S);
560 }
561};
562
563/// Helper class for generic replication: function returned -> cs returned.
564template <typename AAType, typename BaseType,
565 typename StateType = typename BaseType::StateType,
566 bool IntroduceCallBaseContext = false,
567 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
568struct AACalleeToCallSite : public BaseType {
569 AACalleeToCallSite(const IRPosition &IRP, Attributor &A) : BaseType(IRP, A) {}
570
571 /// See AbstractAttribute::updateImpl(...).
572 ChangeStatus updateImpl(Attributor &A) override {
573 auto IRPKind = this->getIRPosition().getPositionKind();
574 assert((IRPKind == IRPosition::IRP_CALL_SITE_RETURNED ||
575 IRPKind == IRPosition::IRP_CALL_SITE) &&
576 "Can only wrap function returned positions for call site "
577 "returned positions!");
578 auto &S = this->getState();
579
580 CallBase &CB = cast<CallBase>(this->getAnchorValue());
581 if (IntroduceCallBaseContext)
582 LLVM_DEBUG(dbgs() << "[Attributor] Introducing call base context:" << CB
583 << "\n");
584
585 ChangeStatus Changed = ChangeStatus::UNCHANGED;
586 auto CalleePred = [&](ArrayRef<const Function *> Callees) {
587 for (const Function *Callee : Callees) {
588 IRPosition FnPos =
589 IRPKind == llvm::IRPosition::IRP_CALL_SITE_RETURNED
590 ? IRPosition::returned(F: *Callee,
591 CBContext: IntroduceCallBaseContext ? &CB : nullptr)
592 : IRPosition::function(
593 F: *Callee, CBContext: IntroduceCallBaseContext ? &CB : nullptr);
594 // If possible, use the hasAssumedIRAttr interface.
595 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
596 bool IsKnown;
597 if (!AA::hasAssumedIRAttr<IRAttributeKind>(
598 A, this, FnPos, DepClassTy::REQUIRED, IsKnown))
599 return false;
600 continue;
601 }
602
603 const AAType *AA =
604 A.getAAFor<AAType>(*this, FnPos, DepClassTy::REQUIRED);
605 if (!AA)
606 return false;
607 Changed |= clampStateAndIndicateChange(S, AA->getState());
608 if (S.isAtFixpoint())
609 return S.isValidState();
610 }
611 return true;
612 };
613 if (!A.checkForAllCallees(Pred: CalleePred, QueryingAA: *this, CB))
614 return S.indicatePessimisticFixpoint();
615 return Changed;
616 }
617};
618
619/// Helper function to accumulate uses.
620template <class AAType, typename StateType = typename AAType::StateType>
621static void followUsesInContext(AAType &AA, Attributor &A,
622 MustBeExecutedContextExplorer &Explorer,
623 const Instruction *CtxI,
624 SetVector<const Use *> &Uses,
625 StateType &State) {
626 auto EIt = Explorer.begin(PP: CtxI), EEnd = Explorer.end(CtxI);
627 for (unsigned u = 0; u < Uses.size(); ++u) {
628 const Use *U = Uses[u];
629 if (const Instruction *UserI = dyn_cast<Instruction>(Val: U->getUser())) {
630 bool Found = Explorer.findInContextOf(I: UserI, EIt, EEnd);
631 if (Found && AA.followUseInMBEC(A, U, UserI, State))
632 Uses.insert_range(R: llvm::make_pointer_range(Range: UserI->uses()));
633 }
634 }
635}
636
637/// Use the must-be-executed-context around \p I to add information into \p S.
638/// The AAType class is required to have `followUseInMBEC` method with the
639/// following signature and behaviour:
640///
641/// bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I)
642/// U - Underlying use.
643/// I - The user of the \p U.
644/// Returns true if the value should be tracked transitively.
645///
646template <class AAType, typename StateType = typename AAType::StateType>
647static void followUsesInMBEC(AAType &AA, Attributor &A, StateType &S,
648 Instruction &CtxI) {
649 const Value &Val = AA.getIRPosition().getAssociatedValue();
650 if (isa<ConstantData>(Val))
651 return;
652
653 MustBeExecutedContextExplorer *Explorer =
654 A.getInfoCache().getMustBeExecutedContextExplorer();
655 if (!Explorer)
656 return;
657
658 // Container for (transitive) uses of the associated value.
659 SetVector<const Use *> Uses;
660 for (const Use &U : Val.uses())
661 Uses.insert(X: &U);
662
663 followUsesInContext<AAType>(AA, A, *Explorer, &CtxI, Uses, S);
664
665 if (S.isAtFixpoint())
666 return;
667
668 SmallVector<const CondBrInst *, 4> BrInsts;
669 auto Pred = [&](const Instruction *I) {
670 if (const CondBrInst *Br = dyn_cast<CondBrInst>(Val: I))
671 BrInsts.push_back(Elt: Br);
672 return true;
673 };
674
675 // Here, accumulate conditional branch instructions in the context. We
676 // explore the child paths and collect the known states. The disjunction of
677 // those states can be merged to its own state. Let ParentState_i be a state
678 // to indicate the known information for an i-th branch instruction in the
679 // context. ChildStates are created for its successors respectively.
680 //
681 // ParentS_1 = ChildS_{1, 1} /\ ChildS_{1, 2} /\ ... /\ ChildS_{1, n_1}
682 // ParentS_2 = ChildS_{2, 1} /\ ChildS_{2, 2} /\ ... /\ ChildS_{2, n_2}
683 // ...
684 // ParentS_m = ChildS_{m, 1} /\ ChildS_{m, 2} /\ ... /\ ChildS_{m, n_m}
685 //
686 // Known State |= ParentS_1 \/ ParentS_2 \/... \/ ParentS_m
687 //
688 // FIXME: Currently, recursive branches are not handled. For example, we
689 // can't deduce that ptr must be dereferenced in below function.
690 //
691 // void f(int a, int c, int *ptr) {
692 // if(a)
693 // if (b) {
694 // *ptr = 0;
695 // } else {
696 // *ptr = 1;
697 // }
698 // else {
699 // if (b) {
700 // *ptr = 0;
701 // } else {
702 // *ptr = 1;
703 // }
704 // }
705 // }
706
707 Explorer->checkForAllContext(PP: &CtxI, Pred);
708 for (const CondBrInst *Br : BrInsts) {
709 StateType ParentState;
710
711 // The known state of the parent state is a conjunction of children's
712 // known states so it is initialized with a best state.
713 ParentState.indicateOptimisticFixpoint();
714
715 for (const BasicBlock *BB : Br->successors()) {
716 StateType ChildState;
717
718 size_t BeforeSize = Uses.size();
719 followUsesInContext(AA, A, *Explorer, &BB->front(), Uses, ChildState);
720
721 // Erase uses which only appear in the child.
722 for (auto It = Uses.begin() + BeforeSize; It != Uses.end();)
723 It = Uses.erase(I: It);
724
725 ParentState &= ChildState;
726 }
727
728 // Use only known state.
729 S += ParentState;
730 }
731}
732} // namespace
733
734/// ------------------------ PointerInfo ---------------------------------------
735
736namespace llvm {
737namespace AA {
738namespace PointerInfo {
739
740struct State;
741
742} // namespace PointerInfo
743} // namespace AA
744
745/// Helper for AA::PointerInfo::Access DenseMap/Set usage.
746template <>
747struct DenseMapInfo<AAPointerInfo::Access> : DenseMapInfo<Instruction *> {
748 using Access = AAPointerInfo::Access;
749 static unsigned getHashValue(const Access &A);
750 static bool isEqual(const Access &LHS, const Access &RHS);
751};
752
753/// Helper that allows RangeTy as a key in a DenseMap.
754template <> struct DenseMapInfo<AA::RangeTy> {
755 static unsigned getHashValue(const AA::RangeTy &Range) {
756 return detail::combineHashValue(
757 a: DenseMapInfo<int64_t>::getHashValue(Val: Range.Offset),
758 b: DenseMapInfo<int64_t>::getHashValue(Val: Range.Size));
759 }
760
761 static bool isEqual(const AA::RangeTy &A, const AA::RangeTy B) {
762 return A == B;
763 }
764};
765
766} // namespace llvm
767
768/// A type to track pointer/struct usage and accesses for AAPointerInfo.
769struct AA::PointerInfo::State : public AbstractState {
770 /// Return the best possible representable state.
771 static State getBestState(const State &SIS) { return State(); }
772
773 /// Return the worst possible representable state.
774 static State getWorstState(const State &SIS) {
775 State R;
776 R.indicatePessimisticFixpoint();
777 return R;
778 }
779
780 State() = default;
781 State(State &&SIS) = default;
782
783 const State &getAssumed() const { return *this; }
784
785 /// See AbstractState::isValidState().
786 bool isValidState() const override { return BS.isValidState(); }
787
788 /// See AbstractState::isAtFixpoint().
789 bool isAtFixpoint() const override { return BS.isAtFixpoint(); }
790
791 /// See AbstractState::indicateOptimisticFixpoint().
792 ChangeStatus indicateOptimisticFixpoint() override {
793 BS.indicateOptimisticFixpoint();
794 return ChangeStatus::UNCHANGED;
795 }
796
797 /// See AbstractState::indicatePessimisticFixpoint().
798 ChangeStatus indicatePessimisticFixpoint() override {
799 BS.indicatePessimisticFixpoint();
800 return ChangeStatus::CHANGED;
801 }
802
803 State &operator=(const State &R) {
804 if (this == &R)
805 return *this;
806 BS = R.BS;
807 AccessList = R.AccessList;
808 OffsetBins = R.OffsetBins;
809 RemoteIMap = R.RemoteIMap;
810 ReturnedOffsets = R.ReturnedOffsets;
811 return *this;
812 }
813
814 State &operator=(State &&R) {
815 if (this == &R)
816 return *this;
817 std::swap(a&: BS, b&: R.BS);
818 std::swap(LHS&: AccessList, RHS&: R.AccessList);
819 std::swap(a&: OffsetBins, b&: R.OffsetBins);
820 std::swap(a&: RemoteIMap, b&: R.RemoteIMap);
821 std::swap(a&: ReturnedOffsets, b&: R.ReturnedOffsets);
822 return *this;
823 }
824
825 /// Add a new Access to the state at offset \p Offset and with size \p Size.
826 /// The access is associated with \p I, writes \p Content (if anything), and
827 /// is of kind \p Kind. If an Access already exists for the same \p I and same
828 /// \p RemoteI, the two are combined, potentially losing information about
829 /// offset and size. The resulting access must now be moved from its original
830 /// OffsetBin to the bin for its new offset.
831 ///
832 /// \Returns CHANGED, if the state changed, UNCHANGED otherwise.
833 ChangeStatus addAccess(Attributor &A, const AAPointerInfo::RangeList &Ranges,
834 Instruction &I, std::optional<Value *> Content,
835 AAPointerInfo::AccessKind Kind, Type *Ty,
836 Instruction *RemoteI = nullptr);
837
838 AAPointerInfo::const_bin_iterator begin() const { return OffsetBins.begin(); }
839 AAPointerInfo::const_bin_iterator end() const { return OffsetBins.end(); }
840 int64_t numOffsetBins() const { return OffsetBins.size(); }
841
842 const AAPointerInfo::Access &getAccess(unsigned Index) const {
843 return AccessList[Index];
844 }
845
846protected:
847 // Every memory instruction results in an Access object. We maintain a list of
848 // all Access objects that we own, along with the following maps:
849 //
850 // - OffsetBins: RangeTy -> { Access }
851 // - RemoteIMap: RemoteI x LocalI -> Access
852 //
853 // A RemoteI is any instruction that accesses memory. RemoteI is different
854 // from LocalI if and only if LocalI is a call; then RemoteI is some
855 // instruction in the callgraph starting from LocalI. Multiple paths in the
856 // callgraph from LocalI to RemoteI may produce multiple accesses, but these
857 // are all combined into a single Access object. This may result in loss of
858 // information in RangeTy in the Access object.
859 SmallVector<AAPointerInfo::Access> AccessList;
860 AAPointerInfo::OffsetBinsTy OffsetBins;
861 DenseMap<const Instruction *, SmallVector<unsigned>> RemoteIMap;
862
863 /// Flag to determine if the underlying pointer is reaching a return statement
864 /// in the associated function or not. Returns in other functions cause
865 /// invalidation.
866 AAPointerInfo::OffsetInfo ReturnedOffsets;
867
868 /// See AAPointerInfo::forallInterferingAccesses.
869 template <typename F>
870 bool forallInterferingAccesses(AA::RangeTy Range, F CB) const {
871 if (!isValidState() || !ReturnedOffsets.isUnassigned())
872 return false;
873
874 for (const auto &It : OffsetBins) {
875 AA::RangeTy ItRange = It.getFirst();
876 if (!Range.mayOverlap(Range: ItRange))
877 continue;
878 bool IsExact = Range == ItRange && !Range.offsetOrSizeAreUnknown();
879 for (auto Index : It.getSecond()) {
880 auto &Access = AccessList[Index];
881 if (!CB(Access, IsExact))
882 return false;
883 }
884 }
885 return true;
886 }
887
888 /// See AAPointerInfo::forallInterferingAccesses.
889 template <typename F>
890 bool forallInterferingAccesses(Instruction &I, F CB,
891 AA::RangeTy &Range) const {
892 if (!isValidState() || !ReturnedOffsets.isUnassigned())
893 return false;
894
895 auto LocalList = RemoteIMap.find(Val: &I);
896 if (LocalList == RemoteIMap.end()) {
897 return true;
898 }
899
900 for (unsigned Index : LocalList->getSecond()) {
901 for (auto &R : AccessList[Index]) {
902 Range &= R;
903 if (Range.offsetAndSizeAreUnknown())
904 break;
905 }
906 }
907 return forallInterferingAccesses(Range, CB);
908 }
909
910private:
911 /// State to track fixpoint and validity.
912 BooleanState BS;
913};
914
915ChangeStatus AA::PointerInfo::State::addAccess(
916 Attributor &A, const AAPointerInfo::RangeList &Ranges, Instruction &I,
917 std::optional<Value *> Content, AAPointerInfo::AccessKind Kind, Type *Ty,
918 Instruction *RemoteI) {
919 RemoteI = RemoteI ? RemoteI : &I;
920
921 // Check if we have an access for this instruction, if not, simply add it.
922 auto &LocalList = RemoteIMap[RemoteI];
923 bool AccExists = false;
924 unsigned AccIndex = AccessList.size();
925 for (auto Index : LocalList) {
926 auto &A = AccessList[Index];
927 if (A.getLocalInst() == &I) {
928 AccExists = true;
929 AccIndex = Index;
930 break;
931 }
932 }
933
934 auto AddToBins = [&](const AAPointerInfo::RangeList &ToAdd) {
935 LLVM_DEBUG(if (ToAdd.size()) dbgs()
936 << "[AAPointerInfo] Inserting access in new offset bins\n";);
937
938 for (auto Key : ToAdd) {
939 LLVM_DEBUG(dbgs() << " key " << Key << "\n");
940 OffsetBins[Key].insert(V: AccIndex);
941 }
942 };
943
944 if (!AccExists) {
945 AccessList.emplace_back(Args: &I, Args&: RemoteI, Args: Ranges, Args&: Content, Args&: Kind, Args&: Ty);
946 assert((AccessList.size() == AccIndex + 1) &&
947 "New Access should have been at AccIndex");
948 LocalList.push_back(Elt: AccIndex);
949 AddToBins(AccessList[AccIndex].getRanges());
950 return ChangeStatus::CHANGED;
951 }
952
953 // Combine the new Access with the existing Access, and then update the
954 // mapping in the offset bins.
955 AAPointerInfo::Access Acc(&I, RemoteI, Ranges, Content, Kind, Ty);
956 auto &Current = AccessList[AccIndex];
957 auto Before = Current;
958 Current &= Acc;
959 if (Current == Before)
960 return ChangeStatus::UNCHANGED;
961
962 auto &ExistingRanges = Before.getRanges();
963 auto &NewRanges = Current.getRanges();
964
965 // Ranges that are in the old access but not the new access need to be removed
966 // from the offset bins.
967 AAPointerInfo::RangeList ToRemove;
968 AAPointerInfo::RangeList::set_difference(L: ExistingRanges, R: NewRanges, D&: ToRemove);
969 LLVM_DEBUG(if (ToRemove.size()) dbgs()
970 << "[AAPointerInfo] Removing access from old offset bins\n";);
971
972 for (auto Key : ToRemove) {
973 LLVM_DEBUG(dbgs() << " key " << Key << "\n");
974 assert(OffsetBins.count(Key) && "Existing Access must be in some bin.");
975 auto &Bin = OffsetBins[Key];
976 assert(Bin.count(AccIndex) &&
977 "Expected bin to actually contain the Access.");
978 Bin.erase(V: AccIndex);
979 }
980
981 // Ranges that are in the new access but not the old access need to be added
982 // to the offset bins.
983 AAPointerInfo::RangeList ToAdd;
984 AAPointerInfo::RangeList::set_difference(L: NewRanges, R: ExistingRanges, D&: ToAdd);
985 AddToBins(ToAdd);
986 return ChangeStatus::CHANGED;
987}
988
989namespace {
990
991#ifndef NDEBUG
992static raw_ostream &operator<<(raw_ostream &OS,
993 const AAPointerInfo::OffsetInfo &OI) {
994 OS << llvm::interleaved_array(OI);
995 return OS;
996}
997#endif // NDEBUG
998
999struct AAPointerInfoImpl
1000 : public StateWrapper<AA::PointerInfo::State, AAPointerInfo> {
1001 using BaseTy = StateWrapper<AA::PointerInfo::State, AAPointerInfo>;
1002 AAPointerInfoImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
1003
1004 /// See AbstractAttribute::getAsStr().
1005 const std::string getAsStr(Attributor *A) const override {
1006 return std::string("PointerInfo ") +
1007 (isValidState() ? (std::string("#") +
1008 std::to_string(val: OffsetBins.size()) + " bins")
1009 : "<invalid>") +
1010 (reachesReturn()
1011 ? (" (returned:" +
1012 join(R: map_range(C: ReturnedOffsets,
1013 F: [](int64_t O) { return std::to_string(val: O); }),
1014 Separator: ", ") +
1015 ")")
1016 : "");
1017 }
1018
1019 /// See AbstractAttribute::manifest(...).
1020 ChangeStatus manifest(Attributor &A) override {
1021 return AAPointerInfo::manifest(A);
1022 }
1023
1024 const_bin_iterator begin() const override { return State::begin(); }
1025 const_bin_iterator end() const override { return State::end(); }
1026 int64_t numOffsetBins() const override { return State::numOffsetBins(); }
1027 bool reachesReturn() const override {
1028 return !ReturnedOffsets.isUnassigned();
1029 }
1030 void addReturnedOffsetsTo(OffsetInfo &OI) const override {
1031 if (ReturnedOffsets.isUnknown()) {
1032 OI.setUnknown();
1033 return;
1034 }
1035
1036 OffsetInfo MergedOI;
1037 for (auto Offset : ReturnedOffsets) {
1038 OffsetInfo TmpOI = OI;
1039 TmpOI.addToAll(Inc: Offset);
1040 MergedOI.merge(R: TmpOI);
1041 }
1042 OI = std::move(MergedOI);
1043 }
1044
1045 ChangeStatus setReachesReturn(const OffsetInfo &ReachedReturnedOffsets) {
1046 if (ReturnedOffsets.isUnknown())
1047 return ChangeStatus::UNCHANGED;
1048 if (ReachedReturnedOffsets.isUnknown()) {
1049 ReturnedOffsets.setUnknown();
1050 return ChangeStatus::CHANGED;
1051 }
1052 if (ReturnedOffsets.merge(R: ReachedReturnedOffsets))
1053 return ChangeStatus::CHANGED;
1054 return ChangeStatus::UNCHANGED;
1055 }
1056
1057 bool forallInterferingAccesses(
1058 AA::RangeTy Range,
1059 function_ref<bool(const AAPointerInfo::Access &, bool)> CB)
1060 const override {
1061 return State::forallInterferingAccesses(Range, CB);
1062 }
1063
1064 bool forallInterferingAccesses(
1065 Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I,
1066 bool FindInterferingWrites, bool FindInterferingReads,
1067 function_ref<bool(const Access &, bool)> UserCB, bool &HasBeenWrittenTo,
1068 AA::RangeTy &Range,
1069 function_ref<bool(const Access &)> SkipCB) const override {
1070 HasBeenWrittenTo = false;
1071
1072 SmallPtrSet<const Access *, 8> DominatingWrites;
1073 SmallVector<std::pair<const Access *, bool>, 8> InterferingAccesses;
1074
1075 Function &Scope = *I.getFunction();
1076 bool IsKnownNoSync;
1077 bool IsAssumedNoSync = AA::hasAssumedIRAttr<Attribute::NoSync>(
1078 A, QueryingAA: &QueryingAA, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL,
1079 IsKnown&: IsKnownNoSync);
1080 const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
1081 IRP: IRPosition::function(F: Scope), QueryingAA: &QueryingAA, DepClass: DepClassTy::NONE);
1082 bool AllInSameNoSyncFn = IsAssumedNoSync;
1083 bool InstIsExecutedByInitialThreadOnly =
1084 ExecDomainAA && ExecDomainAA->isExecutedByInitialThreadOnly(I);
1085
1086 // If the function is not ending in aligned barriers, we need the stores to
1087 // be in aligned barriers. The load being in one is not sufficient since the
1088 // store might be executed by a thread that disappears after, causing the
1089 // aligned barrier guarding the load to unblock and the load to read a value
1090 // that has no CFG path to the load.
1091 bool InstIsExecutedInAlignedRegion =
1092 FindInterferingReads && ExecDomainAA &&
1093 ExecDomainAA->isExecutedInAlignedRegion(A, I);
1094
1095 if (InstIsExecutedInAlignedRegion || InstIsExecutedByInitialThreadOnly)
1096 A.recordDependence(FromAA: *ExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1097
1098 InformationCache &InfoCache = A.getInfoCache();
1099 bool IsThreadLocalObj =
1100 AA::isAssumedThreadLocalObject(A, Obj&: getAssociatedValue(), QueryingAA: *this);
1101
1102 // Helper to determine if we need to consider threading, which we cannot
1103 // right now. However, if the function is (assumed) nosync or the thread
1104 // executing all instructions is the main thread only we can ignore
1105 // threading. Also, thread-local objects do not require threading reasoning.
1106 // Finally, we can ignore threading if either access is executed in an
1107 // aligned region.
1108 auto CanIgnoreThreadingForInst = [&](const Instruction &I) -> bool {
1109 if (IsThreadLocalObj || AllInSameNoSyncFn)
1110 return true;
1111 const auto *FnExecDomainAA =
1112 I.getFunction() == &Scope
1113 ? ExecDomainAA
1114 : A.lookupAAFor<AAExecutionDomain>(
1115 IRP: IRPosition::function(F: *I.getFunction()), QueryingAA: &QueryingAA,
1116 DepClass: DepClassTy::NONE);
1117 if (!FnExecDomainAA)
1118 return false;
1119 if (InstIsExecutedInAlignedRegion ||
1120 (FindInterferingWrites &&
1121 FnExecDomainAA->isExecutedInAlignedRegion(A, I))) {
1122 A.recordDependence(FromAA: *FnExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1123 return true;
1124 }
1125 if (InstIsExecutedByInitialThreadOnly &&
1126 FnExecDomainAA->isExecutedByInitialThreadOnly(I)) {
1127 A.recordDependence(FromAA: *FnExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1128 return true;
1129 }
1130 return false;
1131 };
1132
1133 // Helper to determine if the access is executed by the same thread as the
1134 // given instruction, for now it is sufficient to avoid any potential
1135 // threading effects as we cannot deal with them anyway.
1136 auto CanIgnoreThreading = [&](const Access &Acc) -> bool {
1137 return CanIgnoreThreadingForInst(*Acc.getRemoteInst()) ||
1138 (Acc.getRemoteInst() != Acc.getLocalInst() &&
1139 CanIgnoreThreadingForInst(*Acc.getLocalInst()));
1140 };
1141
1142 // TODO: Use inter-procedural reachability and dominance.
1143 bool IsKnownNoRecurse;
1144 AA::hasAssumedIRAttr<Attribute::NoRecurse>(
1145 A, QueryingAA: this, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL,
1146 IsKnown&: IsKnownNoRecurse);
1147
1148 // TODO: Use reaching kernels from AAKernelInfo (or move it to
1149 // AAExecutionDomain) such that we allow scopes other than kernels as long
1150 // as the reaching kernels are disjoint.
1151 bool InstInKernel = A.getInfoCache().isKernel(F: Scope);
1152 bool ObjHasKernelLifetime = false;
1153 const bool UseDominanceReasoning =
1154 FindInterferingWrites && IsKnownNoRecurse;
1155 const DominatorTree *DT =
1156 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: Scope);
1157
1158 // Helper to check if a value has "kernel lifetime", that is it will not
1159 // outlive a GPU kernel. This is true for shared, constant, and local
1160 // globals on AMD and NVIDIA GPUs.
1161 auto HasKernelLifetime = [&](Value *V, Module &M) {
1162 if (!AA::isGPU(M))
1163 return false;
1164 unsigned VAS = V->getType()->getPointerAddressSpace();
1165 return AA::isGPUSharedAddressSpace(M, AS: VAS) ||
1166 AA::isGPUConstantAddressSpace(M, AS: VAS) ||
1167 AA::isGPULocalAddressSpace(M, AS: VAS);
1168 };
1169
1170 // The IsLiveInCalleeCB will be used by the AA::isPotentiallyReachable query
1171 // to determine if we should look at reachability from the callee. For
1172 // certain pointers we know the lifetime and we do not have to step into the
1173 // callee to determine reachability as the pointer would be dead in the
1174 // callee. See the conditional initialization below.
1175 std::function<bool(const Function &)> IsLiveInCalleeCB;
1176
1177 if (auto *AI = dyn_cast<AllocaInst>(Val: &getAssociatedValue())) {
1178 // If the alloca containing function is not recursive the alloca
1179 // must be dead in the callee.
1180 const Function *AIFn = AI->getFunction();
1181 ObjHasKernelLifetime = A.getInfoCache().isKernel(F: *AIFn);
1182 bool IsKnownNoRecurse;
1183 if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
1184 A, QueryingAA: this, IRP: IRPosition::function(F: *AIFn), DepClass: DepClassTy::OPTIONAL,
1185 IsKnown&: IsKnownNoRecurse)) {
1186 IsLiveInCalleeCB = [AIFn](const Function &Fn) { return AIFn != &Fn; };
1187 }
1188 } else if (auto *GV = dyn_cast<GlobalValue>(Val: &getAssociatedValue())) {
1189 // If the global has kernel lifetime we can stop if we reach a kernel
1190 // as it is "dead" in the (unknown) callees.
1191 ObjHasKernelLifetime = HasKernelLifetime(GV, *GV->getParent());
1192 if (ObjHasKernelLifetime)
1193 IsLiveInCalleeCB = [&A](const Function &Fn) {
1194 return !A.getInfoCache().isKernel(F: Fn);
1195 };
1196 }
1197
1198 // Set of accesses/instructions that will overwrite the result and are
1199 // therefore blockers in the reachability traversal.
1200 AA::InstExclusionSetTy ExclusionSet;
1201
1202 auto AccessCB = [&](const Access &Acc, bool Exact) {
1203 Function *AccScope = Acc.getRemoteInst()->getFunction();
1204 bool AccInSameScope = AccScope == &Scope;
1205
1206 // If the object has kernel lifetime we can ignore accesses only reachable
1207 // by other kernels. For now we only skip accesses *in* other kernels.
1208 if (InstInKernel && ObjHasKernelLifetime && !AccInSameScope &&
1209 A.getInfoCache().isKernel(F: *AccScope))
1210 return true;
1211
1212 if (Exact && Acc.isMustAccess() && Acc.getRemoteInst() != &I) {
1213 if (Acc.isWrite() || (isa<LoadInst>(Val: I) && Acc.isWriteOrAssumption()))
1214 ExclusionSet.insert(Ptr: Acc.getRemoteInst());
1215 }
1216
1217 if ((!FindInterferingWrites || !Acc.isWriteOrAssumption()) &&
1218 (!FindInterferingReads || !Acc.isRead()))
1219 return true;
1220
1221 bool Dominates = FindInterferingWrites && DT && Exact &&
1222 Acc.isMustAccess() && AccInSameScope &&
1223 DT->dominates(Def: Acc.getRemoteInst(), User: &I);
1224 if (Dominates)
1225 DominatingWrites.insert(Ptr: &Acc);
1226
1227 // Track if all interesting accesses are in the same `nosync` function as
1228 // the given instruction.
1229 AllInSameNoSyncFn &= Acc.getRemoteInst()->getFunction() == &Scope;
1230
1231 InterferingAccesses.push_back(Elt: {&Acc, Exact});
1232 return true;
1233 };
1234 if (!State::forallInterferingAccesses(I, CB: AccessCB, Range))
1235 return false;
1236
1237 HasBeenWrittenTo = !DominatingWrites.empty();
1238
1239 // Dominating writes form a chain, find the least/lowest member.
1240 Instruction *LeastDominatingWriteInst = nullptr;
1241 for (const Access *Acc : DominatingWrites) {
1242 if (!LeastDominatingWriteInst) {
1243 LeastDominatingWriteInst = Acc->getRemoteInst();
1244 } else if (DT->dominates(Def: LeastDominatingWriteInst,
1245 User: Acc->getRemoteInst())) {
1246 LeastDominatingWriteInst = Acc->getRemoteInst();
1247 }
1248 }
1249
1250 // Helper to determine if we can skip a specific write access.
1251 auto CanSkipAccess = [&](const Access &Acc, bool Exact) {
1252 if (SkipCB && SkipCB(Acc))
1253 return true;
1254 if (!CanIgnoreThreading(Acc))
1255 return false;
1256
1257 // Check read (RAW) dependences and write (WAR) dependences as necessary.
1258 // If we successfully excluded all effects we are interested in, the
1259 // access can be skipped.
1260 bool ReadChecked = !FindInterferingReads;
1261 bool WriteChecked = !FindInterferingWrites;
1262
1263 // If the instruction cannot reach the access, the former does not
1264 // interfere with what the access reads.
1265 if (!ReadChecked) {
1266 if (!AA::isPotentiallyReachable(A, FromI: I, ToI: *Acc.getRemoteInst(), QueryingAA,
1267 ExclusionSet: &ExclusionSet, GoBackwardsCB: IsLiveInCalleeCB))
1268 ReadChecked = true;
1269 }
1270 // If the instruction cannot be reach from the access, the latter does not
1271 // interfere with what the instruction reads.
1272 if (!WriteChecked) {
1273 if (!AA::isPotentiallyReachable(A, FromI: *Acc.getRemoteInst(), ToI: I, QueryingAA,
1274 ExclusionSet: &ExclusionSet, GoBackwardsCB: IsLiveInCalleeCB))
1275 WriteChecked = true;
1276 }
1277
1278 // If we still might be affected by the write of the access but there are
1279 // dominating writes in the function of the instruction
1280 // (HasBeenWrittenTo), we can try to reason that the access is overwritten
1281 // by them. This would have happend above if they are all in the same
1282 // function, so we only check the inter-procedural case. Effectively, we
1283 // want to show that there is no call after the dominting write that might
1284 // reach the access, and when it returns reach the instruction with the
1285 // updated value. To this end, we iterate all call sites, check if they
1286 // might reach the instruction without going through another access
1287 // (ExclusionSet) and at the same time might reach the access. However,
1288 // that is all part of AAInterFnReachability.
1289 if (!WriteChecked && HasBeenWrittenTo &&
1290 Acc.getRemoteInst()->getFunction() != &Scope) {
1291
1292 const auto *FnReachabilityAA = A.getAAFor<AAInterFnReachability>(
1293 QueryingAA, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL);
1294 if (FnReachabilityAA) {
1295 // Without going backwards in the call tree, can we reach the access
1296 // from the least dominating write. Do not allow to pass the
1297 // instruction itself either.
1298 bool Inserted = ExclusionSet.insert(Ptr: &I).second;
1299
1300 if (!FnReachabilityAA->instructionCanReach(
1301 A, Inst: *LeastDominatingWriteInst,
1302 Fn: *Acc.getRemoteInst()->getFunction(), ExclusionSet: &ExclusionSet))
1303 WriteChecked = true;
1304
1305 if (Inserted)
1306 ExclusionSet.erase(Ptr: &I);
1307 }
1308 }
1309
1310 if (ReadChecked && WriteChecked)
1311 return true;
1312
1313 if (!DT || !UseDominanceReasoning)
1314 return false;
1315 if (!DominatingWrites.count(Ptr: &Acc))
1316 return false;
1317 return LeastDominatingWriteInst != Acc.getRemoteInst();
1318 };
1319
1320 // Run the user callback on all accesses we cannot skip and return if
1321 // that succeeded for all or not.
1322 for (auto &It : InterferingAccesses) {
1323 if ((!AllInSameNoSyncFn && !IsThreadLocalObj && !ExecDomainAA) ||
1324 !CanSkipAccess(*It.first, It.second)) {
1325 if (!UserCB(*It.first, It.second))
1326 return false;
1327 }
1328 }
1329 return true;
1330 }
1331
1332 ChangeStatus translateAndAddStateFromCallee(Attributor &A,
1333 const AAPointerInfo &OtherAA,
1334 CallBase &CB) {
1335 using namespace AA::PointerInfo;
1336 if (!OtherAA.getState().isValidState() || !isValidState())
1337 return indicatePessimisticFixpoint();
1338
1339 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1340 const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1341 bool IsByval = OtherAAImpl.getAssociatedArgument()->hasByValAttr();
1342 Changed |= setReachesReturn(OtherAAImpl.ReturnedOffsets);
1343
1344 // Combine the accesses bin by bin.
1345 const auto &State = OtherAAImpl.getState();
1346 for (const auto &It : State) {
1347 for (auto Index : It.getSecond()) {
1348 const auto &RAcc = State.getAccess(Index);
1349 if (IsByval && !RAcc.isRead())
1350 continue;
1351 bool UsedAssumedInformation = false;
1352 AccessKind AK = RAcc.getKind();
1353 auto Content = A.translateArgumentToCallSiteContent(
1354 V: RAcc.getContent(), CB, AA: *this, UsedAssumedInformation);
1355 AK = AccessKind(AK & (IsByval ? AccessKind::AK_R : AccessKind::AK_RW));
1356 AK = AccessKind(AK | (RAcc.isMayAccess() ? AK_MAY : AK_MUST));
1357
1358 Changed |= addAccess(A, Ranges: RAcc.getRanges(), I&: CB, Content, Kind: AK,
1359 Ty: RAcc.getType(), RemoteI: RAcc.getRemoteInst());
1360 }
1361 }
1362 return Changed;
1363 }
1364
1365 ChangeStatus translateAndAddState(Attributor &A, const AAPointerInfo &OtherAA,
1366 const OffsetInfo &Offsets, CallBase &CB,
1367 bool IsMustAcc) {
1368 using namespace AA::PointerInfo;
1369 if (!OtherAA.getState().isValidState() || !isValidState())
1370 return indicatePessimisticFixpoint();
1371
1372 const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1373
1374 // Combine the accesses bin by bin.
1375 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1376 const auto &State = OtherAAImpl.getState();
1377 for (const auto &It : State) {
1378 for (auto Index : It.getSecond()) {
1379 const auto &RAcc = State.getAccess(Index);
1380 if (!IsMustAcc && RAcc.isAssumption())
1381 continue;
1382 for (auto Offset : Offsets) {
1383 auto NewRanges = Offset == AA::RangeTy::Unknown
1384 ? AA::RangeTy::getUnknown()
1385 : RAcc.getRanges();
1386 if (!NewRanges.isUnknown()) {
1387 NewRanges.addToAllOffsets(Inc: Offset);
1388 }
1389 AccessKind AK = RAcc.getKind();
1390 if (!IsMustAcc)
1391 AK = AccessKind((AK & ~AK_MUST) | AK_MAY);
1392 Changed |= addAccess(A, Ranges: NewRanges, I&: CB, Content: RAcc.getContent(), Kind: AK,
1393 Ty: RAcc.getType(), RemoteI: RAcc.getRemoteInst());
1394 }
1395 }
1396 }
1397 return Changed;
1398 }
1399
1400 /// Statistic tracking for all AAPointerInfo implementations.
1401 /// See AbstractAttribute::trackStatistics().
1402 void trackPointerInfoStatistics(const IRPosition &IRP) const {}
1403
1404 /// Dump the state into \p O.
1405 void dumpState(raw_ostream &O) {
1406 for (auto &It : OffsetBins) {
1407 O << "[" << It.first.Offset << "-" << It.first.Offset + It.first.Size
1408 << "] : " << It.getSecond().size() << "\n";
1409 for (auto AccIndex : It.getSecond()) {
1410 auto &Acc = AccessList[AccIndex];
1411 O << " - " << Acc.getKind() << " - " << *Acc.getLocalInst() << "\n";
1412 if (Acc.getLocalInst() != Acc.getRemoteInst())
1413 O << " --> " << *Acc.getRemoteInst()
1414 << "\n";
1415 if (!Acc.isWrittenValueYetUndetermined()) {
1416 if (isa_and_nonnull<Function>(Val: Acc.getWrittenValue()))
1417 O << " - c: func " << Acc.getWrittenValue()->getName()
1418 << "\n";
1419 else if (Acc.getWrittenValue())
1420 O << " - c: " << *Acc.getWrittenValue() << "\n";
1421 else
1422 O << " - c: <unknown>\n";
1423 }
1424 }
1425 }
1426 }
1427};
1428
1429struct AAPointerInfoFloating : public AAPointerInfoImpl {
1430 using AccessKind = AAPointerInfo::AccessKind;
1431 AAPointerInfoFloating(const IRPosition &IRP, Attributor &A)
1432 : AAPointerInfoImpl(IRP, A) {}
1433
1434 /// Deal with an access and signal if it was handled successfully.
1435 bool handleAccess(Attributor &A, Instruction &I,
1436 std::optional<Value *> Content, AccessKind Kind,
1437 OffsetInfo::VecTy &Offsets, ChangeStatus &Changed,
1438 Type &Ty) {
1439 using namespace AA::PointerInfo;
1440 auto Size = AA::RangeTy::Unknown;
1441 const DataLayout &DL = A.getDataLayout();
1442 TypeSize AccessSize = DL.getTypeStoreSize(Ty: &Ty);
1443 if (!AccessSize.isScalable())
1444 Size = AccessSize.getFixedValue();
1445
1446 // Make a strictly ascending list of offsets as required by addAccess()
1447 SmallVector<int64_t> OffsetsSorted(Offsets.begin(), Offsets.end());
1448 llvm::sort(C&: OffsetsSorted);
1449
1450 VectorType *VT = dyn_cast<VectorType>(Val: &Ty);
1451 if (!VT || VT->getElementCount().isScalable() ||
1452 !Content.value_or(u: nullptr) || !isa<Constant>(Val: *Content) ||
1453 (*Content)->getType() != VT ||
1454 DL.getTypeStoreSize(Ty: VT->getElementType()).isScalable()) {
1455 Changed =
1456 Changed | addAccess(A, Ranges: {OffsetsSorted, Size}, I, Content, Kind, Ty: &Ty);
1457 } else {
1458 // Handle vector stores with constant content element-wise.
1459 // TODO: We could look for the elements or create instructions
1460 // representing them.
1461 // TODO: We need to push the Content into the range abstraction
1462 // (AA::RangeTy) to allow different content values for different
1463 // ranges. ranges. Hence, support vectors storing different values.
1464 Type *ElementType = VT->getElementType();
1465 int64_t ElementSize = DL.getTypeStoreSize(Ty: ElementType).getFixedValue();
1466 auto *ConstContent = cast<Constant>(Val: *Content);
1467 Type *Int32Ty = Type::getInt32Ty(C&: ElementType->getContext());
1468 SmallVector<int64_t> ElementOffsets(Offsets.begin(), Offsets.end());
1469
1470 for (int i = 0, e = VT->getElementCount().getFixedValue(); i != e; ++i) {
1471 Value *ElementContent = ConstantExpr::getExtractElement(
1472 Vec: ConstContent, Idx: ConstantInt::get(Ty: Int32Ty, V: i));
1473
1474 // Add the element access.
1475 Changed = Changed | addAccess(A, Ranges: {ElementOffsets, ElementSize}, I,
1476 Content: ElementContent, Kind, Ty: ElementType);
1477
1478 // Advance the offsets for the next element.
1479 for (auto &ElementOffset : ElementOffsets)
1480 ElementOffset += ElementSize;
1481 }
1482 }
1483 return true;
1484 };
1485
1486 /// See AbstractAttribute::updateImpl(...).
1487 ChangeStatus updateImpl(Attributor &A) override;
1488
1489 /// If the indices to \p GEP can be traced to constants, incorporate all
1490 /// of these into \p UsrOI.
1491 ///
1492 /// \return true iff \p UsrOI is updated.
1493 bool collectConstantsForGEP(Attributor &A, const DataLayout &DL,
1494 OffsetInfo &UsrOI, const OffsetInfo &PtrOI,
1495 const GEPOperator *GEP);
1496
1497 /// See AbstractAttribute::trackStatistics()
1498 void trackStatistics() const override {
1499 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1500 }
1501};
1502
1503bool AAPointerInfoFloating::collectConstantsForGEP(Attributor &A,
1504 const DataLayout &DL,
1505 OffsetInfo &UsrOI,
1506 const OffsetInfo &PtrOI,
1507 const GEPOperator *GEP) {
1508 unsigned BitWidth = DL.getIndexTypeSizeInBits(Ty: GEP->getType());
1509 SmallMapVector<Value *, APInt, 4> VariableOffsets;
1510 APInt ConstantOffset(BitWidth, 0);
1511
1512 assert(!UsrOI.isUnknown() && !PtrOI.isUnknown() &&
1513 "Don't look for constant values if the offset has already been "
1514 "determined to be unknown.");
1515
1516 if (!GEP->collectOffset(DL, BitWidth, VariableOffsets, ConstantOffset)) {
1517 UsrOI.setUnknown();
1518 return true;
1519 }
1520
1521 LLVM_DEBUG(dbgs() << "[AAPointerInfo] GEP offset is "
1522 << (VariableOffsets.empty() ? "" : "not") << " constant "
1523 << *GEP << "\n");
1524
1525 auto Union = PtrOI;
1526 Union.addToAll(Inc: ConstantOffset.getSExtValue());
1527
1528 // Each VI in VariableOffsets has a set of potential constant values. Every
1529 // combination of elements, picked one each from these sets, is separately
1530 // added to the original set of offsets, thus resulting in more offsets.
1531 for (const auto &VI : VariableOffsets) {
1532 auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
1533 QueryingAA: *this, IRP: IRPosition::value(V: *VI.first), DepClass: DepClassTy::OPTIONAL);
1534 if (!PotentialConstantsAA || !PotentialConstantsAA->isValidState()) {
1535 UsrOI.setUnknown();
1536 return true;
1537 }
1538
1539 // UndefValue is treated as a zero, which leaves Union as is.
1540 if (PotentialConstantsAA->undefIsContained())
1541 continue;
1542
1543 // We need at least one constant in every set to compute an actual offset.
1544 // Otherwise, we end up pessimizing AAPointerInfo by respecting offsets that
1545 // don't actually exist. In other words, the absence of constant values
1546 // implies that the operation can be assumed dead for now.
1547 auto &AssumedSet = PotentialConstantsAA->getAssumedSet();
1548 if (AssumedSet.empty())
1549 return false;
1550
1551 OffsetInfo Product;
1552 for (const auto &ConstOffset : AssumedSet) {
1553 auto CopyPerOffset = Union;
1554 CopyPerOffset.addToAll(Inc: ConstOffset.getSExtValue() *
1555 VI.second.getZExtValue());
1556 Product.merge(R: CopyPerOffset);
1557 }
1558 Union = Product;
1559 }
1560
1561 UsrOI = std::move(Union);
1562 return true;
1563}
1564
1565ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
1566 using namespace AA::PointerInfo;
1567 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1568 const DataLayout &DL = A.getDataLayout();
1569 Value &AssociatedValue = getAssociatedValue();
1570
1571 DenseMap<Value *, OffsetInfo> OffsetInfoMap;
1572 OffsetInfoMap[&AssociatedValue].insert(Offset: 0);
1573
1574 auto HandlePassthroughUser = [&](Value *Usr, Value *CurPtr, bool &Follow) {
1575 // One does not simply walk into a map and assign a reference to a possibly
1576 // new location. That can cause an invalidation before the assignment
1577 // happens, like so:
1578 //
1579 // OffsetInfoMap[Usr] = OffsetInfoMap[CurPtr]; /* bad idea! */
1580 //
1581 // The RHS is a reference that may be invalidated by an insertion caused by
1582 // the LHS. So we ensure that the side-effect of the LHS happens first.
1583
1584 assert(OffsetInfoMap.contains(CurPtr) &&
1585 "CurPtr does not exist in the map!");
1586
1587 auto &UsrOI = OffsetInfoMap[Usr];
1588 auto &PtrOI = OffsetInfoMap[CurPtr];
1589 assert(!PtrOI.isUnassigned() &&
1590 "Cannot pass through if the input Ptr was not visited!");
1591 UsrOI.merge(R: PtrOI);
1592 Follow = true;
1593 return true;
1594 };
1595
1596 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
1597 Value *CurPtr = U.get();
1598 User *Usr = U.getUser();
1599 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Analyze " << *CurPtr << " in " << *Usr
1600 << "\n");
1601 assert(OffsetInfoMap.count(CurPtr) &&
1602 "The current pointer offset should have been seeded!");
1603 assert(!OffsetInfoMap[CurPtr].isUnassigned() &&
1604 "Current pointer should be assigned");
1605
1606 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: Usr)) {
1607 if (CE->isCast())
1608 return HandlePassthroughUser(Usr, CurPtr, Follow);
1609 if (!isa<GEPOperator>(Val: CE)) {
1610 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled constant user " << *CE
1611 << "\n");
1612 return false;
1613 }
1614 }
1615 if (auto *GEP = dyn_cast<GEPOperator>(Val: Usr)) {
1616 // Note the order here, the Usr access might change the map, CurPtr is
1617 // already in it though.
1618 auto &UsrOI = OffsetInfoMap[Usr];
1619 auto &PtrOI = OffsetInfoMap[CurPtr];
1620
1621 if (UsrOI.isUnknown())
1622 return true;
1623
1624 if (PtrOI.isUnknown()) {
1625 Follow = true;
1626 UsrOI.setUnknown();
1627 return true;
1628 }
1629
1630 Follow = collectConstantsForGEP(A, DL, UsrOI, PtrOI, GEP);
1631 return true;
1632 }
1633 if (isa<PtrToIntInst>(Val: Usr))
1634 return false;
1635 if (isa<CastInst>(Val: Usr) || isa<SelectInst>(Val: Usr))
1636 return HandlePassthroughUser(Usr, CurPtr, Follow);
1637 // Returns are allowed if they are in the associated functions. Users can
1638 // then check the call site return. Returns from other functions can't be
1639 // tracked and are cause for invalidation.
1640 if (auto *RI = dyn_cast<ReturnInst>(Val: Usr)) {
1641 if (RI->getFunction() == getAssociatedFunction()) {
1642 auto &PtrOI = OffsetInfoMap[CurPtr];
1643 Changed |= setReachesReturn(PtrOI);
1644 return true;
1645 }
1646 return false;
1647 }
1648
1649 // For PHIs we need to take care of the recurrence explicitly as the value
1650 // might change while we iterate through a loop. For now, we give up if
1651 // the PHI is not invariant.
1652 if (auto *PHI = dyn_cast<PHINode>(Val: Usr)) {
1653 // Note the order here, the Usr access might change the map, CurPtr is
1654 // already in it though.
1655 auto [PhiIt, IsFirstPHIUser] = OffsetInfoMap.try_emplace(Key: PHI);
1656 auto &UsrOI = PhiIt->second;
1657 auto &PtrOI = OffsetInfoMap[CurPtr];
1658
1659 // Check if the PHI operand has already an unknown offset as we can't
1660 // improve on that anymore.
1661 if (PtrOI.isUnknown()) {
1662 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand offset unknown "
1663 << *CurPtr << " in " << *PHI << "\n");
1664 Follow = !UsrOI.isUnknown();
1665 UsrOI.setUnknown();
1666 return true;
1667 }
1668
1669 // Check if the PHI is invariant (so far).
1670 if (UsrOI == PtrOI) {
1671 assert(!PtrOI.isUnassigned() &&
1672 "Cannot assign if the current Ptr was not visited!");
1673 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant (so far)");
1674 return true;
1675 }
1676
1677 // Check if the PHI operand can be traced back to AssociatedValue.
1678 APInt Offset(
1679 DL.getIndexSizeInBits(AS: CurPtr->getType()->getPointerAddressSpace()),
1680 0);
1681 Value *CurPtrBase = CurPtr->stripAndAccumulateConstantOffsets(
1682 DL, Offset, /* AllowNonInbounds */ true);
1683 auto It = OffsetInfoMap.find(Val: CurPtrBase);
1684 if (It == OffsetInfoMap.end()) {
1685 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand is too complex "
1686 << *CurPtr << " in " << *PHI
1687 << " (base: " << *CurPtrBase << ")\n");
1688 UsrOI.setUnknown();
1689 Follow = true;
1690 return true;
1691 }
1692
1693 // Check if the PHI operand is not dependent on the PHI itself. Every
1694 // recurrence is a cyclic net of PHIs in the data flow, and has an
1695 // equivalent Cycle in the control flow. One of those PHIs must be in the
1696 // header of that control flow Cycle. This is independent of the choice of
1697 // Cycles reported by CycleInfo. It is sufficient to check the PHIs in
1698 // every Cycle header; if such a node is marked unknown, this will
1699 // eventually propagate through the whole net of PHIs in the recurrence.
1700 const auto *CI =
1701 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
1702 F: *PHI->getFunction());
1703 if (mayBeInCycle(CI, I: cast<Instruction>(Val: Usr), /* HeaderOnly */ true)) {
1704 auto BaseOI = It->getSecond();
1705 BaseOI.addToAll(Inc: Offset.getZExtValue());
1706 if (IsFirstPHIUser || BaseOI == UsrOI) {
1707 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant " << *CurPtr
1708 << " in " << *Usr << "\n");
1709 return HandlePassthroughUser(Usr, CurPtr, Follow);
1710 }
1711
1712 LLVM_DEBUG(
1713 dbgs() << "[AAPointerInfo] PHI operand pointer offset mismatch "
1714 << *CurPtr << " in " << *PHI << "\n");
1715 UsrOI.setUnknown();
1716 Follow = true;
1717 return true;
1718 }
1719
1720 UsrOI.merge(R: PtrOI);
1721 Follow = true;
1722 return true;
1723 }
1724
1725 if (auto *LoadI = dyn_cast<LoadInst>(Val: Usr)) {
1726 // If the access is to a pointer that may or may not be the associated
1727 // value, e.g. due to a PHI, we cannot assume it will be read.
1728 AccessKind AK = AccessKind::AK_R;
1729 if (getUnderlyingObject(V: CurPtr) == &AssociatedValue)
1730 AK = AccessKind(AK | AccessKind::AK_MUST);
1731 else
1732 AK = AccessKind(AK | AccessKind::AK_MAY);
1733 if (!handleAccess(A, I&: *LoadI, /* Content */ nullptr, Kind: AK,
1734 Offsets&: OffsetInfoMap[CurPtr].Offsets, Changed,
1735 Ty&: *LoadI->getType()))
1736 return false;
1737
1738 auto IsAssumption = [](Instruction &I) {
1739 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
1740 return II->isAssumeLikeIntrinsic();
1741 return false;
1742 };
1743
1744 auto IsImpactedInRange = [&](Instruction *FromI, Instruction *ToI) {
1745 // Check if the assumption and the load are executed together without
1746 // memory modification.
1747 do {
1748 if (FromI->mayWriteToMemory() && !IsAssumption(*FromI))
1749 return true;
1750 FromI = FromI->getNextNode();
1751 } while (FromI && FromI != ToI);
1752 return false;
1753 };
1754
1755 BasicBlock *BB = LoadI->getParent();
1756 auto IsValidAssume = [&](IntrinsicInst &IntrI) {
1757 if (IntrI.getIntrinsicID() != Intrinsic::assume)
1758 return false;
1759 BasicBlock *IntrBB = IntrI.getParent();
1760 if (IntrI.getParent() == BB) {
1761 if (IsImpactedInRange(LoadI->getNextNode(), &IntrI))
1762 return false;
1763 } else {
1764 auto PredIt = pred_begin(BB: IntrBB);
1765 if (PredIt == pred_end(BB: IntrBB))
1766 return false;
1767 if ((*PredIt) != BB)
1768 return false;
1769 if (++PredIt != pred_end(BB: IntrBB))
1770 return false;
1771 for (auto *SuccBB : successors(BB)) {
1772 if (SuccBB == IntrBB)
1773 continue;
1774 if (isa<UnreachableInst>(Val: SuccBB->getTerminator()))
1775 continue;
1776 return false;
1777 }
1778 if (IsImpactedInRange(LoadI->getNextNode(), BB->getTerminator()))
1779 return false;
1780 if (IsImpactedInRange(&IntrBB->front(), &IntrI))
1781 return false;
1782 }
1783 return true;
1784 };
1785
1786 std::pair<Value *, IntrinsicInst *> Assumption;
1787 for (const Use &LoadU : LoadI->uses()) {
1788 if (auto *CmpI = dyn_cast<CmpInst>(Val: LoadU.getUser())) {
1789 if (!CmpI->isEquality() || !CmpI->isTrueWhenEqual())
1790 continue;
1791 for (const Use &CmpU : CmpI->uses()) {
1792 if (auto *IntrI = dyn_cast<IntrinsicInst>(Val: CmpU.getUser())) {
1793 if (!IsValidAssume(*IntrI))
1794 continue;
1795 int Idx = CmpI->getOperandUse(i: 0) == LoadU;
1796 Assumption = {CmpI->getOperand(i_nocapture: Idx), IntrI};
1797 break;
1798 }
1799 }
1800 }
1801 if (Assumption.first)
1802 break;
1803 }
1804
1805 // Check if we found an assumption associated with this load.
1806 if (!Assumption.first || !Assumption.second)
1807 return true;
1808
1809 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Assumption found "
1810 << *Assumption.second << ": " << *LoadI
1811 << " == " << *Assumption.first << "\n");
1812 bool UsedAssumedInformation = false;
1813 std::optional<Value *> Content = nullptr;
1814 if (Assumption.first)
1815 Content =
1816 A.getAssumedSimplified(V: *Assumption.first, AA: *this,
1817 UsedAssumedInformation, S: AA::Interprocedural);
1818 return handleAccess(
1819 A, I&: *Assumption.second, Content, Kind: AccessKind::AK_ASSUMPTION,
1820 Offsets&: OffsetInfoMap[CurPtr].Offsets, Changed, Ty&: *LoadI->getType());
1821 }
1822
1823 auto HandleStoreLike = [&](Instruction &I, Value *ValueOp, Type &ValueTy,
1824 ArrayRef<Value *> OtherOps, AccessKind AK) {
1825 for (auto *OtherOp : OtherOps) {
1826 if (OtherOp == CurPtr) {
1827 LLVM_DEBUG(
1828 dbgs()
1829 << "[AAPointerInfo] Escaping use in store like instruction " << I
1830 << "\n");
1831 return false;
1832 }
1833 }
1834
1835 // If the access is to a pointer that may or may not be the associated
1836 // value, e.g. due to a PHI, we cannot assume it will be written.
1837 if (getUnderlyingObject(V: CurPtr) == &AssociatedValue)
1838 AK = AccessKind(AK | AccessKind::AK_MUST);
1839 else
1840 AK = AccessKind(AK | AccessKind::AK_MAY);
1841 bool UsedAssumedInformation = false;
1842 std::optional<Value *> Content = nullptr;
1843 if (ValueOp)
1844 Content = A.getAssumedSimplified(
1845 V: *ValueOp, AA: *this, UsedAssumedInformation, S: AA::Interprocedural);
1846 return handleAccess(A, I, Content, Kind: AK, Offsets&: OffsetInfoMap[CurPtr].Offsets,
1847 Changed, Ty&: ValueTy);
1848 };
1849
1850 if (auto *StoreI = dyn_cast<StoreInst>(Val: Usr))
1851 return HandleStoreLike(*StoreI, StoreI->getValueOperand(),
1852 *StoreI->getValueOperand()->getType(),
1853 {StoreI->getValueOperand()}, AccessKind::AK_W);
1854 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: Usr))
1855 return HandleStoreLike(*RMWI, nullptr, *RMWI->getValOperand()->getType(),
1856 {RMWI->getValOperand()}, AccessKind::AK_RW);
1857 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: Usr))
1858 return HandleStoreLike(
1859 *CXI, nullptr, *CXI->getNewValOperand()->getType(),
1860 {CXI->getCompareOperand(), CXI->getNewValOperand()},
1861 AccessKind::AK_RW);
1862
1863 if (auto *CB = dyn_cast<CallBase>(Val: Usr)) {
1864 if (CB->isLifetimeStartOrEnd())
1865 return true;
1866 const auto *TLI =
1867 A.getInfoCache().getTargetLibraryInfoForFunction(F: *CB->getFunction());
1868 if (getFreedOperand(CB, TLI) == U)
1869 return true;
1870 if (CB->isArgOperand(U: &U)) {
1871 unsigned ArgNo = CB->getArgOperandNo(U: &U);
1872 const auto *CSArgPI = A.getAAFor<AAPointerInfo>(
1873 QueryingAA: *this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
1874 DepClass: DepClassTy::REQUIRED);
1875 if (!CSArgPI)
1876 return false;
1877 bool IsArgMustAcc = (getUnderlyingObject(V: CurPtr) == &AssociatedValue);
1878 Changed = translateAndAddState(A, OtherAA: *CSArgPI, Offsets: OffsetInfoMap[CurPtr], CB&: *CB,
1879 IsMustAcc: IsArgMustAcc) |
1880 Changed;
1881 if (!CSArgPI->reachesReturn())
1882 return isValidState();
1883
1884 Function *Callee = CB->getCalledFunction();
1885 if (!Callee || Callee->arg_size() <= ArgNo)
1886 return false;
1887 bool UsedAssumedInformation = false;
1888 auto ReturnedValue = A.getAssumedSimplified(
1889 IRP: IRPosition::returned(F: *Callee), AA: *this, UsedAssumedInformation,
1890 S: AA::ValueScope::Intraprocedural);
1891 auto *ReturnedArg =
1892 dyn_cast_or_null<Argument>(Val: ReturnedValue.value_or(u: nullptr));
1893 auto *Arg = Callee->getArg(i: ArgNo);
1894 if (ReturnedArg && Arg != ReturnedArg)
1895 return true;
1896 bool IsRetMustAcc = IsArgMustAcc && (ReturnedArg == Arg);
1897 const auto *CSRetPI = A.getAAFor<AAPointerInfo>(
1898 QueryingAA: *this, IRP: IRPosition::callsite_returned(CB: *CB), DepClass: DepClassTy::REQUIRED);
1899 if (!CSRetPI)
1900 return false;
1901 OffsetInfo OI = OffsetInfoMap[CurPtr];
1902 CSArgPI->addReturnedOffsetsTo(OI);
1903 Changed =
1904 translateAndAddState(A, OtherAA: *CSRetPI, Offsets: OI, CB&: *CB, IsMustAcc: IsRetMustAcc) | Changed;
1905 return isValidState();
1906 }
1907 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Call user not handled " << *CB
1908 << "\n");
1909 return false;
1910 }
1911
1912 LLVM_DEBUG(dbgs() << "[AAPointerInfo] User not handled " << *Usr << "\n");
1913 return false;
1914 };
1915 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
1916 assert(OffsetInfoMap.count(OldU) && "Old use should be known already!");
1917 assert(!OffsetInfoMap[OldU].isUnassigned() && "Old use should be assinged");
1918 if (OffsetInfoMap.count(Val: NewU)) {
1919 LLVM_DEBUG({
1920 if (!(OffsetInfoMap[NewU] == OffsetInfoMap[OldU])) {
1921 dbgs() << "[AAPointerInfo] Equivalent use callback failed: "
1922 << OffsetInfoMap[NewU] << " vs " << OffsetInfoMap[OldU]
1923 << "\n";
1924 }
1925 });
1926 return OffsetInfoMap[NewU] == OffsetInfoMap[OldU];
1927 }
1928 bool Unused;
1929 return HandlePassthroughUser(NewU.get(), OldU.get(), Unused);
1930 };
1931 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: AssociatedValue,
1932 /* CheckBBLivenessOnly */ true, LivenessDepClass: DepClassTy::OPTIONAL,
1933 /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
1934 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Check for all uses failed, abort!\n");
1935 return indicatePessimisticFixpoint();
1936 }
1937
1938 LLVM_DEBUG({
1939 dbgs() << "Accesses by bin after update:\n";
1940 dumpState(dbgs());
1941 });
1942
1943 return Changed;
1944}
1945
1946struct AAPointerInfoReturned final : AAPointerInfoImpl {
1947 AAPointerInfoReturned(const IRPosition &IRP, Attributor &A)
1948 : AAPointerInfoImpl(IRP, A) {}
1949
1950 /// See AbstractAttribute::updateImpl(...).
1951 ChangeStatus updateImpl(Attributor &A) override {
1952 return indicatePessimisticFixpoint();
1953 }
1954
1955 /// See AbstractAttribute::trackStatistics()
1956 void trackStatistics() const override {
1957 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1958 }
1959};
1960
1961struct AAPointerInfoArgument final : AAPointerInfoFloating {
1962 AAPointerInfoArgument(const IRPosition &IRP, Attributor &A)
1963 : AAPointerInfoFloating(IRP, A) {}
1964
1965 /// See AbstractAttribute::trackStatistics()
1966 void trackStatistics() const override {
1967 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1968 }
1969};
1970
1971struct AAPointerInfoCallSiteArgument final : AAPointerInfoFloating {
1972 AAPointerInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
1973 : AAPointerInfoFloating(IRP, A) {}
1974
1975 /// See AbstractAttribute::updateImpl(...).
1976 ChangeStatus updateImpl(Attributor &A) override {
1977 using namespace AA::PointerInfo;
1978 // We handle memory intrinsics explicitly, at least the first (=
1979 // destination) and second (=source) arguments as we know how they are
1980 // accessed.
1981 if (auto *MI = dyn_cast_or_null<MemIntrinsic>(Val: getCtxI())) {
1982 int64_t LengthVal = AA::RangeTy::Unknown;
1983 if (auto Length = MI->getLengthInBytes())
1984 LengthVal = Length->getSExtValue();
1985 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
1986 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1987 if (ArgNo > 1) {
1988 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled memory intrinsic "
1989 << *MI << "\n");
1990 return indicatePessimisticFixpoint();
1991 } else {
1992 auto Kind =
1993 ArgNo == 0 ? AccessKind::AK_MUST_WRITE : AccessKind::AK_MUST_READ;
1994 Changed =
1995 Changed | addAccess(A, Ranges: {0, LengthVal}, I&: *MI, Content: nullptr, Kind, Ty: nullptr);
1996 }
1997 LLVM_DEBUG({
1998 dbgs() << "Accesses by bin after update:\n";
1999 dumpState(dbgs());
2000 });
2001
2002 return Changed;
2003 }
2004
2005 // TODO: Once we have call site specific value information we can provide
2006 // call site specific liveness information and then it makes
2007 // sense to specialize attributes for call sites arguments instead of
2008 // redirecting requests to the callee argument.
2009 Argument *Arg = getAssociatedArgument();
2010 if (Arg) {
2011 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
2012 auto *ArgAA =
2013 A.getAAFor<AAPointerInfo>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
2014 if (ArgAA && ArgAA->getState().isValidState())
2015 return translateAndAddStateFromCallee(A, OtherAA: *ArgAA,
2016 CB&: *cast<CallBase>(Val: getCtxI()));
2017 if (!Arg->getParent()->isDeclaration())
2018 return indicatePessimisticFixpoint();
2019 }
2020
2021 bool IsKnownNoCapture;
2022 if (!AA::hasAssumedIRAttr<Attribute::Captures>(
2023 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture))
2024 return indicatePessimisticFixpoint();
2025
2026 bool IsKnown = false;
2027 if (AA::isAssumedReadNone(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
2028 return ChangeStatus::UNCHANGED;
2029 bool ReadOnly = AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown);
2030 auto Kind =
2031 ReadOnly ? AccessKind::AK_MAY_READ : AccessKind::AK_MAY_READ_WRITE;
2032 return addAccess(A, Ranges: AA::RangeTy::getUnknown(), I&: *getCtxI(), Content: nullptr, Kind,
2033 Ty: nullptr);
2034 }
2035
2036 /// See AbstractAttribute::trackStatistics()
2037 void trackStatistics() const override {
2038 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
2039 }
2040};
2041
2042struct AAPointerInfoCallSiteReturned final : AAPointerInfoFloating {
2043 AAPointerInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
2044 : AAPointerInfoFloating(IRP, A) {}
2045
2046 /// See AbstractAttribute::trackStatistics()
2047 void trackStatistics() const override {
2048 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
2049 }
2050};
2051} // namespace
2052
2053/// -----------------------NoUnwind Function Attribute--------------------------
2054
2055namespace {
2056struct AANoUnwindImpl : AANoUnwind {
2057 AANoUnwindImpl(const IRPosition &IRP, Attributor &A) : AANoUnwind(IRP, A) {}
2058
2059 /// See AbstractAttribute::initialize(...).
2060 void initialize(Attributor &A) override {
2061 bool IsKnown;
2062 assert(!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
2063 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2064 (void)IsKnown;
2065 }
2066
2067 const std::string getAsStr(Attributor *A) const override {
2068 return getAssumed() ? "nounwind" : "may-unwind";
2069 }
2070
2071 /// See AbstractAttribute::updateImpl(...).
2072 ChangeStatus updateImpl(Attributor &A) override {
2073 auto Opcodes = {
2074 (unsigned)Instruction::Invoke, (unsigned)Instruction::CallBr,
2075 (unsigned)Instruction::Call, (unsigned)Instruction::CleanupRet,
2076 (unsigned)Instruction::CatchSwitch, (unsigned)Instruction::Resume};
2077
2078 auto CheckForNoUnwind = [&](Instruction &I) {
2079 if (!I.mayThrow(/* IncludePhaseOneUnwind */ true))
2080 return true;
2081
2082 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
2083 bool IsKnownNoUnwind;
2084 return AA::hasAssumedIRAttr<Attribute::NoUnwind>(
2085 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::REQUIRED,
2086 IsKnown&: IsKnownNoUnwind);
2087 }
2088 return false;
2089 };
2090
2091 bool UsedAssumedInformation = false;
2092 if (!A.checkForAllInstructions(Pred: CheckForNoUnwind, QueryingAA: *this, Opcodes,
2093 UsedAssumedInformation))
2094 return indicatePessimisticFixpoint();
2095
2096 return ChangeStatus::UNCHANGED;
2097 }
2098};
2099
2100struct AANoUnwindFunction final : public AANoUnwindImpl {
2101 AANoUnwindFunction(const IRPosition &IRP, Attributor &A)
2102 : AANoUnwindImpl(IRP, A) {}
2103
2104 /// See AbstractAttribute::trackStatistics()
2105 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nounwind) }
2106};
2107
2108/// NoUnwind attribute deduction for a call sites.
2109struct AANoUnwindCallSite final
2110 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl> {
2111 AANoUnwindCallSite(const IRPosition &IRP, Attributor &A)
2112 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl>(IRP, A) {}
2113
2114 /// See AbstractAttribute::trackStatistics()
2115 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nounwind); }
2116};
2117} // namespace
2118
2119/// ------------------------ NoSync Function Attribute -------------------------
2120
2121bool AANoSync::isAlignedBarrier(const CallBase &CB, bool ExecutedAligned) {
2122 switch (CB.getIntrinsicID()) {
2123 case Intrinsic::nvvm_barrier_cta_sync_aligned_all:
2124 case Intrinsic::nvvm_barrier_cta_sync_aligned_count:
2125 case Intrinsic::nvvm_barrier_cta_red_and_aligned_all:
2126 case Intrinsic::nvvm_barrier_cta_red_and_aligned_count:
2127 case Intrinsic::nvvm_barrier_cta_red_or_aligned_all:
2128 case Intrinsic::nvvm_barrier_cta_red_or_aligned_count:
2129 case Intrinsic::nvvm_barrier_cta_red_popc_aligned_all:
2130 case Intrinsic::nvvm_barrier_cta_red_popc_aligned_count:
2131 return true;
2132 case Intrinsic::amdgcn_s_barrier:
2133 if (ExecutedAligned)
2134 return true;
2135 break;
2136 default:
2137 break;
2138 }
2139 return hasAssumption(CB, AssumptionStr: KnownAssumptionString("ompx_aligned_barrier"));
2140}
2141
2142bool AANoSync::isNonRelaxedAtomic(const Instruction *I) {
2143 if (!I->isAtomic())
2144 return false;
2145
2146 if (auto *FI = dyn_cast<FenceInst>(Val: I))
2147 // All legal orderings for fence are stronger than monotonic.
2148 return FI->getSyncScopeID() != SyncScope::SingleThread;
2149 if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
2150 // Unordered is not a legal ordering for cmpxchg.
2151 return (AI->getSuccessOrdering() != AtomicOrdering::Monotonic ||
2152 AI->getFailureOrdering() != AtomicOrdering::Monotonic);
2153 }
2154
2155 AtomicOrdering Ordering;
2156 switch (I->getOpcode()) {
2157 case Instruction::AtomicRMW:
2158 Ordering = cast<AtomicRMWInst>(Val: I)->getOrdering();
2159 break;
2160 case Instruction::Store:
2161 Ordering = cast<StoreInst>(Val: I)->getOrdering();
2162 break;
2163 case Instruction::Load:
2164 Ordering = cast<LoadInst>(Val: I)->getOrdering();
2165 break;
2166 default:
2167 llvm_unreachable(
2168 "New atomic operations need to be known in the attributor.");
2169 }
2170
2171 return (Ordering != AtomicOrdering::Unordered &&
2172 Ordering != AtomicOrdering::Monotonic);
2173}
2174
2175namespace {
2176struct AANoSyncImpl : AANoSync {
2177 AANoSyncImpl(const IRPosition &IRP, Attributor &A) : AANoSync(IRP, A) {}
2178
2179 /// See AbstractAttribute::initialize(...).
2180 void initialize(Attributor &A) override {
2181 bool IsKnown;
2182 assert(!AA::hasAssumedIRAttr<Attribute::NoSync>(A, nullptr, getIRPosition(),
2183 DepClassTy::NONE, IsKnown));
2184 (void)IsKnown;
2185 }
2186
2187 const std::string getAsStr(Attributor *A) const override {
2188 return getAssumed() ? "nosync" : "may-sync";
2189 }
2190
2191 /// See AbstractAttribute::updateImpl(...).
2192 ChangeStatus updateImpl(Attributor &A) override;
2193};
2194
2195ChangeStatus AANoSyncImpl::updateImpl(Attributor &A) {
2196
2197 auto CheckRWInstForNoSync = [&](Instruction &I) {
2198 return AA::isNoSyncInst(A, I, QueryingAA: *this);
2199 };
2200
2201 auto CheckForNoSync = [&](Instruction &I) {
2202 // At this point we handled all read/write effects and they are all
2203 // nosync, so they can be skipped.
2204 if (I.mayReadOrWriteMemory())
2205 return true;
2206
2207 bool IsKnown;
2208 CallBase &CB = cast<CallBase>(Val&: I);
2209 if (AA::hasAssumedIRAttr<Attribute::NoSync>(
2210 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::OPTIONAL,
2211 IsKnown))
2212 return true;
2213
2214 // non-convergent and readnone imply nosync.
2215 return !CB.isConvergent();
2216 };
2217
2218 bool UsedAssumedInformation = false;
2219 if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInstForNoSync, QueryingAA&: *this,
2220 UsedAssumedInformation) ||
2221 !A.checkForAllCallLikeInstructions(Pred: CheckForNoSync, QueryingAA: *this,
2222 UsedAssumedInformation))
2223 return indicatePessimisticFixpoint();
2224
2225 return ChangeStatus::UNCHANGED;
2226}
2227
2228struct AANoSyncFunction final : public AANoSyncImpl {
2229 AANoSyncFunction(const IRPosition &IRP, Attributor &A)
2230 : AANoSyncImpl(IRP, A) {}
2231
2232 /// See AbstractAttribute::trackStatistics()
2233 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nosync) }
2234};
2235
2236/// NoSync attribute deduction for a call sites.
2237struct AANoSyncCallSite final : AACalleeToCallSite<AANoSync, AANoSyncImpl> {
2238 AANoSyncCallSite(const IRPosition &IRP, Attributor &A)
2239 : AACalleeToCallSite<AANoSync, AANoSyncImpl>(IRP, A) {}
2240
2241 /// See AbstractAttribute::trackStatistics()
2242 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nosync); }
2243};
2244} // namespace
2245
2246/// ------------------------ No-Free Attributes ----------------------------
2247
2248namespace {
2249struct AANoFreeImpl : public AANoFree {
2250 AANoFreeImpl(const IRPosition &IRP, Attributor &A) : AANoFree(IRP, A) {}
2251
2252 /// See AbstractAttribute::initialize(...).
2253 void initialize(Attributor &A) override {
2254 bool IsKnown;
2255 assert(!AA::hasAssumedIRAttr<Attribute::NoFree>(A, nullptr, getIRPosition(),
2256 DepClassTy::NONE, IsKnown));
2257 (void)IsKnown;
2258 }
2259
2260 /// See AbstractAttribute::updateImpl(...).
2261 ChangeStatus updateImpl(Attributor &A) override {
2262 auto CheckForNoFree = [&](Instruction &I) {
2263 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
2264 bool IsKnown;
2265 return AA::hasAssumedIRAttr<Attribute::NoFree>(
2266 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::REQUIRED,
2267 IsKnown);
2268 }
2269 // Make sure that synchronization cannot establish happens-before with a
2270 // free on another thread.
2271 return AA::isNoSyncInst(A, I, QueryingAA: *this);
2272 };
2273
2274 bool UsedAssumedInformation = false;
2275 if (!A.checkForAllReadWriteInstructions(Pred: CheckForNoFree, QueryingAA&: *this,
2276 UsedAssumedInformation) ||
2277 !A.checkForAllCallLikeInstructions(Pred: CheckForNoFree, QueryingAA: *this,
2278 UsedAssumedInformation))
2279 return indicatePessimisticFixpoint();
2280
2281 return ChangeStatus::UNCHANGED;
2282 }
2283
2284 /// See AbstractAttribute::getAsStr().
2285 const std::string getAsStr(Attributor *A) const override {
2286 return getAssumed() ? "nofree" : "may-free";
2287 }
2288};
2289
2290struct AANoFreeFunction final : public AANoFreeImpl {
2291 AANoFreeFunction(const IRPosition &IRP, Attributor &A)
2292 : AANoFreeImpl(IRP, A) {}
2293
2294 /// See AbstractAttribute::trackStatistics()
2295 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nofree) }
2296};
2297
2298/// NoFree attribute deduction for a call sites.
2299struct AANoFreeCallSite final : AACalleeToCallSite<AANoFree, AANoFreeImpl> {
2300 AANoFreeCallSite(const IRPosition &IRP, Attributor &A)
2301 : AACalleeToCallSite<AANoFree, AANoFreeImpl>(IRP, A) {}
2302
2303 /// See AbstractAttribute::trackStatistics()
2304 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nofree); }
2305};
2306
2307/// NoFree attribute for floating values.
2308struct AANoFreeFloating : AANoFreeImpl {
2309 AANoFreeFloating(const IRPosition &IRP, Attributor &A)
2310 : AANoFreeImpl(IRP, A) {}
2311
2312 /// See AbstractAttribute::trackStatistics()
2313 void trackStatistics() const override{STATS_DECLTRACK_FLOATING_ATTR(nofree)}
2314
2315 /// See Abstract Attribute::updateImpl(...).
2316 ChangeStatus updateImpl(Attributor &A) override {
2317 const IRPosition &IRP = getIRPosition();
2318
2319 bool IsKnown;
2320 if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, QueryingAA: this,
2321 IRP: IRPosition::function_scope(IRP),
2322 DepClass: DepClassTy::OPTIONAL, IsKnown))
2323 return ChangeStatus::UNCHANGED;
2324
2325 Value &AssociatedValue = getIRPosition().getAssociatedValue();
2326 auto Pred = [&](const Use &U, bool &Follow) -> bool {
2327 Instruction *UserI = cast<Instruction>(Val: U.getUser());
2328 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
2329 if (CB->isBundleOperand(U: &U))
2330 return false;
2331 if (!CB->isArgOperand(U: &U))
2332 return true;
2333 unsigned ArgNo = CB->getArgOperandNo(U: &U);
2334
2335 // Even if the argument is nofree, we still need to check for nocapture,
2336 // as the call may capture the argument without freeing it, and the
2337 // captured argument is freed later.
2338 bool IsKnown;
2339 if (!AA::hasAssumedIRAttr<Attribute::NoFree>(
2340 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
2341 DepClass: DepClassTy::REQUIRED, IsKnown))
2342 return false;
2343
2344 const AANoCapture *NoCaptureAA = nullptr;
2345 if (!AA::hasAssumedIRAttr<Attribute::Captures>(
2346 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
2347 DepClass: DepClassTy::REQUIRED, IsKnown,
2348 /*IgnoreSubsumingPositions=*/false, AAPtr: &NoCaptureAA)) {
2349 if (NoCaptureAA && NoCaptureAA->isAssumedNoCaptureMaybeReturned()) {
2350 Follow = true;
2351 return true;
2352 }
2353 return false;
2354 }
2355
2356 return true;
2357 }
2358
2359 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
2360 if (!capturesAnyProvenance(CC: CI))
2361 return true;
2362 if (capturesAnyProvenance(CC: CI.ResultCC)) {
2363 Follow = true;
2364 return true;
2365 }
2366
2367 if (isa<ReturnInst>(Val: UserI) && getIRPosition().isArgumentPosition())
2368 return true;
2369
2370 // Capturing user.
2371 return false;
2372 };
2373 if (!A.checkForAllUses(Pred, QueryingAA: *this, V: AssociatedValue))
2374 return indicatePessimisticFixpoint();
2375
2376 return ChangeStatus::UNCHANGED;
2377 }
2378};
2379
2380/// NoFree attribute for a call site argument.
2381struct AANoFreeArgument final : AANoFreeFloating {
2382 AANoFreeArgument(const IRPosition &IRP, Attributor &A)
2383 : AANoFreeFloating(IRP, A) {}
2384
2385 /// See AbstractAttribute::trackStatistics()
2386 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofree) }
2387};
2388
2389/// NoFree attribute for call site arguments.
2390struct AANoFreeCallSiteArgument final : AANoFreeFloating {
2391 AANoFreeCallSiteArgument(const IRPosition &IRP, Attributor &A)
2392 : AANoFreeFloating(IRP, A) {}
2393
2394 /// See AbstractAttribute::updateImpl(...).
2395 ChangeStatus updateImpl(Attributor &A) override {
2396 // TODO: Once we have call site specific value information we can provide
2397 // call site specific liveness information and then it makes
2398 // sense to specialize attributes for call sites arguments instead of
2399 // redirecting requests to the callee argument.
2400 Argument *Arg = getAssociatedArgument();
2401 if (!Arg)
2402 return indicatePessimisticFixpoint();
2403 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
2404 bool IsKnown;
2405 if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, QueryingAA: this, IRP: ArgPos,
2406 DepClass: DepClassTy::REQUIRED, IsKnown))
2407 return ChangeStatus::UNCHANGED;
2408 return indicatePessimisticFixpoint();
2409 }
2410
2411 /// See AbstractAttribute::trackStatistics()
2412 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nofree) };
2413};
2414
2415/// NoFree attribute for function return value.
2416struct AANoFreeReturned final : AANoFreeFloating {
2417 AANoFreeReturned(const IRPosition &IRP, Attributor &A)
2418 : AANoFreeFloating(IRP, A) {
2419 llvm_unreachable("NoFree is not applicable to function returns!");
2420 }
2421
2422 /// See AbstractAttribute::initialize(...).
2423 void initialize(Attributor &A) override {
2424 llvm_unreachable("NoFree is not applicable to function returns!");
2425 }
2426
2427 /// See AbstractAttribute::updateImpl(...).
2428 ChangeStatus updateImpl(Attributor &A) override {
2429 llvm_unreachable("NoFree is not applicable to function returns!");
2430 }
2431
2432 /// See AbstractAttribute::trackStatistics()
2433 void trackStatistics() const override {}
2434};
2435
2436/// NoFree attribute deduction for a call site return value.
2437struct AANoFreeCallSiteReturned final : AANoFreeFloating {
2438 AANoFreeCallSiteReturned(const IRPosition &IRP, Attributor &A)
2439 : AANoFreeFloating(IRP, A) {}
2440
2441 ChangeStatus manifest(Attributor &A) override {
2442 return ChangeStatus::UNCHANGED;
2443 }
2444 /// See AbstractAttribute::trackStatistics()
2445 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nofree) }
2446};
2447} // namespace
2448
2449/// ------------------------ NonNull Argument Attribute ------------------------
2450
2451bool AANonNull::isImpliedByIR(Attributor &A, const IRPosition &IRP,
2452 Attribute::AttrKind ImpliedAttributeKind,
2453 bool IgnoreSubsumingPositions) {
2454 SmallVector<Attribute::AttrKind, 2> AttrKinds;
2455 AttrKinds.push_back(Elt: Attribute::NonNull);
2456 if (!NullPointerIsDefined(F: IRP.getAnchorScope(),
2457 AS: IRP.getAssociatedType()->getPointerAddressSpace()))
2458 AttrKinds.push_back(Elt: Attribute::Dereferenceable);
2459 if (A.hasAttr(IRP, AKs: AttrKinds, IgnoreSubsumingPositions, ImpliedAttributeKind: Attribute::NonNull))
2460 return true;
2461
2462 DominatorTree *DT = nullptr;
2463 AssumptionCache *AC = nullptr;
2464 InformationCache &InfoCache = A.getInfoCache();
2465 if (const Function *Fn = IRP.getAnchorScope()) {
2466 if (!Fn->isDeclaration()) {
2467 DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *Fn);
2468 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *Fn);
2469 }
2470 }
2471
2472 SmallVector<AA::ValueAndContext> Worklist;
2473 if (IRP.getPositionKind() != IRP_RETURNED) {
2474 Worklist.push_back(Elt: {IRP.getAssociatedValue(), IRP.getCtxI()});
2475 } else {
2476 bool UsedAssumedInformation = false;
2477 if (!A.checkForAllInstructions(
2478 Pred: [&](Instruction &I) {
2479 Worklist.push_back(Elt: {*cast<ReturnInst>(Val&: I).getReturnValue(), &I});
2480 return true;
2481 },
2482 Fn: IRP.getAssociatedFunction(), QueryingAA: nullptr, Opcodes: {Instruction::Ret},
2483 UsedAssumedInformation, CheckBBLivenessOnly: false, /*CheckPotentiallyDead=*/true))
2484 return false;
2485 }
2486
2487 if (llvm::any_of(Range&: Worklist, P: [&](AA::ValueAndContext VAC) {
2488 return !isKnownNonZero(
2489 V: VAC.getValue(),
2490 Q: SimplifyQuery(A.getDataLayout(), DT, AC, VAC.getCtxI()));
2491 }))
2492 return false;
2493
2494 A.manifestAttrs(IRP, DeducedAttrs: {Attribute::get(Context&: IRP.getAnchorValue().getContext(),
2495 Kind: Attribute::NonNull)});
2496 return true;
2497}
2498
2499namespace {
2500static int64_t getKnownNonNullAndDerefBytesForUse(
2501 Attributor &A, const AbstractAttribute &QueryingAA, Value &AssociatedValue,
2502 const Use *U, const Instruction *I, bool &IsNonNull, bool &TrackUse) {
2503 TrackUse = false;
2504
2505 const Value *UseV = U->get();
2506 if (!UseV->getType()->isPointerTy())
2507 return 0;
2508
2509 // We need to follow common pointer manipulation uses to the accesses they
2510 // feed into. We can try to be smart to avoid looking through things we do not
2511 // like for now, e.g., non-inbounds GEPs.
2512 if (isa<CastInst>(Val: I)) {
2513 TrackUse = true;
2514 return 0;
2515 }
2516
2517 if (isa<GetElementPtrInst>(Val: I)) {
2518 TrackUse = true;
2519 return 0;
2520 }
2521
2522 Type *PtrTy = UseV->getType();
2523 const Function *F = I->getFunction();
2524 bool NullPointerIsDefined =
2525 F ? llvm::NullPointerIsDefined(F, AS: PtrTy->getPointerAddressSpace()) : true;
2526 const DataLayout &DL = A.getInfoCache().getDL();
2527 if (const auto *CB = dyn_cast<CallBase>(Val: I)) {
2528 if (CB->isBundleOperand(U)) {
2529 if (RetainedKnowledge RK = getKnowledgeFromUse(
2530 U, AttrKinds: {Attribute::NonNull, Attribute::Dereferenceable})) {
2531 IsNonNull |=
2532 (RK.AttrKind == Attribute::NonNull || !NullPointerIsDefined);
2533 return RK.ArgValue;
2534 }
2535 return 0;
2536 }
2537
2538 if (CB->isCallee(U)) {
2539 IsNonNull |= !NullPointerIsDefined;
2540 return 0;
2541 }
2542
2543 unsigned ArgNo = CB->getArgOperandNo(U);
2544 IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
2545 // As long as we only use known information there is no need to track
2546 // dependences here.
2547 bool IsKnownNonNull;
2548 AA::hasAssumedIRAttr<Attribute::NonNull>(A, QueryingAA: &QueryingAA, IRP,
2549 DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
2550 IsNonNull |= IsKnownNonNull;
2551 auto *DerefAA =
2552 A.getAAFor<AADereferenceable>(QueryingAA, IRP, DepClass: DepClassTy::NONE);
2553 return DerefAA ? DerefAA->getKnownDereferenceableBytes() : 0;
2554 }
2555
2556 std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(Inst: I);
2557 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() ||
2558 Loc->Size.isScalable() || I->isVolatile())
2559 return 0;
2560
2561 int64_t Offset;
2562 const Value *Base =
2563 getMinimalBaseOfPointer(A, QueryingAA, Ptr: Loc->Ptr, BytesOffset&: Offset, DL);
2564 if (Base && Base == &AssociatedValue) {
2565 int64_t DerefBytes = Loc->Size.getValue() + Offset;
2566 IsNonNull |= !NullPointerIsDefined;
2567 return std::max(a: int64_t(0), b: DerefBytes);
2568 }
2569
2570 /// Corner case when an offset is 0.
2571 Base = GetPointerBaseWithConstantOffset(Ptr: Loc->Ptr, Offset, DL,
2572 /*AllowNonInbounds*/ true);
2573 if (Base && Base == &AssociatedValue && Offset == 0) {
2574 int64_t DerefBytes = Loc->Size.getValue();
2575 IsNonNull |= !NullPointerIsDefined;
2576 return std::max(a: int64_t(0), b: DerefBytes);
2577 }
2578
2579 return 0;
2580}
2581
2582struct AANonNullImpl : AANonNull {
2583 AANonNullImpl(const IRPosition &IRP, Attributor &A) : AANonNull(IRP, A) {}
2584
2585 /// See AbstractAttribute::initialize(...).
2586 void initialize(Attributor &A) override {
2587 Value &V = *getAssociatedValue().stripPointerCasts();
2588 if (isa<ConstantPointerNull>(Val: V)) {
2589 indicatePessimisticFixpoint();
2590 return;
2591 }
2592
2593 if (Instruction *CtxI = getCtxI())
2594 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
2595 }
2596
2597 /// See followUsesInMBEC
2598 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
2599 AANonNull::StateType &State) {
2600 bool IsNonNull = false;
2601 bool TrackUse = false;
2602 getKnownNonNullAndDerefBytesForUse(A, QueryingAA: *this, AssociatedValue&: getAssociatedValue(), U, I,
2603 IsNonNull, TrackUse);
2604 State.setKnown(IsNonNull);
2605 return TrackUse;
2606 }
2607
2608 /// See AbstractAttribute::getAsStr().
2609 const std::string getAsStr(Attributor *A) const override {
2610 return getAssumed() ? "nonnull" : "may-null";
2611 }
2612};
2613
2614/// NonNull attribute for a floating value.
2615struct AANonNullFloating : public AANonNullImpl {
2616 AANonNullFloating(const IRPosition &IRP, Attributor &A)
2617 : AANonNullImpl(IRP, A) {}
2618
2619 /// See AbstractAttribute::updateImpl(...).
2620 ChangeStatus updateImpl(Attributor &A) override {
2621 auto CheckIRP = [&](const IRPosition &IRP) {
2622 bool IsKnownNonNull;
2623 return AA::hasAssumedIRAttr<Attribute::NonNull>(
2624 A, QueryingAA: *this, IRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNonNull);
2625 };
2626
2627 bool Stripped;
2628 bool UsedAssumedInformation = false;
2629 Value *AssociatedValue = &getAssociatedValue();
2630 SmallVector<AA::ValueAndContext> Values;
2631 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
2632 S: AA::AnyScope, UsedAssumedInformation))
2633 Stripped = false;
2634 else
2635 Stripped =
2636 Values.size() != 1 || Values.front().getValue() != AssociatedValue;
2637
2638 if (!Stripped) {
2639 bool IsKnown;
2640 if (auto *PHI = dyn_cast<PHINode>(Val: AssociatedValue))
2641 if (llvm::all_of(Range: PHI->incoming_values(), P: [&](Value *Op) {
2642 return AA::hasAssumedIRAttr<Attribute::NonNull>(
2643 A, QueryingAA: this, IRP: IRPosition::value(V: *Op), DepClass: DepClassTy::OPTIONAL,
2644 IsKnown);
2645 }))
2646 return ChangeStatus::UNCHANGED;
2647 if (auto *Select = dyn_cast<SelectInst>(Val: AssociatedValue))
2648 if (AA::hasAssumedIRAttr<Attribute::NonNull>(
2649 A, QueryingAA: this, IRP: IRPosition::value(V: *Select->getFalseValue()),
2650 DepClass: DepClassTy::OPTIONAL, IsKnown) &&
2651 AA::hasAssumedIRAttr<Attribute::NonNull>(
2652 A, QueryingAA: this, IRP: IRPosition::value(V: *Select->getTrueValue()),
2653 DepClass: DepClassTy::OPTIONAL, IsKnown))
2654 return ChangeStatus::UNCHANGED;
2655
2656 // If we haven't stripped anything we might still be able to use a
2657 // different AA, but only if the IRP changes. Effectively when we
2658 // interpret this not as a call site value but as a floating/argument
2659 // value.
2660 const IRPosition AVIRP = IRPosition::value(V: *AssociatedValue);
2661 if (AVIRP == getIRPosition() || !CheckIRP(AVIRP))
2662 return indicatePessimisticFixpoint();
2663 return ChangeStatus::UNCHANGED;
2664 }
2665
2666 for (const auto &VAC : Values)
2667 if (!CheckIRP(IRPosition::value(V: *VAC.getValue())))
2668 return indicatePessimisticFixpoint();
2669
2670 return ChangeStatus::UNCHANGED;
2671 }
2672
2673 /// See AbstractAttribute::trackStatistics()
2674 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2675};
2676
2677/// NonNull attribute for function return value.
2678struct AANonNullReturned final
2679 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2680 false, AANonNull::IRAttributeKind, false> {
2681 AANonNullReturned(const IRPosition &IRP, Attributor &A)
2682 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2683 false, Attribute::NonNull, false>(IRP, A) {
2684 }
2685
2686 /// See AbstractAttribute::getAsStr().
2687 const std::string getAsStr(Attributor *A) const override {
2688 return getAssumed() ? "nonnull" : "may-null";
2689 }
2690
2691 /// See AbstractAttribute::trackStatistics()
2692 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2693};
2694
2695/// NonNull attribute for function argument.
2696struct AANonNullArgument final
2697 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl> {
2698 AANonNullArgument(const IRPosition &IRP, Attributor &A)
2699 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl>(IRP, A) {}
2700
2701 /// See AbstractAttribute::trackStatistics()
2702 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nonnull) }
2703};
2704
2705struct AANonNullCallSiteArgument final : AANonNullFloating {
2706 AANonNullCallSiteArgument(const IRPosition &IRP, Attributor &A)
2707 : AANonNullFloating(IRP, A) {}
2708
2709 /// See AbstractAttribute::trackStatistics()
2710 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nonnull) }
2711};
2712
2713/// NonNull attribute for a call site return position.
2714struct AANonNullCallSiteReturned final
2715 : AACalleeToCallSite<AANonNull, AANonNullImpl> {
2716 AANonNullCallSiteReturned(const IRPosition &IRP, Attributor &A)
2717 : AACalleeToCallSite<AANonNull, AANonNullImpl>(IRP, A) {}
2718
2719 /// See AbstractAttribute::trackStatistics()
2720 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nonnull) }
2721};
2722} // namespace
2723
2724/// ------------------------ Must-Progress Attributes --------------------------
2725namespace {
2726struct AAMustProgressImpl : public AAMustProgress {
2727 AAMustProgressImpl(const IRPosition &IRP, Attributor &A)
2728 : AAMustProgress(IRP, A) {}
2729
2730 /// See AbstractAttribute::initialize(...).
2731 void initialize(Attributor &A) override {
2732 bool IsKnown;
2733 assert(!AA::hasAssumedIRAttr<Attribute::MustProgress>(
2734 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2735 (void)IsKnown;
2736 }
2737
2738 /// See AbstractAttribute::getAsStr()
2739 const std::string getAsStr(Attributor *A) const override {
2740 return getAssumed() ? "mustprogress" : "may-not-progress";
2741 }
2742};
2743
2744struct AAMustProgressFunction final : AAMustProgressImpl {
2745 AAMustProgressFunction(const IRPosition &IRP, Attributor &A)
2746 : AAMustProgressImpl(IRP, A) {}
2747
2748 /// See AbstractAttribute::updateImpl(...).
2749 ChangeStatus updateImpl(Attributor &A) override {
2750 bool IsKnown;
2751 if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
2752 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown)) {
2753 if (IsKnown)
2754 return indicateOptimisticFixpoint();
2755 return ChangeStatus::UNCHANGED;
2756 }
2757
2758 auto CheckForMustProgress = [&](AbstractCallSite ACS) {
2759 IRPosition IPos = IRPosition::callsite_function(CB: *ACS.getInstruction());
2760 bool IsKnownMustProgress;
2761 return AA::hasAssumedIRAttr<Attribute::MustProgress>(
2762 A, QueryingAA: this, IRP: IPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownMustProgress,
2763 /* IgnoreSubsumingPositions */ true);
2764 };
2765
2766 bool AllCallSitesKnown = true;
2767 if (!A.checkForAllCallSites(Pred: CheckForMustProgress, QueryingAA: *this,
2768 /* RequireAllCallSites */ true,
2769 UsedAssumedInformation&: AllCallSitesKnown))
2770 return indicatePessimisticFixpoint();
2771
2772 return ChangeStatus::UNCHANGED;
2773 }
2774
2775 /// See AbstractAttribute::trackStatistics()
2776 void trackStatistics() const override {
2777 STATS_DECLTRACK_FN_ATTR(mustprogress)
2778 }
2779};
2780
2781/// MustProgress attribute deduction for a call sites.
2782struct AAMustProgressCallSite final : AAMustProgressImpl {
2783 AAMustProgressCallSite(const IRPosition &IRP, Attributor &A)
2784 : AAMustProgressImpl(IRP, A) {}
2785
2786 /// See AbstractAttribute::updateImpl(...).
2787 ChangeStatus updateImpl(Attributor &A) override {
2788 // TODO: Once we have call site specific value information we can provide
2789 // call site specific liveness information and then it makes
2790 // sense to specialize attributes for call sites arguments instead of
2791 // redirecting requests to the callee argument.
2792 const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
2793 bool IsKnownMustProgress;
2794 if (!AA::hasAssumedIRAttr<Attribute::MustProgress>(
2795 A, QueryingAA: this, IRP: FnPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownMustProgress))
2796 return indicatePessimisticFixpoint();
2797 return ChangeStatus::UNCHANGED;
2798 }
2799
2800 /// See AbstractAttribute::trackStatistics()
2801 void trackStatistics() const override {
2802 STATS_DECLTRACK_CS_ATTR(mustprogress);
2803 }
2804};
2805} // namespace
2806
2807/// ------------------------ No-Recurse Attributes ----------------------------
2808
2809namespace {
2810struct AANoRecurseImpl : public AANoRecurse {
2811 AANoRecurseImpl(const IRPosition &IRP, Attributor &A) : AANoRecurse(IRP, A) {}
2812
2813 /// See AbstractAttribute::initialize(...).
2814 void initialize(Attributor &A) override {
2815 bool IsKnown;
2816 assert(!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
2817 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2818 (void)IsKnown;
2819 }
2820
2821 /// See AbstractAttribute::getAsStr()
2822 const std::string getAsStr(Attributor *A) const override {
2823 return getAssumed() ? "norecurse" : "may-recurse";
2824 }
2825};
2826
2827struct AANoRecurseFunction final : AANoRecurseImpl {
2828 AANoRecurseFunction(const IRPosition &IRP, Attributor &A)
2829 : AANoRecurseImpl(IRP, A) {}
2830
2831 /// See AbstractAttribute::updateImpl(...).
2832 ChangeStatus updateImpl(Attributor &A) override {
2833
2834 // If all live call sites are known to be no-recurse, we are as well.
2835 auto CallSitePred = [&](AbstractCallSite ACS) {
2836 bool IsKnownNoRecurse;
2837 if (!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
2838 A, QueryingAA: this,
2839 IRP: IRPosition::function(F: *ACS.getInstruction()->getFunction()),
2840 DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoRecurse))
2841 return false;
2842 return IsKnownNoRecurse;
2843 };
2844 bool UsedAssumedInformation = false;
2845 if (A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this, RequireAllCallSites: true,
2846 UsedAssumedInformation)) {
2847 // If we know all call sites and all are known no-recurse, we are done.
2848 // If all known call sites, which might not be all that exist, are known
2849 // to be no-recurse, we are not done but we can continue to assume
2850 // no-recurse. If one of the call sites we have not visited will become
2851 // live, another update is triggered.
2852 if (!UsedAssumedInformation)
2853 indicateOptimisticFixpoint();
2854 return ChangeStatus::UNCHANGED;
2855 }
2856
2857 const AAInterFnReachability *EdgeReachability =
2858 A.getAAFor<AAInterFnReachability>(QueryingAA: *this, IRP: getIRPosition(),
2859 DepClass: DepClassTy::REQUIRED);
2860 if (EdgeReachability && EdgeReachability->canReach(A, Fn: *getAnchorScope()))
2861 return indicatePessimisticFixpoint();
2862 return ChangeStatus::UNCHANGED;
2863 }
2864
2865 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(norecurse) }
2866};
2867
2868/// NoRecurse attribute deduction for a call sites.
2869struct AANoRecurseCallSite final
2870 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl> {
2871 AANoRecurseCallSite(const IRPosition &IRP, Attributor &A)
2872 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl>(IRP, A) {}
2873
2874 /// See AbstractAttribute::trackStatistics()
2875 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(norecurse); }
2876};
2877} // namespace
2878
2879/// ------------------------ No-Convergent Attribute --------------------------
2880
2881namespace {
2882struct AANonConvergentImpl : public AANonConvergent {
2883 AANonConvergentImpl(const IRPosition &IRP, Attributor &A)
2884 : AANonConvergent(IRP, A) {}
2885
2886 /// See AbstractAttribute::getAsStr()
2887 const std::string getAsStr(Attributor *A) const override {
2888 return getAssumed() ? "non-convergent" : "may-be-convergent";
2889 }
2890};
2891
2892struct AANonConvergentFunction final : AANonConvergentImpl {
2893 AANonConvergentFunction(const IRPosition &IRP, Attributor &A)
2894 : AANonConvergentImpl(IRP, A) {}
2895
2896 /// See AbstractAttribute::updateImpl(...).
2897 ChangeStatus updateImpl(Attributor &A) override {
2898 // If all function calls are known to not be convergent, we are not
2899 // convergent.
2900 auto CalleeIsNotConvergent = [&](Instruction &Inst) {
2901 CallBase &CB = cast<CallBase>(Val&: Inst);
2902 auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand());
2903 if (!Callee || Callee->isIntrinsic()) {
2904 return false;
2905 }
2906 if (Callee->isDeclaration()) {
2907 return !Callee->hasFnAttribute(Kind: Attribute::Convergent);
2908 }
2909 const auto *ConvergentAA = A.getAAFor<AANonConvergent>(
2910 QueryingAA: *this, IRP: IRPosition::function(F: *Callee), DepClass: DepClassTy::REQUIRED);
2911 return ConvergentAA && ConvergentAA->isAssumedNotConvergent();
2912 };
2913
2914 bool UsedAssumedInformation = false;
2915 if (!A.checkForAllCallLikeInstructions(Pred: CalleeIsNotConvergent, QueryingAA: *this,
2916 UsedAssumedInformation)) {
2917 return indicatePessimisticFixpoint();
2918 }
2919 return ChangeStatus::UNCHANGED;
2920 }
2921
2922 ChangeStatus manifest(Attributor &A) override {
2923 if (isKnownNotConvergent() &&
2924 A.hasAttr(IRP: getIRPosition(), AKs: Attribute::Convergent)) {
2925 A.removeAttrs(IRP: getIRPosition(), AttrKinds: {Attribute::Convergent});
2926 return ChangeStatus::CHANGED;
2927 }
2928 return ChangeStatus::UNCHANGED;
2929 }
2930
2931 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(convergent) }
2932};
2933} // namespace
2934
2935/// -------------------- Undefined-Behavior Attributes ------------------------
2936
2937namespace {
2938struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior {
2939 AAUndefinedBehaviorImpl(const IRPosition &IRP, Attributor &A)
2940 : AAUndefinedBehavior(IRP, A) {}
2941
2942 struct UBInfo {
2943 enum Kind {
2944 NullPtrAccess,
2945 UndefPtrAccess,
2946 UndefBranchCondition,
2947 UndefReturnValue,
2948 NullReturnViolatesNonNull,
2949 UndefCallArgument,
2950 NullArgViolatesNonNull,
2951 };
2952
2953 Kind K;
2954 std::optional<unsigned> ArgNo;
2955
2956 UBInfo(Kind K) : K(K), ArgNo(std::nullopt) {}
2957
2958 UBInfo(Kind K, std::optional<unsigned> ArgNo) : K(K), ArgNo(ArgNo) {}
2959 };
2960
2961 /// See AbstractAttribute::updateImpl(...).
2962 // through a pointer (i.e. also branches etc.)
2963 ChangeStatus updateImpl(Attributor &A) override {
2964 const size_t UBPrevSize = KnownUBInsts.size();
2965 const size_t NoUBPrevSize = AssumedNoUBInsts.size();
2966
2967 auto InspectMemAccessInstForUB = [&](Instruction &I) {
2968 // Volatile accesses on null are not necessarily UB.
2969 if (I.isVolatile())
2970 return true;
2971
2972 // Skip instructions that are already saved.
2973 if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Key: &I))
2974 return true;
2975
2976 // If we reach here, we know we have an instruction
2977 // that accesses memory through a pointer operand,
2978 // for which getPointerOperand() should give it to us.
2979 Value *PtrOp =
2980 const_cast<Value *>(getPointerOperand(I: &I, /* AllowVolatile */ true));
2981 assert(PtrOp &&
2982 "Expected pointer operand of memory accessing instruction");
2983
2984 // Either we stopped and the appropriate action was taken,
2985 // or we got back a simplified value to continue.
2986 std::optional<Value *> SimplifiedPtrOp =
2987 stopOnUndefOrAssumed(A, V: PtrOp, I: &I, K: UBInfo::UndefPtrAccess);
2988 if (!SimplifiedPtrOp || !*SimplifiedPtrOp)
2989 return true;
2990 const Value *PtrOpVal = *SimplifiedPtrOp;
2991
2992 // A memory access through a pointer is considered UB
2993 // only if the pointer has constant null value.
2994 // TODO: Expand it to not only check constant values.
2995 if (!isa<ConstantPointerNull>(Val: PtrOpVal)) {
2996 AssumedNoUBInsts.insert(Ptr: &I);
2997 return true;
2998 }
2999 const Type *PtrTy = PtrOpVal->getType();
3000
3001 // Because we only consider instructions inside functions,
3002 // assume that a parent function exists.
3003 const Function *F = I.getFunction();
3004
3005 // A memory access using constant null pointer is only considered UB
3006 // if null pointer is _not_ defined for the target platform.
3007 if (llvm::NullPointerIsDefined(F, AS: PtrTy->getPointerAddressSpace()))
3008 AssumedNoUBInsts.insert(Ptr: &I);
3009 else
3010 KnownUBInsts.try_emplace(Key: &I, Args: UBInfo::NullPtrAccess);
3011 return true;
3012 };
3013
3014 auto InspectBrInstForUB = [&](Instruction &I) {
3015 // A conditional branch instruction is considered UB if it has `undef`
3016 // condition.
3017
3018 // Skip instructions that are already saved.
3019 if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Key: &I))
3020 return true;
3021
3022 // We know we have a branch instruction.
3023 auto *BrInst = cast<CondBrInst>(Val: &I);
3024
3025 // Either we stopped and the appropriate action was taken,
3026 // or we got back a simplified value to continue.
3027 std::optional<Value *> SimplifiedCond = stopOnUndefOrAssumed(
3028 A, V: BrInst->getCondition(), I: BrInst, K: UBInfo::UndefBranchCondition);
3029 if (!SimplifiedCond || !*SimplifiedCond)
3030 return true;
3031 AssumedNoUBInsts.insert(Ptr: &I);
3032 return true;
3033 };
3034
3035 auto InspectCallSiteForUB = [&](Instruction &I) {
3036 // Check whether a callsite always cause UB or not
3037
3038 // Skip instructions that are already saved.
3039 if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Key: &I))
3040 return true;
3041
3042 // Check nonnull and noundef argument attribute violation for each
3043 // callsite.
3044 CallBase &CB = cast<CallBase>(Val&: I);
3045 auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand());
3046 if (!Callee)
3047 return true;
3048 for (unsigned idx = 0; idx < CB.arg_size(); idx++) {
3049 // If current argument is known to be simplified to null pointer and the
3050 // corresponding argument position is known to have nonnull attribute,
3051 // the argument is poison. Furthermore, if the argument is poison and
3052 // the position is known to have noundef attriubte, this callsite is
3053 // considered UB.
3054 if (idx >= Callee->arg_size())
3055 break;
3056 Value *ArgVal = CB.getArgOperand(i: idx);
3057 if (!ArgVal)
3058 continue;
3059 // Here, we handle three cases.
3060 // (1) Not having a value means it is dead. (we can replace the value
3061 // with undef)
3062 // (2) Simplified to undef. The argument violate noundef attriubte.
3063 // (3) Simplified to null pointer where known to be nonnull.
3064 // The argument is a poison value and violate noundef attribute.
3065 IRPosition CalleeArgumentIRP = IRPosition::callsite_argument(CB, ArgNo: idx);
3066 bool IsKnownNoUndef;
3067 AA::hasAssumedIRAttr<Attribute::NoUndef>(
3068 A, QueryingAA: this, IRP: CalleeArgumentIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoUndef);
3069 if (!IsKnownNoUndef)
3070 continue;
3071 bool UsedAssumedInformation = false;
3072 std::optional<Value *> SimplifiedVal =
3073 A.getAssumedSimplified(IRP: IRPosition::value(V: *ArgVal), AA: *this,
3074 UsedAssumedInformation, S: AA::Interprocedural);
3075 if (UsedAssumedInformation)
3076 continue;
3077 if (SimplifiedVal && !*SimplifiedVal)
3078 return true;
3079 if (!SimplifiedVal || isa<UndefValue>(Val: **SimplifiedVal)) {
3080 KnownUBInsts.try_emplace(Key: &I, Args: UBInfo(UBInfo::UndefCallArgument, idx));
3081 continue;
3082 }
3083 if (!ArgVal->getType()->isPointerTy() ||
3084 !isa<ConstantPointerNull>(Val: **SimplifiedVal))
3085 continue;
3086 bool IsKnownNonNull;
3087 AA::hasAssumedIRAttr<Attribute::NonNull>(
3088 A, QueryingAA: this, IRP: CalleeArgumentIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
3089 if (IsKnownNonNull)
3090 KnownUBInsts.try_emplace(Key: &I,
3091 Args: UBInfo(UBInfo::NullArgViolatesNonNull, idx));
3092 }
3093 return true;
3094 };
3095
3096 auto InspectReturnInstForUB = [&](Instruction &I) {
3097 auto &RI = cast<ReturnInst>(Val&: I);
3098 // Either we stopped and the appropriate action was taken,
3099 // or we got back a simplified return value to continue.
3100 std::optional<Value *> SimplifiedRetValue = stopOnUndefOrAssumed(
3101 A, V: RI.getReturnValue(), I: &I, K: UBInfo::UndefReturnValue);
3102 if (!SimplifiedRetValue || !*SimplifiedRetValue)
3103 return true;
3104
3105 // Check if a return instruction always cause UB or not
3106 // Note: It is guaranteed that the returned position of the anchor
3107 // scope has noundef attribute when this is called.
3108 // We also ensure the return position is not "assumed dead"
3109 // because the returned value was then potentially simplified to
3110 // `undef` in AAReturnedValues without removing the `noundef`
3111 // attribute yet.
3112
3113 // When the returned position has noundef attriubte, UB occurs in the
3114 // following cases.
3115 // (1) Returned value is known to be undef.
3116 // (2) The value is known to be a null pointer and the returned
3117 // position has nonnull attribute (because the returned value is
3118 // poison).
3119 if (isa<ConstantPointerNull>(Val: *SimplifiedRetValue)) {
3120 bool IsKnownNonNull;
3121 AA::hasAssumedIRAttr<Attribute::NonNull>(
3122 A, QueryingAA: this, IRP: IRPosition::returned(F: *getAnchorScope()), DepClass: DepClassTy::NONE,
3123 IsKnown&: IsKnownNonNull);
3124 if (IsKnownNonNull)
3125 KnownUBInsts.try_emplace(Key: &I, Args: UBInfo::NullReturnViolatesNonNull);
3126 }
3127
3128 return true;
3129 };
3130
3131 bool UsedAssumedInformation = false;
3132 A.checkForAllInstructions(Pred: InspectMemAccessInstForUB, QueryingAA: *this,
3133 Opcodes: {Instruction::Load, Instruction::Store,
3134 Instruction::AtomicCmpXchg,
3135 Instruction::AtomicRMW},
3136 UsedAssumedInformation,
3137 /* CheckBBLivenessOnly */ true);
3138 A.checkForAllInstructions(Pred: InspectBrInstForUB, QueryingAA: *this, Opcodes: {Instruction::CondBr},
3139 UsedAssumedInformation,
3140 /* CheckBBLivenessOnly */ true);
3141 A.checkForAllCallLikeInstructions(Pred: InspectCallSiteForUB, QueryingAA: *this,
3142 UsedAssumedInformation);
3143
3144 // If the returned position of the anchor scope has noundef attriubte, check
3145 // all returned instructions.
3146 if (!getAnchorScope()->getReturnType()->isVoidTy()) {
3147 const IRPosition &ReturnIRP = IRPosition::returned(F: *getAnchorScope());
3148 if (!A.isAssumedDead(IRP: ReturnIRP, QueryingAA: this, FnLivenessAA: nullptr, UsedAssumedInformation)) {
3149 bool IsKnownNoUndef;
3150 AA::hasAssumedIRAttr<Attribute::NoUndef>(
3151 A, QueryingAA: this, IRP: ReturnIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoUndef);
3152 if (IsKnownNoUndef)
3153 A.checkForAllInstructions(Pred: InspectReturnInstForUB, QueryingAA: *this,
3154 Opcodes: {Instruction::Ret}, UsedAssumedInformation,
3155 /* CheckBBLivenessOnly */ true);
3156 }
3157 }
3158
3159 if (NoUBPrevSize != AssumedNoUBInsts.size() ||
3160 UBPrevSize != KnownUBInsts.size())
3161 return ChangeStatus::CHANGED;
3162 return ChangeStatus::UNCHANGED;
3163 }
3164
3165 bool isKnownToCauseUB(Instruction *I) const override {
3166 return KnownUBInsts.count(Key: I);
3167 }
3168
3169 bool isAssumedToCauseUB(Instruction *I) const override {
3170 // In simple words, if an instruction is not in the assumed to _not_
3171 // cause UB, then it is assumed UB (that includes those
3172 // in the KnownUBInsts set). The rest is boilerplate
3173 // is to ensure that it is one of the instructions we test
3174 // for UB.
3175
3176 switch (I->getOpcode()) {
3177 case Instruction::Load:
3178 case Instruction::Store:
3179 case Instruction::AtomicCmpXchg:
3180 case Instruction::AtomicRMW:
3181 case Instruction::CondBr:
3182 return !AssumedNoUBInsts.count(Ptr: I);
3183 default:
3184 return false;
3185 }
3186 return false;
3187 }
3188
3189 /// Emit an optimization remark explaining why \p I is known to cause UB,
3190 /// per \p Info, right before it is replaced with 'unreachable'.
3191 static void emitUBRemark(Attributor &A, Instruction *I, const UBInfo &Info) {
3192 auto Remark = [&](OptimizationRemark OR) {
3193 switch (Info.K) {
3194 case UBInfo::NullPtrAccess:
3195 case UBInfo::UndefPtrAccess: {
3196 return OR << "Memory access through a pointer known to be "
3197 << ore::NV("Pointer",
3198 getPointerOperand(I, /*AllowVolatile*/ true))
3199 << " is undefined behavior; replacing with 'unreachable'.";
3200 }
3201 case UBInfo::UndefBranchCondition:
3202 return OR << "Branch condition known to be "
3203 << ore::NV("Condition", cast<CondBrInst>(Val: I)->getCondition())
3204 << " is undefined behavior; replacing with 'unreachable'.";
3205 case UBInfo::UndefReturnValue:
3206 case UBInfo::NullReturnViolatesNonNull:
3207 return OR << "Value returned known to be "
3208 << ore::NV("ReturnValue",
3209 cast<ReturnInst>(Val: I)->getReturnValue())
3210 << " is undefined behavior; replacing with 'unreachable'.";
3211 case UBInfo::UndefCallArgument:
3212 case UBInfo::NullArgViolatesNonNull: {
3213 bool IsUndef = Info.K == UBInfo::UndefCallArgument;
3214 CallBase &CB = *cast<CallBase>(Val: I);
3215 OR << "Argument " << ore::NV("ArgNo", *Info.ArgNo)
3216 << " passed to parameter of ";
3217 if (auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand()))
3218 OR << ore::NV("Callee", Callee);
3219 else
3220 OR << "the callee";
3221 return OR << " known to be "
3222 << ore::NV("Argument", IsUndef ? "undef" : "null")
3223 << " is undefined behavior; replacing with 'unreachable'.";
3224 }
3225 }
3226 llvm_unreachable("Unknown UBInfo::Kind");
3227 };
3228 A.emitRemark<OptimizationRemark>(I, RemarkName: "UndefinedBehavior", RemarkCB&: Remark);
3229 }
3230
3231 ChangeStatus manifest(Attributor &A) override {
3232 if (KnownUBInsts.empty())
3233 return ChangeStatus::UNCHANGED;
3234 for (const auto &[I, Info] : KnownUBInsts) {
3235 emitUBRemark(A, I, Info);
3236 A.changeToUnreachableAfterManifest(I);
3237 }
3238 return ChangeStatus::CHANGED;
3239 }
3240
3241 /// See AbstractAttribute::getAsStr()
3242 const std::string getAsStr(Attributor *A) const override {
3243 return getAssumed() ? "undefined-behavior" : "no-ub";
3244 }
3245
3246 /// Note: The correctness of this analysis depends on the fact that the
3247 /// following 2 sets will stop changing after some point.
3248 /// "Change" here means that their size changes.
3249 /// The size of each set is monotonically increasing
3250 /// (we only add items to them) and it is upper bounded by the number of
3251 /// instructions in the processed function (we can never save more
3252 /// elements in either set than this number). Hence, at some point,
3253 /// they will stop increasing.
3254 /// Consequently, at some point, both sets will have stopped
3255 /// changing, effectively making the analysis reach a fixpoint.
3256
3257 /// Note: These 2 sets are disjoint and an instruction can be considered
3258 /// one of 3 things:
3259 /// 1) Known to cause UB (AAUndefinedBehavior could prove it) and put it in
3260 /// the KnownUBInsts set.
3261 /// 2) Assumed to cause UB (in every updateImpl, AAUndefinedBehavior
3262 /// has a reason to assume it).
3263 /// 3) Assumed to not cause UB. very other instruction - AAUndefinedBehavior
3264 /// could not find a reason to assume or prove that it can cause UB,
3265 /// hence it assumes it doesn't. We have a set for these instructions
3266 /// so that we don't reprocess them in every update.
3267 /// Note however that instructions in this set may cause UB.
3268
3269protected:
3270 /// A map from all live instructions _known_ to cause UB to the reason why,
3271 /// used to build actionable optimization remarks in manifest().
3272 MapVector<Instruction *, UBInfo> KnownUBInsts;
3273
3274private:
3275 /// A set of all the (live) instructions that are assumed to _not_ cause UB.
3276 SmallPtrSet<Instruction *, 8> AssumedNoUBInsts;
3277
3278 // Should be called on updates in which if we're processing an instruction
3279 // \p I that depends on a value \p V, one of the following has to happen:
3280 // - If the value is assumed, then stop.
3281 // - If the value is known but undef, then consider it UB for \p K.
3282 // - Otherwise, do specific processing with the simplified value.
3283 // We return std::nullopt in the first 2 cases to signify that an appropriate
3284 // action was taken and the caller should stop.
3285 // Otherwise, we return the simplified value that the caller should
3286 // use for specific processing.
3287 std::optional<Value *> stopOnUndefOrAssumed(Attributor &A, Value *V,
3288 Instruction *I, UBInfo::Kind K) {
3289 bool UsedAssumedInformation = false;
3290 std::optional<Value *> SimplifiedV =
3291 A.getAssumedSimplified(IRP: IRPosition::value(V: *V), AA: *this,
3292 UsedAssumedInformation, S: AA::Interprocedural);
3293 if (!UsedAssumedInformation) {
3294 // Don't depend on assumed values.
3295 if (!SimplifiedV) {
3296 // If it is known (which we tested above) but it doesn't have a value,
3297 // then we can assume `undef` and hence the instruction is UB.
3298 KnownUBInsts.try_emplace(Key: I, Args&: K);
3299 return std::nullopt;
3300 }
3301 if (!*SimplifiedV)
3302 return nullptr;
3303 V = *SimplifiedV;
3304 }
3305 if (isa<UndefValue>(Val: V)) {
3306 KnownUBInsts.try_emplace(Key: I, Args&: K);
3307 return std::nullopt;
3308 }
3309 return V;
3310 }
3311};
3312
3313struct AAUndefinedBehaviorFunction final : AAUndefinedBehaviorImpl {
3314 AAUndefinedBehaviorFunction(const IRPosition &IRP, Attributor &A)
3315 : AAUndefinedBehaviorImpl(IRP, A) {}
3316
3317 /// See AbstractAttribute::trackStatistics()
3318 void trackStatistics() const override {
3319 STATS_DECL(UndefinedBehaviorInstruction, Instruction,
3320 "Number of instructions known to have UB");
3321 BUILD_STAT_NAME(UndefinedBehaviorInstruction, Instruction) +=
3322 KnownUBInsts.size();
3323 }
3324};
3325} // namespace
3326
3327/// ------------------------ Will-Return Attributes ----------------------------
3328
3329namespace {
3330// Helper function that checks whether a function has any cycle which we don't
3331// know if it is bounded or not.
3332// Loops with maximum trip count are considered bounded, any other cycle not.
3333static bool mayContainUnboundedCycle(Function &F, Attributor &A) {
3334 ScalarEvolution *SE =
3335 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(F);
3336 LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F);
3337 // If either SCEV or LoopInfo is not available for the function then we assume
3338 // any cycle to be unbounded cycle.
3339 // We use scc_iterator which uses Tarjan algorithm to find all the maximal
3340 // SCCs.To detect if there's a cycle, we only need to find the maximal ones.
3341 if (!SE || !LI) {
3342 for (scc_iterator<Function *> SCCI = scc_begin(G: &F); !SCCI.isAtEnd(); ++SCCI)
3343 if (SCCI.hasCycle())
3344 return true;
3345 return false;
3346 }
3347
3348 // If there's irreducible control, the function may contain non-loop cycles.
3349 if (mayContainIrreducibleControl(F, LI))
3350 return true;
3351
3352 // Any loop that does not have a max trip count is considered unbounded cycle.
3353 for (auto *L : LI->getLoopsInPreorder()) {
3354 if (!SE->getSmallConstantMaxTripCount(L))
3355 return true;
3356 }
3357 return false;
3358}
3359
3360struct AAWillReturnImpl : public AAWillReturn {
3361 AAWillReturnImpl(const IRPosition &IRP, Attributor &A)
3362 : AAWillReturn(IRP, A) {}
3363
3364 /// See AbstractAttribute::initialize(...).
3365 void initialize(Attributor &A) override {
3366 bool IsKnown;
3367 assert(!AA::hasAssumedIRAttr<Attribute::WillReturn>(
3368 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
3369 (void)IsKnown;
3370 }
3371
3372 /// Check for `mustprogress` and `readonly` as they imply `willreturn`.
3373 bool isImpliedByMustprogressAndReadonly(Attributor &A, bool KnownOnly) {
3374 if (!A.hasAttr(IRP: getIRPosition(), AKs: {Attribute::MustProgress}))
3375 return false;
3376
3377 bool IsKnown;
3378 if (AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
3379 return IsKnown || !KnownOnly;
3380 return false;
3381 }
3382
3383 /// See AbstractAttribute::updateImpl(...).
3384 ChangeStatus updateImpl(Attributor &A) override {
3385 if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3386 return ChangeStatus::UNCHANGED;
3387
3388 auto CheckForWillReturn = [&](Instruction &I) {
3389 IRPosition IPos = IRPosition::callsite_function(CB: cast<CallBase>(Val&: I));
3390 bool IsKnown;
3391 if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
3392 A, QueryingAA: this, IRP: IPos, DepClass: DepClassTy::REQUIRED, IsKnown)) {
3393 if (IsKnown)
3394 return true;
3395 } else {
3396 return false;
3397 }
3398 bool IsKnownNoRecurse;
3399 return AA::hasAssumedIRAttr<Attribute::NoRecurse>(
3400 A, QueryingAA: this, IRP: IPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoRecurse);
3401 };
3402
3403 bool UsedAssumedInformation = false;
3404 if (!A.checkForAllCallLikeInstructions(Pred: CheckForWillReturn, QueryingAA: *this,
3405 UsedAssumedInformation))
3406 return indicatePessimisticFixpoint();
3407
3408 auto CheckForVolatile = [&](Instruction &I) {
3409 // Volatile operations are not willreturn.
3410 return !I.isVolatile();
3411 };
3412 if (!A.checkForAllInstructions(Pred: CheckForVolatile, QueryingAA: *this,
3413 Opcodes: {Instruction::Load, Instruction::Store,
3414 Instruction::AtomicCmpXchg,
3415 Instruction::AtomicRMW},
3416 UsedAssumedInformation))
3417 return indicatePessimisticFixpoint();
3418
3419 return ChangeStatus::UNCHANGED;
3420 }
3421
3422 /// See AbstractAttribute::getAsStr()
3423 const std::string getAsStr(Attributor *A) const override {
3424 return getAssumed() ? "willreturn" : "may-noreturn";
3425 }
3426};
3427
3428struct AAWillReturnFunction final : AAWillReturnImpl {
3429 AAWillReturnFunction(const IRPosition &IRP, Attributor &A)
3430 : AAWillReturnImpl(IRP, A) {}
3431
3432 /// See AbstractAttribute::initialize(...).
3433 void initialize(Attributor &A) override {
3434 AAWillReturnImpl::initialize(A);
3435
3436 Function *F = getAnchorScope();
3437 assert(F && "Did expect an anchor function");
3438 if (F->isDeclaration() || mayContainUnboundedCycle(F&: *F, A))
3439 indicatePessimisticFixpoint();
3440 }
3441
3442 /// See AbstractAttribute::trackStatistics()
3443 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(willreturn) }
3444};
3445
3446/// WillReturn attribute deduction for a call sites.
3447struct AAWillReturnCallSite final
3448 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl> {
3449 AAWillReturnCallSite(const IRPosition &IRP, Attributor &A)
3450 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl>(IRP, A) {}
3451
3452 /// See AbstractAttribute::updateImpl(...).
3453 ChangeStatus updateImpl(Attributor &A) override {
3454 if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3455 return ChangeStatus::UNCHANGED;
3456
3457 return AACalleeToCallSite::updateImpl(A);
3458 }
3459
3460 /// See AbstractAttribute::trackStatistics()
3461 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(willreturn); }
3462};
3463} // namespace
3464
3465/// -------------------AAIntraFnReachability Attribute--------------------------
3466
3467/// All information associated with a reachability query. This boilerplate code
3468/// is used by both AAIntraFnReachability and AAInterFnReachability, with
3469/// different \p ToTy values.
3470template <typename ToTy> struct ReachabilityQueryInfo {
3471 enum class Reachable {
3472 No,
3473 Yes,
3474 };
3475
3476 /// Start here,
3477 const Instruction *From = nullptr;
3478 /// reach this place,
3479 const ToTy *To = nullptr;
3480 /// without going through any of these instructions,
3481 const AA::InstExclusionSetTy *ExclusionSet = nullptr;
3482 /// and remember if it worked:
3483 Reachable Result = Reachable::No;
3484
3485 /// Precomputed hash for this RQI.
3486 unsigned Hash = 0;
3487
3488 unsigned computeHashValue() const {
3489 assert(Hash == 0 && "Computed hash twice!");
3490 using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;
3491 using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;
3492 return const_cast<ReachabilityQueryInfo<ToTy> *>(this)->Hash =
3493 detail::combineHashValue(a: PairDMI ::getHashValue({From, To}),
3494 b: InstSetDMI::getHashValue(BES: ExclusionSet));
3495 }
3496
3497 ReachabilityQueryInfo(const Instruction *From, const ToTy *To)
3498 : From(From), To(To) {}
3499
3500 /// Constructor replacement to ensure unique and stable sets are used for the
3501 /// cache.
3502 ReachabilityQueryInfo(Attributor &A, const Instruction &From, const ToTy &To,
3503 const AA::InstExclusionSetTy *ES, bool MakeUnique)
3504 : From(&From), To(&To), ExclusionSet(ES) {
3505
3506 if (!ES || ES->empty()) {
3507 ExclusionSet = nullptr;
3508 } else if (MakeUnique) {
3509 ExclusionSet = A.getInfoCache().getOrCreateUniqueBlockExecutionSet(BES: ES);
3510 }
3511 }
3512
3513 ReachabilityQueryInfo(const ReachabilityQueryInfo &RQI)
3514 : From(RQI.From), To(RQI.To), ExclusionSet(RQI.ExclusionSet) {}
3515};
3516
3517namespace llvm {
3518template <typename ToTy> struct DenseMapInfo<ReachabilityQueryInfo<ToTy> *> {
3519 using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;
3520 using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;
3521
3522 static unsigned getHashValue(const ReachabilityQueryInfo<ToTy> *RQI) {
3523 return RQI->Hash ? RQI->Hash : RQI->computeHashValue();
3524 }
3525 static bool isEqual(const ReachabilityQueryInfo<ToTy> *LHS,
3526 const ReachabilityQueryInfo<ToTy> *RHS) {
3527 if (!PairDMI::isEqual({LHS->From, LHS->To}, {RHS->From, RHS->To}))
3528 return false;
3529 return InstSetDMI::isEqual(LHS: LHS->ExclusionSet, RHS: RHS->ExclusionSet);
3530 }
3531};
3532
3533} // namespace llvm
3534
3535namespace {
3536
3537template <typename BaseTy, typename ToTy>
3538struct CachedReachabilityAA : public BaseTy {
3539 using RQITy = ReachabilityQueryInfo<ToTy>;
3540
3541 CachedReachabilityAA(const IRPosition &IRP, Attributor &A) : BaseTy(IRP, A) {}
3542
3543 /// See AbstractAttribute::isQueryAA.
3544 bool isQueryAA() const override { return true; }
3545
3546 /// See AbstractAttribute::updateImpl(...).
3547 ChangeStatus updateImpl(Attributor &A) override {
3548 ChangeStatus Changed = ChangeStatus::UNCHANGED;
3549 for (unsigned u = 0, e = QueryVector.size(); u < e; ++u) {
3550 RQITy *RQI = QueryVector[u];
3551 if (RQI->Result == RQITy::Reachable::No &&
3552 isReachableImpl(A, RQI&: *RQI, /*IsTemporaryRQI=*/false))
3553 Changed = ChangeStatus::CHANGED;
3554 }
3555 return Changed;
3556 }
3557
3558 virtual bool isReachableImpl(Attributor &A, RQITy &RQI,
3559 bool IsTemporaryRQI) = 0;
3560
3561 bool rememberResult(Attributor &A, typename RQITy::Reachable Result,
3562 RQITy &RQI, bool UsedExclusionSet, bool IsTemporaryRQI) {
3563 RQI.Result = Result;
3564
3565 // Remove the temporary RQI from the cache.
3566 if (IsTemporaryRQI)
3567 QueryCache.erase(&RQI);
3568
3569 // Insert a plain RQI (w/o exclusion set) if that makes sense. Two options:
3570 // 1) If it is reachable, it doesn't matter if we have an exclusion set for
3571 // this query. 2) We did not use the exclusion set, potentially because
3572 // there is none.
3573 if (Result == RQITy::Reachable::Yes || !UsedExclusionSet) {
3574 RQITy PlainRQI(RQI.From, RQI.To);
3575 if (!QueryCache.count(&PlainRQI)) {
3576 RQITy *RQIPtr = new (A.Allocator) RQITy(RQI.From, RQI.To);
3577 RQIPtr->Result = Result;
3578 QueryVector.push_back(RQIPtr);
3579 QueryCache.insert(RQIPtr);
3580 }
3581 }
3582
3583 // Check if we need to insert a new permanent RQI with the exclusion set.
3584 if (IsTemporaryRQI && Result != RQITy::Reachable::Yes && UsedExclusionSet) {
3585 assert((!RQI.ExclusionSet || !RQI.ExclusionSet->empty()) &&
3586 "Did not expect empty set!");
3587 RQITy *RQIPtr = new (A.Allocator)
3588 RQITy(A, *RQI.From, *RQI.To, RQI.ExclusionSet, true);
3589 assert(RQIPtr->Result == RQITy::Reachable::No && "Already reachable?");
3590 RQIPtr->Result = Result;
3591 assert(!QueryCache.count(RQIPtr));
3592 QueryVector.push_back(RQIPtr);
3593 QueryCache.insert(RQIPtr);
3594 }
3595
3596 if (Result == RQITy::Reachable::No && IsTemporaryRQI)
3597 A.registerForUpdate(AA&: *this);
3598 return Result == RQITy::Reachable::Yes;
3599 }
3600
3601 const std::string getAsStr(Attributor *A) const override {
3602 // TODO: Return the number of reachable queries.
3603 return "#queries(" + std::to_string(QueryVector.size()) + ")";
3604 }
3605
3606 bool checkQueryCache(Attributor &A, RQITy &StackRQI,
3607 typename RQITy::Reachable &Result) {
3608 if (!this->getState().isValidState()) {
3609 Result = RQITy::Reachable::Yes;
3610 return true;
3611 }
3612
3613 // If we have an exclusion set we might be able to find our answer by
3614 // ignoring it first.
3615 if (StackRQI.ExclusionSet) {
3616 RQITy PlainRQI(StackRQI.From, StackRQI.To);
3617 auto It = QueryCache.find(&PlainRQI);
3618 if (It != QueryCache.end() && (*It)->Result == RQITy::Reachable::No) {
3619 Result = RQITy::Reachable::No;
3620 return true;
3621 }
3622 }
3623
3624 auto It = QueryCache.find(&StackRQI);
3625 if (It != QueryCache.end()) {
3626 Result = (*It)->Result;
3627 return true;
3628 }
3629
3630 // Insert a temporary for recursive queries. We will replace it with a
3631 // permanent entry later.
3632 QueryCache.insert(&StackRQI);
3633 return false;
3634 }
3635
3636private:
3637 SmallVector<RQITy *> QueryVector;
3638 DenseSet<RQITy *> QueryCache;
3639};
3640
3641struct AAIntraFnReachabilityFunction final
3642 : public CachedReachabilityAA<AAIntraFnReachability, Instruction> {
3643 using Base = CachedReachabilityAA<AAIntraFnReachability, Instruction>;
3644 AAIntraFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
3645 : Base(IRP, A) {
3646 DT = A.getInfoCache().getAnalysisResultForFunction<DominatorTreeAnalysis>(
3647 F: *IRP.getAssociatedFunction());
3648 }
3649
3650 bool isAssumedReachable(
3651 Attributor &A, const Instruction &From, const Instruction &To,
3652 const AA::InstExclusionSetTy *ExclusionSet) const override {
3653 auto *NonConstThis = const_cast<AAIntraFnReachabilityFunction *>(this);
3654 if (&From == &To)
3655 return true;
3656
3657 RQITy StackRQI(A, From, To, ExclusionSet, false);
3658 RQITy::Reachable Result;
3659 if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
3660 return NonConstThis->isReachableImpl(A, RQI&: StackRQI,
3661 /*IsTemporaryRQI=*/true);
3662 return Result == RQITy::Reachable::Yes;
3663 }
3664
3665 ChangeStatus updateImpl(Attributor &A) override {
3666 // We only depend on liveness. DeadEdges is all we care about, check if any
3667 // of them changed.
3668 auto *LivenessAA =
3669 A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL);
3670 if (LivenessAA &&
3671 llvm::all_of(Range&: DeadEdges,
3672 P: [&](const auto &DeadEdge) {
3673 return LivenessAA->isEdgeDead(From: DeadEdge.first,
3674 To: DeadEdge.second);
3675 }) &&
3676 llvm::all_of(Range&: DeadBlocks, P: [&](const BasicBlock *BB) {
3677 return LivenessAA->isAssumedDead(BB);
3678 })) {
3679 return ChangeStatus::UNCHANGED;
3680 }
3681 DeadEdges.clear();
3682 DeadBlocks.clear();
3683 return Base::updateImpl(A);
3684 }
3685
3686 bool isReachableImpl(Attributor &A, RQITy &RQI,
3687 bool IsTemporaryRQI) override {
3688 const Instruction *Origin = RQI.From;
3689 bool UsedExclusionSet = false;
3690
3691 auto WillReachInBlock = [&](const Instruction &From, const Instruction &To,
3692 const AA::InstExclusionSetTy *ExclusionSet) {
3693 const Instruction *IP = &From;
3694 while (IP && IP != &To) {
3695 if (ExclusionSet && IP != Origin && ExclusionSet->count(Ptr: IP)) {
3696 UsedExclusionSet = true;
3697 break;
3698 }
3699 IP = IP->getNextNode();
3700 }
3701 return IP == &To;
3702 };
3703
3704 const BasicBlock *FromBB = RQI.From->getParent();
3705 const BasicBlock *ToBB = RQI.To->getParent();
3706 assert(FromBB->getParent() == ToBB->getParent() &&
3707 "Not an intra-procedural query!");
3708
3709 // Check intra-block reachability, however, other reaching paths are still
3710 // possible.
3711 if (FromBB == ToBB &&
3712 WillReachInBlock(*RQI.From, *RQI.To, RQI.ExclusionSet))
3713 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3714 IsTemporaryRQI);
3715
3716 // Check if reaching the ToBB block is sufficient or if even that would not
3717 // ensure reaching the target. In the latter case we are done.
3718 if (!WillReachInBlock(ToBB->front(), *RQI.To, RQI.ExclusionSet))
3719 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3720 IsTemporaryRQI);
3721
3722 const Function *Fn = FromBB->getParent();
3723 SmallPtrSet<const BasicBlock *, 16> ExclusionBlocks;
3724 if (RQI.ExclusionSet)
3725 for (auto *I : *RQI.ExclusionSet)
3726 if (I->getFunction() == Fn)
3727 ExclusionBlocks.insert(Ptr: I->getParent());
3728
3729 // Check if we make it out of the FromBB block at all.
3730 if (ExclusionBlocks.count(Ptr: FromBB) &&
3731 !WillReachInBlock(*RQI.From, *FromBB->getTerminator(),
3732 RQI.ExclusionSet))
3733 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet: true, IsTemporaryRQI);
3734
3735 auto *LivenessAA =
3736 A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL);
3737 if (LivenessAA && LivenessAA->isAssumedDead(BB: ToBB)) {
3738 DeadBlocks.insert(V: ToBB);
3739 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3740 IsTemporaryRQI);
3741 }
3742
3743 SmallPtrSet<const BasicBlock *, 16> Visited;
3744 SmallVector<const BasicBlock *, 16> Worklist;
3745 Worklist.push_back(Elt: FromBB);
3746
3747 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> LocalDeadEdges;
3748 while (!Worklist.empty()) {
3749 const BasicBlock *BB = Worklist.pop_back_val();
3750 if (!Visited.insert(Ptr: BB).second)
3751 continue;
3752 for (const BasicBlock *SuccBB : successors(BB)) {
3753 if (LivenessAA && LivenessAA->isEdgeDead(From: BB, To: SuccBB)) {
3754 LocalDeadEdges.insert(V: {BB, SuccBB});
3755 continue;
3756 }
3757 // We checked before if we just need to reach the ToBB block.
3758 if (SuccBB == ToBB)
3759 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3760 IsTemporaryRQI);
3761 if (DT && ExclusionBlocks.empty() && DT->dominates(A: BB, B: ToBB))
3762 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3763 IsTemporaryRQI);
3764
3765 if (ExclusionBlocks.count(Ptr: SuccBB)) {
3766 UsedExclusionSet = true;
3767 continue;
3768 }
3769 Worklist.push_back(Elt: SuccBB);
3770 }
3771 }
3772
3773 DeadEdges.insert_range(R&: LocalDeadEdges);
3774 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3775 IsTemporaryRQI);
3776 }
3777
3778 /// See AbstractAttribute::trackStatistics()
3779 void trackStatistics() const override {}
3780
3781private:
3782 // Set of assumed dead blocks we used in the last query. If any changes we
3783 // update the state.
3784 DenseSet<const BasicBlock *> DeadBlocks;
3785
3786 // Set of assumed dead edges we used in the last query. If any changes we
3787 // update the state.
3788 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> DeadEdges;
3789
3790 /// The dominator tree of the function to short-circuit reasoning.
3791 const DominatorTree *DT = nullptr;
3792};
3793} // namespace
3794
3795/// ------------------------ NoAlias Argument Attribute ------------------------
3796
3797bool AANoAlias::isImpliedByIR(Attributor &A, const IRPosition &IRP,
3798 Attribute::AttrKind ImpliedAttributeKind,
3799 bool IgnoreSubsumingPositions) {
3800 assert(ImpliedAttributeKind == Attribute::NoAlias &&
3801 "Unexpected attribute kind");
3802 Value *Val = &IRP.getAssociatedValue();
3803 if (IRP.getPositionKind() != IRP_CALL_SITE_ARGUMENT) {
3804 if (isa<AllocaInst>(Val))
3805 return true;
3806 } else {
3807 IgnoreSubsumingPositions = true;
3808 }
3809
3810 if (isa<UndefValue>(Val))
3811 return true;
3812
3813 if (isa<ConstantPointerNull>(Val) &&
3814 !NullPointerIsDefined(F: IRP.getAnchorScope(),
3815 AS: Val->getType()->getPointerAddressSpace()))
3816 return true;
3817
3818 if (A.hasAttr(IRP, AKs: {Attribute::ByVal, Attribute::NoAlias},
3819 IgnoreSubsumingPositions, ImpliedAttributeKind: Attribute::NoAlias))
3820 return true;
3821
3822 return false;
3823}
3824
3825namespace {
3826struct AANoAliasImpl : AANoAlias {
3827 AANoAliasImpl(const IRPosition &IRP, Attributor &A) : AANoAlias(IRP, A) {
3828 assert(getAssociatedType()->isPointerTy() &&
3829 "Noalias is a pointer attribute");
3830 }
3831
3832 const std::string getAsStr(Attributor *A) const override {
3833 return getAssumed() ? "noalias" : "may-alias";
3834 }
3835};
3836
3837/// NoAlias attribute for a floating value.
3838struct AANoAliasFloating final : AANoAliasImpl {
3839 AANoAliasFloating(const IRPosition &IRP, Attributor &A)
3840 : AANoAliasImpl(IRP, A) {}
3841
3842 /// See AbstractAttribute::updateImpl(...).
3843 ChangeStatus updateImpl(Attributor &A) override {
3844 // TODO: Implement this.
3845 return indicatePessimisticFixpoint();
3846 }
3847
3848 /// See AbstractAttribute::trackStatistics()
3849 void trackStatistics() const override {
3850 STATS_DECLTRACK_FLOATING_ATTR(noalias)
3851 }
3852};
3853
3854/// NoAlias attribute for an argument.
3855struct AANoAliasArgument final
3856 : AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl> {
3857 using Base = AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl>;
3858 AANoAliasArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
3859
3860 /// See AbstractAttribute::update(...).
3861 ChangeStatus updateImpl(Attributor &A) override {
3862 // We have to make sure no-alias on the argument does not break
3863 // synchronization when this is a callback argument, see also [1] below.
3864 // If synchronization cannot be affected, we delegate to the base updateImpl
3865 // function, otherwise we give up for now.
3866
3867 // If the function is no-sync, no-alias cannot break synchronization.
3868 bool IsKnownNoSycn;
3869 if (AA::hasAssumedIRAttr<Attribute::NoSync>(
3870 A, QueryingAA: this, IRP: IRPosition::function_scope(IRP: getIRPosition()),
3871 DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoSycn))
3872 return Base::updateImpl(A);
3873
3874 // If the argument is read-only, no-alias cannot break synchronization.
3875 bool IsKnown;
3876 if (AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
3877 return Base::updateImpl(A);
3878
3879 // If the argument is never passed through callbacks, no-alias cannot break
3880 // synchronization.
3881 bool UsedAssumedInformation = false;
3882 if (A.checkForAllCallSites(
3883 Pred: [](AbstractCallSite ACS) { return !ACS.isCallbackCall(); }, QueryingAA: *this,
3884 RequireAllCallSites: true, UsedAssumedInformation))
3885 return Base::updateImpl(A);
3886
3887 // TODO: add no-alias but make sure it doesn't break synchronization by
3888 // introducing fake uses. See:
3889 // [1] Compiler Optimizations for OpenMP, J. Doerfert and H. Finkel,
3890 // International Workshop on OpenMP 2018,
3891 // http://compilers.cs.uni-saarland.de/people/doerfert/par_opt18.pdf
3892
3893 return indicatePessimisticFixpoint();
3894 }
3895
3896 /// See AbstractAttribute::trackStatistics()
3897 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noalias) }
3898};
3899
3900struct AANoAliasCallSiteArgument final : AANoAliasImpl {
3901 AANoAliasCallSiteArgument(const IRPosition &IRP, Attributor &A)
3902 : AANoAliasImpl(IRP, A) {}
3903
3904 /// Determine if the underlying value may alias with the call site argument
3905 /// \p OtherArgNo of \p ICS (= the underlying call site).
3906 bool mayAliasWithArgument(Attributor &A, AAResults *&AAR,
3907 const AAMemoryBehavior &MemBehaviorAA,
3908 const CallBase &CB, unsigned OtherArgNo) {
3909 // We do not need to worry about aliasing with the underlying IRP.
3910 if (this->getCallSiteArgNo() == (int)OtherArgNo)
3911 return false;
3912
3913 // If it is not a pointer or pointer vector we do not alias.
3914 const Value *ArgOp = CB.getArgOperand(i: OtherArgNo);
3915 if (!ArgOp->getType()->isPtrOrPtrVectorTy())
3916 return false;
3917
3918 auto *CBArgMemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
3919 QueryingAA: *this, IRP: IRPosition::callsite_argument(CB, ArgNo: OtherArgNo), DepClass: DepClassTy::NONE);
3920
3921 // If the argument is readnone, there is no read-write aliasing.
3922 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadNone()) {
3923 A.recordDependence(FromAA: *CBArgMemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3924 return false;
3925 }
3926
3927 // If the argument is readonly and the underlying value is readonly, there
3928 // is no read-write aliasing.
3929 bool IsReadOnly = MemBehaviorAA.isAssumedReadOnly();
3930 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadOnly() &&
3931 IsReadOnly) {
3932 A.recordDependence(FromAA: MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3933 A.recordDependence(FromAA: *CBArgMemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3934 return false;
3935 }
3936
3937 // We have to utilize actual alias analysis queries so we need the object.
3938 if (!AAR)
3939 AAR = A.getInfoCache().getAnalysisResultForFunction<AAManager>(
3940 F: *getAnchorScope());
3941
3942 // Try to rule it out at the call site.
3943 bool IsAliasing = !AAR || !AAR->isNoAlias(V1: &getAssociatedValue(), V2: ArgOp);
3944 LLVM_DEBUG(dbgs() << "[NoAliasCSArg] Check alias between "
3945 "callsite arguments: "
3946 << getAssociatedValue() << " " << *ArgOp << " => "
3947 << (IsAliasing ? "" : "no-") << "alias \n");
3948
3949 return IsAliasing;
3950 }
3951
3952 bool isKnownNoAliasDueToNoAliasPreservation(
3953 Attributor &A, AAResults *&AAR, const AAMemoryBehavior &MemBehaviorAA) {
3954 // We can deduce "noalias" if the following conditions hold.
3955 // (i) Associated value is assumed to be noalias in the definition.
3956 // (ii) Associated value is assumed to be no-capture in all the uses
3957 // possibly executed before this callsite.
3958 // (iii) There is no other pointer argument which could alias with the
3959 // value.
3960
3961 const IRPosition &VIRP = IRPosition::value(V: getAssociatedValue());
3962 const Function *ScopeFn = VIRP.getAnchorScope();
3963 // Check whether the value is captured in the scope using AANoCapture.
3964 // Look at CFG and check only uses possibly executed before this
3965 // callsite.
3966 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
3967 Instruction *UserI = cast<Instruction>(Val: U.getUser());
3968
3969 // If UserI is the curr instruction and there is a single potential use of
3970 // the value in UserI we allow the use.
3971 // TODO: We should inspect the operands and allow those that cannot alias
3972 // with the value.
3973 if (UserI == getCtxI() && UserI->getNumOperands() == 1)
3974 return true;
3975
3976 if (ScopeFn) {
3977 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
3978 if (CB->isArgOperand(U: &U)) {
3979
3980 unsigned ArgNo = CB->getArgOperandNo(U: &U);
3981
3982 bool IsKnownNoCapture;
3983 if (AA::hasAssumedIRAttr<Attribute::Captures>(
3984 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
3985 DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture))
3986 return true;
3987 }
3988 }
3989
3990 if (!AA::isPotentiallyReachable(
3991 A, FromI: *UserI, ToI: *getCtxI(), QueryingAA: *this, /* ExclusionSet */ nullptr,
3992 GoBackwardsCB: [ScopeFn](const Function &Fn) { return &Fn != ScopeFn; }))
3993 return true;
3994 }
3995
3996 // TODO: We should track the capturing uses in AANoCapture but the problem
3997 // is CGSCC runs. For those we would need to "allow" AANoCapture for
3998 // a value in the module slice.
3999 // TODO(captures): Make this more precise.
4000 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
4001 if (capturesNothing(CC: CI))
4002 return true;
4003 if (CI.isPassthrough()) {
4004 Follow = true;
4005 return true;
4006 }
4007 LLVM_DEBUG(dbgs() << "[AANoAliasCSArg] Unknown user: " << *UserI << "\n");
4008 return false;
4009 };
4010
4011 bool IsKnownNoCapture;
4012 const AANoCapture *NoCaptureAA = nullptr;
4013 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
4014 A, QueryingAA: this, IRP: VIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false, AAPtr: &NoCaptureAA);
4015 if (!IsAssumedNoCapture &&
4016 (!NoCaptureAA || !NoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
4017 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: getAssociatedValue())) {
4018 LLVM_DEBUG(
4019 dbgs() << "[AANoAliasCSArg] " << getAssociatedValue()
4020 << " cannot be noalias as it is potentially captured\n");
4021 return false;
4022 }
4023 }
4024 if (NoCaptureAA)
4025 A.recordDependence(FromAA: *NoCaptureAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
4026
4027 // Check there is no other pointer argument which could alias with the
4028 // value passed at this call site.
4029 // TODO: AbstractCallSite
4030 const auto &CB = cast<CallBase>(Val&: getAnchorValue());
4031 for (unsigned OtherArgNo = 0; OtherArgNo < CB.arg_size(); OtherArgNo++)
4032 if (mayAliasWithArgument(A, AAR, MemBehaviorAA, CB, OtherArgNo))
4033 return false;
4034
4035 return true;
4036 }
4037
4038 /// See AbstractAttribute::updateImpl(...).
4039 ChangeStatus updateImpl(Attributor &A) override {
4040 // If the argument is readnone we are done as there are no accesses via the
4041 // argument.
4042 auto *MemBehaviorAA =
4043 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::NONE);
4044 if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
4045 A.recordDependence(FromAA: *MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
4046 return ChangeStatus::UNCHANGED;
4047 }
4048
4049 bool IsKnownNoAlias;
4050 const IRPosition &VIRP = IRPosition::value(V: getAssociatedValue());
4051 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
4052 A, QueryingAA: this, IRP: VIRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoAlias)) {
4053 LLVM_DEBUG(dbgs() << "[AANoAlias] " << getAssociatedValue()
4054 << " is not no-alias at the definition\n");
4055 return indicatePessimisticFixpoint();
4056 }
4057
4058 AAResults *AAR = nullptr;
4059 if (MemBehaviorAA &&
4060 isKnownNoAliasDueToNoAliasPreservation(A, AAR, MemBehaviorAA: *MemBehaviorAA)) {
4061 LLVM_DEBUG(
4062 dbgs() << "[AANoAlias] No-Alias deduced via no-alias preservation\n");
4063 return ChangeStatus::UNCHANGED;
4064 }
4065
4066 return indicatePessimisticFixpoint();
4067 }
4068
4069 /// See AbstractAttribute::trackStatistics()
4070 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noalias) }
4071};
4072
4073/// NoAlias attribute for function return value.
4074struct AANoAliasReturned final : AANoAliasImpl {
4075 AANoAliasReturned(const IRPosition &IRP, Attributor &A)
4076 : AANoAliasImpl(IRP, A) {}
4077
4078 /// See AbstractAttribute::updateImpl(...).
4079 ChangeStatus updateImpl(Attributor &A) override {
4080
4081 auto CheckReturnValue = [&](Value &RV) -> bool {
4082 if (Constant *C = dyn_cast<Constant>(Val: &RV))
4083 if (C->isNullValue() || isa<UndefValue>(Val: C))
4084 return true;
4085
4086 /// For now, we can only deduce noalias if we have call sites.
4087 /// FIXME: add more support.
4088 if (!isa<CallBase>(Val: &RV))
4089 return false;
4090
4091 const IRPosition &RVPos = IRPosition::value(V: RV);
4092 bool IsKnownNoAlias;
4093 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
4094 A, QueryingAA: this, IRP: RVPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoAlias))
4095 return false;
4096
4097 bool IsKnownNoCapture;
4098 const AANoCapture *NoCaptureAA = nullptr;
4099 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
4100 A, QueryingAA: this, IRP: RVPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
4101 AAPtr: &NoCaptureAA);
4102 return IsAssumedNoCapture ||
4103 (NoCaptureAA && NoCaptureAA->isAssumedNoCaptureMaybeReturned());
4104 };
4105
4106 if (!A.checkForAllReturnedValues(Pred: CheckReturnValue, QueryingAA: *this))
4107 return indicatePessimisticFixpoint();
4108
4109 return ChangeStatus::UNCHANGED;
4110 }
4111
4112 /// See AbstractAttribute::trackStatistics()
4113 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noalias) }
4114};
4115
4116/// NoAlias attribute deduction for a call site return value.
4117struct AANoAliasCallSiteReturned final
4118 : AACalleeToCallSite<AANoAlias, AANoAliasImpl> {
4119 AANoAliasCallSiteReturned(const IRPosition &IRP, Attributor &A)
4120 : AACalleeToCallSite<AANoAlias, AANoAliasImpl>(IRP, A) {}
4121
4122 /// See AbstractAttribute::trackStatistics()
4123 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noalias); }
4124};
4125} // namespace
4126
4127/// -------------------AAIsDead Function Attribute-----------------------
4128
4129namespace {
4130struct AAIsDeadValueImpl : public AAIsDead {
4131 AAIsDeadValueImpl(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4132
4133 /// See AAIsDead::isAssumedDead().
4134 bool isAssumedDead() const override { return isAssumed(BitsEncoding: IS_DEAD); }
4135
4136 /// See AAIsDead::isKnownDead().
4137 bool isKnownDead() const override { return isKnown(BitsEncoding: IS_DEAD); }
4138
4139 /// See AAIsDead::isAssumedDead(BasicBlock *).
4140 bool isAssumedDead(const BasicBlock *BB) const override { return false; }
4141
4142 /// See AAIsDead::isKnownDead(BasicBlock *).
4143 bool isKnownDead(const BasicBlock *BB) const override { return false; }
4144
4145 /// See AAIsDead::isAssumedDead(Instruction *I).
4146 bool isAssumedDead(const Instruction *I) const override {
4147 return I == getCtxI() && isAssumedDead();
4148 }
4149
4150 /// See AAIsDead::isKnownDead(Instruction *I).
4151 bool isKnownDead(const Instruction *I) const override {
4152 return isAssumedDead(I) && isKnownDead();
4153 }
4154
4155 /// See AbstractAttribute::getAsStr().
4156 const std::string getAsStr(Attributor *A) const override {
4157 return isAssumedDead() ? "assumed-dead" : "assumed-live";
4158 }
4159
4160 /// Check if all uses are assumed dead.
4161 bool areAllUsesAssumedDead(Attributor &A, Value &V) {
4162 // Callers might not check the type, void has no uses.
4163 if (V.getType()->isVoidTy() || V.use_empty())
4164 return true;
4165
4166 // If we replace a value with a constant there are no uses left afterwards.
4167 if (!isa<Constant>(Val: V)) {
4168 if (auto *I = dyn_cast<Instruction>(Val: &V))
4169 if (!A.isRunOn(Fn&: *I->getFunction()))
4170 return false;
4171 bool UsedAssumedInformation = false;
4172 std::optional<Constant *> C =
4173 A.getAssumedConstant(V, AA: *this, UsedAssumedInformation);
4174 if (!C || *C)
4175 return true;
4176 }
4177
4178 auto UsePred = [&](const Use &U, bool &Follow) { return false; };
4179 // Explicitly set the dependence class to required because we want a long
4180 // chain of N dependent instructions to be considered live as soon as one is
4181 // without going through N update cycles. This is not required for
4182 // correctness.
4183 return A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V, /* CheckBBLivenessOnly */ false,
4184 LivenessDepClass: DepClassTy::REQUIRED,
4185 /* IgnoreDroppableUses */ false);
4186 }
4187
4188 /// Determine if \p I is assumed to be side-effect free.
4189 bool isAssumedSideEffectFree(Attributor &A, Instruction *I) {
4190 if (!I || wouldInstructionBeTriviallyDead(I))
4191 return true;
4192
4193 if (!I->isTerminator() && !I->mayHaveSideEffects())
4194 return true;
4195
4196 auto *CB = dyn_cast<CallBase>(Val: I);
4197 if (!CB || isa<IntrinsicInst>(Val: CB))
4198 return false;
4199
4200 const IRPosition &CallIRP = IRPosition::callsite_function(CB: *CB);
4201
4202 bool IsKnownNoUnwind;
4203 if (!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
4204 A, QueryingAA: this, IRP: CallIRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoUnwind))
4205 return false;
4206
4207 bool IsKnown;
4208 return AA::isAssumedReadOnly(A, IRP: CallIRP, QueryingAA: *this, IsKnown);
4209 }
4210};
4211
4212struct AAIsDeadFloating : public AAIsDeadValueImpl {
4213 AAIsDeadFloating(const IRPosition &IRP, Attributor &A)
4214 : AAIsDeadValueImpl(IRP, A) {}
4215
4216 /// See AbstractAttribute::initialize(...).
4217 void initialize(Attributor &A) override {
4218 AAIsDeadValueImpl::initialize(A);
4219
4220 if (isa<UndefValue>(Val: getAssociatedValue())) {
4221 indicatePessimisticFixpoint();
4222 return;
4223 }
4224
4225 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4226 if (!isAssumedSideEffectFree(A, I)) {
4227 if (!isa_and_nonnull<StoreInst>(Val: I) && !isa_and_nonnull<FenceInst>(Val: I))
4228 indicatePessimisticFixpoint();
4229 else
4230 removeAssumedBits(BitsEncoding: HAS_NO_EFFECT);
4231 }
4232 }
4233
4234 bool isDeadFence(Attributor &A, FenceInst &FI) {
4235 const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
4236 IRP: IRPosition::function(F: *FI.getFunction()), QueryingAA: *this, DepClass: DepClassTy::NONE);
4237 if (!ExecDomainAA || !ExecDomainAA->isNoOpFence(FI))
4238 return false;
4239 A.recordDependence(FromAA: *ExecDomainAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
4240 return true;
4241 }
4242
4243 bool isDeadStore(Attributor &A, StoreInst &SI,
4244 SmallSetVector<Instruction *, 8> *AssumeOnlyInst = nullptr) {
4245 // Lang ref now states volatile store is not UB/dead, let's skip them.
4246 if (SI.isVolatile())
4247 return false;
4248
4249 // If we are collecting assumes to be deleted we are in the manifest stage.
4250 // It's problematic to collect the potential copies again now so we use the
4251 // cached ones.
4252 bool UsedAssumedInformation = false;
4253 if (!AssumeOnlyInst) {
4254 PotentialCopies.clear();
4255 if (!AA::getPotentialCopiesOfStoredValue(A, SI, PotentialCopies, QueryingAA: *this,
4256 UsedAssumedInformation)) {
4257 LLVM_DEBUG(
4258 dbgs()
4259 << "[AAIsDead] Could not determine potential copies of store!\n");
4260 return false;
4261 }
4262 }
4263 LLVM_DEBUG(dbgs() << "[AAIsDead] Store has " << PotentialCopies.size()
4264 << " potential copies.\n");
4265
4266 InformationCache &InfoCache = A.getInfoCache();
4267 return llvm::all_of(Range&: PotentialCopies, P: [&](Value *V) {
4268 if (A.isAssumedDead(IRP: IRPosition::value(V: *V), QueryingAA: this, FnLivenessAA: nullptr,
4269 UsedAssumedInformation))
4270 return true;
4271 if (auto *LI = dyn_cast<LoadInst>(Val: V)) {
4272 if (llvm::all_of(Range: LI->uses(), P: [&](const Use &U) {
4273 auto &UserI = cast<Instruction>(Val&: *U.getUser());
4274 if (InfoCache.isOnlyUsedByAssume(I: UserI)) {
4275 if (AssumeOnlyInst)
4276 AssumeOnlyInst->insert(X: &UserI);
4277 return true;
4278 }
4279 return A.isAssumedDead(U, QueryingAA: this, FnLivenessAA: nullptr, UsedAssumedInformation);
4280 })) {
4281 return true;
4282 }
4283 }
4284 LLVM_DEBUG(dbgs() << "[AAIsDead] Potential copy " << *V
4285 << " is assumed live!\n");
4286 return false;
4287 });
4288 }
4289
4290 /// See AbstractAttribute::getAsStr().
4291 const std::string getAsStr(Attributor *A) const override {
4292 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4293 if (isa_and_nonnull<StoreInst>(Val: I))
4294 if (isValidState())
4295 return "assumed-dead-store";
4296 if (isa_and_nonnull<FenceInst>(Val: I))
4297 if (isValidState())
4298 return "assumed-dead-fence";
4299 return AAIsDeadValueImpl::getAsStr(A);
4300 }
4301
4302 /// See AbstractAttribute::updateImpl(...).
4303 ChangeStatus updateImpl(Attributor &A) override {
4304 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4305 if (auto *SI = dyn_cast_or_null<StoreInst>(Val: I)) {
4306 if (!isDeadStore(A, SI&: *SI))
4307 return indicatePessimisticFixpoint();
4308 } else if (auto *FI = dyn_cast_or_null<FenceInst>(Val: I)) {
4309 if (!isDeadFence(A, FI&: *FI))
4310 return indicatePessimisticFixpoint();
4311 } else {
4312 if (!isAssumedSideEffectFree(A, I))
4313 return indicatePessimisticFixpoint();
4314 if (!areAllUsesAssumedDead(A, V&: getAssociatedValue()))
4315 return indicatePessimisticFixpoint();
4316 }
4317 return ChangeStatus::UNCHANGED;
4318 }
4319
4320 bool isRemovableStore() const override {
4321 return isAssumed(BitsEncoding: IS_REMOVABLE) && isa<StoreInst>(Val: &getAssociatedValue());
4322 }
4323
4324 /// See AbstractAttribute::manifest(...).
4325 ChangeStatus manifest(Attributor &A) override {
4326 Value &V = getAssociatedValue();
4327 if (auto *I = dyn_cast<Instruction>(Val: &V)) {
4328 // If we get here we basically know the users are all dead. We check if
4329 // isAssumedSideEffectFree returns true here again because it might not be
4330 // the case and only the users are dead but the instruction (=call) is
4331 // still needed.
4332 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
4333 SmallSetVector<Instruction *, 8> AssumeOnlyInst;
4334 bool IsDead = isDeadStore(A, SI&: *SI, AssumeOnlyInst: &AssumeOnlyInst);
4335 (void)IsDead;
4336 assert(IsDead && "Store was assumed to be dead!");
4337 A.deleteAfterManifest(I&: *I);
4338 for (size_t i = 0; i < AssumeOnlyInst.size(); ++i) {
4339 Instruction *AOI = AssumeOnlyInst[i];
4340 for (auto *Usr : AOI->users())
4341 AssumeOnlyInst.insert(X: cast<Instruction>(Val: Usr));
4342 A.deleteAfterManifest(I&: *AOI);
4343 }
4344 return ChangeStatus::CHANGED;
4345 }
4346 if (auto *FI = dyn_cast<FenceInst>(Val: I)) {
4347 assert(isDeadFence(A, *FI));
4348 A.deleteAfterManifest(I&: *FI);
4349 return ChangeStatus::CHANGED;
4350 }
4351 if (isAssumedSideEffectFree(A, I) && !I->isTerminator()) {
4352 A.deleteAfterManifest(I&: *I);
4353 return ChangeStatus::CHANGED;
4354 }
4355 }
4356 return ChangeStatus::UNCHANGED;
4357 }
4358
4359 /// See AbstractAttribute::trackStatistics()
4360 void trackStatistics() const override {
4361 STATS_DECLTRACK_FLOATING_ATTR(IsDead)
4362 }
4363
4364private:
4365 // The potential copies of a dead store, used for deletion during manifest.
4366 SmallSetVector<Value *, 4> PotentialCopies;
4367};
4368
4369struct AAIsDeadArgument : public AAIsDeadFloating {
4370 AAIsDeadArgument(const IRPosition &IRP, Attributor &A)
4371 : AAIsDeadFloating(IRP, A) {}
4372
4373 /// See AbstractAttribute::manifest(...).
4374 ChangeStatus manifest(Attributor &A) override {
4375 Argument &Arg = *getAssociatedArgument();
4376 if (A.isValidFunctionSignatureRewrite(Arg, /* ReplacementTypes */ {}))
4377 if (A.registerFunctionSignatureRewrite(
4378 Arg, /* ReplacementTypes */ {},
4379 CalleeRepairCB: Attributor::ArgumentReplacementInfo::CalleeRepairCBTy{},
4380 ACSRepairCB: Attributor::ArgumentReplacementInfo::ACSRepairCBTy{})) {
4381 return ChangeStatus::CHANGED;
4382 }
4383 return ChangeStatus::UNCHANGED;
4384 }
4385
4386 /// See AbstractAttribute::trackStatistics()
4387 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(IsDead) }
4388};
4389
4390struct AAIsDeadCallSiteArgument : public AAIsDeadValueImpl {
4391 AAIsDeadCallSiteArgument(const IRPosition &IRP, Attributor &A)
4392 : AAIsDeadValueImpl(IRP, A) {}
4393
4394 /// See AbstractAttribute::initialize(...).
4395 void initialize(Attributor &A) override {
4396 AAIsDeadValueImpl::initialize(A);
4397 if (isa<UndefValue>(Val: getAssociatedValue()))
4398 indicatePessimisticFixpoint();
4399 }
4400
4401 /// See AbstractAttribute::updateImpl(...).
4402 ChangeStatus updateImpl(Attributor &A) override {
4403 // TODO: Once we have call site specific value information we can provide
4404 // call site specific liveness information and then it makes
4405 // sense to specialize attributes for call sites arguments instead of
4406 // redirecting requests to the callee argument.
4407 Argument *Arg = getAssociatedArgument();
4408 if (!Arg)
4409 return indicatePessimisticFixpoint();
4410 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
4411 auto *ArgAA = A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
4412 if (!ArgAA)
4413 return indicatePessimisticFixpoint();
4414 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
4415 }
4416
4417 /// See AbstractAttribute::manifest(...).
4418 ChangeStatus manifest(Attributor &A) override {
4419 CallBase &CB = cast<CallBase>(Val&: getAnchorValue());
4420 Use &U = CB.getArgOperandUse(i: getCallSiteArgNo());
4421 assert(!isa<UndefValue>(U.get()) &&
4422 "Expected undef values to be filtered out!");
4423 UndefValue &UV = *UndefValue::get(T: U->getType());
4424 if (A.changeUseAfterManifest(U, NV&: UV))
4425 return ChangeStatus::CHANGED;
4426 return ChangeStatus::UNCHANGED;
4427 }
4428
4429 /// See AbstractAttribute::trackStatistics()
4430 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(IsDead) }
4431};
4432
4433struct AAIsDeadCallSiteReturned : public AAIsDeadFloating {
4434 AAIsDeadCallSiteReturned(const IRPosition &IRP, Attributor &A)
4435 : AAIsDeadFloating(IRP, A) {}
4436
4437 /// See AAIsDead::isAssumedDead().
4438 bool isAssumedDead() const override {
4439 return AAIsDeadFloating::isAssumedDead() && IsAssumedSideEffectFree;
4440 }
4441
4442 /// See AbstractAttribute::initialize(...).
4443 void initialize(Attributor &A) override {
4444 AAIsDeadFloating::initialize(A);
4445 if (isa<UndefValue>(Val: getAssociatedValue())) {
4446 indicatePessimisticFixpoint();
4447 return;
4448 }
4449
4450 // We track this separately as a secondary state.
4451 IsAssumedSideEffectFree = isAssumedSideEffectFree(A, I: getCtxI());
4452 }
4453
4454 /// See AbstractAttribute::updateImpl(...).
4455 ChangeStatus updateImpl(Attributor &A) override {
4456 ChangeStatus Changed = ChangeStatus::UNCHANGED;
4457 if (IsAssumedSideEffectFree && !isAssumedSideEffectFree(A, I: getCtxI())) {
4458 IsAssumedSideEffectFree = false;
4459 Changed = ChangeStatus::CHANGED;
4460 }
4461 if (!areAllUsesAssumedDead(A, V&: getAssociatedValue()))
4462 return indicatePessimisticFixpoint();
4463 return Changed;
4464 }
4465
4466 /// See AbstractAttribute::trackStatistics()
4467 void trackStatistics() const override {
4468 if (IsAssumedSideEffectFree)
4469 STATS_DECLTRACK_CSRET_ATTR(IsDead)
4470 else
4471 STATS_DECLTRACK_CSRET_ATTR(UnusedResult)
4472 }
4473
4474 /// See AbstractAttribute::getAsStr().
4475 const std::string getAsStr(Attributor *A) const override {
4476 return isAssumedDead()
4477 ? "assumed-dead"
4478 : (getAssumed() ? "assumed-dead-users" : "assumed-live");
4479 }
4480
4481private:
4482 bool IsAssumedSideEffectFree = true;
4483};
4484
4485struct AAIsDeadReturned : public AAIsDeadValueImpl {
4486 AAIsDeadReturned(const IRPosition &IRP, Attributor &A)
4487 : AAIsDeadValueImpl(IRP, A) {}
4488
4489 /// See AbstractAttribute::updateImpl(...).
4490 ChangeStatus updateImpl(Attributor &A) override {
4491
4492 bool UsedAssumedInformation = false;
4493 A.checkForAllInstructions(Pred: [](Instruction &) { return true; }, QueryingAA: *this,
4494 Opcodes: {Instruction::Ret}, UsedAssumedInformation);
4495
4496 auto PredForCallSite = [&](AbstractCallSite ACS) {
4497 if (ACS.isCallbackCall() || !ACS.getInstruction())
4498 return false;
4499 return areAllUsesAssumedDead(A, V&: *ACS.getInstruction());
4500 };
4501
4502 if (!A.checkForAllCallSites(Pred: PredForCallSite, QueryingAA: *this, RequireAllCallSites: true,
4503 UsedAssumedInformation))
4504 return indicatePessimisticFixpoint();
4505
4506 return ChangeStatus::UNCHANGED;
4507 }
4508
4509 /// See AbstractAttribute::manifest(...).
4510 ChangeStatus manifest(Attributor &A) override {
4511 // TODO: Rewrite the signature to return void?
4512 bool AnyChange = false;
4513 UndefValue &UV = *UndefValue::get(T: getAssociatedFunction()->getReturnType());
4514 auto RetInstPred = [&](Instruction &I) {
4515 ReturnInst &RI = cast<ReturnInst>(Val&: I);
4516 if (!isa<UndefValue>(Val: RI.getReturnValue()))
4517 AnyChange |= A.changeUseAfterManifest(U&: RI.getOperandUse(i: 0), NV&: UV);
4518 return true;
4519 };
4520 bool UsedAssumedInformation = false;
4521 A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
4522 UsedAssumedInformation);
4523 return AnyChange ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
4524 }
4525
4526 /// See AbstractAttribute::trackStatistics()
4527 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(IsDead) }
4528};
4529
4530struct AAIsDeadFunction : public AAIsDead {
4531 AAIsDeadFunction(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4532
4533 /// See AbstractAttribute::initialize(...).
4534 void initialize(Attributor &A) override {
4535 Function *F = getAnchorScope();
4536 assert(F && "Did expect an anchor function");
4537 if (!isAssumedDeadInternalFunction(A)) {
4538 ToBeExploredFrom.insert(X: &F->getEntryBlock().front());
4539 assumeLive(A, BB: F->getEntryBlock());
4540 }
4541 }
4542
4543 bool isAssumedDeadInternalFunction(Attributor &A) {
4544 if (!getAnchorScope()->hasLocalLinkage())
4545 return false;
4546 bool UsedAssumedInformation = false;
4547 return A.checkForAllCallSites(Pred: [](AbstractCallSite) { return false; }, QueryingAA: *this,
4548 RequireAllCallSites: true, UsedAssumedInformation);
4549 }
4550
4551 /// See AbstractAttribute::getAsStr().
4552 const std::string getAsStr(Attributor *A) const override {
4553 return "Live[#BB " + std::to_string(val: AssumedLiveBlocks.size()) + "/" +
4554 std::to_string(val: getAnchorScope()->size()) + "][#TBEP " +
4555 std::to_string(val: ToBeExploredFrom.size()) + "][#KDE " +
4556 std::to_string(val: KnownDeadEnds.size()) + "]";
4557 }
4558
4559 /// See AbstractAttribute::manifest(...).
4560 ChangeStatus manifest(Attributor &A) override {
4561 assert(getState().isValidState() &&
4562 "Attempted to manifest an invalid state!");
4563
4564 ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
4565 Function &F = *getAnchorScope();
4566
4567 if (AssumedLiveBlocks.empty()) {
4568 A.deleteAfterManifest(F);
4569 return ChangeStatus::CHANGED;
4570 }
4571
4572 // Flag to determine if we can change an invoke to a call assuming the
4573 // callee is nounwind. This is not possible if the personality of the
4574 // function allows to catch asynchronous exceptions.
4575 bool Invoke2CallAllowed = !mayCatchAsynchronousExceptions(F);
4576
4577 KnownDeadEnds.set_union(ToBeExploredFrom);
4578 for (const Instruction *DeadEndI : KnownDeadEnds) {
4579 auto *CB = dyn_cast<CallBase>(Val: DeadEndI);
4580 if (!CB)
4581 continue;
4582 bool IsKnownNoReturn;
4583 bool MayReturn = !AA::hasAssumedIRAttr<Attribute::NoReturn>(
4584 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL,
4585 IsKnown&: IsKnownNoReturn);
4586 if (MayReturn && (!Invoke2CallAllowed || !isa<InvokeInst>(Val: CB)))
4587 continue;
4588
4589 if (auto *II = dyn_cast<InvokeInst>(Val: DeadEndI))
4590 A.registerInvokeWithDeadSuccessor(II&: const_cast<InvokeInst &>(*II));
4591 else
4592 A.changeToUnreachableAfterManifest(
4593 I: const_cast<Instruction *>(DeadEndI->getNextNode()));
4594 HasChanged = ChangeStatus::CHANGED;
4595 }
4596
4597 STATS_DECL(AAIsDead, BasicBlock, "Number of dead basic blocks deleted.");
4598 for (BasicBlock &BB : F)
4599 if (!AssumedLiveBlocks.count(V: &BB)) {
4600 A.deleteAfterManifest(BB);
4601 ++BUILD_STAT_NAME(AAIsDead, BasicBlock);
4602 HasChanged = ChangeStatus::CHANGED;
4603 }
4604
4605 return HasChanged;
4606 }
4607
4608 /// See AbstractAttribute::updateImpl(...).
4609 ChangeStatus updateImpl(Attributor &A) override;
4610
4611 bool isEdgeDead(const BasicBlock *From, const BasicBlock *To) const override {
4612 assert(From->getParent() == getAnchorScope() &&
4613 To->getParent() == getAnchorScope() &&
4614 "Used AAIsDead of the wrong function");
4615 return isValidState() && !AssumedLiveEdges.count(V: std::make_pair(x&: From, y&: To));
4616 }
4617
4618 /// See AbstractAttribute::trackStatistics()
4619 void trackStatistics() const override {}
4620
4621 /// Returns true if the function is assumed dead.
4622 bool isAssumedDead() const override { return false; }
4623
4624 /// See AAIsDead::isKnownDead().
4625 bool isKnownDead() const override { return false; }
4626
4627 /// See AAIsDead::isAssumedDead(BasicBlock *).
4628 bool isAssumedDead(const BasicBlock *BB) const override {
4629 assert(BB->getParent() == getAnchorScope() &&
4630 "BB must be in the same anchor scope function.");
4631
4632 if (!getAssumed())
4633 return false;
4634 return !AssumedLiveBlocks.count(V: BB);
4635 }
4636
4637 /// See AAIsDead::isKnownDead(BasicBlock *).
4638 bool isKnownDead(const BasicBlock *BB) const override {
4639 return getKnown() && isAssumedDead(BB);
4640 }
4641
4642 /// See AAIsDead::isAssumed(Instruction *I).
4643 bool isAssumedDead(const Instruction *I) const override {
4644 assert(I->getParent()->getParent() == getAnchorScope() &&
4645 "Instruction must be in the same anchor scope function.");
4646
4647 if (!getAssumed())
4648 return false;
4649
4650 // If it is not in AssumedLiveBlocks then it for sure dead.
4651 // Otherwise, it can still be after noreturn call in a live block.
4652 if (!AssumedLiveBlocks.count(V: I->getParent()))
4653 return true;
4654
4655 // If it is not after a liveness barrier it is live.
4656 const Instruction *PrevI = I->getPrevNode();
4657 while (PrevI) {
4658 if (KnownDeadEnds.count(key: PrevI) || ToBeExploredFrom.count(key: PrevI))
4659 return true;
4660 PrevI = PrevI->getPrevNode();
4661 }
4662 return false;
4663 }
4664
4665 /// See AAIsDead::isKnownDead(Instruction *I).
4666 bool isKnownDead(const Instruction *I) const override {
4667 return getKnown() && isAssumedDead(I);
4668 }
4669
4670 /// Assume \p BB is (partially) live now and indicate to the Attributor \p A
4671 /// that internal function called from \p BB should now be looked at.
4672 bool assumeLive(Attributor &A, const BasicBlock &BB) {
4673 if (!AssumedLiveBlocks.insert(V: &BB).second)
4674 return false;
4675
4676 if (!A.isDuringDeduction())
4677 return true;
4678
4679 // We assume that all of BB is (probably) live now and if there are calls to
4680 // internal functions we will assume that those are now live as well. This
4681 // is a performance optimization for blocks with calls to a lot of internal
4682 // functions. It can however cause dead functions to be treated as live.
4683 for (const Instruction &I : BB)
4684 if (const auto *CB = dyn_cast<CallBase>(Val: &I))
4685 if (auto *F = dyn_cast_if_present<Function>(Val: CB->getCalledOperand()))
4686 if (F->hasLocalLinkage()) {
4687 LLVM_DEBUG({
4688 dbgs() << "[AAIsDead] Seeding live internal callee ";
4689 F->printAsOperand(dbgs(), /*PrintType=*/false);
4690 dbgs() << " from ";
4691 BB.getParent()->printAsOperand(dbgs(), /*PrintType=*/false);
4692 dbgs() << "\n";
4693 });
4694 A.markLiveInternalFunction(F: *F);
4695 }
4696 return true;
4697 }
4698
4699 /// Collection of instructions that need to be explored again, e.g., we
4700 /// did assume they do not transfer control to (one of their) successors.
4701 SmallSetVector<const Instruction *, 8> ToBeExploredFrom;
4702
4703 /// Collection of instructions that are known to not transfer control.
4704 SmallSetVector<const Instruction *, 8> KnownDeadEnds;
4705
4706 /// Collection of all assumed live edges
4707 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> AssumedLiveEdges;
4708
4709 /// Collection of all assumed live BasicBlocks.
4710 DenseSet<const BasicBlock *> AssumedLiveBlocks;
4711};
4712
4713static bool
4714identifyAliveSuccessors(Attributor &A, const CallBase &CB,
4715 AbstractAttribute &AA,
4716 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4717 const IRPosition &IPos = IRPosition::callsite_function(CB);
4718
4719 bool IsKnownNoReturn;
4720 if (AA::hasAssumedIRAttr<Attribute::NoReturn>(
4721 A, QueryingAA: &AA, IRP: IPos, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoReturn))
4722 return !IsKnownNoReturn;
4723 if (CB.isTerminator())
4724 AliveSuccessors.push_back(Elt: &CB.getSuccessor(Idx: 0)->front());
4725 else
4726 AliveSuccessors.push_back(Elt: CB.getNextNode());
4727 return false;
4728}
4729
4730static bool
4731identifyAliveSuccessors(Attributor &A, const InvokeInst &II,
4732 AbstractAttribute &AA,
4733 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4734 bool UsedAssumedInformation =
4735 identifyAliveSuccessors(A, CB: cast<CallBase>(Val: II), AA, AliveSuccessors);
4736
4737 // First, determine if we can change an invoke to a call assuming the
4738 // callee is nounwind. This is not possible if the personality of the
4739 // function allows to catch asynchronous exceptions.
4740 if (AAIsDeadFunction::mayCatchAsynchronousExceptions(F: *II.getFunction())) {
4741 AliveSuccessors.push_back(Elt: &II.getUnwindDest()->front());
4742 } else {
4743 const IRPosition &IPos = IRPosition::callsite_function(CB: II);
4744
4745 bool IsKnownNoUnwind;
4746 if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
4747 A, QueryingAA: &AA, IRP: IPos, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoUnwind)) {
4748 UsedAssumedInformation |= !IsKnownNoUnwind;
4749 } else {
4750 AliveSuccessors.push_back(Elt: &II.getUnwindDest()->front());
4751 }
4752 }
4753 return UsedAssumedInformation;
4754}
4755
4756static bool
4757identifyAliveSuccessors(Attributor &, const UncondBrInst &BI,
4758 AbstractAttribute &,
4759 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4760 AliveSuccessors.push_back(Elt: &BI.getSuccessor()->front());
4761 return false;
4762}
4763
4764static bool
4765identifyAliveSuccessors(Attributor &A, const CondBrInst &BI,
4766 AbstractAttribute &AA,
4767 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4768 bool UsedAssumedInformation = false;
4769 std::optional<Constant *> C =
4770 A.getAssumedConstant(V: *BI.getCondition(), AA, UsedAssumedInformation);
4771 if (!C || isa_and_nonnull<UndefValue>(Val: *C)) {
4772 // No value yet, assume both edges are dead.
4773 } else if (isa_and_nonnull<ConstantInt>(Val: *C)) {
4774 const BasicBlock *SuccBB =
4775 BI.getSuccessor(i: 1 - cast<ConstantInt>(Val: *C)->getValue().getZExtValue());
4776 AliveSuccessors.push_back(Elt: &SuccBB->front());
4777 } else {
4778 AliveSuccessors.push_back(Elt: &BI.getSuccessor(i: 0)->front());
4779 AliveSuccessors.push_back(Elt: &BI.getSuccessor(i: 1)->front());
4780 UsedAssumedInformation = false;
4781 }
4782 return UsedAssumedInformation;
4783}
4784
4785static bool
4786identifyAliveSuccessors(Attributor &A, const SwitchInst &SI,
4787 AbstractAttribute &AA,
4788 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4789 bool UsedAssumedInformation = false;
4790 SmallVector<AA::ValueAndContext> Values;
4791 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *SI.getCondition()), AA: &AA,
4792 Values, S: AA::AnyScope,
4793 UsedAssumedInformation)) {
4794 // Something went wrong, assume all successors are live.
4795 for (const BasicBlock *SuccBB : successors(BB: SI.getParent()))
4796 AliveSuccessors.push_back(Elt: &SuccBB->front());
4797 return false;
4798 }
4799
4800 if (Values.empty() ||
4801 (Values.size() == 1 &&
4802 isa_and_nonnull<UndefValue>(Val: Values.front().getValue()))) {
4803 // No valid value yet, assume all edges are dead.
4804 return UsedAssumedInformation;
4805 }
4806
4807 Type &Ty = *SI.getCondition()->getType();
4808 SmallPtrSet<ConstantInt *, 8> Constants;
4809 auto CheckForConstantInt = [&](Value *V) {
4810 if (auto *CI = dyn_cast_if_present<ConstantInt>(Val: AA::getWithType(V&: *V, Ty))) {
4811 Constants.insert(Ptr: CI);
4812 return true;
4813 }
4814 return false;
4815 };
4816
4817 if (!all_of(Range&: Values, P: [&](AA::ValueAndContext &VAC) {
4818 return CheckForConstantInt(VAC.getValue());
4819 })) {
4820 for (const BasicBlock *SuccBB : successors(BB: SI.getParent()))
4821 AliveSuccessors.push_back(Elt: &SuccBB->front());
4822 return UsedAssumedInformation;
4823 }
4824
4825 unsigned MatchedCases = 0;
4826 for (const auto &CaseIt : SI.cases()) {
4827 if (Constants.count(Ptr: CaseIt.getCaseValue())) {
4828 ++MatchedCases;
4829 AliveSuccessors.push_back(Elt: &CaseIt.getCaseSuccessor()->front());
4830 }
4831 }
4832
4833 // If all potential values have been matched, we will not visit the default
4834 // case.
4835 if (MatchedCases < Constants.size())
4836 AliveSuccessors.push_back(Elt: &SI.getDefaultDest()->front());
4837 return UsedAssumedInformation;
4838}
4839
4840ChangeStatus AAIsDeadFunction::updateImpl(Attributor &A) {
4841 ChangeStatus Change = ChangeStatus::UNCHANGED;
4842
4843 if (AssumedLiveBlocks.empty()) {
4844 if (isAssumedDeadInternalFunction(A))
4845 return ChangeStatus::UNCHANGED;
4846
4847 Function *F = getAnchorScope();
4848 ToBeExploredFrom.insert(X: &F->getEntryBlock().front());
4849 assumeLive(A, BB: F->getEntryBlock());
4850 Change = ChangeStatus::CHANGED;
4851 }
4852
4853 LLVM_DEBUG(dbgs() << "[AAIsDead] Live [" << AssumedLiveBlocks.size() << "/"
4854 << getAnchorScope()->size() << "] BBs and "
4855 << ToBeExploredFrom.size() << " exploration points and "
4856 << KnownDeadEnds.size() << " known dead ends\n");
4857
4858 // Copy and clear the list of instructions we need to explore from. It is
4859 // refilled with instructions the next update has to look at.
4860 SmallVector<const Instruction *, 8> Worklist(ToBeExploredFrom.begin(),
4861 ToBeExploredFrom.end());
4862 decltype(ToBeExploredFrom) NewToBeExploredFrom;
4863
4864 SmallVector<const Instruction *, 8> AliveSuccessors;
4865 while (!Worklist.empty()) {
4866 const Instruction *I = Worklist.pop_back_val();
4867 LLVM_DEBUG(dbgs() << "[AAIsDead] Exploration inst: " << *I << "\n");
4868
4869 // Fast forward for uninteresting instructions. We could look for UB here
4870 // though.
4871 while (!I->isTerminator() && !isa<CallBase>(Val: I))
4872 I = I->getNextNode();
4873
4874 AliveSuccessors.clear();
4875
4876 bool UsedAssumedInformation = false;
4877 switch (I->getOpcode()) {
4878 // TODO: look for (assumed) UB to backwards propagate "deadness".
4879 default:
4880 assert(I->isTerminator() &&
4881 "Expected non-terminators to be handled already!");
4882 for (const BasicBlock *SuccBB : successors(BB: I->getParent()))
4883 AliveSuccessors.push_back(Elt: &SuccBB->front());
4884 break;
4885 case Instruction::Call:
4886 UsedAssumedInformation = identifyAliveSuccessors(A, CB: cast<CallInst>(Val: *I),
4887 AA&: *this, AliveSuccessors);
4888 break;
4889 case Instruction::Invoke:
4890 UsedAssumedInformation = identifyAliveSuccessors(A, II: cast<InvokeInst>(Val: *I),
4891 AA&: *this, AliveSuccessors);
4892 break;
4893 case Instruction::UncondBr:
4894 UsedAssumedInformation = identifyAliveSuccessors(
4895 A, BI: cast<UncondBrInst>(Val: *I), *this, AliveSuccessors);
4896 break;
4897 case Instruction::CondBr:
4898 UsedAssumedInformation = identifyAliveSuccessors(A, BI: cast<CondBrInst>(Val: *I),
4899 AA&: *this, AliveSuccessors);
4900 break;
4901 case Instruction::Switch:
4902 UsedAssumedInformation = identifyAliveSuccessors(A, SI: cast<SwitchInst>(Val: *I),
4903 AA&: *this, AliveSuccessors);
4904 break;
4905 }
4906
4907 if (UsedAssumedInformation) {
4908 NewToBeExploredFrom.insert(X: I);
4909 } else if (AliveSuccessors.empty() ||
4910 (I->isTerminator() &&
4911 AliveSuccessors.size() < I->getNumSuccessors())) {
4912 if (KnownDeadEnds.insert(X: I))
4913 Change = ChangeStatus::CHANGED;
4914 }
4915
4916 LLVM_DEBUG(dbgs() << "[AAIsDead] #AliveSuccessors: "
4917 << AliveSuccessors.size() << " UsedAssumedInformation: "
4918 << UsedAssumedInformation << "\n");
4919
4920 for (const Instruction *AliveSuccessor : AliveSuccessors) {
4921 if (!I->isTerminator()) {
4922 assert(AliveSuccessors.size() == 1 &&
4923 "Non-terminator expected to have a single successor!");
4924 Worklist.push_back(Elt: AliveSuccessor);
4925 } else {
4926 // record the assumed live edge
4927 auto Edge = std::make_pair(x: I->getParent(), y: AliveSuccessor->getParent());
4928 if (AssumedLiveEdges.insert(V: Edge).second)
4929 Change = ChangeStatus::CHANGED;
4930 if (assumeLive(A, BB: *AliveSuccessor->getParent()))
4931 Worklist.push_back(Elt: AliveSuccessor);
4932 }
4933 }
4934 }
4935
4936 // Check if the content of ToBeExploredFrom changed, ignore the order.
4937 if (NewToBeExploredFrom.size() != ToBeExploredFrom.size() ||
4938 llvm::any_of(Range&: NewToBeExploredFrom, P: [&](const Instruction *I) {
4939 return !ToBeExploredFrom.count(key: I);
4940 })) {
4941 Change = ChangeStatus::CHANGED;
4942 ToBeExploredFrom = std::move(NewToBeExploredFrom);
4943 }
4944
4945 // If we know everything is live there is no need to query for liveness.
4946 // Instead, indicating a pessimistic fixpoint will cause the state to be
4947 // "invalid" and all queries to be answered conservatively without lookups.
4948 // To be in this state we have to (1) finished the exploration and (3) not
4949 // discovered any non-trivial dead end and (2) not ruled unreachable code
4950 // dead.
4951 if (ToBeExploredFrom.empty() &&
4952 getAnchorScope()->size() == AssumedLiveBlocks.size() &&
4953 llvm::all_of(Range&: KnownDeadEnds, P: [](const Instruction *DeadEndI) {
4954 return DeadEndI->isTerminator() && DeadEndI->getNumSuccessors() == 0;
4955 }))
4956 return indicatePessimisticFixpoint();
4957 return Change;
4958}
4959
4960/// Liveness information for a call sites.
4961struct AAIsDeadCallSite final : AAIsDeadFunction {
4962 AAIsDeadCallSite(const IRPosition &IRP, Attributor &A)
4963 : AAIsDeadFunction(IRP, A) {}
4964
4965 /// See AbstractAttribute::initialize(...).
4966 void initialize(Attributor &A) override {
4967 // TODO: Once we have call site specific value information we can provide
4968 // call site specific liveness information and then it makes
4969 // sense to specialize attributes for call sites instead of
4970 // redirecting requests to the callee.
4971 llvm_unreachable("Abstract attributes for liveness are not "
4972 "supported for call sites yet!");
4973 }
4974
4975 /// See AbstractAttribute::updateImpl(...).
4976 ChangeStatus updateImpl(Attributor &A) override {
4977 return indicatePessimisticFixpoint();
4978 }
4979
4980 /// See AbstractAttribute::trackStatistics()
4981 void trackStatistics() const override {}
4982};
4983} // namespace
4984
4985/// -------------------- Dereferenceable Argument Attribute --------------------
4986
4987namespace {
4988struct AADereferenceableImpl : AADereferenceable {
4989 AADereferenceableImpl(const IRPosition &IRP, Attributor &A)
4990 : AADereferenceable(IRP, A) {}
4991 using StateType = DerefState;
4992
4993 /// See AbstractAttribute::initialize(...).
4994 void initialize(Attributor &A) override {
4995 Value &V = *getAssociatedValue().stripPointerCasts();
4996 SmallVector<Attribute, 4> Attrs;
4997 A.getAttrs(IRP: getIRPosition(),
4998 AKs: {Attribute::Dereferenceable, Attribute::DereferenceableOrNull},
4999 Attrs, /* IgnoreSubsumingPositions */ false);
5000 for (const Attribute &Attr : Attrs)
5001 takeKnownDerefBytesMaximum(Bytes: Attr.getValueAsInt());
5002
5003 // Ensure we initialize the non-null AA (if necessary).
5004 bool IsKnownNonNull;
5005 AA::hasAssumedIRAttr<Attribute::NonNull>(
5006 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNonNull);
5007
5008 bool CanBeNull;
5009 takeKnownDerefBytesMaximum(Bytes: V.getPointerDereferenceableBytes(
5010 DL: A.getDataLayout(), CanBeNull, /*CanBeFreed=*/nullptr));
5011
5012 if (Instruction *CtxI = getCtxI())
5013 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
5014 }
5015
5016 /// See AbstractAttribute::getState()
5017 /// {
5018 StateType &getState() override { return *this; }
5019 const StateType &getState() const override { return *this; }
5020 /// }
5021
5022 /// Helper function for collecting accessed bytes in must-be-executed-context
5023 void addAccessedBytesForUse(Attributor &A, const Use *U, const Instruction *I,
5024 DerefState &State) {
5025 const Value *UseV = U->get();
5026 if (!UseV->getType()->isPointerTy())
5027 return;
5028
5029 std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(Inst: I);
5030 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() || I->isVolatile())
5031 return;
5032
5033 int64_t Offset;
5034 const Value *Base = GetPointerBaseWithConstantOffset(
5035 Ptr: Loc->Ptr, Offset, DL: A.getDataLayout(), /*AllowNonInbounds*/ true);
5036 if (Base && Base == &getAssociatedValue())
5037 State.addAccessedBytes(Offset, Size: Loc->Size.getValue());
5038 }
5039
5040 /// See followUsesInMBEC
5041 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
5042 AADereferenceable::StateType &State) {
5043 bool IsNonNull = false;
5044 bool TrackUse = false;
5045 int64_t DerefBytes = getKnownNonNullAndDerefBytesForUse(
5046 A, QueryingAA: *this, AssociatedValue&: getAssociatedValue(), U, I, IsNonNull, TrackUse);
5047 LLVM_DEBUG(dbgs() << "[AADereferenceable] Deref bytes: " << DerefBytes
5048 << " for instruction " << *I << "\n");
5049
5050 addAccessedBytesForUse(A, U, I, State);
5051 State.takeKnownDerefBytesMaximum(Bytes: DerefBytes);
5052 return TrackUse;
5053 }
5054
5055 /// See AbstractAttribute::manifest(...).
5056 ChangeStatus manifest(Attributor &A) override {
5057 ChangeStatus Change = AADereferenceable::manifest(A);
5058 bool IsKnownNonNull;
5059 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
5060 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
5061 if (IsAssumedNonNull &&
5062 A.hasAttr(IRP: getIRPosition(), AKs: Attribute::DereferenceableOrNull)) {
5063 A.removeAttrs(IRP: getIRPosition(), AttrKinds: {Attribute::DereferenceableOrNull});
5064 return ChangeStatus::CHANGED;
5065 }
5066 return Change;
5067 }
5068
5069 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5070 SmallVectorImpl<Attribute> &Attrs) const override {
5071 // TODO: Add *_globally support
5072 bool IsKnownNonNull;
5073 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
5074 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
5075 if (IsAssumedNonNull)
5076 Attrs.emplace_back(Args: Attribute::getWithDereferenceableBytes(
5077 Context&: Ctx, Bytes: getAssumedDereferenceableBytes()));
5078 else
5079 Attrs.emplace_back(Args: Attribute::getWithDereferenceableOrNullBytes(
5080 Context&: Ctx, Bytes: getAssumedDereferenceableBytes()));
5081 }
5082
5083 /// See AbstractAttribute::getAsStr().
5084 const std::string getAsStr(Attributor *A) const override {
5085 if (!getAssumedDereferenceableBytes())
5086 return "unknown-dereferenceable";
5087 bool IsKnownNonNull;
5088 bool IsAssumedNonNull = false;
5089 if (A)
5090 IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
5091 A&: *A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
5092 return std::string("dereferenceable") +
5093 (IsAssumedNonNull ? "" : "_or_null") +
5094 (isAssumedGlobal() ? "_globally" : "") + "<" +
5095 std::to_string(val: getKnownDereferenceableBytes()) + "-" +
5096 std::to_string(val: getAssumedDereferenceableBytes()) + ">" +
5097 (!A ? " [non-null is unknown]" : "");
5098 }
5099};
5100
5101/// Dereferenceable attribute for a floating value.
5102struct AADereferenceableFloating : AADereferenceableImpl {
5103 AADereferenceableFloating(const IRPosition &IRP, Attributor &A)
5104 : AADereferenceableImpl(IRP, A) {}
5105
5106 /// See AbstractAttribute::updateImpl(...).
5107 ChangeStatus updateImpl(Attributor &A) override {
5108 bool Stripped;
5109 bool UsedAssumedInformation = false;
5110 SmallVector<AA::ValueAndContext> Values;
5111 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
5112 S: AA::AnyScope, UsedAssumedInformation)) {
5113 Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
5114 Stripped = false;
5115 } else {
5116 Stripped = Values.size() != 1 ||
5117 Values.front().getValue() != &getAssociatedValue();
5118 }
5119
5120 const DataLayout &DL = A.getDataLayout();
5121 DerefState T;
5122
5123 auto VisitValueCB = [&](const Value &V) -> bool {
5124 unsigned IdxWidth =
5125 DL.getIndexSizeInBits(AS: V.getType()->getPointerAddressSpace());
5126 APInt Offset(IdxWidth, 0);
5127 const Value *Base = stripAndAccumulateOffsets(
5128 A, QueryingAA: *this, Val: &V, DL, Offset, /* GetMinOffset */ false,
5129 /* AllowNonInbounds */ true);
5130
5131 const auto *AA = A.getAAFor<AADereferenceable>(
5132 QueryingAA: *this, IRP: IRPosition::value(V: *Base), DepClass: DepClassTy::REQUIRED);
5133 int64_t DerefBytes = 0;
5134 if (!AA || (!Stripped && this == AA)) {
5135 // Use IR information if we did not strip anything.
5136 // TODO: track globally.
5137 bool CanBeNull;
5138 DerefBytes = Base->getPointerDereferenceableBytes(
5139 DL, CanBeNull, /*CanBeFreed=*/nullptr);
5140 T.GlobalState.indicatePessimisticFixpoint();
5141 } else {
5142 const DerefState &DS = AA->getState();
5143 DerefBytes = DS.DerefBytesState.getAssumed();
5144 T.GlobalState &= DS.GlobalState;
5145 }
5146
5147 // For now we do not try to "increase" dereferenceability due to negative
5148 // indices as we first have to come up with code to deal with loops and
5149 // for overflows of the dereferenceable bytes.
5150 int64_t OffsetSExt = Offset.getSExtValue();
5151 if (OffsetSExt < 0)
5152 OffsetSExt = 0;
5153
5154 T.takeAssumedDerefBytesMinimum(
5155 Bytes: std::max(a: int64_t(0), b: DerefBytes - OffsetSExt));
5156
5157 if (this == AA) {
5158 if (!Stripped) {
5159 // If nothing was stripped IR information is all we got.
5160 T.takeKnownDerefBytesMaximum(
5161 Bytes: std::max(a: int64_t(0), b: DerefBytes - OffsetSExt));
5162 T.indicatePessimisticFixpoint();
5163 } else if (OffsetSExt > 0) {
5164 // If something was stripped but there is circular reasoning we look
5165 // for the offset. If it is positive we basically decrease the
5166 // dereferenceable bytes in a circular loop now, which will simply
5167 // drive them down to the known value in a very slow way which we
5168 // can accelerate.
5169 T.indicatePessimisticFixpoint();
5170 }
5171 }
5172
5173 return T.isValidState();
5174 };
5175
5176 for (const auto &VAC : Values)
5177 if (!VisitValueCB(*VAC.getValue()))
5178 return indicatePessimisticFixpoint();
5179
5180 return clampStateAndIndicateChange(S&: getState(), R: T);
5181 }
5182
5183 /// See AbstractAttribute::trackStatistics()
5184 void trackStatistics() const override {
5185 STATS_DECLTRACK_FLOATING_ATTR(dereferenceable)
5186 }
5187};
5188
5189/// Dereferenceable attribute for a return value.
5190struct AADereferenceableReturned final
5191 : AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl> {
5192 using Base =
5193 AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl>;
5194 AADereferenceableReturned(const IRPosition &IRP, Attributor &A)
5195 : Base(IRP, A) {}
5196
5197 /// See AbstractAttribute::trackStatistics()
5198 void trackStatistics() const override {
5199 STATS_DECLTRACK_FNRET_ATTR(dereferenceable)
5200 }
5201};
5202
5203/// Dereferenceable attribute for an argument
5204struct AADereferenceableArgument final
5205 : AAArgumentFromCallSiteArguments<AADereferenceable,
5206 AADereferenceableImpl> {
5207 using Base =
5208 AAArgumentFromCallSiteArguments<AADereferenceable, AADereferenceableImpl>;
5209 AADereferenceableArgument(const IRPosition &IRP, Attributor &A)
5210 : Base(IRP, A) {}
5211
5212 /// See AbstractAttribute::trackStatistics()
5213 void trackStatistics() const override {
5214 STATS_DECLTRACK_ARG_ATTR(dereferenceable)
5215 }
5216};
5217
5218/// Dereferenceable attribute for a call site argument.
5219struct AADereferenceableCallSiteArgument final : AADereferenceableFloating {
5220 AADereferenceableCallSiteArgument(const IRPosition &IRP, Attributor &A)
5221 : AADereferenceableFloating(IRP, A) {}
5222
5223 /// See AbstractAttribute::trackStatistics()
5224 void trackStatistics() const override {
5225 STATS_DECLTRACK_CSARG_ATTR(dereferenceable)
5226 }
5227};
5228
5229/// Dereferenceable attribute deduction for a call site return value.
5230struct AADereferenceableCallSiteReturned final
5231 : AACalleeToCallSite<AADereferenceable, AADereferenceableImpl> {
5232 using Base = AACalleeToCallSite<AADereferenceable, AADereferenceableImpl>;
5233 AADereferenceableCallSiteReturned(const IRPosition &IRP, Attributor &A)
5234 : Base(IRP, A) {}
5235
5236 /// See AbstractAttribute::trackStatistics()
5237 void trackStatistics() const override {
5238 STATS_DECLTRACK_CS_ATTR(dereferenceable);
5239 }
5240};
5241} // namespace
5242
5243// ------------------------ Align Argument Attribute ------------------------
5244
5245namespace {
5246
5247static unsigned getKnownAlignForUse(Attributor &A, AAAlign &QueryingAA,
5248 Value &AssociatedValue, const Use *U,
5249 const Instruction *I, bool &TrackUse) {
5250 // We need to follow common pointer manipulation uses to the accesses they
5251 // feed into.
5252 if (isa<CastInst>(Val: I)) {
5253 // Follow all but ptr2int casts.
5254 TrackUse = !isa<PtrToIntInst>(Val: I);
5255 return 0;
5256 }
5257 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: I)) {
5258 if (GEP->hasAllConstantIndices())
5259 TrackUse = true;
5260 return 0;
5261 }
5262 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I))
5263 switch (II->getIntrinsicID()) {
5264 case Intrinsic::ptrmask: {
5265 // Is it appropriate to pull attribute in initialization?
5266 const auto *ConstVals = A.getAAFor<AAPotentialConstantValues>(
5267 QueryingAA, IRP: IRPosition::value(V: *II->getOperand(i_nocapture: 1)), DepClass: DepClassTy::NONE);
5268 const auto *AlignAA = A.getAAFor<AAAlign>(
5269 QueryingAA, IRP: IRPosition::value(V: *II), DepClass: DepClassTy::NONE);
5270 if (ConstVals && ConstVals->isValidState() && ConstVals->isAtFixpoint()) {
5271 unsigned ShiftValue = std::min(a: ConstVals->getAssumedMinTrailingZeros(),
5272 b: Value::MaxAlignmentExponent);
5273 Align ConstAlign(UINT64_C(1) << ShiftValue);
5274 if (ConstAlign >= AlignAA->getKnownAlign())
5275 return Align(1).value();
5276 }
5277 if (AlignAA)
5278 return AlignAA->getKnownAlign().value();
5279 break;
5280 }
5281 case Intrinsic::amdgcn_make_buffer_rsrc: {
5282 const auto *AlignAA = A.getAAFor<AAAlign>(
5283 QueryingAA, IRP: IRPosition::value(V: *II), DepClass: DepClassTy::NONE);
5284 if (AlignAA)
5285 return AlignAA->getKnownAlign().value();
5286 break;
5287 }
5288 default:
5289 break;
5290 }
5291
5292 MaybeAlign MA;
5293 if (const auto *CB = dyn_cast<CallBase>(Val: I)) {
5294 if (CB->isBundleOperand(U) || CB->isCallee(U))
5295 return 0;
5296
5297 unsigned ArgNo = CB->getArgOperandNo(U);
5298 IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
5299 // As long as we only use known information there is no need to track
5300 // dependences here.
5301 auto *AlignAA = A.getAAFor<AAAlign>(QueryingAA, IRP, DepClass: DepClassTy::NONE);
5302 if (AlignAA)
5303 MA = MaybeAlign(AlignAA->getKnownAlign());
5304 }
5305
5306 const DataLayout &DL = A.getDataLayout();
5307 const Value *UseV = U->get();
5308 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
5309 if (SI->getPointerOperand() == UseV)
5310 MA = SI->getAlign();
5311 } else if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
5312 if (LI->getPointerOperand() == UseV)
5313 MA = LI->getAlign();
5314 } else if (auto *AI = dyn_cast<AtomicRMWInst>(Val: I)) {
5315 if (AI->getPointerOperand() == UseV)
5316 MA = AI->getAlign();
5317 } else if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
5318 if (AI->getPointerOperand() == UseV)
5319 MA = AI->getAlign();
5320 }
5321
5322 if (!MA || *MA <= QueryingAA.getKnownAlign())
5323 return 0;
5324
5325 unsigned Alignment = MA->value();
5326 int64_t Offset;
5327
5328 if (const Value *Base = GetPointerBaseWithConstantOffset(Ptr: UseV, Offset, DL)) {
5329 if (Base == &AssociatedValue) {
5330 // BasePointerAddr + Offset = Alignment * Q for some integer Q.
5331 // So we can say that the maximum power of two which is a divisor of
5332 // gcd(Offset, Alignment) is an alignment.
5333
5334 uint32_t gcd = std::gcd(m: uint32_t(abs(x: (int32_t)Offset)), n: Alignment);
5335 Alignment = llvm::bit_floor(Value: gcd);
5336 }
5337 }
5338
5339 return Alignment;
5340}
5341
5342struct AAAlignImpl : AAAlign {
5343 AAAlignImpl(const IRPosition &IRP, Attributor &A) : AAAlign(IRP, A) {}
5344
5345 /// See AbstractAttribute::initialize(...).
5346 void initialize(Attributor &A) override {
5347 SmallVector<Attribute, 4> Attrs;
5348 A.getAttrs(IRP: getIRPosition(), AKs: {Attribute::Alignment}, Attrs);
5349 for (const Attribute &Attr : Attrs)
5350 takeKnownMaximum(Value: Attr.getValueAsInt());
5351
5352 Value &V = *getAssociatedValue().stripPointerCasts();
5353 takeKnownMaximum(Value: V.getPointerAlignment(DL: A.getDataLayout()).value());
5354
5355 if (Instruction *CtxI = getCtxI())
5356 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
5357 }
5358
5359 /// See AbstractAttribute::manifest(...).
5360 ChangeStatus manifest(Attributor &A) override {
5361 ChangeStatus InstrChanged = ChangeStatus::UNCHANGED;
5362
5363 // Check for users that allow alignment annotations.
5364 Value &AssociatedValue = getAssociatedValue();
5365 if (isa<ConstantData>(Val: AssociatedValue))
5366 return ChangeStatus::UNCHANGED;
5367
5368 for (const Use &U : AssociatedValue.uses()) {
5369 if (auto *SI = dyn_cast<StoreInst>(Val: U.getUser())) {
5370 if (SI->getPointerOperand() == &AssociatedValue)
5371 if (SI->getAlign() < getAssumedAlign()) {
5372 STATS_DECLTRACK(AAAlign, Store,
5373 "Number of times alignment added to a store");
5374 SI->setAlignment(getAssumedAlign());
5375 InstrChanged = ChangeStatus::CHANGED;
5376 }
5377 } else if (auto *LI = dyn_cast<LoadInst>(Val: U.getUser())) {
5378 if (LI->getPointerOperand() == &AssociatedValue)
5379 if (LI->getAlign() < getAssumedAlign()) {
5380 LI->setAlignment(getAssumedAlign());
5381 STATS_DECLTRACK(AAAlign, Load,
5382 "Number of times alignment added to a load");
5383 InstrChanged = ChangeStatus::CHANGED;
5384 }
5385 } else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: U.getUser())) {
5386 if (RMW->getPointerOperand() == &AssociatedValue) {
5387 if (RMW->getAlign() < getAssumedAlign()) {
5388 STATS_DECLTRACK(AAAlign, AtomicRMW,
5389 "Number of times alignment added to atomicrmw");
5390
5391 RMW->setAlignment(getAssumedAlign());
5392 InstrChanged = ChangeStatus::CHANGED;
5393 }
5394 }
5395 } else if (auto *CAS = dyn_cast<AtomicCmpXchgInst>(Val: U.getUser())) {
5396 if (CAS->getPointerOperand() == &AssociatedValue) {
5397 if (CAS->getAlign() < getAssumedAlign()) {
5398 STATS_DECLTRACK(AAAlign, AtomicCmpXchg,
5399 "Number of times alignment added to cmpxchg");
5400 CAS->setAlignment(getAssumedAlign());
5401 InstrChanged = ChangeStatus::CHANGED;
5402 }
5403 }
5404 }
5405 }
5406
5407 ChangeStatus Changed = AAAlign::manifest(A);
5408
5409 Align InheritAlign =
5410 getAssociatedValue().getPointerAlignment(DL: A.getDataLayout());
5411 if (InheritAlign >= getAssumedAlign())
5412 return InstrChanged;
5413 return Changed | InstrChanged;
5414 }
5415
5416 // TODO: Provide a helper to determine the implied ABI alignment and check in
5417 // the existing manifest method and a new one for AAAlignImpl that value
5418 // to avoid making the alignment explicit if it did not improve.
5419
5420 /// See AbstractAttribute::getDeducedAttributes
5421 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5422 SmallVectorImpl<Attribute> &Attrs) const override {
5423 if (getAssumedAlign() > 1)
5424 Attrs.emplace_back(
5425 Args: Attribute::getWithAlignment(Context&: Ctx, Alignment: Align(getAssumedAlign())));
5426 }
5427
5428 /// See followUsesInMBEC
5429 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
5430 AAAlign::StateType &State) {
5431 bool TrackUse = false;
5432
5433 unsigned int KnownAlign =
5434 getKnownAlignForUse(A, QueryingAA&: *this, AssociatedValue&: getAssociatedValue(), U, I, TrackUse);
5435 State.takeKnownMaximum(Value: KnownAlign);
5436
5437 return TrackUse;
5438 }
5439
5440 /// See AbstractAttribute::getAsStr().
5441 const std::string getAsStr(Attributor *A) const override {
5442 return "align<" + std::to_string(val: getKnownAlign().value()) + "-" +
5443 std::to_string(val: getAssumedAlign().value()) + ">";
5444 }
5445};
5446
5447/// Align attribute for a floating value.
5448struct AAAlignFloating : AAAlignImpl {
5449 AAAlignFloating(const IRPosition &IRP, Attributor &A) : AAAlignImpl(IRP, A) {}
5450
5451 /// See AbstractAttribute::updateImpl(...).
5452 ChangeStatus updateImpl(Attributor &A) override {
5453 const DataLayout &DL = A.getDataLayout();
5454
5455 bool Stripped;
5456 bool UsedAssumedInformation = false;
5457 SmallVector<AA::ValueAndContext> Values;
5458 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
5459 S: AA::AnyScope, UsedAssumedInformation)) {
5460 Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
5461 Stripped = false;
5462 } else {
5463 Stripped = Values.size() != 1 ||
5464 Values.front().getValue() != &getAssociatedValue();
5465 }
5466
5467 StateType T;
5468 auto VisitValueCB = [&](Value &V) -> bool {
5469 if (isa<UndefValue>(Val: V) || isa<ConstantPointerNull>(Val: V))
5470 return true;
5471 const auto *AA = A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V),
5472 DepClass: DepClassTy::REQUIRED);
5473 if (!AA || (!Stripped && this == AA)) {
5474 int64_t Offset;
5475 unsigned Alignment = 1;
5476 if (const Value *Base =
5477 GetPointerBaseWithConstantOffset(Ptr: &V, Offset, DL)) {
5478 // TODO: Use AAAlign for the base too.
5479 Align PA = Base->getPointerAlignment(DL);
5480 // BasePointerAddr + Offset = Alignment * Q for some integer Q.
5481 // So we can say that the maximum power of two which is a divisor of
5482 // gcd(Offset, Alignment) is an alignment.
5483
5484 uint32_t gcd =
5485 std::gcd(m: uint32_t(abs(x: (int32_t)Offset)), n: uint32_t(PA.value()));
5486 Alignment = llvm::bit_floor(Value: gcd);
5487 } else {
5488 Alignment = V.getPointerAlignment(DL).value();
5489 }
5490 // Use only IR information if we did not strip anything.
5491 T.takeKnownMaximum(Value: Alignment);
5492 T.indicatePessimisticFixpoint();
5493 } else {
5494 // Use abstract attribute information.
5495 const AAAlign::StateType &DS = AA->getState();
5496 T ^= DS;
5497 }
5498 return T.isValidState();
5499 };
5500
5501 for (const auto &VAC : Values) {
5502 if (!VisitValueCB(*VAC.getValue()))
5503 return indicatePessimisticFixpoint();
5504 }
5505
5506 // TODO: If we know we visited all incoming values, thus no are assumed
5507 // dead, we can take the known information from the state T.
5508 return clampStateAndIndicateChange(S&: getState(), R: T);
5509 }
5510
5511 /// See AbstractAttribute::trackStatistics()
5512 void trackStatistics() const override { STATS_DECLTRACK_FLOATING_ATTR(align) }
5513};
5514
5515/// Align attribute for function return value.
5516struct AAAlignReturned final
5517 : AAReturnedFromReturnedValues<AAAlign, AAAlignImpl> {
5518 using Base = AAReturnedFromReturnedValues<AAAlign, AAAlignImpl>;
5519 AAAlignReturned(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5520
5521 /// See AbstractAttribute::trackStatistics()
5522 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(aligned) }
5523};
5524
5525/// Align attribute for function argument.
5526struct AAAlignArgument final
5527 : AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl> {
5528 using Base = AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl>;
5529 AAAlignArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5530
5531 /// See AbstractAttribute::manifest(...).
5532 ChangeStatus manifest(Attributor &A) override {
5533 // If the associated argument is involved in a must-tail call we give up
5534 // because we would need to keep the argument alignments of caller and
5535 // callee in-sync. Just does not seem worth the trouble right now.
5536 if (A.getInfoCache().isInvolvedInMustTailCall(Arg: *getAssociatedArgument()))
5537 return ChangeStatus::UNCHANGED;
5538 return Base::manifest(A);
5539 }
5540
5541 /// See AbstractAttribute::trackStatistics()
5542 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(aligned) }
5543};
5544
5545struct AAAlignCallSiteArgument final : AAAlignFloating {
5546 AAAlignCallSiteArgument(const IRPosition &IRP, Attributor &A)
5547 : AAAlignFloating(IRP, A) {}
5548
5549 /// See AbstractAttribute::manifest(...).
5550 ChangeStatus manifest(Attributor &A) override {
5551 // If the associated argument is involved in a must-tail call we give up
5552 // because we would need to keep the argument alignments of caller and
5553 // callee in-sync. Just does not seem worth the trouble right now.
5554 if (Argument *Arg = getAssociatedArgument())
5555 if (A.getInfoCache().isInvolvedInMustTailCall(Arg: *Arg))
5556 return ChangeStatus::UNCHANGED;
5557 ChangeStatus Changed = AAAlignImpl::manifest(A);
5558 Align InheritAlign =
5559 getAssociatedValue().getPointerAlignment(DL: A.getDataLayout());
5560 if (InheritAlign >= getAssumedAlign())
5561 Changed = ChangeStatus::UNCHANGED;
5562 return Changed;
5563 }
5564
5565 /// See AbstractAttribute::updateImpl(Attributor &A).
5566 ChangeStatus updateImpl(Attributor &A) override {
5567 ChangeStatus Changed = AAAlignFloating::updateImpl(A);
5568 if (Argument *Arg = getAssociatedArgument()) {
5569 // We only take known information from the argument
5570 // so we do not need to track a dependence.
5571 const auto *ArgAlignAA = A.getAAFor<AAAlign>(
5572 QueryingAA: *this, IRP: IRPosition::argument(Arg: *Arg), DepClass: DepClassTy::NONE);
5573 if (ArgAlignAA)
5574 takeKnownMaximum(Value: ArgAlignAA->getKnownAlign().value());
5575 }
5576 return Changed;
5577 }
5578
5579 /// See AbstractAttribute::trackStatistics()
5580 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(aligned) }
5581};
5582
5583/// Align attribute deduction for a call site return value.
5584struct AAAlignCallSiteReturned final
5585 : AACalleeToCallSite<AAAlign, AAAlignImpl> {
5586 using Base = AACalleeToCallSite<AAAlign, AAAlignImpl>;
5587 AAAlignCallSiteReturned(const IRPosition &IRP, Attributor &A)
5588 : Base(IRP, A) {}
5589
5590 ChangeStatus updateImpl(Attributor &A) override {
5591 Instruction *I = getIRPosition().getCtxI();
5592 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
5593 switch (II->getIntrinsicID()) {
5594 case Intrinsic::ptrmask: {
5595 Align Alignment;
5596 bool Valid = false;
5597
5598 const auto *ConstVals = A.getAAFor<AAPotentialConstantValues>(
5599 QueryingAA: *this, IRP: IRPosition::value(V: *II->getOperand(i_nocapture: 1)), DepClass: DepClassTy::REQUIRED);
5600 if (ConstVals && ConstVals->isValidState()) {
5601 unsigned ShiftValue =
5602 std::min(a: ConstVals->getAssumedMinTrailingZeros(),
5603 b: Value::MaxAlignmentExponent);
5604 Alignment = Align(UINT64_C(1) << ShiftValue);
5605 Valid = true;
5606 }
5607
5608 const auto *AlignAA =
5609 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *(II->getOperand(i_nocapture: 0))),
5610 DepClass: DepClassTy::REQUIRED);
5611 if (AlignAA) {
5612 Alignment = std::max(a: AlignAA->getAssumedAlign(), b: Alignment);
5613 Valid = true;
5614 }
5615
5616 if (Valid)
5617 return clampStateAndIndicateChange<StateType>(
5618 S&: this->getState(),
5619 R: std::min(a: this->getAssumedAlign(), b: Alignment).value());
5620 break;
5621 }
5622 // FIXME: Should introduce target specific sub-attributes and letting
5623 // getAAfor<AAAlign> lead to create sub-attribute to handle target
5624 // specific intrinsics.
5625 case Intrinsic::amdgcn_make_buffer_rsrc: {
5626 const auto *AlignAA =
5627 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *(II->getOperand(i_nocapture: 0))),
5628 DepClass: DepClassTy::REQUIRED);
5629 if (AlignAA)
5630 return clampStateAndIndicateChange<StateType>(
5631 S&: this->getState(), R: AlignAA->getAssumedAlign().value());
5632 break;
5633 }
5634 default:
5635 break;
5636 }
5637 }
5638 return Base::updateImpl(A);
5639 };
5640 /// See AbstractAttribute::trackStatistics()
5641 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(align); }
5642};
5643} // namespace
5644
5645/// ------------------ Function No-Return Attribute ----------------------------
5646namespace {
5647struct AANoReturnImpl : public AANoReturn {
5648 AANoReturnImpl(const IRPosition &IRP, Attributor &A) : AANoReturn(IRP, A) {}
5649
5650 /// See AbstractAttribute::initialize(...).
5651 void initialize(Attributor &A) override {
5652 bool IsKnown;
5653 assert(!AA::hasAssumedIRAttr<Attribute::NoReturn>(
5654 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5655 (void)IsKnown;
5656 }
5657
5658 /// See AbstractAttribute::getAsStr().
5659 const std::string getAsStr(Attributor *A) const override {
5660 return getAssumed() ? "noreturn" : "may-return";
5661 }
5662
5663 /// See AbstractAttribute::updateImpl(Attributor &A).
5664 ChangeStatus updateImpl(Attributor &A) override {
5665 auto CheckForNoReturn = [](Instruction &) { return false; };
5666 bool UsedAssumedInformation = false;
5667 if (!A.checkForAllInstructions(Pred: CheckForNoReturn, QueryingAA: *this,
5668 Opcodes: {(unsigned)Instruction::Ret},
5669 UsedAssumedInformation))
5670 return indicatePessimisticFixpoint();
5671 return ChangeStatus::UNCHANGED;
5672 }
5673};
5674
5675struct AANoReturnFunction final : AANoReturnImpl {
5676 AANoReturnFunction(const IRPosition &IRP, Attributor &A)
5677 : AANoReturnImpl(IRP, A) {}
5678
5679 /// See AbstractAttribute::trackStatistics()
5680 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(noreturn) }
5681};
5682
5683/// NoReturn attribute deduction for a call sites.
5684struct AANoReturnCallSite final
5685 : AACalleeToCallSite<AANoReturn, AANoReturnImpl> {
5686 AANoReturnCallSite(const IRPosition &IRP, Attributor &A)
5687 : AACalleeToCallSite<AANoReturn, AANoReturnImpl>(IRP, A) {}
5688
5689 /// See AbstractAttribute::trackStatistics()
5690 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(noreturn); }
5691};
5692} // namespace
5693
5694/// ----------------------- Instance Info ---------------------------------
5695
5696namespace {
5697/// A class to hold the state of for no-capture attributes.
5698struct AAInstanceInfoImpl : public AAInstanceInfo {
5699 AAInstanceInfoImpl(const IRPosition &IRP, Attributor &A)
5700 : AAInstanceInfo(IRP, A) {}
5701
5702 /// See AbstractAttribute::initialize(...).
5703 void initialize(Attributor &A) override {
5704 Value &V = getAssociatedValue();
5705 if (auto *C = dyn_cast<Constant>(Val: &V)) {
5706 if (C->isThreadDependent())
5707 indicatePessimisticFixpoint();
5708 else
5709 indicateOptimisticFixpoint();
5710 return;
5711 }
5712 if (auto *CB = dyn_cast<CallBase>(Val: &V))
5713 if (CB->arg_size() == 0 && !CB->mayHaveSideEffects() &&
5714 !CB->mayReadFromMemory()) {
5715 indicateOptimisticFixpoint();
5716 return;
5717 }
5718 if (auto *I = dyn_cast<Instruction>(Val: &V)) {
5719 const auto *CI =
5720 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
5721 F: *I->getFunction());
5722 if (mayBeInCycle(CI, I, /* HeaderOnly */ false)) {
5723 indicatePessimisticFixpoint();
5724 return;
5725 }
5726 }
5727 }
5728
5729 /// See AbstractAttribute::updateImpl(...).
5730 ChangeStatus updateImpl(Attributor &A) override {
5731 ChangeStatus Changed = ChangeStatus::UNCHANGED;
5732
5733 Value &V = getAssociatedValue();
5734 const Function *Scope = nullptr;
5735 if (auto *I = dyn_cast<Instruction>(Val: &V))
5736 Scope = I->getFunction();
5737 if (auto *A = dyn_cast<Argument>(Val: &V)) {
5738 Scope = A->getParent();
5739 if (!Scope->hasLocalLinkage())
5740 return Changed;
5741 }
5742 if (!Scope)
5743 return indicateOptimisticFixpoint();
5744
5745 bool IsKnownNoRecurse;
5746 if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
5747 A, QueryingAA: this, IRP: IRPosition::function(F: *Scope), DepClass: DepClassTy::OPTIONAL,
5748 IsKnown&: IsKnownNoRecurse))
5749 return Changed;
5750
5751 auto UsePred = [&](const Use &U, bool &Follow) {
5752 const Instruction *UserI = dyn_cast<Instruction>(Val: U.getUser());
5753 if (!UserI || isa<GetElementPtrInst>(Val: UserI) || isa<CastInst>(Val: UserI) ||
5754 isa<PHINode>(Val: UserI) || isa<SelectInst>(Val: UserI)) {
5755 Follow = true;
5756 return true;
5757 }
5758 if (isa<LoadInst>(Val: UserI) || isa<CmpInst>(Val: UserI) ||
5759 (isa<StoreInst>(Val: UserI) &&
5760 cast<StoreInst>(Val: UserI)->getValueOperand() != U.get()))
5761 return true;
5762 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
5763 // This check is not guaranteeing uniqueness but for now that we cannot
5764 // end up with two versions of \p U thinking it was one.
5765 auto *Callee = dyn_cast_if_present<Function>(Val: CB->getCalledOperand());
5766 if (!Callee || !Callee->hasLocalLinkage())
5767 return true;
5768 if (!CB->isArgOperand(U: &U))
5769 return false;
5770 const auto *ArgInstanceInfoAA = A.getAAFor<AAInstanceInfo>(
5771 QueryingAA: *this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U)),
5772 DepClass: DepClassTy::OPTIONAL);
5773 if (!ArgInstanceInfoAA ||
5774 !ArgInstanceInfoAA->isAssumedUniqueForAnalysis())
5775 return false;
5776 // If this call base might reach the scope again we might forward the
5777 // argument back here. This is very conservative.
5778 if (AA::isPotentiallyReachable(
5779 A, FromI: *CB, ToFn: *Scope, QueryingAA: *this, /* ExclusionSet */ nullptr,
5780 GoBackwardsCB: [Scope](const Function &Fn) { return &Fn != Scope; }))
5781 return false;
5782 return true;
5783 }
5784 return false;
5785 };
5786
5787 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
5788 if (auto *SI = dyn_cast<StoreInst>(Val: OldU.getUser())) {
5789 auto *Ptr = SI->getPointerOperand()->stripPointerCasts();
5790 if ((isa<AllocaInst>(Val: Ptr) || isNoAliasCall(V: Ptr)) &&
5791 AA::isDynamicallyUnique(A, QueryingAA: *this, V: *Ptr))
5792 return true;
5793 }
5794 return false;
5795 };
5796
5797 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V, /* CheckBBLivenessOnly */ true,
5798 LivenessDepClass: DepClassTy::OPTIONAL,
5799 /* IgnoreDroppableUses */ true, EquivalentUseCB))
5800 return indicatePessimisticFixpoint();
5801
5802 return Changed;
5803 }
5804
5805 /// See AbstractState::getAsStr().
5806 const std::string getAsStr(Attributor *A) const override {
5807 return isAssumedUniqueForAnalysis() ? "<unique [fAa]>" : "<unknown>";
5808 }
5809
5810 /// See AbstractAttribute::trackStatistics()
5811 void trackStatistics() const override {}
5812};
5813
5814/// InstanceInfo attribute for floating values.
5815struct AAInstanceInfoFloating : AAInstanceInfoImpl {
5816 AAInstanceInfoFloating(const IRPosition &IRP, Attributor &A)
5817 : AAInstanceInfoImpl(IRP, A) {}
5818};
5819
5820/// NoCapture attribute for function arguments.
5821struct AAInstanceInfoArgument final : AAInstanceInfoFloating {
5822 AAInstanceInfoArgument(const IRPosition &IRP, Attributor &A)
5823 : AAInstanceInfoFloating(IRP, A) {}
5824};
5825
5826/// InstanceInfo attribute for call site arguments.
5827struct AAInstanceInfoCallSiteArgument final : AAInstanceInfoImpl {
5828 AAInstanceInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
5829 : AAInstanceInfoImpl(IRP, A) {}
5830
5831 /// See AbstractAttribute::updateImpl(...).
5832 ChangeStatus updateImpl(Attributor &A) override {
5833 // TODO: Once we have call site specific value information we can provide
5834 // call site specific liveness information and then it makes
5835 // sense to specialize attributes for call sites arguments instead of
5836 // redirecting requests to the callee argument.
5837 Argument *Arg = getAssociatedArgument();
5838 if (!Arg)
5839 return indicatePessimisticFixpoint();
5840 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
5841 auto *ArgAA =
5842 A.getAAFor<AAInstanceInfo>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
5843 if (!ArgAA)
5844 return indicatePessimisticFixpoint();
5845 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
5846 }
5847};
5848
5849/// InstanceInfo attribute for function return value.
5850struct AAInstanceInfoReturned final : AAInstanceInfoImpl {
5851 AAInstanceInfoReturned(const IRPosition &IRP, Attributor &A)
5852 : AAInstanceInfoImpl(IRP, A) {
5853 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5854 }
5855
5856 /// See AbstractAttribute::initialize(...).
5857 void initialize(Attributor &A) override {
5858 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5859 }
5860
5861 /// See AbstractAttribute::updateImpl(...).
5862 ChangeStatus updateImpl(Attributor &A) override {
5863 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5864 }
5865};
5866
5867/// InstanceInfo attribute deduction for a call site return value.
5868struct AAInstanceInfoCallSiteReturned final : AAInstanceInfoFloating {
5869 AAInstanceInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
5870 : AAInstanceInfoFloating(IRP, A) {}
5871};
5872} // namespace
5873
5874/// ----------------------- Variable Capturing ---------------------------------
5875bool AANoCapture::isImpliedByIR(Attributor &A, const IRPosition &IRP,
5876 Attribute::AttrKind ImpliedAttributeKind,
5877 bool IgnoreSubsumingPositions) {
5878 assert(ImpliedAttributeKind == Attribute::Captures &&
5879 "Unexpected attribute kind");
5880 Value &V = IRP.getAssociatedValue();
5881 if (!isa<Constant>(Val: V) && !IRP.isArgumentPosition())
5882 return V.use_empty();
5883
5884 // You cannot "capture" null in the default address space.
5885 //
5886 // FIXME: This should use NullPointerIsDefined to account for the function
5887 // attribute.
5888 if (isa<UndefValue>(Val: V) || (isa<ConstantPointerNull>(Val: V) &&
5889 V.getType()->getPointerAddressSpace() == 0)) {
5890 return true;
5891 }
5892
5893 SmallVector<Attribute, 1> Attrs;
5894 A.getAttrs(IRP, AKs: {Attribute::Captures}, Attrs,
5895 /* IgnoreSubsumingPositions */ true);
5896 for (const Attribute &Attr : Attrs)
5897 if (capturesNothing(CC: Attr.getCaptureInfo()))
5898 return true;
5899
5900 if (IRP.getPositionKind() == IRP_CALL_SITE_ARGUMENT)
5901 if (Argument *Arg = IRP.getAssociatedArgument()) {
5902 SmallVector<Attribute, 1> Attrs;
5903 A.getAttrs(IRP: IRPosition::argument(Arg: *Arg),
5904 AKs: {Attribute::Captures, Attribute::ByVal}, Attrs,
5905 /* IgnoreSubsumingPositions */ true);
5906 bool ArgNoCapture = any_of(Range&: Attrs, P: [](Attribute Attr) {
5907 return Attr.getKindAsEnum() == Attribute::ByVal ||
5908 capturesNothing(CC: Attr.getCaptureInfo());
5909 });
5910 if (ArgNoCapture) {
5911 A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithCaptureInfo(
5912 Context&: V.getContext(), CI: CaptureInfo::none()));
5913 return true;
5914 }
5915 }
5916
5917 if (const Function *F = IRP.getAssociatedFunction()) {
5918 // Check what state the associated function can actually capture.
5919 AANoCapture::StateType State;
5920 determineFunctionCaptureCapabilities(IRP, F: *F, State);
5921 if (State.isKnown(BitsEncoding: NO_CAPTURE)) {
5922 A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithCaptureInfo(Context&: V.getContext(),
5923 CI: CaptureInfo::none()));
5924 return true;
5925 }
5926 }
5927
5928 return false;
5929}
5930
5931/// Set the NOT_CAPTURED_IN_MEM and NOT_CAPTURED_IN_RET bits in \p Known
5932/// depending on the ability of the function associated with \p IRP to capture
5933/// state in memory and through "returning/throwing", respectively.
5934void AANoCapture::determineFunctionCaptureCapabilities(const IRPosition &IRP,
5935 const Function &F,
5936 BitIntegerState &State) {
5937 // TODO: Once we have memory behavior attributes we should use them here.
5938
5939 // If we know we cannot communicate or write to memory, we do not care about
5940 // ptr2int anymore.
5941 bool ReadOnly = F.onlyReadsMemory();
5942 bool NoThrow = F.doesNotThrow();
5943 bool IsVoidReturn = F.getReturnType()->isVoidTy();
5944 if (ReadOnly && NoThrow && IsVoidReturn) {
5945 State.addKnownBits(Bits: NO_CAPTURE);
5946 return;
5947 }
5948
5949 // A function cannot capture state in memory if it only reads memory, it can
5950 // however return/throw state and the state might be influenced by the
5951 // pointer value, e.g., loading from a returned pointer might reveal a bit.
5952 if (ReadOnly)
5953 State.addKnownBits(Bits: NOT_CAPTURED_IN_MEM);
5954
5955 // A function cannot communicate state back if it does not through
5956 // exceptions and doesn not return values.
5957 if (NoThrow && IsVoidReturn)
5958 State.addKnownBits(Bits: NOT_CAPTURED_IN_RET);
5959
5960 // Check existing "returned" attributes.
5961 int ArgNo = IRP.getCalleeArgNo();
5962 if (!NoThrow || ArgNo < 0 ||
5963 !F.getAttributes().hasAttrSomewhere(Kind: Attribute::Returned))
5964 return;
5965
5966 for (unsigned U = 0, E = F.arg_size(); U < E; ++U)
5967 if (F.hasParamAttribute(ArgNo: U, Kind: Attribute::Returned)) {
5968 if (U == unsigned(ArgNo))
5969 State.removeAssumedBits(BitsEncoding: NOT_CAPTURED_IN_RET);
5970 else if (ReadOnly)
5971 State.addKnownBits(Bits: NO_CAPTURE);
5972 else
5973 State.addKnownBits(Bits: NOT_CAPTURED_IN_RET);
5974 break;
5975 }
5976}
5977
5978namespace {
5979/// A class to hold the state of for no-capture attributes.
5980struct AANoCaptureImpl : public AANoCapture {
5981 AANoCaptureImpl(const IRPosition &IRP, Attributor &A) : AANoCapture(IRP, A) {}
5982
5983 /// See AbstractAttribute::initialize(...).
5984 void initialize(Attributor &A) override {
5985 bool IsKnown;
5986 assert(!AA::hasAssumedIRAttr<Attribute::Captures>(
5987 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5988 (void)IsKnown;
5989 }
5990
5991 /// See AbstractAttribute::updateImpl(...).
5992 ChangeStatus updateImpl(Attributor &A) override;
5993
5994 /// see AbstractAttribute::isAssumedNoCaptureMaybeReturned(...).
5995 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5996 SmallVectorImpl<Attribute> &Attrs) const override {
5997 if (!isAssumedNoCaptureMaybeReturned())
5998 return;
5999
6000 if (isArgumentPosition()) {
6001 if (isAssumedNoCapture())
6002 Attrs.emplace_back(Args: Attribute::get(Context&: Ctx, Kind: Attribute::Captures));
6003 else if (ManifestInternal)
6004 Attrs.emplace_back(Args: Attribute::get(Context&: Ctx, Kind: "no-capture-maybe-returned"));
6005 }
6006 }
6007
6008 /// See AbstractState::getAsStr().
6009 const std::string getAsStr(Attributor *A) const override {
6010 if (isKnownNoCapture())
6011 return "known not-captured";
6012 if (isAssumedNoCapture())
6013 return "assumed not-captured";
6014 if (isKnownNoCaptureMaybeReturned())
6015 return "known not-captured-maybe-returned";
6016 if (isAssumedNoCaptureMaybeReturned())
6017 return "assumed not-captured-maybe-returned";
6018 return "assumed-captured";
6019 }
6020
6021 /// Check the use \p U and update \p State accordingly. Return true if we
6022 /// should continue to update the state.
6023 bool checkUse(Attributor &A, AANoCapture::StateType &State, const Use &U,
6024 bool &Follow) {
6025 Instruction *UInst = cast<Instruction>(Val: U.getUser());
6026 LLVM_DEBUG(dbgs() << "[AANoCapture] Check use: " << *U.get() << " in "
6027 << *UInst << "\n");
6028
6029 // Deal with ptr2int by following uses.
6030 if (isa<PtrToIntInst>(Val: UInst)) {
6031 LLVM_DEBUG(dbgs() << " - ptr2int assume the worst!\n");
6032 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
6033 /* Return */ CapturedInRet: true);
6034 }
6035
6036 // For stores we already checked if we can follow them, if they make it
6037 // here we give up.
6038 if (isa<StoreInst>(Val: UInst))
6039 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
6040 /* Return */ CapturedInRet: true);
6041
6042 // Explicitly catch return instructions.
6043 if (isa<ReturnInst>(Val: UInst)) {
6044 if (UInst->getFunction() == getAnchorScope())
6045 return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
6046 /* Return */ CapturedInRet: true);
6047 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
6048 /* Return */ CapturedInRet: true);
6049 }
6050
6051 // For now we only use special logic for call sites. However, the tracker
6052 // itself knows about a lot of other non-capturing cases already.
6053 auto *CB = dyn_cast<CallBase>(Val: UInst);
6054 if (!CB || !CB->isArgOperand(U: &U))
6055 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
6056 /* Return */ CapturedInRet: true);
6057
6058 unsigned ArgNo = CB->getArgOperandNo(U: &U);
6059 const IRPosition &CSArgPos = IRPosition::callsite_argument(CB: *CB, ArgNo);
6060 // If we have a abstract no-capture attribute for the argument we can use
6061 // it to justify a non-capture attribute here. This allows recursion!
6062 bool IsKnownNoCapture;
6063 const AANoCapture *ArgNoCaptureAA = nullptr;
6064 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
6065 A, QueryingAA: this, IRP: CSArgPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
6066 AAPtr: &ArgNoCaptureAA);
6067 if (IsAssumedNoCapture)
6068 return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
6069 /* Return */ CapturedInRet: false);
6070 if (ArgNoCaptureAA && ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned()) {
6071 Follow = true;
6072 return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
6073 /* Return */ CapturedInRet: false);
6074 }
6075
6076 // Lastly, we could not find a reason no-capture can be assumed so we don't.
6077 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
6078 /* Return */ CapturedInRet: true);
6079 }
6080
6081 /// Update \p State according to \p CapturedInMem, \p CapturedInInt, and
6082 /// \p CapturedInRet, then return true if we should continue updating the
6083 /// state.
6084 static bool isCapturedIn(AANoCapture::StateType &State, bool CapturedInMem,
6085 bool CapturedInInt, bool CapturedInRet) {
6086 LLVM_DEBUG(dbgs() << " - captures [Mem " << CapturedInMem << "|Int "
6087 << CapturedInInt << "|Ret " << CapturedInRet << "]\n");
6088 if (CapturedInMem)
6089 State.removeAssumedBits(BitsEncoding: AANoCapture::NOT_CAPTURED_IN_MEM);
6090 if (CapturedInInt)
6091 State.removeAssumedBits(BitsEncoding: AANoCapture::NOT_CAPTURED_IN_INT);
6092 if (CapturedInRet)
6093 State.removeAssumedBits(BitsEncoding: AANoCapture::NOT_CAPTURED_IN_RET);
6094 return State.isAssumed(BitsEncoding: AANoCapture::NO_CAPTURE_MAYBE_RETURNED);
6095 }
6096};
6097
6098ChangeStatus AANoCaptureImpl::updateImpl(Attributor &A) {
6099 const IRPosition &IRP = getIRPosition();
6100 Value *V = isArgumentPosition() ? IRP.getAssociatedArgument()
6101 : &IRP.getAssociatedValue();
6102 if (!V)
6103 return indicatePessimisticFixpoint();
6104
6105 const Function *F =
6106 isArgumentPosition() ? IRP.getAssociatedFunction() : IRP.getAnchorScope();
6107
6108 // TODO: Is the checkForAllUses below useful for constants?
6109 if (!F)
6110 return indicatePessimisticFixpoint();
6111
6112 AANoCapture::StateType T;
6113 const IRPosition &FnPos = IRPosition::function(F: *F);
6114
6115 // Readonly means we cannot capture through memory.
6116 bool IsKnown;
6117 if (AA::isAssumedReadOnly(A, IRP: FnPos, QueryingAA: *this, IsKnown)) {
6118 T.addKnownBits(Bits: NOT_CAPTURED_IN_MEM);
6119 if (IsKnown)
6120 addKnownBits(Bits: NOT_CAPTURED_IN_MEM);
6121 }
6122
6123 // Make sure all returned values are different than the underlying value.
6124 // TODO: we could do this in a more sophisticated way inside
6125 // AAReturnedValues, e.g., track all values that escape through returns
6126 // directly somehow.
6127 auto CheckReturnedArgs = [&](bool &UsedAssumedInformation) {
6128 SmallVector<AA::ValueAndContext> Values;
6129 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::returned(F: *F), AA: this, Values,
6130 S: AA::ValueScope::Intraprocedural,
6131 UsedAssumedInformation))
6132 return false;
6133 bool SeenConstant = false;
6134 for (const AA::ValueAndContext &VAC : Values) {
6135 if (isa<Constant>(Val: VAC.getValue())) {
6136 if (SeenConstant)
6137 return false;
6138 SeenConstant = true;
6139 } else if (!isa<Argument>(Val: VAC.getValue()) ||
6140 VAC.getValue() == getAssociatedArgument())
6141 return false;
6142 }
6143 return true;
6144 };
6145
6146 bool IsKnownNoUnwind;
6147 if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
6148 A, QueryingAA: this, IRP: FnPos, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoUnwind)) {
6149 bool IsVoidTy = F->getReturnType()->isVoidTy();
6150 bool UsedAssumedInformation = false;
6151 if (IsVoidTy || CheckReturnedArgs(UsedAssumedInformation)) {
6152 T.addKnownBits(Bits: NOT_CAPTURED_IN_RET);
6153 if (T.isKnown(BitsEncoding: NOT_CAPTURED_IN_MEM))
6154 return ChangeStatus::UNCHANGED;
6155 if (IsKnownNoUnwind && (IsVoidTy || !UsedAssumedInformation)) {
6156 addKnownBits(Bits: NOT_CAPTURED_IN_RET);
6157 if (isKnown(BitsEncoding: NOT_CAPTURED_IN_MEM))
6158 return indicateOptimisticFixpoint();
6159 }
6160 }
6161 }
6162
6163 auto UseCheck = [&](const Use &U, bool &Follow) -> bool {
6164 // TODO(captures): Make this more precise.
6165 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
6166 if (capturesNothing(CC: CI))
6167 return true;
6168 if (CI.isPassthrough()) {
6169 Follow = true;
6170 return true;
6171 }
6172 return checkUse(A, State&: T, U, Follow);
6173 };
6174
6175 if (!A.checkForAllUses(Pred: UseCheck, QueryingAA: *this, V: *V))
6176 return indicatePessimisticFixpoint();
6177
6178 AANoCapture::StateType &S = getState();
6179 auto Assumed = S.getAssumed();
6180 S.intersectAssumedBits(BitsEncoding: T.getAssumed());
6181 if (!isAssumedNoCaptureMaybeReturned())
6182 return indicatePessimisticFixpoint();
6183 return Assumed == S.getAssumed() ? ChangeStatus::UNCHANGED
6184 : ChangeStatus::CHANGED;
6185}
6186
6187/// NoCapture attribute for function arguments.
6188struct AANoCaptureArgument final : AANoCaptureImpl {
6189 AANoCaptureArgument(const IRPosition &IRP, Attributor &A)
6190 : AANoCaptureImpl(IRP, A) {}
6191
6192 /// See AbstractAttribute::trackStatistics()
6193 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nocapture) }
6194};
6195
6196/// NoCapture attribute for call site arguments.
6197struct AANoCaptureCallSiteArgument final : AANoCaptureImpl {
6198 AANoCaptureCallSiteArgument(const IRPosition &IRP, Attributor &A)
6199 : AANoCaptureImpl(IRP, A) {}
6200
6201 /// See AbstractAttribute::updateImpl(...).
6202 ChangeStatus updateImpl(Attributor &A) override {
6203 // TODO: Once we have call site specific value information we can provide
6204 // call site specific liveness information and then it makes
6205 // sense to specialize attributes for call sites arguments instead of
6206 // redirecting requests to the callee argument.
6207 Argument *Arg = getAssociatedArgument();
6208 if (!Arg)
6209 return indicatePessimisticFixpoint();
6210 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
6211 bool IsKnownNoCapture;
6212 const AANoCapture *ArgAA = nullptr;
6213 if (AA::hasAssumedIRAttr<Attribute::Captures>(
6214 A, QueryingAA: this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
6215 AAPtr: &ArgAA))
6216 return ChangeStatus::UNCHANGED;
6217 if (!ArgAA || !ArgAA->isAssumedNoCaptureMaybeReturned())
6218 return indicatePessimisticFixpoint();
6219 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
6220 }
6221
6222 /// See AbstractAttribute::trackStatistics()
6223 void trackStatistics() const override {
6224 STATS_DECLTRACK_CSARG_ATTR(nocapture)
6225 };
6226};
6227
6228/// NoCapture attribute for floating values.
6229struct AANoCaptureFloating final : AANoCaptureImpl {
6230 AANoCaptureFloating(const IRPosition &IRP, Attributor &A)
6231 : AANoCaptureImpl(IRP, A) {}
6232
6233 /// See AbstractAttribute::trackStatistics()
6234 void trackStatistics() const override {
6235 STATS_DECLTRACK_FLOATING_ATTR(nocapture)
6236 }
6237};
6238
6239/// NoCapture attribute for function return value.
6240struct AANoCaptureReturned final : AANoCaptureImpl {
6241 AANoCaptureReturned(const IRPosition &IRP, Attributor &A)
6242 : AANoCaptureImpl(IRP, A) {
6243 llvm_unreachable("NoCapture is not applicable to function returns!");
6244 }
6245
6246 /// See AbstractAttribute::initialize(...).
6247 void initialize(Attributor &A) override {
6248 llvm_unreachable("NoCapture is not applicable to function returns!");
6249 }
6250
6251 /// See AbstractAttribute::updateImpl(...).
6252 ChangeStatus updateImpl(Attributor &A) override {
6253 llvm_unreachable("NoCapture is not applicable to function returns!");
6254 }
6255
6256 /// See AbstractAttribute::trackStatistics()
6257 void trackStatistics() const override {}
6258};
6259
6260/// NoCapture attribute deduction for a call site return value.
6261struct AANoCaptureCallSiteReturned final : AANoCaptureImpl {
6262 AANoCaptureCallSiteReturned(const IRPosition &IRP, Attributor &A)
6263 : AANoCaptureImpl(IRP, A) {}
6264
6265 /// See AbstractAttribute::initialize(...).
6266 void initialize(Attributor &A) override {
6267 const Function *F = getAnchorScope();
6268 // Check what state the associated function can actually capture.
6269 determineFunctionCaptureCapabilities(IRP: getIRPosition(), F: *F, State&: *this);
6270 }
6271
6272 /// See AbstractAttribute::trackStatistics()
6273 void trackStatistics() const override {
6274 STATS_DECLTRACK_CSRET_ATTR(nocapture)
6275 }
6276};
6277} // namespace
6278
6279/// ------------------ Value Simplify Attribute ----------------------------
6280
6281bool ValueSimplifyStateType::unionAssumed(std::optional<Value *> Other) {
6282 // FIXME: Add a typecast support.
6283 SimplifiedAssociatedValue = AA::combineOptionalValuesInAAValueLatice(
6284 A: SimplifiedAssociatedValue, B: Other, Ty);
6285 if (SimplifiedAssociatedValue == std::optional<Value *>(nullptr))
6286 return false;
6287
6288 LLVM_DEBUG({
6289 if (SimplifiedAssociatedValue)
6290 dbgs() << "[ValueSimplify] is assumed to be "
6291 << **SimplifiedAssociatedValue << "\n";
6292 else
6293 dbgs() << "[ValueSimplify] is assumed to be <none>\n";
6294 });
6295 return true;
6296}
6297
6298namespace {
6299struct AAValueSimplifyImpl : AAValueSimplify {
6300 AAValueSimplifyImpl(const IRPosition &IRP, Attributor &A)
6301 : AAValueSimplify(IRP, A) {}
6302
6303 /// See AbstractAttribute::initialize(...).
6304 void initialize(Attributor &A) override {
6305 if (getAssociatedValue().getType()->isVoidTy())
6306 indicatePessimisticFixpoint();
6307 if (A.hasSimplificationCallback(IRP: getIRPosition()))
6308 indicatePessimisticFixpoint();
6309 }
6310
6311 /// See AbstractAttribute::getAsStr().
6312 const std::string getAsStr(Attributor *A) const override {
6313 LLVM_DEBUG({
6314 dbgs() << "SAV: " << (bool)SimplifiedAssociatedValue << " ";
6315 if (SimplifiedAssociatedValue && *SimplifiedAssociatedValue)
6316 dbgs() << "SAV: " << **SimplifiedAssociatedValue << " ";
6317 });
6318 return isValidState() ? (isAtFixpoint() ? "simplified" : "maybe-simple")
6319 : "not-simple";
6320 }
6321
6322 /// See AbstractAttribute::trackStatistics()
6323 void trackStatistics() const override {}
6324
6325 /// See AAValueSimplify::getAssumedSimplifiedValue()
6326 std::optional<Value *>
6327 getAssumedSimplifiedValue(Attributor &A) const override {
6328 return SimplifiedAssociatedValue;
6329 }
6330
6331 /// Ensure the return value is \p V with type \p Ty, if not possible return
6332 /// nullptr. If \p Check is true we will only verify such an operation would
6333 /// suceed and return a non-nullptr value if that is the case. No IR is
6334 /// generated or modified.
6335 static Value *ensureType(Attributor &A, Value &V, Type &Ty, Instruction *CtxI,
6336 bool Check) {
6337 if (auto *TypedV = AA::getWithType(V, Ty))
6338 return TypedV;
6339 if (CtxI && V.getType()->canLosslesslyBitCastTo(Ty: &Ty))
6340 return Check ? &V
6341 : BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6342 S: &V, Ty: &Ty, Name: "", InsertBefore: CtxI->getIterator());
6343 return nullptr;
6344 }
6345
6346 /// Reproduce \p I with type \p Ty or return nullptr if that is not posisble.
6347 /// If \p Check is true we will only verify such an operation would suceed and
6348 /// return a non-nullptr value if that is the case. No IR is generated or
6349 /// modified.
6350 static Value *reproduceInst(Attributor &A,
6351 const AbstractAttribute &QueryingAA,
6352 Instruction &I, Type &Ty, Instruction *CtxI,
6353 bool Check, ValueToValueMapTy &VMap) {
6354 assert(CtxI && "Cannot reproduce an instruction without context!");
6355 if (Check && (I.mayReadFromMemory() ||
6356 !isSafeToSpeculativelyExecute(I: &I, CtxI, /* DT */ AC: nullptr,
6357 /* TLI */ DT: nullptr)))
6358 return nullptr;
6359 for (Value *Op : I.operands()) {
6360 Value *NewOp = reproduceValue(A, QueryingAA, V&: *Op, Ty, CtxI, Check, VMap);
6361 if (!NewOp) {
6362 assert(Check && "Manifest of new value unexpectedly failed!");
6363 return nullptr;
6364 }
6365 if (!Check)
6366 VMap[Op] = NewOp;
6367 }
6368 if (Check)
6369 return &I;
6370
6371 Instruction *CloneI = I.clone();
6372 // TODO: Try to salvage debug information here.
6373 CloneI->setDebugLoc(DebugLoc());
6374 VMap[&I] = CloneI;
6375 CloneI->insertBefore(InsertPos: CtxI->getIterator());
6376 RemapInstruction(I: CloneI, VM&: VMap);
6377 return CloneI;
6378 }
6379
6380 /// Reproduce \p V with type \p Ty or return nullptr if that is not posisble.
6381 /// If \p Check is true we will only verify such an operation would suceed and
6382 /// return a non-nullptr value if that is the case. No IR is generated or
6383 /// modified.
6384 static Value *reproduceValue(Attributor &A,
6385 const AbstractAttribute &QueryingAA, Value &V,
6386 Type &Ty, Instruction *CtxI, bool Check,
6387 ValueToValueMapTy &VMap) {
6388 if (const auto &NewV = VMap.lookup(Val: &V))
6389 return NewV;
6390 bool UsedAssumedInformation = false;
6391 std::optional<Value *> SimpleV = A.getAssumedSimplified(
6392 V, AA: QueryingAA, UsedAssumedInformation, S: AA::Interprocedural);
6393 if (!SimpleV.has_value())
6394 return PoisonValue::get(T: &Ty);
6395 Value *EffectiveV = &V;
6396 if (*SimpleV)
6397 EffectiveV = *SimpleV;
6398 if (auto *C = dyn_cast<Constant>(Val: EffectiveV))
6399 return C;
6400 if (CtxI && AA::isValidAtPosition(VAC: AA::ValueAndContext(*EffectiveV, *CtxI),
6401 InfoCache&: A.getInfoCache()))
6402 return ensureType(A, V&: *EffectiveV, Ty, CtxI, Check);
6403 if (auto *I = dyn_cast<Instruction>(Val: EffectiveV))
6404 if (Value *NewV = reproduceInst(A, QueryingAA, I&: *I, Ty, CtxI, Check, VMap))
6405 return ensureType(A, V&: *NewV, Ty, CtxI, Check);
6406 return nullptr;
6407 }
6408
6409 /// Return a value we can use as replacement for the associated one, or
6410 /// nullptr if we don't have one that makes sense.
6411 Value *manifestReplacementValue(Attributor &A, Instruction *CtxI) const {
6412 Value *NewV = SimplifiedAssociatedValue
6413 ? *SimplifiedAssociatedValue
6414 : UndefValue::get(T: getAssociatedType());
6415 if (NewV && NewV != &getAssociatedValue()) {
6416 ValueToValueMapTy VMap;
6417 // First verify we can reprduce the value with the required type at the
6418 // context location before we actually start modifying the IR.
6419 if (reproduceValue(A, QueryingAA: *this, V&: *NewV, Ty&: *getAssociatedType(), CtxI,
6420 /* CheckOnly */ Check: true, VMap))
6421 return reproduceValue(A, QueryingAA: *this, V&: *NewV, Ty&: *getAssociatedType(), CtxI,
6422 /* CheckOnly */ Check: false, VMap);
6423 }
6424 return nullptr;
6425 }
6426
6427 /// Helper function for querying AAValueSimplify and updating candidate.
6428 /// \param IRP The value position we are trying to unify with SimplifiedValue
6429 bool checkAndUpdate(Attributor &A, const AbstractAttribute &QueryingAA,
6430 const IRPosition &IRP, bool Simplify = true) {
6431 bool UsedAssumedInformation = false;
6432 std::optional<Value *> QueryingValueSimplified = &IRP.getAssociatedValue();
6433 if (Simplify)
6434 QueryingValueSimplified = A.getAssumedSimplified(
6435 IRP, AA: QueryingAA, UsedAssumedInformation, S: AA::Interprocedural);
6436 return unionAssumed(Other: QueryingValueSimplified);
6437 }
6438
6439 /// Returns a candidate is found or not
6440 template <typename AAType> bool askSimplifiedValueFor(Attributor &A) {
6441 if (!getAssociatedValue().getType()->isIntegerTy())
6442 return false;
6443
6444 // This will also pass the call base context.
6445 const auto *AA =
6446 A.getAAFor<AAType>(*this, getIRPosition(), DepClassTy::NONE);
6447 if (!AA)
6448 return false;
6449
6450 std::optional<Constant *> COpt = AA->getAssumedConstant(A);
6451
6452 if (!COpt) {
6453 SimplifiedAssociatedValue = std::nullopt;
6454 A.recordDependence(FromAA: *AA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
6455 return true;
6456 }
6457 if (auto *C = *COpt) {
6458 SimplifiedAssociatedValue = C;
6459 A.recordDependence(FromAA: *AA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
6460 return true;
6461 }
6462 return false;
6463 }
6464
6465 bool askSimplifiedValueForOtherAAs(Attributor &A) {
6466 if (askSimplifiedValueFor<AAValueConstantRange>(A))
6467 return true;
6468 if (askSimplifiedValueFor<AAPotentialConstantValues>(A))
6469 return true;
6470 return false;
6471 }
6472
6473 /// See AbstractAttribute::manifest(...).
6474 ChangeStatus manifest(Attributor &A) override {
6475 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6476 for (auto &U : getAssociatedValue().uses()) {
6477 // Check if we need to adjust the insertion point to make sure the IR is
6478 // valid.
6479 Instruction *IP = dyn_cast<Instruction>(Val: U.getUser());
6480 if (auto *PHI = dyn_cast_or_null<PHINode>(Val: IP))
6481 IP = PHI->getIncomingBlock(U)->getTerminator();
6482 if (auto *NewV = manifestReplacementValue(A, CtxI: IP)) {
6483 LLVM_DEBUG(dbgs() << "[ValueSimplify] " << getAssociatedValue()
6484 << " -> " << *NewV << " :: " << *this << "\n");
6485 if (A.changeUseAfterManifest(U, NV&: *NewV))
6486 Changed = ChangeStatus::CHANGED;
6487 }
6488 }
6489
6490 return Changed | AAValueSimplify::manifest(A);
6491 }
6492
6493 /// See AbstractState::indicatePessimisticFixpoint(...).
6494 ChangeStatus indicatePessimisticFixpoint() override {
6495 SimplifiedAssociatedValue = &getAssociatedValue();
6496 return AAValueSimplify::indicatePessimisticFixpoint();
6497 }
6498};
6499
6500struct AAValueSimplifyArgument final : AAValueSimplifyImpl {
6501 AAValueSimplifyArgument(const IRPosition &IRP, Attributor &A)
6502 : AAValueSimplifyImpl(IRP, A) {}
6503
6504 void initialize(Attributor &A) override {
6505 AAValueSimplifyImpl::initialize(A);
6506 if (A.hasAttr(IRP: getIRPosition(),
6507 AKs: {Attribute::InAlloca, Attribute::Preallocated,
6508 Attribute::StructRet, Attribute::Nest, Attribute::ByVal},
6509 /* IgnoreSubsumingPositions */ true))
6510 indicatePessimisticFixpoint();
6511 }
6512
6513 /// See AbstractAttribute::updateImpl(...).
6514 ChangeStatus updateImpl(Attributor &A) override {
6515 // Byval is only replacable if it is readonly otherwise we would write into
6516 // the replaced value and not the copy that byval creates implicitly.
6517 Argument *Arg = getAssociatedArgument();
6518 if (Arg->hasByValAttr()) {
6519 // TODO: We probably need to verify synchronization is not an issue, e.g.,
6520 // there is no race by not copying a constant byval.
6521 bool IsKnown;
6522 if (!AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
6523 return indicatePessimisticFixpoint();
6524 }
6525
6526 auto Before = SimplifiedAssociatedValue;
6527
6528 auto PredForCallSite = [&](AbstractCallSite ACS) {
6529 const IRPosition &ACSArgPos =
6530 IRPosition::callsite_argument(ACS, ArgNo: getCallSiteArgNo());
6531 // Check if a coresponding argument was found or if it is on not
6532 // associated (which can happen for callback calls).
6533 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
6534 return false;
6535
6536 // Simplify the argument operand explicitly and check if the result is
6537 // valid in the current scope. This avoids refering to simplified values
6538 // in other functions, e.g., we don't want to say a an argument in a
6539 // static function is actually an argument in a different function.
6540 bool UsedAssumedInformation = false;
6541 std::optional<Constant *> SimpleArgOp =
6542 A.getAssumedConstant(IRP: ACSArgPos, AA: *this, UsedAssumedInformation);
6543 if (!SimpleArgOp)
6544 return true;
6545 if (!*SimpleArgOp)
6546 return false;
6547 if (!AA::isDynamicallyUnique(A, QueryingAA: *this, V: **SimpleArgOp))
6548 return false;
6549 return unionAssumed(Other: *SimpleArgOp);
6550 };
6551
6552 // Generate a answer specific to a call site context.
6553 bool Success;
6554 bool UsedAssumedInformation = false;
6555 if (hasCallBaseContext() &&
6556 getCallBaseContext()->getCalledOperand() == Arg->getParent())
6557 Success = PredForCallSite(
6558 AbstractCallSite(&getCallBaseContext()->getCalledOperandUse()));
6559 else
6560 Success = A.checkForAllCallSites(Pred: PredForCallSite, QueryingAA: *this, RequireAllCallSites: true,
6561 UsedAssumedInformation);
6562
6563 if (!Success)
6564 if (!askSimplifiedValueForOtherAAs(A))
6565 return indicatePessimisticFixpoint();
6566
6567 // If a candidate was found in this update, return CHANGED.
6568 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6569 : ChangeStatus ::CHANGED;
6570 }
6571
6572 /// See AbstractAttribute::trackStatistics()
6573 void trackStatistics() const override {
6574 STATS_DECLTRACK_ARG_ATTR(value_simplify)
6575 }
6576};
6577
6578struct AAValueSimplifyReturned : AAValueSimplifyImpl {
6579 AAValueSimplifyReturned(const IRPosition &IRP, Attributor &A)
6580 : AAValueSimplifyImpl(IRP, A) {}
6581
6582 /// See AAValueSimplify::getAssumedSimplifiedValue()
6583 std::optional<Value *>
6584 getAssumedSimplifiedValue(Attributor &A) const override {
6585 if (!isValidState())
6586 return nullptr;
6587 return SimplifiedAssociatedValue;
6588 }
6589
6590 /// See AbstractAttribute::updateImpl(...).
6591 ChangeStatus updateImpl(Attributor &A) override {
6592 auto Before = SimplifiedAssociatedValue;
6593
6594 auto ReturnInstCB = [&](Instruction &I) {
6595 auto &RI = cast<ReturnInst>(Val&: I);
6596 return checkAndUpdate(
6597 A, QueryingAA: *this,
6598 IRP: IRPosition::value(V: *RI.getReturnValue(), CBContext: getCallBaseContext()));
6599 };
6600
6601 bool UsedAssumedInformation = false;
6602 if (!A.checkForAllInstructions(Pred: ReturnInstCB, QueryingAA: *this, Opcodes: {Instruction::Ret},
6603 UsedAssumedInformation))
6604 if (!askSimplifiedValueForOtherAAs(A))
6605 return indicatePessimisticFixpoint();
6606
6607 // If a candidate was found in this update, return CHANGED.
6608 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6609 : ChangeStatus ::CHANGED;
6610 }
6611
6612 ChangeStatus manifest(Attributor &A) override {
6613 // We queried AAValueSimplify for the returned values so they will be
6614 // replaced if a simplified form was found. Nothing to do here.
6615 return ChangeStatus::UNCHANGED;
6616 }
6617
6618 /// See AbstractAttribute::trackStatistics()
6619 void trackStatistics() const override {
6620 STATS_DECLTRACK_FNRET_ATTR(value_simplify)
6621 }
6622};
6623
6624struct AAValueSimplifyFloating : AAValueSimplifyImpl {
6625 AAValueSimplifyFloating(const IRPosition &IRP, Attributor &A)
6626 : AAValueSimplifyImpl(IRP, A) {}
6627
6628 /// See AbstractAttribute::initialize(...).
6629 void initialize(Attributor &A) override {
6630 AAValueSimplifyImpl::initialize(A);
6631 Value &V = getAnchorValue();
6632
6633 // TODO: add other stuffs
6634 if (isa<Constant>(Val: V))
6635 indicatePessimisticFixpoint();
6636 }
6637
6638 /// See AbstractAttribute::updateImpl(...).
6639 ChangeStatus updateImpl(Attributor &A) override {
6640 auto Before = SimplifiedAssociatedValue;
6641 if (!askSimplifiedValueForOtherAAs(A))
6642 return indicatePessimisticFixpoint();
6643
6644 // If a candidate was found in this update, return CHANGED.
6645 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6646 : ChangeStatus ::CHANGED;
6647 }
6648
6649 /// See AbstractAttribute::trackStatistics()
6650 void trackStatistics() const override {
6651 STATS_DECLTRACK_FLOATING_ATTR(value_simplify)
6652 }
6653};
6654
6655struct AAValueSimplifyFunction : AAValueSimplifyImpl {
6656 AAValueSimplifyFunction(const IRPosition &IRP, Attributor &A)
6657 : AAValueSimplifyImpl(IRP, A) {}
6658
6659 /// See AbstractAttribute::initialize(...).
6660 void initialize(Attributor &A) override {
6661 SimplifiedAssociatedValue = nullptr;
6662 indicateOptimisticFixpoint();
6663 }
6664 /// See AbstractAttribute::initialize(...).
6665 ChangeStatus updateImpl(Attributor &A) override {
6666 llvm_unreachable(
6667 "AAValueSimplify(Function|CallSite)::updateImpl will not be called");
6668 }
6669 /// See AbstractAttribute::trackStatistics()
6670 void trackStatistics() const override {
6671 STATS_DECLTRACK_FN_ATTR(value_simplify)
6672 }
6673};
6674
6675struct AAValueSimplifyCallSite : AAValueSimplifyFunction {
6676 AAValueSimplifyCallSite(const IRPosition &IRP, Attributor &A)
6677 : AAValueSimplifyFunction(IRP, A) {}
6678 /// See AbstractAttribute::trackStatistics()
6679 void trackStatistics() const override {
6680 STATS_DECLTRACK_CS_ATTR(value_simplify)
6681 }
6682};
6683
6684struct AAValueSimplifyCallSiteReturned : AAValueSimplifyImpl {
6685 AAValueSimplifyCallSiteReturned(const IRPosition &IRP, Attributor &A)
6686 : AAValueSimplifyImpl(IRP, A) {}
6687
6688 void initialize(Attributor &A) override {
6689 AAValueSimplifyImpl::initialize(A);
6690 Function *Fn = getAssociatedFunction();
6691 assert(Fn && "Did expect an associted function");
6692 for (Argument &Arg : Fn->args()) {
6693 if (Arg.hasReturnedAttr()) {
6694 auto IRP = IRPosition::callsite_argument(CB: *cast<CallBase>(Val: getCtxI()),
6695 ArgNo: Arg.getArgNo());
6696 if (IRP.getPositionKind() == IRPosition::IRP_CALL_SITE_ARGUMENT &&
6697 checkAndUpdate(A, QueryingAA: *this, IRP))
6698 indicateOptimisticFixpoint();
6699 else
6700 indicatePessimisticFixpoint();
6701 return;
6702 }
6703 }
6704 }
6705
6706 /// See AbstractAttribute::updateImpl(...).
6707 ChangeStatus updateImpl(Attributor &A) override {
6708 return indicatePessimisticFixpoint();
6709 }
6710
6711 void trackStatistics() const override {
6712 STATS_DECLTRACK_CSRET_ATTR(value_simplify)
6713 }
6714};
6715
6716struct AAValueSimplifyCallSiteArgument : AAValueSimplifyFloating {
6717 AAValueSimplifyCallSiteArgument(const IRPosition &IRP, Attributor &A)
6718 : AAValueSimplifyFloating(IRP, A) {}
6719
6720 /// See AbstractAttribute::manifest(...).
6721 ChangeStatus manifest(Attributor &A) override {
6722 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6723 // TODO: We should avoid simplification duplication to begin with.
6724 auto *FloatAA = A.lookupAAFor<AAValueSimplify>(
6725 IRP: IRPosition::value(V: getAssociatedValue()), QueryingAA: this, DepClass: DepClassTy::NONE);
6726 if (FloatAA && FloatAA->getState().isValidState())
6727 return Changed;
6728
6729 if (auto *NewV = manifestReplacementValue(A, CtxI: getCtxI())) {
6730 Use &U = cast<CallBase>(Val: &getAnchorValue())
6731 ->getArgOperandUse(i: getCallSiteArgNo());
6732 if (A.changeUseAfterManifest(U, NV&: *NewV))
6733 Changed = ChangeStatus::CHANGED;
6734 }
6735
6736 return Changed | AAValueSimplify::manifest(A);
6737 }
6738
6739 void trackStatistics() const override {
6740 STATS_DECLTRACK_CSARG_ATTR(value_simplify)
6741 }
6742};
6743} // namespace
6744
6745/// ----------------------- Heap-To-Stack Conversion ---------------------------
6746namespace {
6747struct AAHeapToStackFunction final : public AAHeapToStack {
6748
6749 static bool isGlobalizedLocal(const CallBase &CB) {
6750 Attribute A = CB.getFnAttr(Kind: "alloc-family");
6751 return A.isValid() && A.getValueAsString() == "__kmpc_alloc_shared";
6752 }
6753
6754 struct AllocationInfo {
6755 /// The call that allocates the memory.
6756 CallBase *const CB;
6757
6758 /// Whether this allocation is an OpenMP globalized local variable.
6759 bool IsGlobalizedLocal = false;
6760
6761 /// The status wrt. a rewrite.
6762 enum {
6763 STACK_DUE_TO_USE,
6764 STACK_DUE_TO_FREE,
6765 INVALID,
6766 } Status = STACK_DUE_TO_USE;
6767
6768 /// Flag to indicate if we encountered a use that might free this allocation
6769 /// but which is not in the deallocation infos.
6770 bool HasPotentiallyFreeingUnknownUses = false;
6771
6772 /// Flag to indicate that we should place the new alloca in the function
6773 /// entry block rather than where the call site (CB) is.
6774 bool MoveAllocaIntoEntry = true;
6775
6776 /// The set of free calls that use this allocation.
6777 SmallSetVector<CallBase *, 1> PotentialFreeCalls{};
6778 };
6779
6780 struct DeallocationInfo {
6781 /// The call that deallocates the memory.
6782 CallBase *const CB;
6783 /// The value freed by the call.
6784 Value *FreedOp;
6785
6786 /// Flag to indicate if we don't know all objects this deallocation might
6787 /// free.
6788 bool MightFreeUnknownObjects = false;
6789
6790 /// The set of allocation calls that are potentially freed.
6791 SmallSetVector<CallBase *, 1> PotentialAllocationCalls{};
6792 };
6793
6794 AAHeapToStackFunction(const IRPosition &IRP, Attributor &A)
6795 : AAHeapToStack(IRP, A) {}
6796
6797 ~AAHeapToStackFunction() override {
6798 // Ensure we call the destructor so we release any memory allocated in the
6799 // sets.
6800 for (auto &It : AllocationInfos)
6801 It.second->~AllocationInfo();
6802 for (auto &It : DeallocationInfos)
6803 It.second->~DeallocationInfo();
6804 }
6805
6806 void initialize(Attributor &A) override {
6807 AAHeapToStack::initialize(A);
6808
6809 const Function *F = getAnchorScope();
6810 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6811
6812 auto AllocationIdentifierCB = [&](Instruction &I) {
6813 CallBase *CB = dyn_cast<CallBase>(Val: &I);
6814 if (!CB)
6815 return true;
6816 if (Value *FreedOp = getFreedOperand(CB, TLI)) {
6817 DeallocationInfos[CB] = new (A.Allocator) DeallocationInfo{.CB: CB, .FreedOp: FreedOp};
6818 return true;
6819 }
6820 // To do heap to stack, we need to know that the allocation itself is
6821 // removable once uses are rewritten, and that we can initialize the
6822 // alloca to the same pattern as the original allocation result.
6823 if (isRemovableAlloc(V: CB, TLI)) {
6824 auto *I8Ty = Type::getInt8Ty(C&: CB->getParent()->getContext());
6825 if (nullptr != getInitialValueOfAllocation(V: CB, TLI, Ty: I8Ty)) {
6826 AllocationInfo *AI = new (A.Allocator) AllocationInfo{.CB: CB};
6827 AllocationInfos[CB] = AI;
6828 AI->IsGlobalizedLocal = isGlobalizedLocal(CB: *CB);
6829 }
6830 }
6831 return true;
6832 };
6833
6834 bool UsedAssumedInformation = false;
6835 bool Success = A.checkForAllCallLikeInstructions(
6836 Pred: AllocationIdentifierCB, QueryingAA: *this, UsedAssumedInformation,
6837 /* CheckBBLivenessOnly */ false,
6838 /* CheckPotentiallyDead */ true);
6839 (void)Success;
6840 assert(Success && "Did not expect the call base visit callback to fail!");
6841
6842 Attributor::SimplifictionCallbackTy SCB =
6843 [](const IRPosition &, const AbstractAttribute *,
6844 bool &) -> std::optional<Value *> { return nullptr; };
6845 for (const auto &It : AllocationInfos)
6846 A.registerSimplificationCallback(IRP: IRPosition::callsite_returned(CB: *It.first),
6847 CB: SCB);
6848 for (const auto &It : DeallocationInfos)
6849 A.registerSimplificationCallback(IRP: IRPosition::callsite_returned(CB: *It.first),
6850 CB: SCB);
6851 }
6852
6853 const std::string getAsStr(Attributor *A) const override {
6854 unsigned NumH2SMallocs = 0, NumInvalidMallocs = 0;
6855 for (const auto &It : AllocationInfos) {
6856 if (It.second->Status == AllocationInfo::INVALID)
6857 ++NumInvalidMallocs;
6858 else
6859 ++NumH2SMallocs;
6860 }
6861 return "[H2S] Mallocs Good/Bad: " + std::to_string(val: NumH2SMallocs) + "/" +
6862 std::to_string(val: NumInvalidMallocs);
6863 }
6864
6865 /// See AbstractAttribute::trackStatistics().
6866 void trackStatistics() const override {
6867 STATS_DECL(
6868 MallocCalls, Function,
6869 "Number of malloc/calloc/aligned_alloc calls converted to allocas");
6870 for (const auto &It : AllocationInfos)
6871 if (It.second->Status != AllocationInfo::INVALID)
6872 ++BUILD_STAT_NAME(MallocCalls, Function);
6873 }
6874
6875 bool isAssumedHeapToStack(const CallBase &CB) const override {
6876 if (isValidState())
6877 if (AllocationInfo *AI =
6878 AllocationInfos.lookup(Key: const_cast<CallBase *>(&CB)))
6879 return AI->Status != AllocationInfo::INVALID;
6880 return false;
6881 }
6882
6883 bool isAssumedHeapToStackRemovedFree(CallBase &CB) const override {
6884 if (!isValidState())
6885 return false;
6886
6887 for (const auto &It : AllocationInfos) {
6888 AllocationInfo &AI = *It.second;
6889 if (AI.Status == AllocationInfo::INVALID)
6890 continue;
6891
6892 if (AI.PotentialFreeCalls.count(key: &CB))
6893 return true;
6894 }
6895
6896 return false;
6897 }
6898
6899 ChangeStatus manifest(Attributor &A) override {
6900 assert(getState().isValidState() &&
6901 "Attempted to manifest an invalid state!");
6902
6903 ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
6904 Function *F = getAnchorScope();
6905 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6906
6907 for (auto &It : AllocationInfos) {
6908 AllocationInfo &AI = *It.second;
6909 if (AI.Status == AllocationInfo::INVALID)
6910 continue;
6911
6912 for (CallBase *FreeCall : AI.PotentialFreeCalls) {
6913 LLVM_DEBUG(dbgs() << "H2S: Removing free call: " << *FreeCall << "\n");
6914 A.deleteAfterManifest(I&: *FreeCall);
6915 HasChanged = ChangeStatus::CHANGED;
6916 }
6917
6918 LLVM_DEBUG(dbgs() << "H2S: Removing malloc-like call: " << *AI.CB
6919 << "\n");
6920
6921 auto Remark = [&](OptimizationRemark OR) {
6922 if (AI.IsGlobalizedLocal)
6923 return OR << "Moving globalized variable to the stack.";
6924 return OR << "Moving memory allocation from the heap to the stack.";
6925 };
6926 if (AI.IsGlobalizedLocal)
6927 A.emitRemark<OptimizationRemark>(I: AI.CB, RemarkName: "OMP110", RemarkCB&: Remark);
6928 else
6929 A.emitRemark<OptimizationRemark>(I: AI.CB, RemarkName: "HeapToStack", RemarkCB&: Remark);
6930
6931 const DataLayout &DL = A.getInfoCache().getDL();
6932 Value *Size;
6933 std::optional<APInt> SizeAPI = getSize(A, AA: *this, AI);
6934 if (SizeAPI) {
6935 Size = ConstantInt::get(Context&: AI.CB->getContext(), V: *SizeAPI);
6936 } else {
6937 ObjectSizeOpts Opts;
6938 ObjectSizeOffsetEvaluator Eval(*AI.CB->getModule(), TLI, Opts);
6939 SizeOffsetValue SizeOffsetPair = Eval.compute(V: AI.CB);
6940 assert(SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown() &&
6941 cast<ConstantInt>(SizeOffsetPair.Offset)->isZero());
6942 Size = SizeOffsetPair.Size;
6943 }
6944
6945 BasicBlock::iterator IP = AI.MoveAllocaIntoEntry
6946 ? F->getEntryBlock().begin()
6947 : AI.CB->getIterator();
6948
6949 Align Alignment(1);
6950 if (MaybeAlign RetAlign = AI.CB->getRetAlign())
6951 Alignment = std::max(a: Alignment, b: *RetAlign);
6952 if (Value *Align = getAllocAlignment(V: AI.CB, TLI)) {
6953 std::optional<APInt> AlignmentAPI = getAPInt(A, AA: *this, V&: *Align);
6954 assert(AlignmentAPI && AlignmentAPI->getZExtValue() > 0 &&
6955 "Expected an alignment during manifest!");
6956 Alignment =
6957 std::max(a: Alignment, b: assumeAligned(Value: AlignmentAPI->getZExtValue()));
6958 }
6959
6960 // TODO: Hoist the alloca towards the function entry.
6961 unsigned AS = DL.getAllocaAddrSpace();
6962 Instruction *Alloca =
6963 new AllocaInst(Type::getInt8Ty(C&: F->getContext()), AS, Size, Alignment,
6964 AI.CB->getName() + ".h2s", IP);
6965
6966 if (Alloca->getType() != AI.CB->getType())
6967 Alloca = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6968 S: Alloca, Ty: AI.CB->getType(), Name: "malloc_cast", InsertBefore: AI.CB->getIterator());
6969
6970 auto *I8Ty = Type::getInt8Ty(C&: F->getContext());
6971 auto *InitVal = getInitialValueOfAllocation(V: AI.CB, TLI, Ty: I8Ty);
6972 assert(InitVal &&
6973 "Must be able to materialize initial memory state of allocation");
6974
6975 A.changeAfterManifest(IRP: IRPosition::inst(I: *AI.CB), NV&: *Alloca);
6976
6977 if (auto *II = dyn_cast<InvokeInst>(Val: AI.CB)) {
6978 auto *NBB = II->getNormalDest();
6979 UncondBrInst::Create(Target: NBB, InsertBefore: AI.CB->getParent());
6980 A.deleteAfterManifest(I&: *AI.CB);
6981 } else {
6982 A.deleteAfterManifest(I&: *AI.CB);
6983 }
6984
6985 // Initialize the alloca with the same value as used by the allocation
6986 // function. We can skip undef as the initial value of an alloc is
6987 // undef, and the memset would simply end up being DSEd.
6988 if (!isa<UndefValue>(Val: InitVal)) {
6989 IRBuilder<> Builder(Alloca->getNextNode());
6990 // TODO: Use alignment above if align!=1
6991 Builder.CreateMemSet(Ptr: Alloca, Val: InitVal, Size, Align: std::nullopt);
6992 }
6993 HasChanged = ChangeStatus::CHANGED;
6994 }
6995
6996 return HasChanged;
6997 }
6998
6999 std::optional<APInt> getAPInt(Attributor &A, const AbstractAttribute &AA,
7000 Value &V) {
7001 bool UsedAssumedInformation = false;
7002 std::optional<Constant *> SimpleV =
7003 A.getAssumedConstant(V, AA, UsedAssumedInformation);
7004 if (!SimpleV)
7005 return APInt(64, 0);
7006 if (auto *CI = dyn_cast_or_null<ConstantInt>(Val: *SimpleV))
7007 return CI->getValue();
7008 return std::nullopt;
7009 }
7010
7011 std::optional<APInt> getSize(Attributor &A, const AbstractAttribute &AA,
7012 AllocationInfo &AI) {
7013 auto Mapper = [&](const Value *V) -> const Value * {
7014 bool UsedAssumedInformation = false;
7015 if (std::optional<Constant *> SimpleV =
7016 A.getAssumedConstant(V: *V, AA, UsedAssumedInformation))
7017 if (*SimpleV)
7018 return *SimpleV;
7019 return V;
7020 };
7021
7022 const Function *F = getAnchorScope();
7023 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
7024 return getAllocSize(CB: AI.CB, TLI, Mapper);
7025 }
7026
7027 /// Collection of all malloc-like calls in a function with associated
7028 /// information.
7029 MapVector<CallBase *, AllocationInfo *> AllocationInfos;
7030
7031 /// Collection of all free-like calls in a function with associated
7032 /// information.
7033 MapVector<CallBase *, DeallocationInfo *> DeallocationInfos;
7034
7035 ChangeStatus updateImpl(Attributor &A) override;
7036};
7037
7038ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) {
7039 ChangeStatus Changed = ChangeStatus::UNCHANGED;
7040 const Function *F = getAnchorScope();
7041 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
7042
7043 const auto *LivenessAA =
7044 A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: IRPosition::function(F: *F), DepClass: DepClassTy::NONE);
7045
7046 MustBeExecutedContextExplorer *Explorer =
7047 A.getInfoCache().getMustBeExecutedContextExplorer();
7048
7049 bool StackIsAccessibleByOtherThreads =
7050 A.getInfoCache().stackIsAccessibleByOtherThreads();
7051
7052 LoopInfo *LI =
7053 A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F: *F);
7054 std::optional<bool> MayContainIrreducibleControl;
7055 auto IsInLoop = [&](BasicBlock &BB) {
7056 if (&F->getEntryBlock() == &BB)
7057 return false;
7058 if (!MayContainIrreducibleControl.has_value())
7059 MayContainIrreducibleControl = mayContainIrreducibleControl(F: *F, LI);
7060 if (*MayContainIrreducibleControl)
7061 return true;
7062 if (!LI)
7063 return true;
7064 return LI->getLoopFor(BB: &BB) != nullptr;
7065 };
7066
7067 // Flag to ensure we update our deallocation information at most once per
7068 // updateImpl call and only if we use the free check reasoning.
7069 bool HasUpdatedFrees = false;
7070
7071 auto UpdateFrees = [&]() {
7072 HasUpdatedFrees = true;
7073
7074 for (auto &It : DeallocationInfos) {
7075 DeallocationInfo &DI = *It.second;
7076 // For now we cannot use deallocations that have unknown inputs, skip
7077 // them.
7078 if (DI.MightFreeUnknownObjects)
7079 continue;
7080
7081 // No need to analyze dead calls, ignore them instead.
7082 bool UsedAssumedInformation = false;
7083 if (A.isAssumedDead(I: *DI.CB, QueryingAA: this, LivenessAA, UsedAssumedInformation,
7084 /* CheckBBLivenessOnly */ true))
7085 continue;
7086
7087 // Use the non-optimistic version to get the freed object.
7088 Value *Obj = getUnderlyingObject(V: DI.FreedOp);
7089 if (!Obj) {
7090 LLVM_DEBUG(dbgs() << "[H2S] Unknown underlying object for free!\n");
7091 DI.MightFreeUnknownObjects = true;
7092 continue;
7093 }
7094
7095 // Free of null and undef can be ignored as no-ops (or UB in the latter
7096 // case).
7097 if (isa<ConstantPointerNull>(Val: Obj) || isa<UndefValue>(Val: Obj))
7098 continue;
7099
7100 CallBase *ObjCB = dyn_cast<CallBase>(Val: Obj);
7101 if (!ObjCB) {
7102 LLVM_DEBUG(dbgs() << "[H2S] Free of a non-call object: " << *Obj
7103 << "\n");
7104 DI.MightFreeUnknownObjects = true;
7105 continue;
7106 }
7107
7108 AllocationInfo *AI = AllocationInfos.lookup(Key: ObjCB);
7109 if (!AI) {
7110 LLVM_DEBUG(dbgs() << "[H2S] Free of a non-allocation object: " << *Obj
7111 << "\n");
7112 DI.MightFreeUnknownObjects = true;
7113 continue;
7114 }
7115
7116 DI.PotentialAllocationCalls.insert(X: ObjCB);
7117 }
7118 };
7119
7120 auto FreeCheck = [&](AllocationInfo &AI) {
7121 // If the stack is not accessible by other threads, the "must-free" logic
7122 // doesn't apply as the pointer could be shared and needs to be places in
7123 // "shareable" memory.
7124 if (!StackIsAccessibleByOtherThreads) {
7125 bool IsKnownNoSycn;
7126 if (!AA::hasAssumedIRAttr<Attribute::NoSync>(
7127 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoSycn)) {
7128 LLVM_DEBUG(
7129 dbgs() << "[H2S] found an escaping use, stack is not accessible by "
7130 "other threads and function is not nosync:\n");
7131 return false;
7132 }
7133 }
7134 if (!HasUpdatedFrees)
7135 UpdateFrees();
7136
7137 // TODO: Allow multi exit functions that have different free calls.
7138 if (AI.PotentialFreeCalls.size() != 1) {
7139 LLVM_DEBUG(dbgs() << "[H2S] did not find one free call but "
7140 << AI.PotentialFreeCalls.size() << "\n");
7141 return false;
7142 }
7143 CallBase *UniqueFree = *AI.PotentialFreeCalls.begin();
7144 DeallocationInfo *DI = DeallocationInfos.lookup(Key: UniqueFree);
7145 if (!DI) {
7146 LLVM_DEBUG(
7147 dbgs() << "[H2S] unique free call was not known as deallocation call "
7148 << *UniqueFree << "\n");
7149 return false;
7150 }
7151 if (DI->MightFreeUnknownObjects) {
7152 LLVM_DEBUG(
7153 dbgs() << "[H2S] unique free call might free unknown allocations\n");
7154 return false;
7155 }
7156 if (DI->PotentialAllocationCalls.empty())
7157 return true;
7158 if (DI->PotentialAllocationCalls.size() > 1) {
7159 LLVM_DEBUG(dbgs() << "[H2S] unique free call might free "
7160 << DI->PotentialAllocationCalls.size()
7161 << " different allocations\n");
7162 return false;
7163 }
7164 if (*DI->PotentialAllocationCalls.begin() != AI.CB) {
7165 LLVM_DEBUG(
7166 dbgs()
7167 << "[H2S] unique free call not known to free this allocation but "
7168 << **DI->PotentialAllocationCalls.begin() << "\n");
7169 return false;
7170 }
7171
7172 // __kmpc_alloc_shared and __kmpc_free_shared are by construction matched.
7173 if (!AI.IsGlobalizedLocal) {
7174 Instruction *CtxI = isa<InvokeInst>(Val: AI.CB) ? AI.CB : AI.CB->getNextNode();
7175 if (!Explorer || !Explorer->findInContextOf(I: UniqueFree, PP: CtxI)) {
7176 LLVM_DEBUG(dbgs() << "[H2S] unique free call might not be executed "
7177 "with the allocation "
7178 << *UniqueFree << "\n");
7179 return false;
7180 }
7181 }
7182 return true;
7183 };
7184
7185 auto UsesCheck = [&](AllocationInfo &AI) {
7186 bool ValidUsesOnly = true;
7187
7188 auto Pred = [&](const Use &U, bool &Follow) -> bool {
7189 Instruction *UserI = cast<Instruction>(Val: U.getUser());
7190 if (isa<LoadInst>(Val: UserI))
7191 return true;
7192 if (auto *SI = dyn_cast<StoreInst>(Val: UserI)) {
7193 if (SI->getValueOperand() == U.get()) {
7194 LLVM_DEBUG(dbgs()
7195 << "[H2S] escaping store to memory: " << *UserI << "\n");
7196 ValidUsesOnly = false;
7197 } else {
7198 // A store into the malloc'ed memory is fine.
7199 }
7200 return true;
7201 }
7202 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
7203 if (!CB->isArgOperand(U: &U) || CB->isLifetimeStartOrEnd())
7204 return true;
7205 if (DeallocationInfos.count(Key: CB)) {
7206 AI.PotentialFreeCalls.insert(X: CB);
7207 return true;
7208 }
7209
7210 unsigned ArgNo = CB->getArgOperandNo(U: &U);
7211 auto CBIRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
7212
7213 bool IsKnownNoCapture;
7214 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
7215 A, QueryingAA: this, IRP: CBIRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture);
7216
7217 // If a call site argument use is nofree, we are fine.
7218 bool IsKnownNoFree;
7219 bool IsAssumedNoFree = AA::hasAssumedIRAttr<Attribute::NoFree>(
7220 A, QueryingAA: this, IRP: CBIRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoFree);
7221
7222 if (!IsAssumedNoCapture ||
7223 (!AI.IsGlobalizedLocal && !IsAssumedNoFree)) {
7224 AI.HasPotentiallyFreeingUnknownUses |= !IsAssumedNoFree;
7225
7226 // Emit a missed remark if this is missed OpenMP globalization.
7227 auto Remark = [&](OptimizationRemarkMissed ORM) {
7228 return ORM
7229 << "Could not move globalized variable to the stack. "
7230 "Variable is potentially captured in call. Mark "
7231 "parameter as `__attribute__((noescape))` to override.";
7232 };
7233
7234 if (ValidUsesOnly && AI.IsGlobalizedLocal)
7235 A.emitRemark<OptimizationRemarkMissed>(I: CB, RemarkName: "OMP113", RemarkCB&: Remark);
7236
7237 LLVM_DEBUG(dbgs() << "[H2S] Bad user: " << *UserI << "\n");
7238 ValidUsesOnly = false;
7239 }
7240 return true;
7241 }
7242
7243 if (isa<GetElementPtrInst>(Val: UserI) || isa<BitCastInst>(Val: UserI) ||
7244 isa<PHINode>(Val: UserI) || isa<SelectInst>(Val: UserI)) {
7245 Follow = true;
7246 return true;
7247 }
7248 // Unknown user for which we can not track uses further (in a way that
7249 // makes sense).
7250 LLVM_DEBUG(dbgs() << "[H2S] Unknown user: " << *UserI << "\n");
7251 ValidUsesOnly = false;
7252 return true;
7253 };
7254 if (!A.checkForAllUses(Pred, QueryingAA: *this, V: *AI.CB, /* CheckBBLivenessOnly */ false,
7255 LivenessDepClass: DepClassTy::OPTIONAL, /* IgnoreDroppableUses */ true,
7256 EquivalentUseCB: [&](const Use &OldU, const Use &NewU) {
7257 auto *SI = dyn_cast<StoreInst>(Val: OldU.getUser());
7258 return !SI || StackIsAccessibleByOtherThreads ||
7259 AA::isAssumedThreadLocalObject(
7260 A, Obj&: *SI->getPointerOperand(), QueryingAA: *this);
7261 }))
7262 return false;
7263 return ValidUsesOnly;
7264 };
7265
7266 // The actual update starts here. We look at all allocations and depending on
7267 // their status perform the appropriate check(s).
7268 for (auto &It : AllocationInfos) {
7269 AllocationInfo &AI = *It.second;
7270 if (AI.Status == AllocationInfo::INVALID)
7271 continue;
7272
7273 if (Value *Align = getAllocAlignment(V: AI.CB, TLI)) {
7274 std::optional<APInt> APAlign = getAPInt(A, AA: *this, V&: *Align);
7275 if (!APAlign) {
7276 // Can't generate an alloca which respects the required alignment
7277 // on the allocation.
7278 LLVM_DEBUG(dbgs() << "[H2S] Unknown allocation alignment: " << *AI.CB
7279 << "\n");
7280 AI.Status = AllocationInfo::INVALID;
7281 Changed = ChangeStatus::CHANGED;
7282 continue;
7283 }
7284 if (APAlign->ugt(RHS: llvm::Value::MaximumAlignment) ||
7285 !APAlign->isPowerOf2()) {
7286 LLVM_DEBUG(dbgs() << "[H2S] Invalid allocation alignment: " << APAlign
7287 << "\n");
7288 AI.Status = AllocationInfo::INVALID;
7289 Changed = ChangeStatus::CHANGED;
7290 continue;
7291 }
7292 }
7293
7294 std::optional<APInt> Size = getSize(A, AA: *this, AI);
7295 if (!AI.IsGlobalizedLocal && MaxHeapToStackSize != -1) {
7296 if (!Size || Size->ugt(RHS: MaxHeapToStackSize)) {
7297 LLVM_DEBUG({
7298 if (!Size)
7299 dbgs() << "[H2S] Unknown allocation size: " << *AI.CB << "\n";
7300 else
7301 dbgs() << "[H2S] Allocation size too large: " << *AI.CB << " vs. "
7302 << MaxHeapToStackSize << "\n";
7303 });
7304
7305 AI.Status = AllocationInfo::INVALID;
7306 Changed = ChangeStatus::CHANGED;
7307 continue;
7308 }
7309 }
7310
7311 switch (AI.Status) {
7312 case AllocationInfo::STACK_DUE_TO_USE:
7313 if (UsesCheck(AI))
7314 break;
7315 AI.Status = AllocationInfo::STACK_DUE_TO_FREE;
7316 [[fallthrough]];
7317 case AllocationInfo::STACK_DUE_TO_FREE:
7318 if (FreeCheck(AI))
7319 break;
7320 AI.Status = AllocationInfo::INVALID;
7321 Changed = ChangeStatus::CHANGED;
7322 break;
7323 case AllocationInfo::INVALID:
7324 llvm_unreachable("Invalid allocations should never reach this point!");
7325 };
7326
7327 // Check if we still think we can move it into the entry block. If the
7328 // alloca comes from a converted __kmpc_alloc_shared then we can usually
7329 // ignore the potential complications associated with loops.
7330 bool IsGlobalizedLocal = AI.IsGlobalizedLocal;
7331 if (AI.MoveAllocaIntoEntry &&
7332 (!Size.has_value() ||
7333 (!IsGlobalizedLocal && IsInLoop(*AI.CB->getParent()))))
7334 AI.MoveAllocaIntoEntry = false;
7335 }
7336
7337 return Changed;
7338}
7339} // namespace
7340
7341/// ----------------------- Privatizable Pointers ------------------------------
7342namespace {
7343struct AAPrivatizablePtrImpl : public AAPrivatizablePtr {
7344 AAPrivatizablePtrImpl(const IRPosition &IRP, Attributor &A)
7345 : AAPrivatizablePtr(IRP, A), PrivatizableType(std::nullopt) {}
7346
7347 ChangeStatus indicatePessimisticFixpoint() override {
7348 AAPrivatizablePtr::indicatePessimisticFixpoint();
7349 PrivatizableType = nullptr;
7350 return ChangeStatus::CHANGED;
7351 }
7352
7353 /// Identify the type we can chose for a private copy of the underlying
7354 /// argument. std::nullopt means it is not clear yet, nullptr means there is
7355 /// none.
7356 virtual std::optional<Type *> identifyPrivatizableType(Attributor &A) = 0;
7357
7358 /// Return a privatizable type that encloses both T0 and T1.
7359 /// TODO: This is merely a stub for now as we should manage a mapping as well.
7360 std::optional<Type *> combineTypes(std::optional<Type *> T0,
7361 std::optional<Type *> T1) {
7362 if (!T0)
7363 return T1;
7364 if (!T1)
7365 return T0;
7366 if (T0 == T1)
7367 return T0;
7368 return nullptr;
7369 }
7370
7371 std::optional<Type *> getPrivatizableType() const override {
7372 return PrivatizableType;
7373 }
7374
7375 const std::string getAsStr(Attributor *A) const override {
7376 return isAssumedPrivatizablePtr() ? "[priv]" : "[no-priv]";
7377 }
7378
7379protected:
7380 std::optional<Type *> PrivatizableType;
7381};
7382
7383// TODO: Do this for call site arguments (probably also other values) as well.
7384
7385struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl {
7386 AAPrivatizablePtrArgument(const IRPosition &IRP, Attributor &A)
7387 : AAPrivatizablePtrImpl(IRP, A) {}
7388
7389 /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
7390 std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7391 // If this is a byval argument and we know all the call sites (so we can
7392 // rewrite them), there is no need to check them explicitly.
7393 bool UsedAssumedInformation = false;
7394 SmallVector<Attribute, 1> Attrs;
7395 A.getAttrs(IRP: getIRPosition(), AKs: {Attribute::ByVal}, Attrs,
7396 /* IgnoreSubsumingPositions */ true);
7397 if (!Attrs.empty() &&
7398 A.checkForAllCallSites(Pred: [](AbstractCallSite ACS) { return true; }, QueryingAA: *this,
7399 RequireAllCallSites: true, UsedAssumedInformation))
7400 return Attrs[0].getValueAsType();
7401
7402 std::optional<Type *> Ty;
7403 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
7404
7405 // Make sure the associated call site argument has the same type at all call
7406 // sites and it is an allocation we know is safe to privatize, for now that
7407 // means we only allow alloca instructions.
7408 // TODO: We can additionally analyze the accesses in the callee to create
7409 // the type from that information instead. That is a little more
7410 // involved and will be done in a follow up patch.
7411 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7412 IRPosition ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
7413 // Check if a coresponding argument was found or if it is one not
7414 // associated (which can happen for callback calls).
7415 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
7416 return false;
7417
7418 // Check that all call sites agree on a type.
7419 auto *PrivCSArgAA =
7420 A.getAAFor<AAPrivatizablePtr>(QueryingAA: *this, IRP: ACSArgPos, DepClass: DepClassTy::REQUIRED);
7421 if (!PrivCSArgAA)
7422 return false;
7423 std::optional<Type *> CSTy = PrivCSArgAA->getPrivatizableType();
7424
7425 LLVM_DEBUG({
7426 dbgs() << "[AAPrivatizablePtr] ACSPos: " << ACSArgPos << ", CSTy: ";
7427 if (CSTy && *CSTy)
7428 (*CSTy)->print(dbgs());
7429 else if (CSTy)
7430 dbgs() << "<nullptr>";
7431 else
7432 dbgs() << "<none>";
7433 });
7434
7435 Ty = combineTypes(T0: Ty, T1: CSTy);
7436
7437 LLVM_DEBUG({
7438 dbgs() << " : New Type: ";
7439 if (Ty && *Ty)
7440 (*Ty)->print(dbgs());
7441 else if (Ty)
7442 dbgs() << "<nullptr>";
7443 else
7444 dbgs() << "<none>";
7445 dbgs() << "\n";
7446 });
7447
7448 return !Ty || *Ty;
7449 };
7450
7451 if (!A.checkForAllCallSites(Pred: CallSiteCheck, QueryingAA: *this, RequireAllCallSites: true,
7452 UsedAssumedInformation))
7453 return nullptr;
7454 return Ty;
7455 }
7456
7457 /// See AbstractAttribute::updateImpl(...).
7458 ChangeStatus updateImpl(Attributor &A) override {
7459 PrivatizableType = identifyPrivatizableType(A);
7460 if (!PrivatizableType)
7461 return ChangeStatus::UNCHANGED;
7462 if (!*PrivatizableType)
7463 return indicatePessimisticFixpoint();
7464
7465 // The dependence is optional so we don't give up once we give up on the
7466 // alignment.
7467 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: getAssociatedValue()),
7468 DepClass: DepClassTy::OPTIONAL);
7469
7470 // Avoid arguments with padding for now.
7471 if (!A.hasAttr(IRP: getIRPosition(), AKs: Attribute::ByVal) &&
7472 !isDenselyPacked(Ty: *PrivatizableType, DL: A.getInfoCache().getDL())) {
7473 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Padding detected\n");
7474 return indicatePessimisticFixpoint();
7475 }
7476
7477 // Collect the types that will replace the privatizable type in the function
7478 // signature.
7479 SmallVector<Type *, 16> ReplacementTypes;
7480 identifyReplacementTypes(PrivType: *PrivatizableType, ReplacementTypes);
7481
7482 // Verify callee and caller agree on how the promoted argument would be
7483 // passed.
7484 Function &Fn = *getIRPosition().getAnchorScope();
7485 const auto *TTI =
7486 A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(F: Fn);
7487 if (!TTI) {
7488 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Missing TTI for function "
7489 << Fn.getName() << "\n");
7490 return indicatePessimisticFixpoint();
7491 }
7492
7493 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7494 CallBase *CB = ACS.getInstruction();
7495 return TTI->areTypesABICompatible(
7496 Caller: CB->getCaller(),
7497 Callee: dyn_cast_if_present<Function>(Val: CB->getCalledOperand()),
7498 Types: ReplacementTypes);
7499 };
7500 bool UsedAssumedInformation = false;
7501 if (!A.checkForAllCallSites(Pred: CallSiteCheck, QueryingAA: *this, RequireAllCallSites: true,
7502 UsedAssumedInformation)) {
7503 LLVM_DEBUG(
7504 dbgs() << "[AAPrivatizablePtr] ABI incompatibility detected for "
7505 << Fn.getName() << "\n");
7506 return indicatePessimisticFixpoint();
7507 }
7508
7509 // Register a rewrite of the argument.
7510 Argument *Arg = getAssociatedArgument();
7511 if (!A.isValidFunctionSignatureRewrite(Arg&: *Arg, ReplacementTypes)) {
7512 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Rewrite not valid\n");
7513 return indicatePessimisticFixpoint();
7514 }
7515
7516 unsigned ArgNo = Arg->getArgNo();
7517
7518 // Helper to check if for the given call site the associated argument is
7519 // passed to a callback where the privatization would be different.
7520 auto IsCompatiblePrivArgOfCallback = [&](CallBase &CB) {
7521 SmallVector<const Use *, 4> CallbackUses;
7522 AbstractCallSite::getCallbackUses(CB, CallbackUses);
7523 for (const Use *U : CallbackUses) {
7524 AbstractCallSite CBACS(U);
7525 assert(CBACS && CBACS.isCallbackCall());
7526 for (Argument &CBArg : CBACS.getCalledFunction()->args()) {
7527 int CBArgNo = CBACS.getCallArgOperandNo(Arg&: CBArg);
7528
7529 LLVM_DEBUG({
7530 dbgs()
7531 << "[AAPrivatizablePtr] Argument " << *Arg
7532 << "check if can be privatized in the context of its parent ("
7533 << Arg->getParent()->getName()
7534 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7535 "callback ("
7536 << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7537 << ")\n[AAPrivatizablePtr] " << CBArg << " : "
7538 << CBACS.getCallArgOperand(CBArg) << " vs "
7539 << CB.getArgOperand(ArgNo) << "\n"
7540 << "[AAPrivatizablePtr] " << CBArg << " : "
7541 << CBACS.getCallArgOperandNo(CBArg) << " vs " << ArgNo << "\n";
7542 });
7543
7544 if (CBArgNo != int(ArgNo))
7545 continue;
7546 const auto *CBArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7547 QueryingAA: *this, IRP: IRPosition::argument(Arg: CBArg), DepClass: DepClassTy::REQUIRED);
7548 if (CBArgPrivAA && CBArgPrivAA->isValidState()) {
7549 auto CBArgPrivTy = CBArgPrivAA->getPrivatizableType();
7550 if (!CBArgPrivTy)
7551 continue;
7552 if (*CBArgPrivTy == PrivatizableType)
7553 continue;
7554 }
7555
7556 LLVM_DEBUG({
7557 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7558 << " cannot be privatized in the context of its parent ("
7559 << Arg->getParent()->getName()
7560 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7561 "callback ("
7562 << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7563 << ").\n[AAPrivatizablePtr] for which the argument "
7564 "privatization is not compatible.\n";
7565 });
7566 return false;
7567 }
7568 }
7569 return true;
7570 };
7571
7572 // Helper to check if for the given call site the associated argument is
7573 // passed to a direct call where the privatization would be different.
7574 auto IsCompatiblePrivArgOfDirectCS = [&](AbstractCallSite ACS) {
7575 CallBase *DC = cast<CallBase>(Val: ACS.getInstruction());
7576 int DCArgNo = ACS.getCallArgOperandNo(ArgNo);
7577 assert(DCArgNo >= 0 && unsigned(DCArgNo) < DC->arg_size() &&
7578 "Expected a direct call operand for callback call operand");
7579
7580 Function *DCCallee =
7581 dyn_cast_if_present<Function>(Val: DC->getCalledOperand());
7582 LLVM_DEBUG({
7583 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7584 << " check if be privatized in the context of its parent ("
7585 << Arg->getParent()->getName()
7586 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7587 "direct call of ("
7588 << DCArgNo << "@" << DCCallee->getName() << ").\n";
7589 });
7590
7591 if (unsigned(DCArgNo) < DCCallee->arg_size()) {
7592 const auto *DCArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7593 QueryingAA: *this, IRP: IRPosition::argument(Arg: *DCCallee->getArg(i: DCArgNo)),
7594 DepClass: DepClassTy::REQUIRED);
7595 if (DCArgPrivAA && DCArgPrivAA->isValidState()) {
7596 auto DCArgPrivTy = DCArgPrivAA->getPrivatizableType();
7597 if (!DCArgPrivTy)
7598 return true;
7599 if (*DCArgPrivTy == PrivatizableType)
7600 return true;
7601 }
7602 }
7603
7604 LLVM_DEBUG({
7605 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7606 << " cannot be privatized in the context of its parent ("
7607 << Arg->getParent()->getName()
7608 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7609 "direct call of ("
7610 << ACS.getInstruction()->getCalledOperand()->getName()
7611 << ").\n[AAPrivatizablePtr] for which the argument "
7612 "privatization is not compatible.\n";
7613 });
7614 return false;
7615 };
7616
7617 // Helper to check if the associated argument is used at the given abstract
7618 // call site in a way that is incompatible with the privatization assumed
7619 // here.
7620 auto IsCompatiblePrivArgOfOtherCallSite = [&](AbstractCallSite ACS) {
7621 if (ACS.isDirectCall())
7622 return IsCompatiblePrivArgOfCallback(*ACS.getInstruction());
7623 if (ACS.isCallbackCall())
7624 return IsCompatiblePrivArgOfDirectCS(ACS);
7625 return false;
7626 };
7627
7628 if (!A.checkForAllCallSites(Pred: IsCompatiblePrivArgOfOtherCallSite, QueryingAA: *this, RequireAllCallSites: true,
7629 UsedAssumedInformation))
7630 return indicatePessimisticFixpoint();
7631
7632 return ChangeStatus::UNCHANGED;
7633 }
7634
7635 /// Given a type to private \p PrivType, collect the constituates (which are
7636 /// used) in \p ReplacementTypes.
7637 static void
7638 identifyReplacementTypes(Type *PrivType,
7639 SmallVectorImpl<Type *> &ReplacementTypes) {
7640 // TODO: For now we expand the privatization type to the fullest which can
7641 // lead to dead arguments that need to be removed later.
7642 assert(PrivType && "Expected privatizable type!");
7643
7644 // Traverse the type, extract constituate types on the outermost level.
7645 if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7646 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++)
7647 ReplacementTypes.push_back(Elt: PrivStructType->getElementType(N: u));
7648 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7649 ReplacementTypes.append(NumInputs: PrivArrayType->getNumElements(),
7650 Elt: PrivArrayType->getElementType());
7651 } else {
7652 ReplacementTypes.push_back(Elt: PrivType);
7653 }
7654 }
7655
7656 /// Initialize \p Base according to the type \p PrivType at position \p IP.
7657 /// The values needed are taken from the arguments of \p F starting at
7658 /// position \p ArgNo.
7659 static void createInitialization(Type *PrivType, Value &Base, Function &F,
7660 unsigned ArgNo, BasicBlock::iterator IP) {
7661 assert(PrivType && "Expected privatizable type!");
7662
7663 IRBuilder<NoFolder> IRB(IP);
7664 const DataLayout &DL = F.getDataLayout();
7665
7666 // Traverse the type, build GEPs and stores.
7667 if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7668 const StructLayout *PrivStructLayout = DL.getStructLayout(Ty: PrivStructType);
7669 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7670 Value *Ptr =
7671 constructPointer(Ptr: &Base, Offset: PrivStructLayout->getElementOffset(Idx: u), IRB);
7672 new StoreInst(F.getArg(i: ArgNo + u), Ptr, IP);
7673 }
7674 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7675 Type *PointeeTy = PrivArrayType->getElementType();
7676 uint64_t PointeeTySize = DL.getTypeStoreSize(Ty: PointeeTy);
7677 for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7678 Value *Ptr = constructPointer(Ptr: &Base, Offset: u * PointeeTySize, IRB);
7679 new StoreInst(F.getArg(i: ArgNo + u), Ptr, IP);
7680 }
7681 } else {
7682 new StoreInst(F.getArg(i: ArgNo), &Base, IP);
7683 }
7684 }
7685
7686 /// Extract values from \p Base according to the type \p PrivType at the
7687 /// call position \p ACS. The values are appended to \p ReplacementValues.
7688 void createReplacementValues(Align Alignment, Type *PrivType,
7689 AbstractCallSite ACS, Value *Base,
7690 SmallVectorImpl<Value *> &ReplacementValues) {
7691 assert(Base && "Expected base value!");
7692 assert(PrivType && "Expected privatizable type!");
7693 Instruction *IP = ACS.getInstruction();
7694
7695 IRBuilder<NoFolder> IRB(IP);
7696 const DataLayout &DL = IP->getDataLayout();
7697
7698 // Traverse the type, build GEPs and loads.
7699 if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7700 const StructLayout *PrivStructLayout = DL.getStructLayout(Ty: PrivStructType);
7701 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7702 Type *PointeeTy = PrivStructType->getElementType(N: u);
7703 Value *Ptr =
7704 constructPointer(Ptr: Base, Offset: PrivStructLayout->getElementOffset(Idx: u), IRB);
7705 LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7706 L->setAlignment(Alignment);
7707 ReplacementValues.push_back(Elt: L);
7708 }
7709 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7710 Type *PointeeTy = PrivArrayType->getElementType();
7711 uint64_t PointeeTySize = DL.getTypeStoreSize(Ty: PointeeTy);
7712 for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7713 Value *Ptr = constructPointer(Ptr: Base, Offset: u * PointeeTySize, IRB);
7714 LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7715 L->setAlignment(Alignment);
7716 ReplacementValues.push_back(Elt: L);
7717 }
7718 } else {
7719 LoadInst *L = new LoadInst(PrivType, Base, "", IP->getIterator());
7720 L->setAlignment(Alignment);
7721 ReplacementValues.push_back(Elt: L);
7722 }
7723 }
7724
7725 /// See AbstractAttribute::manifest(...)
7726 ChangeStatus manifest(Attributor &A) override {
7727 if (!PrivatizableType)
7728 return ChangeStatus::UNCHANGED;
7729 assert(*PrivatizableType && "Expected privatizable type!");
7730
7731 // Collect all tail calls in the function as we cannot allow new allocas to
7732 // escape into tail recursion.
7733 // TODO: Be smarter about new allocas escaping into tail calls.
7734 SmallVector<CallInst *, 16> TailCalls;
7735 bool UsedAssumedInformation = false;
7736 if (!A.checkForAllInstructions(
7737 Pred: [&](Instruction &I) {
7738 CallInst &CI = cast<CallInst>(Val&: I);
7739 if (CI.isTailCall())
7740 TailCalls.push_back(Elt: &CI);
7741 return true;
7742 },
7743 QueryingAA: *this, Opcodes: {Instruction::Call}, UsedAssumedInformation))
7744 return ChangeStatus::UNCHANGED;
7745
7746 Argument *Arg = getAssociatedArgument();
7747 // Query AAAlign attribute for alignment of associated argument to
7748 // determine the best alignment of loads.
7749 const auto *AlignAA =
7750 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *Arg), DepClass: DepClassTy::NONE);
7751
7752 // Callback to repair the associated function. A new alloca is placed at the
7753 // beginning and initialized with the values passed through arguments. The
7754 // new alloca replaces the use of the old pointer argument.
7755 Attributor::ArgumentReplacementInfo::CalleeRepairCBTy FnRepairCB =
7756 [=](const Attributor::ArgumentReplacementInfo &ARI,
7757 Function &ReplacementFn, Function::arg_iterator ArgIt) {
7758 BasicBlock &EntryBB = ReplacementFn.getEntryBlock();
7759 BasicBlock::iterator IP = EntryBB.getFirstInsertionPt();
7760 const DataLayout &DL = IP->getDataLayout();
7761 unsigned AS = DL.getAllocaAddrSpace();
7762 Instruction *AI = new AllocaInst(*PrivatizableType, AS,
7763 Arg->getName() + ".priv", IP);
7764 createInitialization(PrivType: *PrivatizableType, Base&: *AI, F&: ReplacementFn,
7765 ArgNo: ArgIt->getArgNo(), IP);
7766
7767 if (AI->getType() != Arg->getType())
7768 AI = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
7769 S: AI, Ty: Arg->getType(), Name: "", InsertBefore: IP);
7770 Arg->replaceAllUsesWith(V: AI);
7771
7772 for (CallInst *CI : TailCalls)
7773 CI->setTailCall(false);
7774 };
7775
7776 // Callback to repair a call site of the associated function. The elements
7777 // of the privatizable type are loaded prior to the call and passed to the
7778 // new function version.
7779 Attributor::ArgumentReplacementInfo::ACSRepairCBTy ACSRepairCB =
7780 [=](const Attributor::ArgumentReplacementInfo &ARI,
7781 AbstractCallSite ACS, SmallVectorImpl<Value *> &NewArgOperands) {
7782 // When no alignment is specified for the load instruction,
7783 // natural alignment is assumed.
7784 createReplacementValues(
7785 Alignment: AlignAA ? AlignAA->getAssumedAlign() : Align(0),
7786 PrivType: *PrivatizableType, ACS,
7787 Base: ACS.getCallArgOperand(ArgNo: ARI.getReplacedArg().getArgNo()),
7788 ReplacementValues&: NewArgOperands);
7789 };
7790
7791 // Collect the types that will replace the privatizable type in the function
7792 // signature.
7793 SmallVector<Type *, 16> ReplacementTypes;
7794 identifyReplacementTypes(PrivType: *PrivatizableType, ReplacementTypes);
7795
7796 // Register a rewrite of the argument.
7797 if (A.registerFunctionSignatureRewrite(Arg&: *Arg, ReplacementTypes,
7798 CalleeRepairCB: std::move(FnRepairCB),
7799 ACSRepairCB: std::move(ACSRepairCB)))
7800 return ChangeStatus::CHANGED;
7801 return ChangeStatus::UNCHANGED;
7802 }
7803
7804 /// See AbstractAttribute::trackStatistics()
7805 void trackStatistics() const override {
7806 STATS_DECLTRACK_ARG_ATTR(privatizable_ptr);
7807 }
7808};
7809
7810struct AAPrivatizablePtrFloating : public AAPrivatizablePtrImpl {
7811 AAPrivatizablePtrFloating(const IRPosition &IRP, Attributor &A)
7812 : AAPrivatizablePtrImpl(IRP, A) {}
7813
7814 /// See AbstractAttribute::initialize(...).
7815 void initialize(Attributor &A) override {
7816 // TODO: We can privatize more than arguments.
7817 indicatePessimisticFixpoint();
7818 }
7819
7820 ChangeStatus updateImpl(Attributor &A) override {
7821 llvm_unreachable("AAPrivatizablePtr(Floating|Returned|CallSiteReturned)::"
7822 "updateImpl will not be called");
7823 }
7824
7825 /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
7826 std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7827 Value *Obj = getUnderlyingObject(V: &getAssociatedValue());
7828 if (!Obj) {
7829 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] No underlying object found!\n");
7830 return nullptr;
7831 }
7832
7833 if (auto *AI = dyn_cast<AllocaInst>(Val: Obj))
7834 if (auto *CI = dyn_cast<ConstantInt>(Val: AI->getArraySize()))
7835 if (CI->isOne())
7836 return AI->getAllocatedType();
7837 if (auto *Arg = dyn_cast<Argument>(Val: Obj)) {
7838 auto *PrivArgAA = A.getAAFor<AAPrivatizablePtr>(
7839 QueryingAA: *this, IRP: IRPosition::argument(Arg: *Arg), DepClass: DepClassTy::REQUIRED);
7840 if (PrivArgAA && PrivArgAA->isAssumedPrivatizablePtr())
7841 return PrivArgAA->getPrivatizableType();
7842 }
7843
7844 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Underlying object neither valid "
7845 "alloca nor privatizable argument: "
7846 << *Obj << "!\n");
7847 return nullptr;
7848 }
7849
7850 /// See AbstractAttribute::trackStatistics()
7851 void trackStatistics() const override {
7852 STATS_DECLTRACK_FLOATING_ATTR(privatizable_ptr);
7853 }
7854};
7855
7856struct AAPrivatizablePtrCallSiteArgument final
7857 : public AAPrivatizablePtrFloating {
7858 AAPrivatizablePtrCallSiteArgument(const IRPosition &IRP, Attributor &A)
7859 : AAPrivatizablePtrFloating(IRP, A) {}
7860
7861 /// See AbstractAttribute::initialize(...).
7862 void initialize(Attributor &A) override {
7863 if (A.hasAttr(IRP: getIRPosition(), AKs: Attribute::ByVal))
7864 indicateOptimisticFixpoint();
7865 }
7866
7867 /// See AbstractAttribute::updateImpl(...).
7868 ChangeStatus updateImpl(Attributor &A) override {
7869 PrivatizableType = identifyPrivatizableType(A);
7870 if (!PrivatizableType)
7871 return ChangeStatus::UNCHANGED;
7872 if (!*PrivatizableType)
7873 return indicatePessimisticFixpoint();
7874
7875 const IRPosition &IRP = getIRPosition();
7876 bool IsKnownNoCapture;
7877 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
7878 A, QueryingAA: this, IRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture);
7879 if (!IsAssumedNoCapture) {
7880 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might be captured!\n");
7881 return indicatePessimisticFixpoint();
7882 }
7883
7884 bool IsKnownNoAlias;
7885 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
7886 A, QueryingAA: this, IRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoAlias)) {
7887 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might alias!\n");
7888 return indicatePessimisticFixpoint();
7889 }
7890
7891 bool IsKnown;
7892 if (!AA::isAssumedReadOnly(A, IRP, QueryingAA: *this, IsKnown)) {
7893 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer is written!\n");
7894 return indicatePessimisticFixpoint();
7895 }
7896
7897 return ChangeStatus::UNCHANGED;
7898 }
7899
7900 /// See AbstractAttribute::trackStatistics()
7901 void trackStatistics() const override {
7902 STATS_DECLTRACK_CSARG_ATTR(privatizable_ptr);
7903 }
7904};
7905
7906struct AAPrivatizablePtrCallSiteReturned final
7907 : public AAPrivatizablePtrFloating {
7908 AAPrivatizablePtrCallSiteReturned(const IRPosition &IRP, Attributor &A)
7909 : AAPrivatizablePtrFloating(IRP, A) {}
7910
7911 /// See AbstractAttribute::initialize(...).
7912 void initialize(Attributor &A) override {
7913 // TODO: We can privatize more than arguments.
7914 indicatePessimisticFixpoint();
7915 }
7916
7917 /// See AbstractAttribute::trackStatistics()
7918 void trackStatistics() const override {
7919 STATS_DECLTRACK_CSRET_ATTR(privatizable_ptr);
7920 }
7921};
7922
7923struct AAPrivatizablePtrReturned final : public AAPrivatizablePtrFloating {
7924 AAPrivatizablePtrReturned(const IRPosition &IRP, Attributor &A)
7925 : AAPrivatizablePtrFloating(IRP, A) {}
7926
7927 /// See AbstractAttribute::initialize(...).
7928 void initialize(Attributor &A) override {
7929 // TODO: We can privatize more than arguments.
7930 indicatePessimisticFixpoint();
7931 }
7932
7933 /// See AbstractAttribute::trackStatistics()
7934 void trackStatistics() const override {
7935 STATS_DECLTRACK_FNRET_ATTR(privatizable_ptr);
7936 }
7937};
7938} // namespace
7939
7940/// -------------------- Memory Behavior Attributes ----------------------------
7941/// Includes read-none, read-only, and write-only.
7942/// ----------------------------------------------------------------------------
7943namespace {
7944struct AAMemoryBehaviorImpl : public AAMemoryBehavior {
7945 AAMemoryBehaviorImpl(const IRPosition &IRP, Attributor &A)
7946 : AAMemoryBehavior(IRP, A) {}
7947
7948 /// See AbstractAttribute::initialize(...).
7949 void initialize(Attributor &A) override {
7950 intersectAssumedBits(BitsEncoding: BEST_STATE);
7951 getKnownStateFromValue(A, IRP: getIRPosition(), State&: getState());
7952 AAMemoryBehavior::initialize(A);
7953 }
7954
7955 /// Return the memory behavior information encoded in the IR for \p IRP.
7956 static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
7957 BitIntegerState &State,
7958 bool IgnoreSubsumingPositions = false) {
7959 SmallVector<Attribute, 2> Attrs;
7960 A.getAttrs(IRP, AKs: AttrKinds, Attrs, IgnoreSubsumingPositions);
7961 for (const Attribute &Attr : Attrs) {
7962 switch (Attr.getKindAsEnum()) {
7963 case Attribute::ReadNone:
7964 State.addKnownBits(Bits: NO_ACCESSES);
7965 break;
7966 case Attribute::ReadOnly:
7967 State.addKnownBits(Bits: NO_WRITES);
7968 break;
7969 case Attribute::WriteOnly:
7970 State.addKnownBits(Bits: NO_READS);
7971 break;
7972 default:
7973 llvm_unreachable("Unexpected attribute!");
7974 }
7975 }
7976
7977 if (auto *I = dyn_cast<Instruction>(Val: &IRP.getAnchorValue())) {
7978 if (!I->mayReadFromMemory())
7979 State.addKnownBits(Bits: NO_READS);
7980 if (!I->mayWriteToMemory())
7981 State.addKnownBits(Bits: NO_WRITES);
7982 }
7983 }
7984
7985 /// See AbstractAttribute::getDeducedAttributes(...).
7986 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
7987 SmallVectorImpl<Attribute> &Attrs) const override {
7988 assert(Attrs.size() == 0);
7989 if (isAssumedReadNone())
7990 Attrs.push_back(Elt: Attribute::get(Context&: Ctx, Kind: Attribute::ReadNone));
7991 else if (isAssumedReadOnly())
7992 Attrs.push_back(Elt: Attribute::get(Context&: Ctx, Kind: Attribute::ReadOnly));
7993 else if (isAssumedWriteOnly())
7994 Attrs.push_back(Elt: Attribute::get(Context&: Ctx, Kind: Attribute::WriteOnly));
7995 assert(Attrs.size() <= 1);
7996 }
7997
7998 /// See AbstractAttribute::manifest(...).
7999 ChangeStatus manifest(Attributor &A) override {
8000 const IRPosition &IRP = getIRPosition();
8001
8002 if (A.hasAttr(IRP, AKs: Attribute::ReadNone,
8003 /* IgnoreSubsumingPositions */ true))
8004 return ChangeStatus::UNCHANGED;
8005
8006 // Check if we would improve the existing attributes first.
8007 SmallVector<Attribute, 4> DeducedAttrs;
8008 getDeducedAttributes(A, Ctx&: IRP.getAnchorValue().getContext(), Attrs&: DeducedAttrs);
8009 if (llvm::all_of(Range&: DeducedAttrs, P: [&](const Attribute &Attr) {
8010 return A.hasAttr(IRP, AKs: Attr.getKindAsEnum(),
8011 /* IgnoreSubsumingPositions */ true);
8012 }))
8013 return ChangeStatus::UNCHANGED;
8014
8015 // Clear existing attributes.
8016 A.removeAttrs(IRP, AttrKinds);
8017 // Clear conflicting writable attribute.
8018 if (isAssumedReadOnly())
8019 A.removeAttrs(IRP, AttrKinds: Attribute::Writable);
8020
8021 // Use the generic manifest method.
8022 return IRAttribute::manifest(A);
8023 }
8024
8025 /// See AbstractState::getAsStr().
8026 const std::string getAsStr(Attributor *A) const override {
8027 if (isAssumedReadNone())
8028 return "readnone";
8029 if (isAssumedReadOnly())
8030 return "readonly";
8031 if (isAssumedWriteOnly())
8032 return "writeonly";
8033 return "may-read/write";
8034 }
8035
8036 /// The set of IR attributes AAMemoryBehavior deals with.
8037 static const Attribute::AttrKind AttrKinds[3];
8038};
8039
8040const Attribute::AttrKind AAMemoryBehaviorImpl::AttrKinds[] = {
8041 Attribute::ReadNone, Attribute::ReadOnly, Attribute::WriteOnly};
8042
8043/// Memory behavior attribute for a floating value.
8044struct AAMemoryBehaviorFloating : AAMemoryBehaviorImpl {
8045 AAMemoryBehaviorFloating(const IRPosition &IRP, Attributor &A)
8046 : AAMemoryBehaviorImpl(IRP, A) {}
8047
8048 /// See AbstractAttribute::updateImpl(...).
8049 ChangeStatus updateImpl(Attributor &A) override;
8050
8051 /// See AbstractAttribute::trackStatistics()
8052 void trackStatistics() const override {
8053 if (isAssumedReadNone())
8054 STATS_DECLTRACK_FLOATING_ATTR(readnone)
8055 else if (isAssumedReadOnly())
8056 STATS_DECLTRACK_FLOATING_ATTR(readonly)
8057 else if (isAssumedWriteOnly())
8058 STATS_DECLTRACK_FLOATING_ATTR(writeonly)
8059 }
8060
8061private:
8062 /// Return true if users of \p UserI might access the underlying
8063 /// variable/location described by \p U and should therefore be analyzed.
8064 bool followUsersOfUseIn(Attributor &A, const Use &U,
8065 const Instruction *UserI);
8066
8067 /// Update the state according to the effect of use \p U in \p UserI.
8068 void analyzeUseIn(Attributor &A, const Use &U, const Instruction *UserI);
8069};
8070
8071/// Memory behavior attribute for function argument.
8072struct AAMemoryBehaviorArgument : AAMemoryBehaviorFloating {
8073 AAMemoryBehaviorArgument(const IRPosition &IRP, Attributor &A)
8074 : AAMemoryBehaviorFloating(IRP, A) {}
8075
8076 /// See AbstractAttribute::initialize(...).
8077 void initialize(Attributor &A) override {
8078 intersectAssumedBits(BitsEncoding: BEST_STATE);
8079 const IRPosition &IRP = getIRPosition();
8080 // TODO: Make IgnoreSubsumingPositions a property of an IRAttribute so we
8081 // can query it when we use has/getAttr. That would allow us to reuse the
8082 // initialize of the base class here.
8083 bool HasByVal = A.hasAttr(IRP, AKs: {Attribute::ByVal},
8084 /* IgnoreSubsumingPositions */ true);
8085 getKnownStateFromValue(A, IRP, State&: getState(),
8086 /* IgnoreSubsumingPositions */ HasByVal);
8087 }
8088
8089 ChangeStatus manifest(Attributor &A) override {
8090 // TODO: Pointer arguments are not supported on vectors of pointers yet.
8091 if (!getAssociatedValue().getType()->isPointerTy())
8092 return ChangeStatus::UNCHANGED;
8093
8094 // TODO: From readattrs.ll: "inalloca parameters are always
8095 // considered written"
8096 if (A.hasAttr(IRP: getIRPosition(),
8097 AKs: {Attribute::InAlloca, Attribute::Preallocated})) {
8098 removeKnownBits(BitsEncoding: NO_WRITES);
8099 removeAssumedBits(BitsEncoding: NO_WRITES);
8100 }
8101 A.removeAttrs(IRP: getIRPosition(), AttrKinds);
8102 return AAMemoryBehaviorFloating::manifest(A);
8103 }
8104
8105 /// See AbstractAttribute::trackStatistics()
8106 void trackStatistics() const override {
8107 if (isAssumedReadNone())
8108 STATS_DECLTRACK_ARG_ATTR(readnone)
8109 else if (isAssumedReadOnly())
8110 STATS_DECLTRACK_ARG_ATTR(readonly)
8111 else if (isAssumedWriteOnly())
8112 STATS_DECLTRACK_ARG_ATTR(writeonly)
8113 }
8114};
8115
8116struct AAMemoryBehaviorCallSiteArgument final : AAMemoryBehaviorArgument {
8117 AAMemoryBehaviorCallSiteArgument(const IRPosition &IRP, Attributor &A)
8118 : AAMemoryBehaviorArgument(IRP, A) {}
8119
8120 /// See AbstractAttribute::initialize(...).
8121 void initialize(Attributor &A) override {
8122 // If we don't have an associated attribute this is either a variadic call
8123 // or an indirect call, either way, nothing to do here.
8124 Argument *Arg = getAssociatedArgument();
8125 if (!Arg) {
8126 indicatePessimisticFixpoint();
8127 return;
8128 }
8129 if (Arg->hasByValAttr()) {
8130 addKnownBits(Bits: NO_WRITES);
8131 removeKnownBits(BitsEncoding: NO_READS);
8132 removeAssumedBits(BitsEncoding: NO_READS);
8133 }
8134 AAMemoryBehaviorArgument::initialize(A);
8135 if (getAssociatedFunction()->isDeclaration())
8136 indicatePessimisticFixpoint();
8137 }
8138
8139 /// See AbstractAttribute::updateImpl(...).
8140 ChangeStatus updateImpl(Attributor &A) override {
8141 // TODO: Once we have call site specific value information we can provide
8142 // call site specific liveness liveness information and then it makes
8143 // sense to specialize attributes for call sites arguments instead of
8144 // redirecting requests to the callee argument.
8145 Argument *Arg = getAssociatedArgument();
8146 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
8147 auto *ArgAA =
8148 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
8149 if (!ArgAA)
8150 return indicatePessimisticFixpoint();
8151 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
8152 }
8153
8154 /// See AbstractAttribute::trackStatistics()
8155 void trackStatistics() const override {
8156 if (isAssumedReadNone())
8157 STATS_DECLTRACK_CSARG_ATTR(readnone)
8158 else if (isAssumedReadOnly())
8159 STATS_DECLTRACK_CSARG_ATTR(readonly)
8160 else if (isAssumedWriteOnly())
8161 STATS_DECLTRACK_CSARG_ATTR(writeonly)
8162 }
8163};
8164
8165/// Memory behavior attribute for a call site return position.
8166struct AAMemoryBehaviorCallSiteReturned final : AAMemoryBehaviorFloating {
8167 AAMemoryBehaviorCallSiteReturned(const IRPosition &IRP, Attributor &A)
8168 : AAMemoryBehaviorFloating(IRP, A) {}
8169
8170 /// See AbstractAttribute::initialize(...).
8171 void initialize(Attributor &A) override {
8172 AAMemoryBehaviorImpl::initialize(A);
8173 }
8174 /// See AbstractAttribute::manifest(...).
8175 ChangeStatus manifest(Attributor &A) override {
8176 // We do not annotate returned values.
8177 return ChangeStatus::UNCHANGED;
8178 }
8179
8180 /// See AbstractAttribute::trackStatistics()
8181 void trackStatistics() const override {}
8182};
8183
8184/// An AA to represent the memory behavior function attributes.
8185struct AAMemoryBehaviorFunction final : public AAMemoryBehaviorImpl {
8186 AAMemoryBehaviorFunction(const IRPosition &IRP, Attributor &A)
8187 : AAMemoryBehaviorImpl(IRP, A) {}
8188
8189 /// See AbstractAttribute::updateImpl(Attributor &A).
8190 ChangeStatus updateImpl(Attributor &A) override;
8191
8192 /// See AbstractAttribute::manifest(...).
8193 ChangeStatus manifest(Attributor &A) override {
8194 // TODO: It would be better to merge this with AAMemoryLocation, so that
8195 // we could determine read/write per location. This would also have the
8196 // benefit of only one place trying to manifest the memory attribute.
8197 Function &F = cast<Function>(Val&: getAnchorValue());
8198 MemoryEffects ME = MemoryEffects::unknown();
8199 if (isAssumedReadNone())
8200 ME = MemoryEffects::none();
8201 else if (isAssumedReadOnly())
8202 ME = MemoryEffects::readOnly();
8203 else if (isAssumedWriteOnly())
8204 ME = MemoryEffects::writeOnly();
8205
8206 A.removeAttrs(IRP: getIRPosition(), AttrKinds);
8207 // Clear conflicting writable attribute.
8208 if (ME.onlyReadsMemory())
8209 for (Argument &Arg : F.args())
8210 A.removeAttrs(IRP: IRPosition::argument(Arg), AttrKinds: Attribute::Writable);
8211 return A.manifestAttrs(IRP: getIRPosition(),
8212 DeducedAttrs: Attribute::getWithMemoryEffects(Context&: F.getContext(), ME));
8213 }
8214
8215 /// See AbstractAttribute::trackStatistics()
8216 void trackStatistics() const override {
8217 if (isAssumedReadNone())
8218 STATS_DECLTRACK_FN_ATTR(readnone)
8219 else if (isAssumedReadOnly())
8220 STATS_DECLTRACK_FN_ATTR(readonly)
8221 else if (isAssumedWriteOnly())
8222 STATS_DECLTRACK_FN_ATTR(writeonly)
8223 }
8224};
8225
8226/// AAMemoryBehavior attribute for call sites.
8227struct AAMemoryBehaviorCallSite final
8228 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl> {
8229 AAMemoryBehaviorCallSite(const IRPosition &IRP, Attributor &A)
8230 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl>(IRP, A) {}
8231
8232 /// See AbstractAttribute::manifest(...).
8233 ChangeStatus manifest(Attributor &A) override {
8234 // TODO: Deduplicate this with AAMemoryBehaviorFunction.
8235 CallBase &CB = cast<CallBase>(Val&: getAnchorValue());
8236 MemoryEffects ME = MemoryEffects::unknown();
8237 if (isAssumedReadNone())
8238 ME = MemoryEffects::none();
8239 else if (isAssumedReadOnly())
8240 ME = MemoryEffects::readOnly();
8241 else if (isAssumedWriteOnly())
8242 ME = MemoryEffects::writeOnly();
8243
8244 A.removeAttrs(IRP: getIRPosition(), AttrKinds);
8245 // Clear conflicting writable attribute.
8246 if (ME.onlyReadsMemory())
8247 for (Use &U : CB.args())
8248 A.removeAttrs(IRP: IRPosition::callsite_argument(CB, ArgNo: U.getOperandNo()),
8249 AttrKinds: Attribute::Writable);
8250 return A.manifestAttrs(
8251 IRP: getIRPosition(), DeducedAttrs: Attribute::getWithMemoryEffects(Context&: CB.getContext(), ME));
8252 }
8253
8254 /// See AbstractAttribute::trackStatistics()
8255 void trackStatistics() const override {
8256 if (isAssumedReadNone())
8257 STATS_DECLTRACK_CS_ATTR(readnone)
8258 else if (isAssumedReadOnly())
8259 STATS_DECLTRACK_CS_ATTR(readonly)
8260 else if (isAssumedWriteOnly())
8261 STATS_DECLTRACK_CS_ATTR(writeonly)
8262 }
8263};
8264
8265ChangeStatus AAMemoryBehaviorFunction::updateImpl(Attributor &A) {
8266
8267 // The current assumed state used to determine a change.
8268 auto AssumedState = getAssumed();
8269
8270 auto CheckRWInst = [&](Instruction &I) {
8271 // If the instruction has an own memory behavior state, use it to restrict
8272 // the local state. No further analysis is required as the other memory
8273 // state is as optimistic as it gets.
8274 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
8275 const auto *MemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
8276 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::REQUIRED);
8277 if (MemBehaviorAA) {
8278 intersectAssumedBits(BitsEncoding: MemBehaviorAA->getAssumed());
8279 return !isAtFixpoint();
8280 }
8281 }
8282
8283 // Remove access kind modifiers if necessary.
8284 if (I.mayReadFromMemory())
8285 removeAssumedBits(BitsEncoding: NO_READS);
8286 if (I.mayWriteToMemory())
8287 removeAssumedBits(BitsEncoding: NO_WRITES);
8288 return !isAtFixpoint();
8289 };
8290
8291 bool UsedAssumedInformation = false;
8292 if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInst, QueryingAA&: *this,
8293 UsedAssumedInformation))
8294 return indicatePessimisticFixpoint();
8295
8296 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8297 : ChangeStatus::UNCHANGED;
8298}
8299
8300ChangeStatus AAMemoryBehaviorFloating::updateImpl(Attributor &A) {
8301
8302 const IRPosition &IRP = getIRPosition();
8303 const IRPosition &FnPos = IRPosition::function_scope(IRP);
8304 AAMemoryBehavior::StateType &S = getState();
8305
8306 // First, check the function scope. We take the known information and we avoid
8307 // work if the assumed information implies the current assumed information for
8308 // this attribute. This is a valid for all but byval arguments.
8309 Argument *Arg = IRP.getAssociatedArgument();
8310 AAMemoryBehavior::base_t FnMemAssumedState =
8311 AAMemoryBehavior::StateType::getWorstState();
8312 if (!Arg || !Arg->hasByValAttr()) {
8313 const auto *FnMemAA =
8314 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::OPTIONAL);
8315 if (FnMemAA) {
8316 FnMemAssumedState = FnMemAA->getAssumed();
8317 S.addKnownBits(Bits: FnMemAA->getKnown());
8318 if ((S.getAssumed() & FnMemAA->getAssumed()) == S.getAssumed())
8319 return ChangeStatus::UNCHANGED;
8320 }
8321 }
8322
8323 // The current assumed state used to determine a change.
8324 auto AssumedState = S.getAssumed();
8325
8326 // Make sure the value is not captured (except through "return"), if
8327 // it is, any information derived would be irrelevant anyway as we cannot
8328 // check the potential aliases introduced by the capture. However, no need
8329 // to fall back to anythign less optimistic than the function state.
8330 bool IsKnownNoCapture;
8331 const AANoCapture *ArgNoCaptureAA = nullptr;
8332 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
8333 A, QueryingAA: this, IRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
8334 AAPtr: &ArgNoCaptureAA);
8335
8336 if (!IsAssumedNoCapture &&
8337 (!ArgNoCaptureAA || !ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
8338 S.intersectAssumedBits(BitsEncoding: FnMemAssumedState);
8339 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8340 : ChangeStatus::UNCHANGED;
8341 }
8342
8343 // Visit and expand uses until all are analyzed or a fixpoint is reached.
8344 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
8345 Instruction *UserI = cast<Instruction>(Val: U.getUser());
8346 LLVM_DEBUG(dbgs() << "[AAMemoryBehavior] Use: " << *U << " in " << *UserI
8347 << " \n");
8348
8349 // Droppable users, e.g., llvm::assume does not actually perform any action.
8350 if (UserI->isDroppable())
8351 return true;
8352
8353 // Check if the users of UserI should also be visited.
8354 Follow = followUsersOfUseIn(A, U, UserI);
8355
8356 // If UserI might touch memory we analyze the use in detail.
8357 if (UserI->mayReadOrWriteMemory())
8358 analyzeUseIn(A, U, UserI);
8359
8360 return !isAtFixpoint();
8361 };
8362
8363 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: getAssociatedValue()))
8364 return indicatePessimisticFixpoint();
8365
8366 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8367 : ChangeStatus::UNCHANGED;
8368}
8369
8370bool AAMemoryBehaviorFloating::followUsersOfUseIn(Attributor &A, const Use &U,
8371 const Instruction *UserI) {
8372 // The loaded value is unrelated to the pointer argument, no need to
8373 // follow the users of the load.
8374 if (isa<LoadInst>(Val: UserI) || isa<ReturnInst>(Val: UserI))
8375 return false;
8376
8377 // By default we follow all uses assuming UserI might leak information on U,
8378 // we have special handling for call sites operands though.
8379 const auto *CB = dyn_cast<CallBase>(Val: UserI);
8380 if (!CB || !CB->isArgOperand(U: &U))
8381 return true;
8382
8383 // If the use is a call argument known not to be captured, the users of
8384 // the call do not need to be visited because they have to be unrelated to
8385 // the input. Note that this check is not trivial even though we disallow
8386 // general capturing of the underlying argument. The reason is that the
8387 // call might the argument "through return", which we allow and for which we
8388 // need to check call users.
8389 if (U.get()->getType()->isPointerTy()) {
8390 unsigned ArgNo = CB->getArgOperandNo(U: &U);
8391 bool IsKnownNoCapture;
8392 return !AA::hasAssumedIRAttr<Attribute::Captures>(
8393 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
8394 DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture);
8395 }
8396
8397 return true;
8398}
8399
8400void AAMemoryBehaviorFloating::analyzeUseIn(Attributor &A, const Use &U,
8401 const Instruction *UserI) {
8402 assert(UserI->mayReadOrWriteMemory());
8403
8404 switch (UserI->getOpcode()) {
8405 default:
8406 // TODO: Handle all atomics and other side-effect operations we know of.
8407 break;
8408 case Instruction::Load:
8409 // Loads cause the NO_READS property to disappear.
8410 removeAssumedBits(BitsEncoding: NO_READS);
8411 return;
8412
8413 case Instruction::Store:
8414 // Stores cause the NO_WRITES property to disappear if the use is the
8415 // pointer operand. Note that while capturing was taken care of somewhere
8416 // else we need to deal with stores of the value that is not looked through.
8417 if (cast<StoreInst>(Val: UserI)->getPointerOperand() == U.get())
8418 removeAssumedBits(BitsEncoding: NO_WRITES);
8419 else
8420 indicatePessimisticFixpoint();
8421 return;
8422
8423 case Instruction::Call:
8424 case Instruction::CallBr:
8425 case Instruction::Invoke: {
8426 // For call sites we look at the argument memory behavior attribute (this
8427 // could be recursive!) in order to restrict our own state.
8428 const auto *CB = cast<CallBase>(Val: UserI);
8429
8430 // Give up on operand bundles.
8431 if (CB->isBundleOperand(U: &U)) {
8432 indicatePessimisticFixpoint();
8433 return;
8434 }
8435
8436 // Calling a function does read the function pointer, maybe write it if the
8437 // function is self-modifying.
8438 if (CB->isCallee(U: &U)) {
8439 removeAssumedBits(BitsEncoding: NO_READS);
8440 break;
8441 }
8442
8443 // Adjust the possible access behavior based on the information on the
8444 // argument.
8445 IRPosition Pos;
8446 if (U.get()->getType()->isPointerTy())
8447 Pos = IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U));
8448 else
8449 Pos = IRPosition::callsite_function(CB: *CB);
8450 const auto *MemBehaviorAA =
8451 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: Pos, DepClass: DepClassTy::OPTIONAL);
8452 if (!MemBehaviorAA)
8453 break;
8454 // "assumed" has at most the same bits as the MemBehaviorAA assumed
8455 // and at least "known".
8456 intersectAssumedBits(BitsEncoding: MemBehaviorAA->getAssumed());
8457 return;
8458 }
8459 };
8460
8461 // Generally, look at the "may-properties" and adjust the assumed state if we
8462 // did not trigger special handling before.
8463 if (UserI->mayReadFromMemory())
8464 removeAssumedBits(BitsEncoding: NO_READS);
8465 if (UserI->mayWriteToMemory())
8466 removeAssumedBits(BitsEncoding: NO_WRITES);
8467}
8468} // namespace
8469
8470/// -------------------- Memory Locations Attributes ---------------------------
8471/// Includes read-none, argmemonly, inaccessiblememonly,
8472/// inaccessiblememorargmemonly
8473/// ----------------------------------------------------------------------------
8474
8475std::string AAMemoryLocation::getMemoryLocationsAsStr(
8476 AAMemoryLocation::MemoryLocationsKind MLK) {
8477 if (0 == (MLK & AAMemoryLocation::NO_LOCATIONS))
8478 return "all memory";
8479 if (MLK == AAMemoryLocation::NO_LOCATIONS)
8480 return "no memory";
8481 std::string S = "memory:";
8482 if (0 == (MLK & AAMemoryLocation::NO_LOCAL_MEM))
8483 S += "stack,";
8484 if (0 == (MLK & AAMemoryLocation::NO_CONST_MEM))
8485 S += "constant,";
8486 if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_INTERNAL_MEM))
8487 S += "internal global,";
8488 if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_EXTERNAL_MEM))
8489 S += "external global,";
8490 if (0 == (MLK & AAMemoryLocation::NO_ARGUMENT_MEM))
8491 S += "argument,";
8492 if (0 == (MLK & AAMemoryLocation::NO_INACCESSIBLE_MEM))
8493 S += "inaccessible,";
8494 if (0 == (MLK & AAMemoryLocation::NO_MALLOCED_MEM))
8495 S += "malloced,";
8496 if (0 == (MLK & AAMemoryLocation::NO_UNKOWN_MEM))
8497 S += "unknown,";
8498 S.pop_back();
8499 return S;
8500}
8501
8502namespace {
8503struct AAMemoryLocationImpl : public AAMemoryLocation {
8504
8505 AAMemoryLocationImpl(const IRPosition &IRP, Attributor &A)
8506 : AAMemoryLocation(IRP, A), Allocator(A.Allocator) {
8507 AccessKind2Accesses.fill(u: nullptr);
8508 }
8509
8510 ~AAMemoryLocationImpl() override {
8511 // The AccessSets are allocated via a BumpPtrAllocator, we call
8512 // the destructor manually.
8513 for (AccessSet *AS : AccessKind2Accesses)
8514 if (AS)
8515 AS->~AccessSet();
8516 }
8517
8518 /// See AbstractAttribute::initialize(...).
8519 void initialize(Attributor &A) override {
8520 intersectAssumedBits(BitsEncoding: BEST_STATE);
8521 getKnownStateFromValue(A, IRP: getIRPosition(), State&: getState());
8522 AAMemoryLocation::initialize(A);
8523 }
8524
8525 /// Return the memory behavior information encoded in the IR for \p IRP.
8526 static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
8527 BitIntegerState &State,
8528 bool IgnoreSubsumingPositions = false) {
8529 // For internal functions we ignore `argmemonly` and
8530 // `inaccessiblememorargmemonly` as we might break it via interprocedural
8531 // constant propagation. It is unclear if this is the best way but it is
8532 // unlikely this will cause real performance problems. If we are deriving
8533 // attributes for the anchor function we even remove the attribute in
8534 // addition to ignoring it.
8535 // TODO: A better way to handle this would be to add ~NO_GLOBAL_MEM /
8536 // MemoryEffects::Other as a possible location.
8537 bool UseArgMemOnly = true;
8538 Function *AnchorFn = IRP.getAnchorScope();
8539 if (AnchorFn && A.isRunOn(Fn&: *AnchorFn))
8540 UseArgMemOnly = !AnchorFn->hasLocalLinkage();
8541
8542 SmallVector<Attribute, 2> Attrs;
8543 A.getAttrs(IRP, AKs: {Attribute::Memory}, Attrs, IgnoreSubsumingPositions);
8544 for (const Attribute &Attr : Attrs) {
8545 // TODO: We can map MemoryEffects to Attributor locations more precisely.
8546 MemoryEffects ME = Attr.getMemoryEffects();
8547 if (ME.doesNotAccessMemory()) {
8548 State.addKnownBits(Bits: NO_LOCAL_MEM | NO_CONST_MEM);
8549 continue;
8550 }
8551 if (ME.onlyAccessesInaccessibleMem()) {
8552 State.addKnownBits(Bits: inverseLocation(Loc: NO_INACCESSIBLE_MEM, AndLocalMem: true, AndConstMem: true));
8553 continue;
8554 }
8555 if (ME.onlyAccessesArgPointees()) {
8556 if (UseArgMemOnly)
8557 State.addKnownBits(Bits: inverseLocation(Loc: NO_ARGUMENT_MEM, AndLocalMem: true, AndConstMem: true));
8558 else {
8559 // Remove location information, only keep read/write info.
8560 ME = MemoryEffects(ME.getModRef());
8561 A.manifestAttrs(IRP,
8562 DeducedAttrs: Attribute::getWithMemoryEffects(
8563 Context&: IRP.getAnchorValue().getContext(), ME),
8564 /*ForceReplace*/ true);
8565 }
8566 continue;
8567 }
8568 if (ME.onlyAccessesInaccessibleOrArgMem()) {
8569 if (UseArgMemOnly)
8570 State.addKnownBits(Bits: inverseLocation(
8571 Loc: NO_INACCESSIBLE_MEM | NO_ARGUMENT_MEM, AndLocalMem: true, AndConstMem: true));
8572 else {
8573 // Remove location information, only keep read/write info.
8574 ME = MemoryEffects(ME.getModRef());
8575 A.manifestAttrs(IRP,
8576 DeducedAttrs: Attribute::getWithMemoryEffects(
8577 Context&: IRP.getAnchorValue().getContext(), ME),
8578 /*ForceReplace*/ true);
8579 }
8580 continue;
8581 }
8582 }
8583 }
8584
8585 /// See AbstractAttribute::getDeducedAttributes(...).
8586 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
8587 SmallVectorImpl<Attribute> &Attrs) const override {
8588 // TODO: We can map Attributor locations to MemoryEffects more precisely.
8589 assert(Attrs.size() == 0);
8590 if (getIRPosition().getPositionKind() == IRPosition::IRP_FUNCTION) {
8591 if (isAssumedReadNone())
8592 Attrs.push_back(
8593 Elt: Attribute::getWithMemoryEffects(Context&: Ctx, ME: MemoryEffects::none()));
8594 else if (isAssumedInaccessibleMemOnly())
8595 Attrs.push_back(Elt: Attribute::getWithMemoryEffects(
8596 Context&: Ctx, ME: MemoryEffects::inaccessibleMemOnly()));
8597 else if (isAssumedArgMemOnly())
8598 Attrs.push_back(
8599 Elt: Attribute::getWithMemoryEffects(Context&: Ctx, ME: MemoryEffects::argMemOnly()));
8600 else if (isAssumedInaccessibleOrArgMemOnly())
8601 Attrs.push_back(Elt: Attribute::getWithMemoryEffects(
8602 Context&: Ctx, ME: MemoryEffects::inaccessibleOrArgMemOnly()));
8603 }
8604 assert(Attrs.size() <= 1);
8605 }
8606
8607 /// See AbstractAttribute::manifest(...).
8608 ChangeStatus manifest(Attributor &A) override {
8609 // TODO: If AAMemoryLocation and AAMemoryBehavior are merged, we could
8610 // provide per-location modref information here.
8611 const IRPosition &IRP = getIRPosition();
8612
8613 SmallVector<Attribute, 1> DeducedAttrs;
8614 getDeducedAttributes(A, Ctx&: IRP.getAnchorValue().getContext(), Attrs&: DeducedAttrs);
8615 if (DeducedAttrs.size() != 1)
8616 return ChangeStatus::UNCHANGED;
8617 MemoryEffects ME = DeducedAttrs[0].getMemoryEffects();
8618
8619 return A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithMemoryEffects(
8620 Context&: IRP.getAnchorValue().getContext(), ME));
8621 }
8622
8623 /// See AAMemoryLocation::checkForAllAccessesToMemoryKind(...).
8624 bool checkForAllAccessesToMemoryKind(
8625 function_ref<bool(const Instruction *, const Value *, AccessKind,
8626 MemoryLocationsKind)>
8627 Pred,
8628 MemoryLocationsKind RequestedMLK) const override {
8629 if (!isValidState())
8630 return false;
8631
8632 MemoryLocationsKind AssumedMLK = getAssumedNotAccessedLocation();
8633 if (AssumedMLK == NO_LOCATIONS)
8634 return true;
8635
8636 unsigned Idx = 0;
8637 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS;
8638 CurMLK *= 2, ++Idx) {
8639 if (CurMLK & RequestedMLK)
8640 continue;
8641
8642 if (const AccessSet *Accesses = AccessKind2Accesses[Idx])
8643 for (const AccessInfo &AI : *Accesses)
8644 if (!Pred(AI.I, AI.Ptr, AI.Kind, CurMLK))
8645 return false;
8646 }
8647
8648 return true;
8649 }
8650
8651 ChangeStatus indicatePessimisticFixpoint() override {
8652 // If we give up and indicate a pessimistic fixpoint this instruction will
8653 // become an access for all potential access kinds:
8654 // TODO: Add pointers for argmemonly and globals to improve the results of
8655 // checkForAllAccessesToMemoryKind.
8656 bool Changed = false;
8657 MemoryLocationsKind KnownMLK = getKnown();
8658 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
8659 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2)
8660 if (!(CurMLK & KnownMLK))
8661 updateStateAndAccessesMap(State&: getState(), MLK: CurMLK, I, Ptr: nullptr, Changed,
8662 AK: getAccessKindFromInst(I));
8663 return AAMemoryLocation::indicatePessimisticFixpoint();
8664 }
8665
8666protected:
8667 /// Helper struct to tie together an instruction that has a read or write
8668 /// effect with the pointer it accesses (if any).
8669 struct AccessInfo {
8670
8671 /// The instruction that caused the access.
8672 const Instruction *I;
8673
8674 /// The base pointer that is accessed, or null if unknown.
8675 const Value *Ptr;
8676
8677 /// The kind of access (read/write/read+write).
8678 AccessKind Kind;
8679
8680 bool operator==(const AccessInfo &RHS) const {
8681 return I == RHS.I && Ptr == RHS.Ptr && Kind == RHS.Kind;
8682 }
8683 bool operator()(const AccessInfo &LHS, const AccessInfo &RHS) const {
8684 if (LHS.I != RHS.I)
8685 return LHS.I < RHS.I;
8686 if (LHS.Ptr != RHS.Ptr)
8687 return LHS.Ptr < RHS.Ptr;
8688 if (LHS.Kind != RHS.Kind)
8689 return LHS.Kind < RHS.Kind;
8690 return false;
8691 }
8692 };
8693
8694 /// Mapping from *single* memory location kinds, e.g., LOCAL_MEM with the
8695 /// value of NO_LOCAL_MEM, to the accesses encountered for this memory kind.
8696 using AccessSet = SmallSet<AccessInfo, 2, AccessInfo>;
8697 std::array<AccessSet *, llvm::ConstantLog2<VALID_STATE>()>
8698 AccessKind2Accesses;
8699
8700 /// Categorize the pointer arguments of CB that might access memory in
8701 /// AccessedLoc and update the state and access map accordingly.
8702 void
8703 categorizeArgumentPointerLocations(Attributor &A, CallBase &CB,
8704 AAMemoryLocation::StateType &AccessedLocs,
8705 bool &Changed);
8706
8707 /// Return the kind(s) of location that may be accessed by \p V.
8708 AAMemoryLocation::MemoryLocationsKind
8709 categorizeAccessedLocations(Attributor &A, Instruction &I, bool &Changed);
8710
8711 /// Return the access kind as determined by \p I.
8712 AccessKind getAccessKindFromInst(const Instruction *I) {
8713 AccessKind AK = READ_WRITE;
8714 if (I) {
8715 AK = I->mayReadFromMemory() ? READ : NONE;
8716 AK = AccessKind(AK | (I->mayWriteToMemory() ? WRITE : NONE));
8717 }
8718 return AK;
8719 }
8720
8721 /// Update the state \p State and the AccessKind2Accesses given that \p I is
8722 /// an access of kind \p AK to a \p MLK memory location with the access
8723 /// pointer \p Ptr.
8724 void updateStateAndAccessesMap(AAMemoryLocation::StateType &State,
8725 MemoryLocationsKind MLK, const Instruction *I,
8726 const Value *Ptr, bool &Changed,
8727 AccessKind AK = READ_WRITE) {
8728
8729 assert(isPowerOf2_32(MLK) && "Expected a single location set!");
8730 auto *&Accesses = AccessKind2Accesses[llvm::Log2_32(Value: MLK)];
8731 if (!Accesses)
8732 Accesses = new (Allocator) AccessSet();
8733 Changed |= Accesses->insert(V: AccessInfo{.I: I, .Ptr: Ptr, .Kind: AK}).second;
8734 if (MLK == NO_UNKOWN_MEM)
8735 MLK = NO_LOCATIONS;
8736 State.removeAssumedBits(BitsEncoding: MLK);
8737 }
8738
8739 /// Determine the underlying locations kinds for \p Ptr, e.g., globals or
8740 /// arguments, and update the state and access map accordingly.
8741 void categorizePtrValue(Attributor &A, const Instruction &I, const Value &Ptr,
8742 AAMemoryLocation::StateType &State, bool &Changed,
8743 unsigned AccessAS = 0);
8744
8745 /// Used to allocate access sets.
8746 BumpPtrAllocator &Allocator;
8747};
8748
8749void AAMemoryLocationImpl::categorizePtrValue(
8750 Attributor &A, const Instruction &I, const Value &Ptr,
8751 AAMemoryLocation::StateType &State, bool &Changed, unsigned AccessAS) {
8752 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize pointer locations for "
8753 << Ptr << " ["
8754 << getMemoryLocationsAsStr(State.getAssumed()) << "]\n");
8755
8756 auto Pred = [&](Value &Obj) {
8757 unsigned ObjectAS = Obj.getType()->getPointerAddressSpace();
8758 // TODO: recognize the TBAA used for constant accesses.
8759 MemoryLocationsKind MLK = NO_LOCATIONS;
8760
8761 // Filter accesses to constant (GPU) memory if we have an AS at the access
8762 // site or the object is known to actually have the associated AS.
8763 if (AA::isGPU(M: A.getModule())) {
8764 if (AA::isGPUConstantAddressSpace(M: A.getModule(), AS: AccessAS) ||
8765 (AA::isGPUConstantAddressSpace(M: A.getModule(), AS: ObjectAS) &&
8766 isIdentifiedObject(V: &Obj)))
8767 return true;
8768 }
8769
8770 if (isa<UndefValue>(Val: &Obj))
8771 return true;
8772 if (isa<Argument>(Val: &Obj)) {
8773 // TODO: For now we do not treat byval arguments as local copies performed
8774 // on the call edge, though, we should. To make that happen we need to
8775 // teach various passes, e.g., DSE, about the copy effect of a byval. That
8776 // would also allow us to mark functions only accessing byval arguments as
8777 // readnone again, arguably their accesses have no effect outside of the
8778 // function, like accesses to allocas.
8779 MLK = NO_ARGUMENT_MEM;
8780 } else if (auto *GV = dyn_cast<GlobalValue>(Val: &Obj)) {
8781 // Reading constant memory is not treated as a read "effect" by the
8782 // function attr pass so we won't neither. Constants defined by TBAA are
8783 // similar. (We know we do not write it because it is constant.)
8784 if (auto *GVar = dyn_cast<GlobalVariable>(Val: GV))
8785 if (GVar->isConstant())
8786 return true;
8787
8788 if (GV->hasLocalLinkage())
8789 MLK = NO_GLOBAL_INTERNAL_MEM;
8790 else
8791 MLK = NO_GLOBAL_EXTERNAL_MEM;
8792 } else if (isa<ConstantPointerNull>(Val: &Obj) &&
8793 (!NullPointerIsDefined(F: getAssociatedFunction(), AS: AccessAS) ||
8794 !NullPointerIsDefined(F: getAssociatedFunction(), AS: ObjectAS))) {
8795 return true;
8796 } else if (isa<AllocaInst>(Val: &Obj)) {
8797 MLK = NO_LOCAL_MEM;
8798 } else if (const auto *CB = dyn_cast<CallBase>(Val: &Obj)) {
8799 bool IsKnownNoAlias;
8800 if (AA::hasAssumedIRAttr<Attribute::NoAlias>(
8801 A, QueryingAA: this, IRP: IRPosition::callsite_returned(CB: *CB), DepClass: DepClassTy::OPTIONAL,
8802 IsKnown&: IsKnownNoAlias))
8803 MLK = NO_MALLOCED_MEM;
8804 else
8805 MLK = NO_UNKOWN_MEM;
8806 } else {
8807 MLK = NO_UNKOWN_MEM;
8808 }
8809
8810 assert(MLK != NO_LOCATIONS && "No location specified!");
8811 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Ptr value can be categorized: "
8812 << Obj << " -> " << getMemoryLocationsAsStr(MLK) << "\n");
8813 updateStateAndAccessesMap(State, MLK, I: &I, Ptr: &Obj, Changed,
8814 AK: getAccessKindFromInst(I: &I));
8815
8816 return true;
8817 };
8818
8819 const auto *AA = A.getAAFor<AAUnderlyingObjects>(
8820 QueryingAA: *this, IRP: IRPosition::value(V: Ptr), DepClass: DepClassTy::OPTIONAL);
8821 if (!AA || !AA->forallUnderlyingObjects(Pred, Scope: AA::Intraprocedural)) {
8822 LLVM_DEBUG(
8823 dbgs() << "[AAMemoryLocation] Pointer locations not categorized\n");
8824 updateStateAndAccessesMap(State, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr, Changed,
8825 AK: getAccessKindFromInst(I: &I));
8826 return;
8827 }
8828
8829 LLVM_DEBUG(
8830 dbgs() << "[AAMemoryLocation] Accessed locations with pointer locations: "
8831 << getMemoryLocationsAsStr(State.getAssumed()) << "\n");
8832}
8833
8834void AAMemoryLocationImpl::categorizeArgumentPointerLocations(
8835 Attributor &A, CallBase &CB, AAMemoryLocation::StateType &AccessedLocs,
8836 bool &Changed) {
8837 for (unsigned ArgNo = 0, E = CB.arg_size(); ArgNo < E; ++ArgNo) {
8838
8839 // Skip non-pointer arguments.
8840 const Value *ArgOp = CB.getArgOperand(i: ArgNo);
8841 if (!ArgOp->getType()->isPtrOrPtrVectorTy())
8842 continue;
8843
8844 // Skip readnone arguments.
8845 const IRPosition &ArgOpIRP = IRPosition::callsite_argument(CB, ArgNo);
8846 const auto *ArgOpMemLocationAA =
8847 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: ArgOpIRP, DepClass: DepClassTy::OPTIONAL);
8848
8849 if (ArgOpMemLocationAA && ArgOpMemLocationAA->isAssumedReadNone())
8850 continue;
8851
8852 // Categorize potentially accessed pointer arguments as if there was an
8853 // access instruction with them as pointer.
8854 categorizePtrValue(A, I: CB, Ptr: *ArgOp, State&: AccessedLocs, Changed);
8855 }
8856}
8857
8858AAMemoryLocation::MemoryLocationsKind
8859AAMemoryLocationImpl::categorizeAccessedLocations(Attributor &A, Instruction &I,
8860 bool &Changed) {
8861 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize accessed locations for "
8862 << I << "\n");
8863
8864 AAMemoryLocation::StateType AccessedLocs;
8865 AccessedLocs.intersectAssumedBits(BitsEncoding: NO_LOCATIONS);
8866
8867 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
8868
8869 // First check if we assume any memory is access is visible.
8870 const auto *CBMemLocationAA = A.getAAFor<AAMemoryLocation>(
8871 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL);
8872 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize call site: " << I
8873 << " [" << CBMemLocationAA << "]\n");
8874 if (!CBMemLocationAA) {
8875 updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr,
8876 Changed, AK: getAccessKindFromInst(I: &I));
8877 return NO_UNKOWN_MEM;
8878 }
8879
8880 if (CBMemLocationAA->isAssumedReadNone())
8881 return NO_LOCATIONS;
8882
8883 if (CBMemLocationAA->isAssumedInaccessibleMemOnly()) {
8884 updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_INACCESSIBLE_MEM, I: &I, Ptr: nullptr,
8885 Changed, AK: getAccessKindFromInst(I: &I));
8886 return AccessedLocs.getAssumed();
8887 }
8888
8889 uint32_t CBAssumedNotAccessedLocs =
8890 CBMemLocationAA->getAssumedNotAccessedLocation();
8891
8892 // Set the argmemonly and global bit as we handle them separately below.
8893 uint32_t CBAssumedNotAccessedLocsNoArgMem =
8894 CBAssumedNotAccessedLocs | NO_ARGUMENT_MEM | NO_GLOBAL_MEM;
8895
8896 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2) {
8897 if (CBAssumedNotAccessedLocsNoArgMem & CurMLK)
8898 continue;
8899 updateStateAndAccessesMap(State&: AccessedLocs, MLK: CurMLK, I: &I, Ptr: nullptr, Changed,
8900 AK: getAccessKindFromInst(I: &I));
8901 }
8902
8903 // Now handle global memory if it might be accessed. This is slightly tricky
8904 // as NO_GLOBAL_MEM has multiple bits set.
8905 bool HasGlobalAccesses = ((~CBAssumedNotAccessedLocs) & NO_GLOBAL_MEM);
8906 if (HasGlobalAccesses) {
8907 auto AccessPred = [&](const Instruction *, const Value *Ptr,
8908 AccessKind Kind, MemoryLocationsKind MLK) {
8909 updateStateAndAccessesMap(State&: AccessedLocs, MLK, I: &I, Ptr, Changed,
8910 AK: getAccessKindFromInst(I: &I));
8911 return true;
8912 };
8913 if (!CBMemLocationAA->checkForAllAccessesToMemoryKind(
8914 Pred: AccessPred, MLK: inverseLocation(Loc: NO_GLOBAL_MEM, AndLocalMem: false, AndConstMem: false)))
8915 return AccessedLocs.getWorstState();
8916 }
8917
8918 LLVM_DEBUG(
8919 dbgs() << "[AAMemoryLocation] Accessed state before argument handling: "
8920 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8921
8922 // Now handle argument memory if it might be accessed.
8923 bool HasArgAccesses = ((~CBAssumedNotAccessedLocs) & NO_ARGUMENT_MEM);
8924 if (HasArgAccesses)
8925 categorizeArgumentPointerLocations(A, CB&: *CB, AccessedLocs, Changed);
8926
8927 LLVM_DEBUG(
8928 dbgs() << "[AAMemoryLocation] Accessed state after argument handling: "
8929 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8930
8931 return AccessedLocs.getAssumed();
8932 }
8933
8934 if (const Value *Ptr = getPointerOperand(I: &I, /* AllowVolatile */ true)) {
8935 LLVM_DEBUG(
8936 dbgs() << "[AAMemoryLocation] Categorize memory access with pointer: "
8937 << I << " [" << *Ptr << "]\n");
8938 categorizePtrValue(A, I, Ptr: *Ptr, State&: AccessedLocs, Changed,
8939 AccessAS: Ptr->getType()->getPointerAddressSpace());
8940 return AccessedLocs.getAssumed();
8941 }
8942
8943 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Failed to categorize instruction: "
8944 << I << "\n");
8945 updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr, Changed,
8946 AK: getAccessKindFromInst(I: &I));
8947 return AccessedLocs.getAssumed();
8948}
8949
8950/// An AA to represent the memory behavior function attributes.
8951struct AAMemoryLocationFunction final : public AAMemoryLocationImpl {
8952 AAMemoryLocationFunction(const IRPosition &IRP, Attributor &A)
8953 : AAMemoryLocationImpl(IRP, A) {}
8954
8955 /// See AbstractAttribute::updateImpl(Attributor &A).
8956 ChangeStatus updateImpl(Attributor &A) override {
8957
8958 const auto *MemBehaviorAA =
8959 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::NONE);
8960 if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
8961 if (MemBehaviorAA->isKnownReadNone())
8962 return indicateOptimisticFixpoint();
8963 assert(isAssumedReadNone() &&
8964 "AAMemoryLocation was not read-none but AAMemoryBehavior was!");
8965 A.recordDependence(FromAA: *MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
8966 return ChangeStatus::UNCHANGED;
8967 }
8968
8969 // The current assumed state used to determine a change.
8970 auto AssumedState = getAssumed();
8971 bool Changed = false;
8972
8973 auto CheckRWInst = [&](Instruction &I) {
8974 MemoryLocationsKind MLK = categorizeAccessedLocations(A, I, Changed);
8975 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Accessed locations for " << I
8976 << ": " << getMemoryLocationsAsStr(MLK) << "\n");
8977 removeAssumedBits(BitsEncoding: inverseLocation(Loc: MLK, AndLocalMem: false, AndConstMem: false));
8978 // Stop once only the valid bit set in the *not assumed location*, thus
8979 // once we don't actually exclude any memory locations in the state.
8980 return getAssumedNotAccessedLocation() != VALID_STATE;
8981 };
8982
8983 bool UsedAssumedInformation = false;
8984 if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInst, QueryingAA&: *this,
8985 UsedAssumedInformation))
8986 return indicatePessimisticFixpoint();
8987
8988 Changed |= AssumedState != getAssumed();
8989 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
8990 }
8991
8992 /// See AbstractAttribute::trackStatistics()
8993 void trackStatistics() const override {
8994 if (isAssumedReadNone())
8995 STATS_DECLTRACK_FN_ATTR(readnone)
8996 else if (isAssumedArgMemOnly())
8997 STATS_DECLTRACK_FN_ATTR(argmemonly)
8998 else if (isAssumedInaccessibleMemOnly())
8999 STATS_DECLTRACK_FN_ATTR(inaccessiblememonly)
9000 else if (isAssumedInaccessibleOrArgMemOnly())
9001 STATS_DECLTRACK_FN_ATTR(inaccessiblememorargmemonly)
9002 }
9003};
9004
9005/// AAMemoryLocation attribute for call sites.
9006struct AAMemoryLocationCallSite final : AAMemoryLocationImpl {
9007 AAMemoryLocationCallSite(const IRPosition &IRP, Attributor &A)
9008 : AAMemoryLocationImpl(IRP, A) {}
9009
9010 /// See AbstractAttribute::updateImpl(...).
9011 ChangeStatus updateImpl(Attributor &A) override {
9012 // TODO: Once we have call site specific value information we can provide
9013 // call site specific liveness liveness information and then it makes
9014 // sense to specialize attributes for call sites arguments instead of
9015 // redirecting requests to the callee argument.
9016 Function *F = getAssociatedFunction();
9017 const IRPosition &FnPos = IRPosition::function(F: *F);
9018 auto *FnAA =
9019 A.getAAFor<AAMemoryLocation>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
9020 if (!FnAA)
9021 return indicatePessimisticFixpoint();
9022 bool Changed = false;
9023 auto AccessPred = [&](const Instruction *I, const Value *Ptr,
9024 AccessKind Kind, MemoryLocationsKind MLK) {
9025 updateStateAndAccessesMap(State&: getState(), MLK, I, Ptr, Changed,
9026 AK: getAccessKindFromInst(I));
9027 return true;
9028 };
9029 if (!FnAA->checkForAllAccessesToMemoryKind(Pred: AccessPred, MLK: ALL_LOCATIONS))
9030 return indicatePessimisticFixpoint();
9031 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
9032 }
9033
9034 /// See AbstractAttribute::trackStatistics()
9035 void trackStatistics() const override {
9036 if (isAssumedReadNone())
9037 STATS_DECLTRACK_CS_ATTR(readnone)
9038 }
9039};
9040} // namespace
9041
9042/// ------------------ denormal-fp-math Attribute -------------------------
9043
9044namespace {
9045struct AADenormalFPMathImpl : public AADenormalFPMath {
9046 AADenormalFPMathImpl(const IRPosition &IRP, Attributor &A)
9047 : AADenormalFPMath(IRP, A) {}
9048
9049 const std::string getAsStr(Attributor *A) const override {
9050 std::string Str("AADenormalFPMath[");
9051 raw_string_ostream OS(Str);
9052
9053 DenormalState Known = getKnown();
9054 if (Known.Mode.isValid())
9055 OS << "denormal-fp-math=" << Known.Mode;
9056 else
9057 OS << "invalid";
9058
9059 if (Known.ModeF32.isValid())
9060 OS << " denormal-fp-math-f32=" << Known.ModeF32;
9061 OS << ']';
9062 return Str;
9063 }
9064};
9065
9066struct AADenormalFPMathFunction final : AADenormalFPMathImpl {
9067 AADenormalFPMathFunction(const IRPosition &IRP, Attributor &A)
9068 : AADenormalFPMathImpl(IRP, A) {}
9069
9070 void initialize(Attributor &A) override {
9071 const Function *F = getAnchorScope();
9072 DenormalFPEnv DenormEnv = F->getDenormalFPEnv();
9073
9074 Known = DenormalState{.Mode: DenormEnv.DefaultMode, .ModeF32: DenormEnv.F32Mode};
9075 if (isModeFixed())
9076 indicateFixpoint();
9077 }
9078
9079 ChangeStatus updateImpl(Attributor &A) override {
9080 ChangeStatus Change = ChangeStatus::UNCHANGED;
9081
9082 auto CheckCallSite = [=, &Change, &A](AbstractCallSite CS) {
9083 Function *Caller = CS.getInstruction()->getFunction();
9084 LLVM_DEBUG(dbgs() << "[AADenormalFPMath] Call " << Caller->getName()
9085 << "->" << getAssociatedFunction()->getName() << '\n');
9086
9087 const auto *CallerInfo = A.getAAFor<AADenormalFPMath>(
9088 QueryingAA: *this, IRP: IRPosition::function(F: *Caller), DepClass: DepClassTy::REQUIRED);
9089 if (!CallerInfo)
9090 return false;
9091
9092 Change = Change | clampStateAndIndicateChange(S&: this->getState(),
9093 R: CallerInfo->getState());
9094 return true;
9095 };
9096
9097 bool AllCallSitesKnown = true;
9098 if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this, RequireAllCallSites: true, UsedAssumedInformation&: AllCallSitesKnown))
9099 return indicatePessimisticFixpoint();
9100
9101 if (Change == ChangeStatus::CHANGED && isModeFixed())
9102 indicateFixpoint();
9103 return Change;
9104 }
9105
9106 ChangeStatus manifest(Attributor &A) override {
9107 LLVMContext &Ctx = getAssociatedFunction()->getContext();
9108
9109 SmallVector<Attribute, 2> AttrToAdd;
9110 SmallVector<Attribute::AttrKind, 2> AttrToRemove;
9111
9112 // TODO: Change to use DenormalFPEnv everywhere.
9113 DenormalFPEnv KnownEnv(Known.Mode, Known.ModeF32);
9114
9115 if (KnownEnv == DenormalFPEnv::getDefault()) {
9116 AttrToRemove.push_back(Elt: Attribute::DenormalFPEnv);
9117 } else {
9118 AttrToAdd.push_back(Elt: Attribute::get(
9119 Context&: Ctx, Kind: Attribute::DenormalFPEnv,
9120 Val: DenormalFPEnv(Known.Mode, Known.ModeF32).toIntValue()));
9121 }
9122
9123 auto &IRP = getIRPosition();
9124
9125 // TODO: There should be a combined add and remove API.
9126 return A.removeAttrs(IRP, AttrKinds: AttrToRemove) |
9127 A.manifestAttrs(IRP, DeducedAttrs: AttrToAdd, /*ForceReplace=*/true);
9128 }
9129
9130 void trackStatistics() const override {
9131 STATS_DECLTRACK_FN_ATTR(denormal_fpenv)
9132 }
9133};
9134} // namespace
9135
9136/// ------------------ Value Constant Range Attribute -------------------------
9137
9138namespace {
9139struct AAValueConstantRangeImpl : AAValueConstantRange {
9140 using StateType = IntegerRangeState;
9141 AAValueConstantRangeImpl(const IRPosition &IRP, Attributor &A)
9142 : AAValueConstantRange(IRP, A) {}
9143
9144 /// See AbstractAttribute::initialize(..).
9145 void initialize(Attributor &A) override {
9146 if (A.hasSimplificationCallback(IRP: getIRPosition())) {
9147 indicatePessimisticFixpoint();
9148 return;
9149 }
9150
9151 // Intersect a range given by SCEV.
9152 intersectKnown(R: getConstantRangeFromSCEV(A, I: getCtxI()));
9153
9154 // Intersect a range given by LVI.
9155 intersectKnown(R: getConstantRangeFromLVI(A, CtxI: getCtxI()));
9156 }
9157
9158 /// See AbstractAttribute::getAsStr().
9159 const std::string getAsStr(Attributor *A) const override {
9160 std::string Str;
9161 llvm::raw_string_ostream OS(Str);
9162 OS << "range(" << getBitWidth() << ")<";
9163 getKnown().print(OS);
9164 OS << " / ";
9165 getAssumed().print(OS);
9166 OS << ">";
9167 return Str;
9168 }
9169
9170 /// Helper function to get a SCEV expr for the associated value at program
9171 /// point \p I.
9172 const SCEV *getSCEV(Attributor &A, const Instruction *I = nullptr) const {
9173 if (!getAnchorScope())
9174 return nullptr;
9175
9176 ScalarEvolution *SE =
9177 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
9178 F: *getAnchorScope());
9179
9180 LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(
9181 F: *getAnchorScope());
9182
9183 if (!SE || !LI)
9184 return nullptr;
9185
9186 const SCEV *S = SE->getSCEV(V: &getAssociatedValue());
9187 if (!I)
9188 return S;
9189
9190 return SE->getSCEVAtScope(S, L: LI->getLoopFor(BB: I->getParent()));
9191 }
9192
9193 /// Helper function to get a range from SCEV for the associated value at
9194 /// program point \p I.
9195 ConstantRange getConstantRangeFromSCEV(Attributor &A,
9196 const Instruction *I = nullptr) const {
9197 if (!getAnchorScope())
9198 return getWorstState(BitWidth: getBitWidth());
9199
9200 ScalarEvolution *SE =
9201 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
9202 F: *getAnchorScope());
9203
9204 const SCEV *S = getSCEV(A, I);
9205 if (!SE || !S)
9206 return getWorstState(BitWidth: getBitWidth());
9207
9208 return SE->getUnsignedRange(S);
9209 }
9210
9211 /// Helper function to get a range from LVI for the associated value at
9212 /// program point \p I.
9213 ConstantRange
9214 getConstantRangeFromLVI(Attributor &A,
9215 const Instruction *CtxI = nullptr) const {
9216 if (!getAnchorScope())
9217 return getWorstState(BitWidth: getBitWidth());
9218
9219 LazyValueInfo *LVI =
9220 A.getInfoCache().getAnalysisResultForFunction<LazyValueAnalysis>(
9221 F: *getAnchorScope());
9222
9223 if (!LVI || !CtxI)
9224 return getWorstState(BitWidth: getBitWidth());
9225 return LVI->getConstantRange(V: &getAssociatedValue(),
9226 CtxI: const_cast<Instruction *>(CtxI),
9227 /*UndefAllowed*/ false);
9228 }
9229
9230 /// Return true if \p CtxI is valid for querying outside analyses.
9231 /// This basically makes sure we do not ask intra-procedural analysis
9232 /// about a context in the wrong function or a context that violates
9233 /// dominance assumptions they might have. The \p AllowAACtxI flag indicates
9234 /// if the original context of this AA is OK or should be considered invalid.
9235 bool isValidCtxInstructionForOutsideAnalysis(Attributor &A,
9236 const Instruction *CtxI,
9237 bool AllowAACtxI) const {
9238 if (!CtxI || (!AllowAACtxI && CtxI == getCtxI()))
9239 return false;
9240
9241 // Our context might be in a different function, neither intra-procedural
9242 // analysis (ScalarEvolution nor LazyValueInfo) can handle that.
9243 if (!AA::isValidInScope(V: getAssociatedValue(), Scope: CtxI->getFunction()))
9244 return false;
9245
9246 // If the context is not dominated by the value there are paths to the
9247 // context that do not define the value. This cannot be handled by
9248 // LazyValueInfo so we need to bail.
9249 if (auto *I = dyn_cast<Instruction>(Val: &getAssociatedValue())) {
9250 InformationCache &InfoCache = A.getInfoCache();
9251 const DominatorTree *DT =
9252 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(
9253 F: *I->getFunction());
9254 return DT && DT->dominates(Def: I, User: CtxI);
9255 }
9256
9257 return true;
9258 }
9259
9260 /// See AAValueConstantRange::getAssumedConstantRange(..).
9261 ConstantRange
9262 getAssumedConstantRange(Attributor &A,
9263 const Instruction *CtxI = nullptr) const override {
9264 // TODO: Make SCEV use Attributor assumption.
9265 // We may be able to bound a variable range via assumptions in
9266 // Attributor. ex.) If x is assumed to be in [1, 3] and y is known to
9267 // evolve to x^2 + x, then we can say that y is in [2, 12].
9268 if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
9269 /* AllowAACtxI */ false))
9270 return getAssumed();
9271
9272 ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
9273 ConstantRange SCEVR = getConstantRangeFromSCEV(A, I: CtxI);
9274 return getAssumed().intersectWith(CR: SCEVR).intersectWith(CR: LVIR);
9275 }
9276
9277 /// Helper function to create MDNode for range metadata.
9278 static MDNode *
9279 getMDNodeForConstantRange(Type *Ty, LLVMContext &Ctx,
9280 const ConstantRange &AssumedConstantRange) {
9281 Metadata *LowAndHigh[] = {ConstantAsMetadata::get(C: ConstantInt::get(
9282 Ty, V: AssumedConstantRange.getLower())),
9283 ConstantAsMetadata::get(C: ConstantInt::get(
9284 Ty, V: AssumedConstantRange.getUpper()))};
9285 return MDNode::get(Context&: Ctx, MDs: LowAndHigh);
9286 }
9287
9288 /// Return true if \p Assumed is included in ranges from instruction \p I.
9289 static bool isBetterRange(const ConstantRange &Assumed,
9290 const Instruction &I) {
9291 if (Assumed.isFullSet())
9292 return false;
9293
9294 std::optional<ConstantRange> Known;
9295
9296 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
9297 Known = CB->getRange();
9298 } else if (MDNode *KnownRanges = I.getMetadata(KindID: LLVMContext::MD_range)) {
9299 // If multiple ranges are annotated in IR, we give up to annotate assumed
9300 // range for now.
9301
9302 // TODO: If there exists a known range which containts assumed range, we
9303 // can say assumed range is better.
9304 if (KnownRanges->getNumOperands() > 2)
9305 return false;
9306
9307 ConstantInt *Lower =
9308 mdconst::extract<ConstantInt>(MD: KnownRanges->getOperand(I: 0));
9309 ConstantInt *Upper =
9310 mdconst::extract<ConstantInt>(MD: KnownRanges->getOperand(I: 1));
9311
9312 Known.emplace(args: Lower->getValue(), args: Upper->getValue());
9313 }
9314 return !Known || (*Known != Assumed && Known->contains(CR: Assumed));
9315 }
9316
9317 /// Helper function to set range metadata.
9318 static bool
9319 setRangeMetadataIfisBetterRange(Instruction *I,
9320 const ConstantRange &AssumedConstantRange) {
9321 if (isBetterRange(Assumed: AssumedConstantRange, I: *I)) {
9322 I->setMetadata(KindID: LLVMContext::MD_range,
9323 Node: getMDNodeForConstantRange(Ty: I->getType(), Ctx&: I->getContext(),
9324 AssumedConstantRange));
9325 return true;
9326 }
9327 return false;
9328 }
9329 /// Helper function to set range return attribute.
9330 static bool
9331 setRangeRetAttrIfisBetterRange(Attributor &A, const IRPosition &IRP,
9332 Instruction *I,
9333 const ConstantRange &AssumedConstantRange) {
9334 if (isBetterRange(Assumed: AssumedConstantRange, I: *I)) {
9335 A.manifestAttrs(IRP,
9336 DeducedAttrs: Attribute::get(Context&: I->getContext(), Kind: Attribute::Range,
9337 CR: AssumedConstantRange),
9338 /*ForceReplace*/ true);
9339 return true;
9340 }
9341 return false;
9342 }
9343
9344 /// See AbstractAttribute::manifest()
9345 ChangeStatus manifest(Attributor &A) override {
9346 ChangeStatus Changed = ChangeStatus::UNCHANGED;
9347 ConstantRange AssumedConstantRange = getAssumedConstantRange(A);
9348 assert(!AssumedConstantRange.isFullSet() && "Invalid state");
9349
9350 auto &V = getAssociatedValue();
9351 if (!AssumedConstantRange.isEmptySet() &&
9352 !AssumedConstantRange.isSingleElement()) {
9353 if (Instruction *I = dyn_cast<Instruction>(Val: &V)) {
9354 assert(I == getCtxI() && "Should not annotate an instruction which is "
9355 "not the context instruction");
9356 if (isa<LoadInst>(Val: I))
9357 if (setRangeMetadataIfisBetterRange(I, AssumedConstantRange))
9358 Changed = ChangeStatus::CHANGED;
9359 if (isa<CallInst>(Val: I))
9360 if (setRangeRetAttrIfisBetterRange(A, IRP: getIRPosition(), I,
9361 AssumedConstantRange))
9362 Changed = ChangeStatus::CHANGED;
9363 }
9364 }
9365
9366 return Changed;
9367 }
9368};
9369
9370struct AAValueConstantRangeArgument final
9371 : AAArgumentFromCallSiteArguments<
9372 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9373 true /* BridgeCallBaseContext */> {
9374 using Base = AAArgumentFromCallSiteArguments<
9375 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9376 true /* BridgeCallBaseContext */>;
9377 AAValueConstantRangeArgument(const IRPosition &IRP, Attributor &A)
9378 : Base(IRP, A) {}
9379
9380 /// See AbstractAttribute::trackStatistics()
9381 void trackStatistics() const override {
9382 STATS_DECLTRACK_ARG_ATTR(value_range)
9383 }
9384};
9385
9386struct AAValueConstantRangeReturned
9387 : AAReturnedFromReturnedValues<AAValueConstantRange,
9388 AAValueConstantRangeImpl,
9389 AAValueConstantRangeImpl::StateType,
9390 /* PropagateCallBaseContext */ true> {
9391 using Base =
9392 AAReturnedFromReturnedValues<AAValueConstantRange,
9393 AAValueConstantRangeImpl,
9394 AAValueConstantRangeImpl::StateType,
9395 /* PropagateCallBaseContext */ true>;
9396 AAValueConstantRangeReturned(const IRPosition &IRP, Attributor &A)
9397 : Base(IRP, A) {}
9398
9399 /// See AbstractAttribute::initialize(...).
9400 void initialize(Attributor &A) override {
9401 if (!A.isFunctionIPOAmendable(F: *getAssociatedFunction()))
9402 indicatePessimisticFixpoint();
9403 }
9404
9405 /// See AbstractAttribute::trackStatistics()
9406 void trackStatistics() const override {
9407 STATS_DECLTRACK_FNRET_ATTR(value_range)
9408 }
9409};
9410
9411struct AAValueConstantRangeFloating : AAValueConstantRangeImpl {
9412 AAValueConstantRangeFloating(const IRPosition &IRP, Attributor &A)
9413 : AAValueConstantRangeImpl(IRP, A) {}
9414
9415 /// See AbstractAttribute::initialize(...).
9416 void initialize(Attributor &A) override {
9417 AAValueConstantRangeImpl::initialize(A);
9418 if (isAtFixpoint())
9419 return;
9420
9421 Value &V = getAssociatedValue();
9422
9423 if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
9424 unionAssumed(R: ConstantRange(C->getValue()));
9425 indicateOptimisticFixpoint();
9426 return;
9427 }
9428
9429 if (isa<UndefValue>(Val: &V)) {
9430 // Collapse the undef state to 0.
9431 unionAssumed(R: ConstantRange(APInt(getBitWidth(), 0)));
9432 indicateOptimisticFixpoint();
9433 return;
9434 }
9435
9436 if (isa<CallBase>(Val: &V))
9437 return;
9438
9439 if (isa<BinaryOperator>(Val: &V) || isa<CmpInst>(Val: &V) || isa<CastInst>(Val: &V))
9440 return;
9441
9442 // If it is a load instruction with range metadata, use it.
9443 if (LoadInst *LI = dyn_cast<LoadInst>(Val: &V))
9444 if (auto *RangeMD = LI->getMetadata(KindID: LLVMContext::MD_range)) {
9445 intersectKnown(R: getConstantRangeFromMetadata(RangeMD: *RangeMD));
9446 return;
9447 }
9448
9449 // We can work with PHI and select instruction as we traverse their operands
9450 // during update.
9451 if (isa<SelectInst>(Val: V) || isa<PHINode>(Val: V))
9452 return;
9453
9454 // Otherwise we give up.
9455 indicatePessimisticFixpoint();
9456
9457 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] We give up: "
9458 << getAssociatedValue() << "\n");
9459 }
9460
9461 bool calculateBinaryOperator(
9462 Attributor &A, BinaryOperator *BinOp, IntegerRangeState &T,
9463 const Instruction *CtxI,
9464 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9465 Value *LHS = BinOp->getOperand(i_nocapture: 0);
9466 Value *RHS = BinOp->getOperand(i_nocapture: 1);
9467
9468 // Simplify the operands first.
9469 bool UsedAssumedInformation = false;
9470 const auto &SimplifiedLHS = A.getAssumedSimplified(
9471 IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()), AA: *this,
9472 UsedAssumedInformation, S: AA::Interprocedural);
9473 if (!SimplifiedLHS.has_value())
9474 return true;
9475 if (!*SimplifiedLHS)
9476 return false;
9477 LHS = *SimplifiedLHS;
9478
9479 const auto &SimplifiedRHS = A.getAssumedSimplified(
9480 IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()), AA: *this,
9481 UsedAssumedInformation, S: AA::Interprocedural);
9482 if (!SimplifiedRHS.has_value())
9483 return true;
9484 if (!*SimplifiedRHS)
9485 return false;
9486 RHS = *SimplifiedRHS;
9487
9488 // TODO: Allow non integers as well.
9489 if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9490 return false;
9491
9492 auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9493 QueryingAA: *this, IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()),
9494 DepClass: DepClassTy::REQUIRED);
9495 if (!LHSAA)
9496 return false;
9497 QuerriedAAs.push_back(Elt: LHSAA);
9498 auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9499
9500 auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9501 QueryingAA: *this, IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()),
9502 DepClass: DepClassTy::REQUIRED);
9503 if (!RHSAA)
9504 return false;
9505 QuerriedAAs.push_back(Elt: RHSAA);
9506 auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9507
9508 auto AssumedRange = LHSAARange.binaryOp(BinOp: BinOp->getOpcode(), Other: RHSAARange);
9509
9510 T.unionAssumed(R: AssumedRange);
9511
9512 // TODO: Track a known state too.
9513
9514 return T.isValidState();
9515 }
9516
9517 bool calculateCastInst(
9518 Attributor &A, CastInst *CastI, IntegerRangeState &T,
9519 const Instruction *CtxI,
9520 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9521 assert(CastI->getNumOperands() == 1 && "Expected cast to be unary!");
9522 // TODO: Allow non integers as well.
9523 Value *OpV = CastI->getOperand(i_nocapture: 0);
9524
9525 // Simplify the operand first.
9526 bool UsedAssumedInformation = false;
9527 const auto &SimplifiedOpV = A.getAssumedSimplified(
9528 IRP: IRPosition::value(V: *OpV, CBContext: getCallBaseContext()), AA: *this,
9529 UsedAssumedInformation, S: AA::Interprocedural);
9530 if (!SimplifiedOpV.has_value())
9531 return true;
9532 if (!*SimplifiedOpV)
9533 return false;
9534 OpV = *SimplifiedOpV;
9535
9536 if (!OpV->getType()->isIntegerTy())
9537 return false;
9538
9539 auto *OpAA = A.getAAFor<AAValueConstantRange>(
9540 QueryingAA: *this, IRP: IRPosition::value(V: *OpV, CBContext: getCallBaseContext()),
9541 DepClass: DepClassTy::REQUIRED);
9542 if (!OpAA)
9543 return false;
9544 QuerriedAAs.push_back(Elt: OpAA);
9545 T.unionAssumed(R: OpAA->getAssumed().castOp(CastOp: CastI->getOpcode(),
9546 BitWidth: getState().getBitWidth()));
9547 return T.isValidState();
9548 }
9549
9550 bool
9551 calculateCmpInst(Attributor &A, CmpInst *CmpI, IntegerRangeState &T,
9552 const Instruction *CtxI,
9553 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9554 Value *LHS = CmpI->getOperand(i_nocapture: 0);
9555 Value *RHS = CmpI->getOperand(i_nocapture: 1);
9556
9557 // Simplify the operands first.
9558 bool UsedAssumedInformation = false;
9559 const auto &SimplifiedLHS = A.getAssumedSimplified(
9560 IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()), AA: *this,
9561 UsedAssumedInformation, S: AA::Interprocedural);
9562 if (!SimplifiedLHS.has_value())
9563 return true;
9564 if (!*SimplifiedLHS)
9565 return false;
9566 LHS = *SimplifiedLHS;
9567
9568 const auto &SimplifiedRHS = A.getAssumedSimplified(
9569 IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()), AA: *this,
9570 UsedAssumedInformation, S: AA::Interprocedural);
9571 if (!SimplifiedRHS.has_value())
9572 return true;
9573 if (!*SimplifiedRHS)
9574 return false;
9575 RHS = *SimplifiedRHS;
9576
9577 // TODO: Allow non integers as well.
9578 if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9579 return false;
9580
9581 auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9582 QueryingAA: *this, IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()),
9583 DepClass: DepClassTy::REQUIRED);
9584 if (!LHSAA)
9585 return false;
9586 QuerriedAAs.push_back(Elt: LHSAA);
9587 auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9588 QueryingAA: *this, IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()),
9589 DepClass: DepClassTy::REQUIRED);
9590 if (!RHSAA)
9591 return false;
9592 QuerriedAAs.push_back(Elt: RHSAA);
9593 auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9594 auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9595
9596 // If one of them is empty set, we can't decide.
9597 if (LHSAARange.isEmptySet() || RHSAARange.isEmptySet())
9598 return true;
9599
9600 bool MustTrue = false, MustFalse = false;
9601
9602 auto AllowedRegion =
9603 ConstantRange::makeAllowedICmpRegion(Pred: CmpI->getPredicate(), Other: RHSAARange);
9604
9605 if (AllowedRegion.intersectWith(CR: LHSAARange).isEmptySet())
9606 MustFalse = true;
9607
9608 if (LHSAARange.icmp(Pred: CmpI->getPredicate(), Other: RHSAARange))
9609 MustTrue = true;
9610
9611 assert((!MustTrue || !MustFalse) &&
9612 "Either MustTrue or MustFalse should be false!");
9613
9614 if (MustTrue)
9615 T.unionAssumed(R: ConstantRange(APInt(/* numBits */ 1, /* val */ 1)));
9616 else if (MustFalse)
9617 T.unionAssumed(R: ConstantRange(APInt(/* numBits */ 1, /* val */ 0)));
9618 else
9619 T.unionAssumed(R: ConstantRange(/* BitWidth */ 1, /* isFullSet */ true));
9620
9621 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] " << *CmpI << " after "
9622 << (MustTrue ? "true" : (MustFalse ? "false" : "unknown"))
9623 << ": " << T << "\n\t" << *LHSAA << "\t<op>\n\t"
9624 << *RHSAA);
9625
9626 // TODO: Track a known state too.
9627 return T.isValidState();
9628 }
9629
9630 /// See AbstractAttribute::updateImpl(...).
9631 ChangeStatus updateImpl(Attributor &A) override {
9632
9633 IntegerRangeState T(getBitWidth());
9634 auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
9635 Instruction *I = dyn_cast<Instruction>(Val: &V);
9636 if (!I || isa<CallBase>(Val: I)) {
9637
9638 // Simplify the operand first.
9639 bool UsedAssumedInformation = false;
9640 const auto &SimplifiedOpV = A.getAssumedSimplified(
9641 IRP: IRPosition::value(V, CBContext: getCallBaseContext()), AA: *this,
9642 UsedAssumedInformation, S: AA::Interprocedural);
9643 if (!SimplifiedOpV.has_value())
9644 return true;
9645 if (!*SimplifiedOpV)
9646 return false;
9647 Value *VPtr = *SimplifiedOpV;
9648
9649 // If the value is not instruction, we query AA to Attributor.
9650 const auto *AA = A.getAAFor<AAValueConstantRange>(
9651 QueryingAA: *this, IRP: IRPosition::value(V: *VPtr, CBContext: getCallBaseContext()),
9652 DepClass: DepClassTy::REQUIRED);
9653
9654 // Clamp operator is not used to utilize a program point CtxI.
9655 if (AA)
9656 T.unionAssumed(R: AA->getAssumedConstantRange(A, CtxI));
9657 else
9658 return false;
9659
9660 return T.isValidState();
9661 }
9662
9663 SmallVector<const AAValueConstantRange *, 4> QuerriedAAs;
9664 if (auto *BinOp = dyn_cast<BinaryOperator>(Val: I)) {
9665 if (!calculateBinaryOperator(A, BinOp, T, CtxI, QuerriedAAs))
9666 return false;
9667 } else if (auto *CmpI = dyn_cast<CmpInst>(Val: I)) {
9668 if (!calculateCmpInst(A, CmpI, T, CtxI, QuerriedAAs))
9669 return false;
9670 } else if (auto *CastI = dyn_cast<CastInst>(Val: I)) {
9671 if (!calculateCastInst(A, CastI, T, CtxI, QuerriedAAs))
9672 return false;
9673 } else {
9674 // Give up with other instructions.
9675 // TODO: Add other instructions
9676
9677 T.indicatePessimisticFixpoint();
9678 return false;
9679 }
9680
9681 // Catch circular reasoning in a pessimistic way for now.
9682 // TODO: Check how the range evolves and if we stripped anything, see also
9683 // AADereferenceable or AAAlign for similar situations.
9684 for (const AAValueConstantRange *QueriedAA : QuerriedAAs) {
9685 if (QueriedAA != this)
9686 continue;
9687 // If we are in a stady state we do not need to worry.
9688 if (T.getAssumed() == getState().getAssumed())
9689 continue;
9690 T.indicatePessimisticFixpoint();
9691 }
9692
9693 return T.isValidState();
9694 };
9695
9696 if (!VisitValueCB(getAssociatedValue(), getCtxI()))
9697 return indicatePessimisticFixpoint();
9698
9699 // Ensure that long def-use chains can't cause circular reasoning either by
9700 // introducing a cutoff below.
9701 if (clampStateAndIndicateChange(S&: getState(), R: T) == ChangeStatus::UNCHANGED)
9702 return ChangeStatus::UNCHANGED;
9703 if (++NumChanges > MaxNumChanges) {
9704 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] performed " << NumChanges
9705 << " but only " << MaxNumChanges
9706 << " are allowed to avoid cyclic reasoning.");
9707 return indicatePessimisticFixpoint();
9708 }
9709 return ChangeStatus::CHANGED;
9710 }
9711
9712 /// See AbstractAttribute::trackStatistics()
9713 void trackStatistics() const override {
9714 STATS_DECLTRACK_FLOATING_ATTR(value_range)
9715 }
9716
9717 /// Tracker to bail after too many widening steps of the constant range.
9718 int NumChanges = 0;
9719
9720 /// Upper bound for the number of allowed changes (=widening steps) for the
9721 /// constant range before we give up.
9722 static constexpr int MaxNumChanges = 5;
9723};
9724
9725struct AAValueConstantRangeFunction : AAValueConstantRangeImpl {
9726 AAValueConstantRangeFunction(const IRPosition &IRP, Attributor &A)
9727 : AAValueConstantRangeImpl(IRP, A) {}
9728
9729 /// See AbstractAttribute::initialize(...).
9730 ChangeStatus updateImpl(Attributor &A) override {
9731 llvm_unreachable("AAValueConstantRange(Function|CallSite)::updateImpl will "
9732 "not be called");
9733 }
9734
9735 /// See AbstractAttribute::trackStatistics()
9736 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(value_range) }
9737};
9738
9739struct AAValueConstantRangeCallSite : AAValueConstantRangeFunction {
9740 AAValueConstantRangeCallSite(const IRPosition &IRP, Attributor &A)
9741 : AAValueConstantRangeFunction(IRP, A) {}
9742
9743 /// See AbstractAttribute::trackStatistics()
9744 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(value_range) }
9745};
9746
9747struct AAValueConstantRangeCallSiteReturned
9748 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9749 AAValueConstantRangeImpl::StateType,
9750 /* IntroduceCallBaseContext */ true> {
9751 AAValueConstantRangeCallSiteReturned(const IRPosition &IRP, Attributor &A)
9752 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9753 AAValueConstantRangeImpl::StateType,
9754 /* IntroduceCallBaseContext */ true>(IRP, A) {}
9755
9756 /// See AbstractAttribute::initialize(...).
9757 void initialize(Attributor &A) override {
9758 // If it is a call instruction with range attribute, use the range.
9759 if (CallInst *CI = dyn_cast<CallInst>(Val: &getAssociatedValue())) {
9760 if (std::optional<ConstantRange> Range = CI->getRange())
9761 intersectKnown(R: *Range);
9762 }
9763
9764 AAValueConstantRangeImpl::initialize(A);
9765 }
9766
9767 /// See AbstractAttribute::trackStatistics()
9768 void trackStatistics() const override {
9769 STATS_DECLTRACK_CSRET_ATTR(value_range)
9770 }
9771};
9772struct AAValueConstantRangeCallSiteArgument : AAValueConstantRangeFloating {
9773 AAValueConstantRangeCallSiteArgument(const IRPosition &IRP, Attributor &A)
9774 : AAValueConstantRangeFloating(IRP, A) {}
9775
9776 /// See AbstractAttribute::manifest()
9777 ChangeStatus manifest(Attributor &A) override {
9778 return ChangeStatus::UNCHANGED;
9779 }
9780
9781 /// See AbstractAttribute::trackStatistics()
9782 void trackStatistics() const override {
9783 STATS_DECLTRACK_CSARG_ATTR(value_range)
9784 }
9785};
9786} // namespace
9787
9788/// ------------------ Potential Values Attribute -------------------------
9789
9790namespace {
9791struct AAPotentialConstantValuesImpl : AAPotentialConstantValues {
9792 using StateType = PotentialConstantIntValuesState;
9793
9794 AAPotentialConstantValuesImpl(const IRPosition &IRP, Attributor &A)
9795 : AAPotentialConstantValues(IRP, A) {}
9796
9797 /// See AbstractAttribute::initialize(..).
9798 void initialize(Attributor &A) override {
9799 if (A.hasSimplificationCallback(IRP: getIRPosition()))
9800 indicatePessimisticFixpoint();
9801 else
9802 AAPotentialConstantValues::initialize(A);
9803 }
9804
9805 bool fillSetWithConstantValues(Attributor &A, const IRPosition &IRP, SetTy &S,
9806 bool &ContainsUndef, bool ForSelf) {
9807 SmallVector<AA::ValueAndContext> Values;
9808 bool UsedAssumedInformation = false;
9809 if (!A.getAssumedSimplifiedValues(IRP, AA: *this, Values, S: AA::Interprocedural,
9810 UsedAssumedInformation)) {
9811 // Avoid recursion when the caller is computing constant values for this
9812 // IRP itself.
9813 if (ForSelf)
9814 return false;
9815 if (!IRP.getAssociatedType()->isIntegerTy())
9816 return false;
9817 auto *PotentialValuesAA = A.getAAFor<AAPotentialConstantValues>(
9818 QueryingAA: *this, IRP, DepClass: DepClassTy::REQUIRED);
9819 if (!PotentialValuesAA || !PotentialValuesAA->getState().isValidState())
9820 return false;
9821 ContainsUndef = PotentialValuesAA->getState().undefIsContained();
9822 S = PotentialValuesAA->getState().getAssumedSet();
9823 return true;
9824 }
9825
9826 // Copy all the constant values, except UndefValue. ContainsUndef is true
9827 // iff Values contains only UndefValue instances. If there are other known
9828 // constants, then UndefValue is dropped.
9829 ContainsUndef = false;
9830 for (auto &It : Values) {
9831 if (isa<UndefValue>(Val: It.getValue())) {
9832 ContainsUndef = true;
9833 continue;
9834 }
9835 auto *CI = dyn_cast<ConstantInt>(Val: It.getValue());
9836 if (!CI)
9837 return false;
9838 S.insert(X: CI->getValue());
9839 }
9840 ContainsUndef &= S.empty();
9841
9842 return true;
9843 }
9844
9845 /// See AbstractAttribute::getAsStr().
9846 const std::string getAsStr(Attributor *A) const override {
9847 std::string Str;
9848 llvm::raw_string_ostream OS(Str);
9849 OS << getState();
9850 return Str;
9851 }
9852
9853 /// See AbstractAttribute::updateImpl(...).
9854 ChangeStatus updateImpl(Attributor &A) override {
9855 return indicatePessimisticFixpoint();
9856 }
9857};
9858
9859struct AAPotentialConstantValuesArgument final
9860 : AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9861 AAPotentialConstantValuesImpl,
9862 PotentialConstantIntValuesState> {
9863 using Base = AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9864 AAPotentialConstantValuesImpl,
9865 PotentialConstantIntValuesState>;
9866 AAPotentialConstantValuesArgument(const IRPosition &IRP, Attributor &A)
9867 : Base(IRP, A) {}
9868
9869 /// See AbstractAttribute::trackStatistics()
9870 void trackStatistics() const override {
9871 STATS_DECLTRACK_ARG_ATTR(potential_values)
9872 }
9873};
9874
9875struct AAPotentialConstantValuesReturned
9876 : AAReturnedFromReturnedValues<AAPotentialConstantValues,
9877 AAPotentialConstantValuesImpl> {
9878 using Base = AAReturnedFromReturnedValues<AAPotentialConstantValues,
9879 AAPotentialConstantValuesImpl>;
9880 AAPotentialConstantValuesReturned(const IRPosition &IRP, Attributor &A)
9881 : Base(IRP, A) {}
9882
9883 void initialize(Attributor &A) override {
9884 if (!A.isFunctionIPOAmendable(F: *getAssociatedFunction()))
9885 indicatePessimisticFixpoint();
9886 Base::initialize(A);
9887 }
9888
9889 /// See AbstractAttribute::trackStatistics()
9890 void trackStatistics() const override {
9891 STATS_DECLTRACK_FNRET_ATTR(potential_values)
9892 }
9893};
9894
9895struct AAPotentialConstantValuesFloating : AAPotentialConstantValuesImpl {
9896 AAPotentialConstantValuesFloating(const IRPosition &IRP, Attributor &A)
9897 : AAPotentialConstantValuesImpl(IRP, A) {}
9898
9899 /// See AbstractAttribute::initialize(..).
9900 void initialize(Attributor &A) override {
9901 AAPotentialConstantValuesImpl::initialize(A);
9902 if (isAtFixpoint())
9903 return;
9904
9905 Value &V = getAssociatedValue();
9906
9907 if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
9908 unionAssumed(C: C->getValue());
9909 indicateOptimisticFixpoint();
9910 return;
9911 }
9912
9913 if (isa<UndefValue>(Val: &V)) {
9914 unionAssumedWithUndef();
9915 indicateOptimisticFixpoint();
9916 return;
9917 }
9918
9919 if (isa<BinaryOperator>(Val: &V) || isa<ICmpInst>(Val: &V) || isa<CastInst>(Val: &V))
9920 return;
9921
9922 if (isa<SelectInst>(Val: V) || isa<PHINode>(Val: V) || isa<LoadInst>(Val: V))
9923 return;
9924
9925 indicatePessimisticFixpoint();
9926
9927 LLVM_DEBUG(dbgs() << "[AAPotentialConstantValues] We give up: "
9928 << getAssociatedValue() << "\n");
9929 }
9930
9931 static bool calculateICmpInst(const ICmpInst *ICI, const APInt &LHS,
9932 const APInt &RHS) {
9933 return ICmpInst::compare(LHS, RHS, Pred: ICI->getPredicate());
9934 }
9935
9936 static APInt calculateCastInst(const CastInst *CI, const APInt &Src,
9937 uint32_t ResultBitWidth) {
9938 Instruction::CastOps CastOp = CI->getOpcode();
9939 switch (CastOp) {
9940 default:
9941 llvm_unreachable("unsupported or not integer cast");
9942 case Instruction::Trunc:
9943 return Src.trunc(width: ResultBitWidth);
9944 case Instruction::SExt:
9945 return Src.sext(width: ResultBitWidth);
9946 case Instruction::ZExt:
9947 return Src.zext(width: ResultBitWidth);
9948 case Instruction::BitCast:
9949 return Src;
9950 }
9951 }
9952
9953 static APInt calculateBinaryOperator(const BinaryOperator *BinOp,
9954 const APInt &LHS, const APInt &RHS,
9955 bool &SkipOperation, bool &Unsupported) {
9956 Instruction::BinaryOps BinOpcode = BinOp->getOpcode();
9957 // Unsupported is set to true when the binary operator is not supported.
9958 // SkipOperation is set to true when UB occur with the given operand pair
9959 // (LHS, RHS).
9960 // TODO: we should look at nsw and nuw keywords to handle operations
9961 // that create poison or undef value.
9962 switch (BinOpcode) {
9963 default:
9964 Unsupported = true;
9965 return LHS;
9966 case Instruction::Add:
9967 return LHS + RHS;
9968 case Instruction::Sub:
9969 return LHS - RHS;
9970 case Instruction::Mul:
9971 return LHS * RHS;
9972 case Instruction::UDiv:
9973 if (RHS.isZero()) {
9974 SkipOperation = true;
9975 return LHS;
9976 }
9977 return LHS.udiv(RHS);
9978 case Instruction::SDiv:
9979 if (RHS.isZero()) {
9980 SkipOperation = true;
9981 return LHS;
9982 }
9983 return LHS.sdiv(RHS);
9984 case Instruction::URem:
9985 if (RHS.isZero()) {
9986 SkipOperation = true;
9987 return LHS;
9988 }
9989 return LHS.urem(RHS);
9990 case Instruction::SRem:
9991 if (RHS.isZero()) {
9992 SkipOperation = true;
9993 return LHS;
9994 }
9995 return LHS.srem(RHS);
9996 case Instruction::Shl:
9997 return LHS.shl(ShiftAmt: RHS);
9998 case Instruction::LShr:
9999 return LHS.lshr(ShiftAmt: RHS);
10000 case Instruction::AShr:
10001 return LHS.ashr(ShiftAmt: RHS);
10002 case Instruction::And:
10003 return LHS & RHS;
10004 case Instruction::Or:
10005 return LHS | RHS;
10006 case Instruction::Xor:
10007 return LHS ^ RHS;
10008 }
10009 }
10010
10011 bool calculateBinaryOperatorAndTakeUnion(const BinaryOperator *BinOp,
10012 const APInt &LHS, const APInt &RHS) {
10013 bool SkipOperation = false;
10014 bool Unsupported = false;
10015 APInt Result =
10016 calculateBinaryOperator(BinOp, LHS, RHS, SkipOperation, Unsupported);
10017 if (Unsupported)
10018 return false;
10019 // If SkipOperation is true, we can ignore this operand pair (L, R).
10020 if (!SkipOperation)
10021 unionAssumed(C: Result);
10022 return isValidState();
10023 }
10024
10025 ChangeStatus updateWithICmpInst(Attributor &A, ICmpInst *ICI) {
10026 auto AssumedBefore = getAssumed();
10027 Value *LHS = ICI->getOperand(i_nocapture: 0);
10028 Value *RHS = ICI->getOperand(i_nocapture: 1);
10029
10030 bool LHSContainsUndef = false, RHSContainsUndef = false;
10031 SetTy LHSAAPVS, RHSAAPVS;
10032 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
10033 ContainsUndef&: LHSContainsUndef, /* ForSelf */ false) ||
10034 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
10035 ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
10036 return indicatePessimisticFixpoint();
10037
10038 // TODO: make use of undef flag to limit potential values aggressively.
10039 bool MaybeTrue = false, MaybeFalse = false;
10040 const APInt Zero(RHS->getType()->getIntegerBitWidth(), 0);
10041 if (LHSContainsUndef && RHSContainsUndef) {
10042 // The result of any comparison between undefs can be soundly replaced
10043 // with undef.
10044 unionAssumedWithUndef();
10045 } else if (LHSContainsUndef) {
10046 for (const APInt &R : RHSAAPVS) {
10047 bool CmpResult = calculateICmpInst(ICI, LHS: Zero, RHS: R);
10048 MaybeTrue |= CmpResult;
10049 MaybeFalse |= !CmpResult;
10050 if (MaybeTrue & MaybeFalse)
10051 return indicatePessimisticFixpoint();
10052 }
10053 } else if (RHSContainsUndef) {
10054 for (const APInt &L : LHSAAPVS) {
10055 bool CmpResult = calculateICmpInst(ICI, LHS: L, RHS: Zero);
10056 MaybeTrue |= CmpResult;
10057 MaybeFalse |= !CmpResult;
10058 if (MaybeTrue & MaybeFalse)
10059 return indicatePessimisticFixpoint();
10060 }
10061 } else {
10062 for (const APInt &L : LHSAAPVS) {
10063 for (const APInt &R : RHSAAPVS) {
10064 bool CmpResult = calculateICmpInst(ICI, LHS: L, RHS: R);
10065 MaybeTrue |= CmpResult;
10066 MaybeFalse |= !CmpResult;
10067 if (MaybeTrue & MaybeFalse)
10068 return indicatePessimisticFixpoint();
10069 }
10070 }
10071 }
10072 if (MaybeTrue)
10073 unionAssumed(C: APInt(/* numBits */ 1, /* val */ 1));
10074 if (MaybeFalse)
10075 unionAssumed(C: APInt(/* numBits */ 1, /* val */ 0));
10076 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10077 : ChangeStatus::CHANGED;
10078 }
10079
10080 ChangeStatus updateWithSelectInst(Attributor &A, SelectInst *SI) {
10081 auto AssumedBefore = getAssumed();
10082 Value *LHS = SI->getTrueValue();
10083 Value *RHS = SI->getFalseValue();
10084
10085 bool UsedAssumedInformation = false;
10086 std::optional<Constant *> C = A.getAssumedConstant(
10087 V: *SI->getCondition(), AA: *this, UsedAssumedInformation);
10088
10089 // Check if we only need one operand.
10090 bool OnlyLeft = false, OnlyRight = false;
10091 if (C && *C && (*C)->isOneValue())
10092 OnlyLeft = true;
10093 else if (C && *C && (*C)->isNullValue())
10094 OnlyRight = true;
10095
10096 bool LHSContainsUndef = false, RHSContainsUndef = false;
10097 SetTy LHSAAPVS, RHSAAPVS;
10098 if (!OnlyRight &&
10099 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
10100 ContainsUndef&: LHSContainsUndef, /* ForSelf */ false))
10101 return indicatePessimisticFixpoint();
10102
10103 if (!OnlyLeft &&
10104 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
10105 ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
10106 return indicatePessimisticFixpoint();
10107
10108 if (OnlyLeft || OnlyRight) {
10109 // select (true/false), lhs, rhs
10110 auto *OpAA = OnlyLeft ? &LHSAAPVS : &RHSAAPVS;
10111 auto Undef = OnlyLeft ? LHSContainsUndef : RHSContainsUndef;
10112
10113 if (Undef)
10114 unionAssumedWithUndef();
10115 else {
10116 for (const auto &It : *OpAA)
10117 unionAssumed(C: It);
10118 }
10119
10120 } else if (LHSContainsUndef && RHSContainsUndef) {
10121 // select i1 *, undef , undef => undef
10122 unionAssumedWithUndef();
10123 } else {
10124 for (const auto &It : LHSAAPVS)
10125 unionAssumed(C: It);
10126 for (const auto &It : RHSAAPVS)
10127 unionAssumed(C: It);
10128 }
10129 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10130 : ChangeStatus::CHANGED;
10131 }
10132
10133 ChangeStatus updateWithCastInst(Attributor &A, CastInst *CI) {
10134 auto AssumedBefore = getAssumed();
10135 if (!CI->isIntegerCast())
10136 return indicatePessimisticFixpoint();
10137 assert(CI->getNumOperands() == 1 && "Expected cast to be unary!");
10138 uint32_t ResultBitWidth = CI->getDestTy()->getIntegerBitWidth();
10139 Value *Src = CI->getOperand(i_nocapture: 0);
10140
10141 bool SrcContainsUndef = false;
10142 SetTy SrcPVS;
10143 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *Src), S&: SrcPVS,
10144 ContainsUndef&: SrcContainsUndef, /* ForSelf */ false))
10145 return indicatePessimisticFixpoint();
10146
10147 if (SrcContainsUndef)
10148 unionAssumedWithUndef();
10149 else {
10150 for (const APInt &S : SrcPVS) {
10151 APInt T = calculateCastInst(CI, Src: S, ResultBitWidth);
10152 unionAssumed(C: T);
10153 }
10154 }
10155 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10156 : ChangeStatus::CHANGED;
10157 }
10158
10159 ChangeStatus updateWithBinaryOperator(Attributor &A, BinaryOperator *BinOp) {
10160 auto AssumedBefore = getAssumed();
10161 Value *LHS = BinOp->getOperand(i_nocapture: 0);
10162 Value *RHS = BinOp->getOperand(i_nocapture: 1);
10163
10164 bool LHSContainsUndef = false, RHSContainsUndef = false;
10165 SetTy LHSAAPVS, RHSAAPVS;
10166 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
10167 ContainsUndef&: LHSContainsUndef, /* ForSelf */ false) ||
10168 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
10169 ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
10170 return indicatePessimisticFixpoint();
10171
10172 const APInt Zero = APInt(LHS->getType()->getIntegerBitWidth(), 0);
10173
10174 // TODO: make use of undef flag to limit potential values aggressively.
10175 if (LHSContainsUndef && RHSContainsUndef) {
10176 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: Zero, RHS: Zero))
10177 return indicatePessimisticFixpoint();
10178 } else if (LHSContainsUndef) {
10179 for (const APInt &R : RHSAAPVS) {
10180 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: Zero, RHS: R))
10181 return indicatePessimisticFixpoint();
10182 }
10183 } else if (RHSContainsUndef) {
10184 for (const APInt &L : LHSAAPVS) {
10185 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: L, RHS: Zero))
10186 return indicatePessimisticFixpoint();
10187 }
10188 } else {
10189 for (const APInt &L : LHSAAPVS) {
10190 for (const APInt &R : RHSAAPVS) {
10191 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: L, RHS: R))
10192 return indicatePessimisticFixpoint();
10193 }
10194 }
10195 }
10196 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10197 : ChangeStatus::CHANGED;
10198 }
10199
10200 ChangeStatus updateWithInstruction(Attributor &A, Instruction *Inst) {
10201 auto AssumedBefore = getAssumed();
10202 SetTy Incoming;
10203 bool ContainsUndef;
10204 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *Inst), S&: Incoming,
10205 ContainsUndef, /* ForSelf */ true))
10206 return indicatePessimisticFixpoint();
10207 if (ContainsUndef) {
10208 unionAssumedWithUndef();
10209 } else {
10210 for (const auto &It : Incoming)
10211 unionAssumed(C: It);
10212 }
10213 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10214 : ChangeStatus::CHANGED;
10215 }
10216
10217 /// See AbstractAttribute::updateImpl(...).
10218 ChangeStatus updateImpl(Attributor &A) override {
10219 Value &V = getAssociatedValue();
10220 Instruction *I = dyn_cast<Instruction>(Val: &V);
10221
10222 if (auto *ICI = dyn_cast<ICmpInst>(Val: I))
10223 return updateWithICmpInst(A, ICI);
10224
10225 if (auto *SI = dyn_cast<SelectInst>(Val: I))
10226 return updateWithSelectInst(A, SI);
10227
10228 if (auto *CI = dyn_cast<CastInst>(Val: I))
10229 return updateWithCastInst(A, CI);
10230
10231 if (auto *BinOp = dyn_cast<BinaryOperator>(Val: I))
10232 return updateWithBinaryOperator(A, BinOp);
10233
10234 if (isa<PHINode>(Val: I) || isa<LoadInst>(Val: I))
10235 return updateWithInstruction(A, Inst: I);
10236
10237 return indicatePessimisticFixpoint();
10238 }
10239
10240 /// See AbstractAttribute::trackStatistics()
10241 void trackStatistics() const override {
10242 STATS_DECLTRACK_FLOATING_ATTR(potential_values)
10243 }
10244};
10245
10246struct AAPotentialConstantValuesFunction : AAPotentialConstantValuesImpl {
10247 AAPotentialConstantValuesFunction(const IRPosition &IRP, Attributor &A)
10248 : AAPotentialConstantValuesImpl(IRP, A) {}
10249
10250 /// See AbstractAttribute::initialize(...).
10251 ChangeStatus updateImpl(Attributor &A) override {
10252 llvm_unreachable(
10253 "AAPotentialConstantValues(Function|CallSite)::updateImpl will "
10254 "not be called");
10255 }
10256
10257 /// See AbstractAttribute::trackStatistics()
10258 void trackStatistics() const override {
10259 STATS_DECLTRACK_FN_ATTR(potential_values)
10260 }
10261};
10262
10263struct AAPotentialConstantValuesCallSite : AAPotentialConstantValuesFunction {
10264 AAPotentialConstantValuesCallSite(const IRPosition &IRP, Attributor &A)
10265 : AAPotentialConstantValuesFunction(IRP, A) {}
10266
10267 /// See AbstractAttribute::trackStatistics()
10268 void trackStatistics() const override {
10269 STATS_DECLTRACK_CS_ATTR(potential_values)
10270 }
10271};
10272
10273struct AAPotentialConstantValuesCallSiteReturned
10274 : AACalleeToCallSite<AAPotentialConstantValues,
10275 AAPotentialConstantValuesImpl> {
10276 AAPotentialConstantValuesCallSiteReturned(const IRPosition &IRP,
10277 Attributor &A)
10278 : AACalleeToCallSite<AAPotentialConstantValues,
10279 AAPotentialConstantValuesImpl>(IRP, A) {}
10280
10281 /// See AbstractAttribute::trackStatistics()
10282 void trackStatistics() const override {
10283 STATS_DECLTRACK_CSRET_ATTR(potential_values)
10284 }
10285};
10286
10287struct AAPotentialConstantValuesCallSiteArgument
10288 : AAPotentialConstantValuesFloating {
10289 AAPotentialConstantValuesCallSiteArgument(const IRPosition &IRP,
10290 Attributor &A)
10291 : AAPotentialConstantValuesFloating(IRP, A) {}
10292
10293 /// See AbstractAttribute::initialize(..).
10294 void initialize(Attributor &A) override {
10295 AAPotentialConstantValuesImpl::initialize(A);
10296 if (isAtFixpoint())
10297 return;
10298
10299 Value &V = getAssociatedValue();
10300
10301 if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
10302 unionAssumed(C: C->getValue());
10303 indicateOptimisticFixpoint();
10304 return;
10305 }
10306
10307 if (isa<UndefValue>(Val: &V)) {
10308 unionAssumedWithUndef();
10309 indicateOptimisticFixpoint();
10310 return;
10311 }
10312 }
10313
10314 /// See AbstractAttribute::updateImpl(...).
10315 ChangeStatus updateImpl(Attributor &A) override {
10316 Value &V = getAssociatedValue();
10317 auto AssumedBefore = getAssumed();
10318 auto *AA = A.getAAFor<AAPotentialConstantValues>(
10319 QueryingAA: *this, IRP: IRPosition::value(V), DepClass: DepClassTy::REQUIRED);
10320 if (!AA)
10321 return indicatePessimisticFixpoint();
10322 const auto &S = AA->getAssumed();
10323 unionAssumed(PVS: S);
10324 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10325 : ChangeStatus::CHANGED;
10326 }
10327
10328 /// See AbstractAttribute::trackStatistics()
10329 void trackStatistics() const override {
10330 STATS_DECLTRACK_CSARG_ATTR(potential_values)
10331 }
10332};
10333} // namespace
10334
10335/// ------------------------ NoUndef Attribute ---------------------------------
10336bool AANoUndef::isImpliedByIR(Attributor &A, const IRPosition &IRP,
10337 Attribute::AttrKind ImpliedAttributeKind,
10338 bool IgnoreSubsumingPositions) {
10339 assert(ImpliedAttributeKind == Attribute::NoUndef &&
10340 "Unexpected attribute kind");
10341 if (A.hasAttr(IRP, AKs: {Attribute::NoUndef}, IgnoreSubsumingPositions,
10342 ImpliedAttributeKind: Attribute::NoUndef))
10343 return true;
10344
10345 Value &Val = IRP.getAssociatedValue();
10346 if (IRP.getPositionKind() != IRPosition::IRP_RETURNED &&
10347 isGuaranteedNotToBeUndefOrPoison(V: &Val)) {
10348 LLVMContext &Ctx = Val.getContext();
10349 A.manifestAttrs(IRP, DeducedAttrs: Attribute::get(Context&: Ctx, Kind: Attribute::NoUndef));
10350 return true;
10351 }
10352
10353 return false;
10354}
10355
10356namespace {
10357struct AANoUndefImpl : AANoUndef {
10358 AANoUndefImpl(const IRPosition &IRP, Attributor &A) : AANoUndef(IRP, A) {}
10359
10360 /// See AbstractAttribute::initialize(...).
10361 void initialize(Attributor &A) override {
10362 Value &V = getAssociatedValue();
10363 if (isa<UndefValue>(Val: V))
10364 indicatePessimisticFixpoint();
10365 assert(!isImpliedByIR(A, getIRPosition(), Attribute::NoUndef));
10366 }
10367
10368 /// See followUsesInMBEC
10369 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10370 AANoUndef::StateType &State) {
10371 const Value *UseV = U->get();
10372 const DominatorTree *DT = nullptr;
10373 AssumptionCache *AC = nullptr;
10374 InformationCache &InfoCache = A.getInfoCache();
10375 if (Function *F = getAnchorScope()) {
10376 DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
10377 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
10378 }
10379 State.setKnown(isGuaranteedNotToBeUndefOrPoison(V: UseV, AC, CtxI: I, DT));
10380 bool TrackUse = false;
10381 // Track use for instructions which must produce undef or poison bits when
10382 // at least one operand contains such bits.
10383 if (isa<CastInst>(Val: *I) || isa<GetElementPtrInst>(Val: *I))
10384 TrackUse = true;
10385 return TrackUse;
10386 }
10387
10388 /// See AbstractAttribute::getAsStr().
10389 const std::string getAsStr(Attributor *A) const override {
10390 return getAssumed() ? "noundef" : "may-undef-or-poison";
10391 }
10392
10393 ChangeStatus manifest(Attributor &A) override {
10394 // We don't manifest noundef attribute for dead positions because the
10395 // associated values with dead positions would be replaced with undef
10396 // values.
10397 bool UsedAssumedInformation = false;
10398 if (A.isAssumedDead(IRP: getIRPosition(), QueryingAA: nullptr, FnLivenessAA: nullptr,
10399 UsedAssumedInformation))
10400 return ChangeStatus::UNCHANGED;
10401 // A position whose simplified value does not have any value is
10402 // considered to be dead. We don't manifest noundef in such positions for
10403 // the same reason above.
10404 if (!A.getAssumedSimplified(IRP: getIRPosition(), AA: *this, UsedAssumedInformation,
10405 S: AA::Interprocedural)
10406 .has_value())
10407 return ChangeStatus::UNCHANGED;
10408 return AANoUndef::manifest(A);
10409 }
10410};
10411
10412struct AANoUndefFloating : public AANoUndefImpl {
10413 AANoUndefFloating(const IRPosition &IRP, Attributor &A)
10414 : AANoUndefImpl(IRP, A) {}
10415
10416 /// See AbstractAttribute::initialize(...).
10417 void initialize(Attributor &A) override {
10418 AANoUndefImpl::initialize(A);
10419 if (!getState().isAtFixpoint() && getAnchorScope() &&
10420 !getAnchorScope()->isDeclaration())
10421 if (Instruction *CtxI = getCtxI())
10422 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
10423 }
10424
10425 /// See AbstractAttribute::updateImpl(...).
10426 ChangeStatus updateImpl(Attributor &A) override {
10427 auto VisitValueCB = [&](const IRPosition &IRP) -> bool {
10428 bool IsKnownNoUndef;
10429 return AA::hasAssumedIRAttr<Attribute::NoUndef>(
10430 A, QueryingAA: this, IRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoUndef);
10431 };
10432
10433 bool Stripped;
10434 bool UsedAssumedInformation = false;
10435 Value *AssociatedValue = &getAssociatedValue();
10436 SmallVector<AA::ValueAndContext> Values;
10437 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
10438 S: AA::AnyScope, UsedAssumedInformation))
10439 Stripped = false;
10440 else
10441 Stripped =
10442 Values.size() != 1 || Values.front().getValue() != AssociatedValue;
10443
10444 if (!Stripped) {
10445 // If we haven't stripped anything we might still be able to use a
10446 // different AA, but only if the IRP changes. Effectively when we
10447 // interpret this not as a call site value but as a floating/argument
10448 // value.
10449 const IRPosition AVIRP = IRPosition::value(V: *AssociatedValue);
10450 if (AVIRP == getIRPosition() || !VisitValueCB(AVIRP))
10451 return indicatePessimisticFixpoint();
10452 return ChangeStatus::UNCHANGED;
10453 }
10454
10455 for (const auto &VAC : Values)
10456 if (!VisitValueCB(IRPosition::value(V: *VAC.getValue())))
10457 return indicatePessimisticFixpoint();
10458
10459 return ChangeStatus::UNCHANGED;
10460 }
10461
10462 /// See AbstractAttribute::trackStatistics()
10463 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10464};
10465
10466struct AANoUndefReturned final
10467 : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl> {
10468 AANoUndefReturned(const IRPosition &IRP, Attributor &A)
10469 : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl>(IRP, A) {}
10470
10471 /// See AbstractAttribute::trackStatistics()
10472 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10473};
10474
10475struct AANoUndefArgument final
10476 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl> {
10477 AANoUndefArgument(const IRPosition &IRP, Attributor &A)
10478 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl>(IRP, A) {}
10479
10480 /// See AbstractAttribute::trackStatistics()
10481 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noundef) }
10482};
10483
10484struct AANoUndefCallSiteArgument final : AANoUndefFloating {
10485 AANoUndefCallSiteArgument(const IRPosition &IRP, Attributor &A)
10486 : AANoUndefFloating(IRP, A) {}
10487
10488 /// See AbstractAttribute::trackStatistics()
10489 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noundef) }
10490};
10491
10492struct AANoUndefCallSiteReturned final
10493 : AACalleeToCallSite<AANoUndef, AANoUndefImpl> {
10494 AANoUndefCallSiteReturned(const IRPosition &IRP, Attributor &A)
10495 : AACalleeToCallSite<AANoUndef, AANoUndefImpl>(IRP, A) {}
10496
10497 /// See AbstractAttribute::trackStatistics()
10498 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noundef) }
10499};
10500
10501/// ------------------------ NoFPClass Attribute -------------------------------
10502
10503struct AANoFPClassImpl : AANoFPClass {
10504 AANoFPClassImpl(const IRPosition &IRP, Attributor &A) : AANoFPClass(IRP, A) {}
10505
10506 void initialize(Attributor &A) override {
10507 const IRPosition &IRP = getIRPosition();
10508
10509 Value &V = IRP.getAssociatedValue();
10510 if (isa<UndefValue>(Val: V)) {
10511 indicateOptimisticFixpoint();
10512 return;
10513 }
10514
10515 SmallVector<Attribute> Attrs;
10516 A.getAttrs(IRP: getIRPosition(), AKs: {Attribute::NoFPClass}, Attrs, IgnoreSubsumingPositions: false);
10517 for (const auto &Attr : Attrs) {
10518 addKnownBits(Bits: Attr.getNoFPClass());
10519 }
10520
10521 Instruction *CtxI = getCtxI();
10522
10523 if (getPositionKind() != IRPosition::IRP_RETURNED) {
10524 const DataLayout &DL = A.getDataLayout();
10525 InformationCache &InfoCache = A.getInfoCache();
10526
10527 const DominatorTree *DT = nullptr;
10528 AssumptionCache *AC = nullptr;
10529 const TargetLibraryInfo *TLI = nullptr;
10530 Function *F = getAnchorScope();
10531 if (F) {
10532 TLI = InfoCache.getTargetLibraryInfoForFunction(F: *F);
10533 if (!F->isDeclaration()) {
10534 DT =
10535 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
10536 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
10537 }
10538 }
10539
10540 SimplifyQuery Q(DL, TLI, DT, AC, CtxI);
10541
10542 KnownFPClass KnownFPClass = computeKnownFPClass(V: &V, InterestedClasses: fcAllFlags, SQ: Q);
10543 addKnownBits(Bits: ~KnownFPClass.getKnownFPClasses());
10544 }
10545
10546 if (CtxI)
10547 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
10548 }
10549
10550 /// See followUsesInMBEC
10551 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10552 AANoFPClass::StateType &State) {
10553 // TODO: Determine what instructions can be looked through.
10554 auto *CB = dyn_cast<CallBase>(Val: I);
10555 if (!CB)
10556 return false;
10557
10558 if (!CB->isArgOperand(U))
10559 return false;
10560
10561 unsigned ArgNo = CB->getArgOperandNo(U);
10562 IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
10563 if (auto *NoFPAA = A.getAAFor<AANoFPClass>(QueryingAA: *this, IRP, DepClass: DepClassTy::NONE))
10564 State.addKnownBits(Bits: NoFPAA->getState().getKnown());
10565 return false;
10566 }
10567
10568 const std::string getAsStr(Attributor *A) const override {
10569 std::string Result = "nofpclass";
10570 raw_string_ostream OS(Result);
10571 OS << getKnownNoFPClass() << '/' << getAssumedNoFPClass();
10572 return Result;
10573 }
10574
10575 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
10576 SmallVectorImpl<Attribute> &Attrs) const override {
10577 Attrs.emplace_back(Args: Attribute::getWithNoFPClass(Context&: Ctx, Mask: getAssumedNoFPClass()));
10578 }
10579};
10580
10581struct AANoFPClassFloating : public AANoFPClassImpl {
10582 AANoFPClassFloating(const IRPosition &IRP, Attributor &A)
10583 : AANoFPClassImpl(IRP, A) {}
10584
10585 /// See AbstractAttribute::updateImpl(...).
10586 ChangeStatus updateImpl(Attributor &A) override {
10587 SmallVector<AA::ValueAndContext> Values;
10588 bool UsedAssumedInformation = false;
10589 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
10590 S: AA::AnyScope, UsedAssumedInformation)) {
10591 Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
10592 }
10593
10594 StateType T;
10595 auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
10596 const auto *AA = A.getAAFor<AANoFPClass>(QueryingAA: *this, IRP: IRPosition::value(V),
10597 DepClass: DepClassTy::REQUIRED);
10598 if (!AA || this == AA) {
10599 T.indicatePessimisticFixpoint();
10600 } else {
10601 const AANoFPClass::StateType &S =
10602 static_cast<const AANoFPClass::StateType &>(AA->getState());
10603 T ^= S;
10604 }
10605 return T.isValidState();
10606 };
10607
10608 for (const auto &VAC : Values)
10609 if (!VisitValueCB(*VAC.getValue(), VAC.getCtxI()))
10610 return indicatePessimisticFixpoint();
10611
10612 return clampStateAndIndicateChange(S&: getState(), R: T);
10613 }
10614
10615 /// See AbstractAttribute::trackStatistics()
10616 void trackStatistics() const override {
10617 STATS_DECLTRACK_FNRET_ATTR(nofpclass)
10618 }
10619};
10620
10621struct AANoFPClassReturned final
10622 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10623 AANoFPClassImpl::StateType, false,
10624 Attribute::None, false> {
10625 AANoFPClassReturned(const IRPosition &IRP, Attributor &A)
10626 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10627 AANoFPClassImpl::StateType, false,
10628 Attribute::None, false>(IRP, A) {}
10629
10630 /// See AbstractAttribute::trackStatistics()
10631 void trackStatistics() const override {
10632 STATS_DECLTRACK_FNRET_ATTR(nofpclass)
10633 }
10634};
10635
10636struct AANoFPClassArgument final
10637 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl> {
10638 AANoFPClassArgument(const IRPosition &IRP, Attributor &A)
10639 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10640
10641 /// See AbstractAttribute::trackStatistics()
10642 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofpclass) }
10643};
10644
10645struct AANoFPClassCallSiteArgument final : AANoFPClassFloating {
10646 AANoFPClassCallSiteArgument(const IRPosition &IRP, Attributor &A)
10647 : AANoFPClassFloating(IRP, A) {}
10648
10649 /// See AbstractAttribute::trackStatistics()
10650 void trackStatistics() const override {
10651 STATS_DECLTRACK_CSARG_ATTR(nofpclass)
10652 }
10653};
10654
10655struct AANoFPClassCallSiteReturned final
10656 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl> {
10657 AANoFPClassCallSiteReturned(const IRPosition &IRP, Attributor &A)
10658 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10659
10660 /// See AbstractAttribute::trackStatistics()
10661 void trackStatistics() const override {
10662 STATS_DECLTRACK_CSRET_ATTR(nofpclass)
10663 }
10664};
10665
10666struct AACallEdgesImpl : public AACallEdges {
10667 AACallEdgesImpl(const IRPosition &IRP, Attributor &A) : AACallEdges(IRP, A) {}
10668
10669 const SetVector<Function *> &getOptimisticEdges() const override {
10670 return CalledFunctions;
10671 }
10672
10673 bool hasUnknownCallee() const override { return HasUnknownCallee; }
10674
10675 bool hasNonAsmUnknownCallee() const override {
10676 return HasUnknownCalleeNonAsm;
10677 }
10678
10679 const std::string getAsStr(Attributor *A) const override {
10680 return "CallEdges[" + std::to_string(val: HasUnknownCallee) + "," +
10681 std::to_string(val: CalledFunctions.size()) + "]";
10682 }
10683
10684 void trackStatistics() const override {}
10685
10686protected:
10687 void addCalledFunction(Function *Fn, ChangeStatus &Change) {
10688 if (CalledFunctions.insert(X: Fn)) {
10689 Change = ChangeStatus::CHANGED;
10690 LLVM_DEBUG(dbgs() << "[AACallEdges] New call edge: " << Fn->getName()
10691 << "\n");
10692 }
10693 }
10694
10695 void setHasUnknownCallee(bool NonAsm, ChangeStatus &Change) {
10696 if (!HasUnknownCallee)
10697 Change = ChangeStatus::CHANGED;
10698 if (NonAsm && !HasUnknownCalleeNonAsm)
10699 Change = ChangeStatus::CHANGED;
10700 HasUnknownCalleeNonAsm |= NonAsm;
10701 HasUnknownCallee = true;
10702 }
10703
10704private:
10705 /// Optimistic set of functions that might be called by this position.
10706 SetVector<Function *> CalledFunctions;
10707
10708 /// Is there any call with a unknown callee.
10709 bool HasUnknownCallee = false;
10710
10711 /// Is there any call with a unknown callee, excluding any inline asm.
10712 bool HasUnknownCalleeNonAsm = false;
10713};
10714
10715struct AACallEdgesCallSite : public AACallEdgesImpl {
10716 AACallEdgesCallSite(const IRPosition &IRP, Attributor &A)
10717 : AACallEdgesImpl(IRP, A) {}
10718 /// See AbstractAttribute::updateImpl(...).
10719 ChangeStatus updateImpl(Attributor &A) override {
10720 ChangeStatus Change = ChangeStatus::UNCHANGED;
10721
10722 auto VisitValue = [&](Value &V, const Instruction *CtxI) -> bool {
10723 if (Function *Fn = dyn_cast<Function>(Val: &V)) {
10724 addCalledFunction(Fn, Change);
10725 } else {
10726 LLVM_DEBUG(dbgs() << "[AACallEdges] Unrecognized value: " << V << "\n");
10727 setHasUnknownCallee(NonAsm: true, Change);
10728 }
10729
10730 // Explore all values.
10731 return true;
10732 };
10733
10734 SmallVector<AA::ValueAndContext> Values;
10735 // Process any value that we might call.
10736 auto ProcessCalledOperand = [&](Value *V, Instruction *CtxI) {
10737 if (isa<Constant>(Val: V)) {
10738 VisitValue(*V, CtxI);
10739 return;
10740 }
10741
10742 bool UsedAssumedInformation = false;
10743 Values.clear();
10744 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *V), AA: *this, Values,
10745 S: AA::AnyScope, UsedAssumedInformation)) {
10746 Values.push_back(Elt: {*V, CtxI});
10747 }
10748 for (auto &VAC : Values)
10749 VisitValue(*VAC.getValue(), VAC.getCtxI());
10750 };
10751
10752 CallBase *CB = cast<CallBase>(Val: getCtxI());
10753
10754 if (auto *IA = dyn_cast<InlineAsm>(Val: CB->getCalledOperand())) {
10755 if (IA->hasSideEffects() &&
10756 !hasAssumption(F: *CB->getCaller(), AssumptionStr: "ompx_no_call_asm") &&
10757 !hasAssumption(CB: *CB, AssumptionStr: "ompx_no_call_asm")) {
10758 setHasUnknownCallee(NonAsm: false, Change);
10759 }
10760 return Change;
10761 }
10762
10763 if (CB->isIndirectCall())
10764 if (auto *IndirectCallAA = A.getAAFor<AAIndirectCallInfo>(
10765 QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL))
10766 if (IndirectCallAA->foreachCallee(
10767 CB: [&](Function *Fn) { return VisitValue(*Fn, CB); }))
10768 return Change;
10769
10770 // The most simple case.
10771 ProcessCalledOperand(CB->getCalledOperand(), CB);
10772
10773 // Process callback functions.
10774 SmallVector<const Use *, 4u> CallbackUses;
10775 AbstractCallSite::getCallbackUses(CB: *CB, CallbackUses);
10776 for (const Use *U : CallbackUses)
10777 ProcessCalledOperand(U->get(), CB);
10778
10779 return Change;
10780 }
10781};
10782
10783struct AACallEdgesFunction : public AACallEdgesImpl {
10784 AACallEdgesFunction(const IRPosition &IRP, Attributor &A)
10785 : AACallEdgesImpl(IRP, A) {}
10786
10787 /// See AbstractAttribute::updateImpl(...).
10788 ChangeStatus updateImpl(Attributor &A) override {
10789 ChangeStatus Change = ChangeStatus::UNCHANGED;
10790
10791 auto ProcessCallInst = [&](Instruction &Inst) {
10792 CallBase &CB = cast<CallBase>(Val&: Inst);
10793
10794 auto *CBEdges = A.getAAFor<AACallEdges>(
10795 QueryingAA: *this, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::REQUIRED);
10796 if (!CBEdges)
10797 return false;
10798 if (CBEdges->hasNonAsmUnknownCallee())
10799 setHasUnknownCallee(NonAsm: true, Change);
10800 if (CBEdges->hasUnknownCallee())
10801 setHasUnknownCallee(NonAsm: false, Change);
10802
10803 for (Function *F : CBEdges->getOptimisticEdges())
10804 addCalledFunction(Fn: F, Change);
10805
10806 return true;
10807 };
10808
10809 // Visit all callable instructions.
10810 bool UsedAssumedInformation = false;
10811 if (!A.checkForAllCallLikeInstructions(Pred: ProcessCallInst, QueryingAA: *this,
10812 UsedAssumedInformation,
10813 /* CheckBBLivenessOnly */ true)) {
10814 // If we haven't looked at all call like instructions, assume that there
10815 // are unknown callees.
10816 setHasUnknownCallee(NonAsm: true, Change);
10817 }
10818
10819 return Change;
10820 }
10821};
10822
10823/// -------------------AAInterFnReachability Attribute--------------------------
10824
10825struct AAInterFnReachabilityFunction
10826 : public CachedReachabilityAA<AAInterFnReachability, Function> {
10827 using Base = CachedReachabilityAA<AAInterFnReachability, Function>;
10828 AAInterFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
10829 : Base(IRP, A) {}
10830
10831 bool instructionCanReach(
10832 Attributor &A, const Instruction &From, const Function &To,
10833 const AA::InstExclusionSetTy *ExclusionSet) const override {
10834 assert(From.getFunction() == getAnchorScope() && "Queried the wrong AA!");
10835 auto *NonConstThis = const_cast<AAInterFnReachabilityFunction *>(this);
10836
10837 RQITy StackRQI(A, From, To, ExclusionSet, false);
10838 RQITy::Reachable Result;
10839 if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
10840 return NonConstThis->isReachableImpl(A, RQI&: StackRQI,
10841 /*IsTemporaryRQI=*/true);
10842 return Result == RQITy::Reachable::Yes;
10843 }
10844
10845 bool isReachableImpl(Attributor &A, RQITy &RQI,
10846 bool IsTemporaryRQI) override {
10847 const Instruction *EntryI =
10848 &RQI.From->getFunction()->getEntryBlock().front();
10849 if (EntryI != RQI.From &&
10850 !instructionCanReach(A, From: *EntryI, To: *RQI.To, ExclusionSet: nullptr))
10851 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet: false,
10852 IsTemporaryRQI);
10853
10854 auto CheckReachableCallBase = [&](CallBase *CB) {
10855 auto *CBEdges = A.getAAFor<AACallEdges>(
10856 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL);
10857 if (!CBEdges || !CBEdges->getState().isValidState())
10858 return false;
10859 // TODO Check To backwards in this case.
10860 if (CBEdges->hasUnknownCallee())
10861 return false;
10862
10863 for (Function *Fn : CBEdges->getOptimisticEdges()) {
10864 if (Fn == RQI.To)
10865 return false;
10866
10867 if (Fn->isDeclaration()) {
10868 if (Fn->hasFnAttribute(Kind: Attribute::NoCallback))
10869 continue;
10870 // TODO Check To backwards in this case.
10871 return false;
10872 }
10873
10874 if (Fn == getAnchorScope()) {
10875 if (EntryI == RQI.From)
10876 continue;
10877 return false;
10878 }
10879
10880 const AAInterFnReachability *InterFnReachability =
10881 A.getAAFor<AAInterFnReachability>(QueryingAA: *this, IRP: IRPosition::function(F: *Fn),
10882 DepClass: DepClassTy::OPTIONAL);
10883
10884 const Instruction &FnFirstInst = Fn->getEntryBlock().front();
10885 if (!InterFnReachability ||
10886 InterFnReachability->instructionCanReach(A, Inst: FnFirstInst, Fn: *RQI.To,
10887 ExclusionSet: RQI.ExclusionSet))
10888 return false;
10889 }
10890 return true;
10891 };
10892
10893 const auto *IntraFnReachability = A.getAAFor<AAIntraFnReachability>(
10894 QueryingAA: *this, IRP: IRPosition::function(F: *RQI.From->getFunction()),
10895 DepClass: DepClassTy::OPTIONAL);
10896
10897 // Determine call like instructions that we can reach from the inst.
10898 auto CheckCallBase = [&](Instruction &CBInst) {
10899 // There are usually less nodes in the call graph, check inter function
10900 // reachability first.
10901 if (CheckReachableCallBase(cast<CallBase>(Val: &CBInst)))
10902 return true;
10903 return IntraFnReachability && !IntraFnReachability->isAssumedReachable(
10904 A, From: *RQI.From, To: CBInst, ExclusionSet: RQI.ExclusionSet);
10905 };
10906
10907 bool UsedExclusionSet = /* conservative */ true;
10908 bool UsedAssumedInformation = false;
10909 if (!A.checkForAllCallLikeInstructions(Pred: CheckCallBase, QueryingAA: *this,
10910 UsedAssumedInformation,
10911 /* CheckBBLivenessOnly */ true))
10912 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
10913 IsTemporaryRQI);
10914
10915 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
10916 IsTemporaryRQI);
10917 }
10918
10919 void trackStatistics() const override {}
10920};
10921} // namespace
10922
10923template <typename AAType>
10924static std::optional<Constant *>
10925askForAssumedConstant(Attributor &A, const AbstractAttribute &QueryingAA,
10926 const IRPosition &IRP, Type &Ty) {
10927 if (!Ty.isIntegerTy())
10928 return nullptr;
10929
10930 // This will also pass the call base context.
10931 const auto *AA = A.getAAFor<AAType>(QueryingAA, IRP, DepClassTy::NONE);
10932 if (!AA)
10933 return nullptr;
10934
10935 std::optional<Constant *> COpt = AA->getAssumedConstant(A);
10936
10937 if (!COpt.has_value()) {
10938 A.recordDependence(FromAA: *AA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
10939 return std::nullopt;
10940 }
10941 if (auto *C = *COpt) {
10942 A.recordDependence(FromAA: *AA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
10943 return C;
10944 }
10945 return nullptr;
10946}
10947
10948Value *AAPotentialValues::getSingleValue(
10949 Attributor &A, const AbstractAttribute &AA, const IRPosition &IRP,
10950 SmallVectorImpl<AA::ValueAndContext> &Values) {
10951 Type &Ty = *IRP.getAssociatedType();
10952 std::optional<Value *> V;
10953 for (auto &It : Values) {
10954 V = AA::combineOptionalValuesInAAValueLatice(A: V, B: It.getValue(), Ty: &Ty);
10955 if (V.has_value() && !*V)
10956 break;
10957 }
10958 if (!V.has_value())
10959 return UndefValue::get(T: &Ty);
10960 return *V;
10961}
10962
10963namespace {
10964struct AAPotentialValuesImpl : AAPotentialValues {
10965 using StateType = PotentialLLVMValuesState;
10966
10967 AAPotentialValuesImpl(const IRPosition &IRP, Attributor &A)
10968 : AAPotentialValues(IRP, A) {}
10969
10970 /// See AbstractAttribute::initialize(..).
10971 void initialize(Attributor &A) override {
10972 if (A.hasSimplificationCallback(IRP: getIRPosition())) {
10973 indicatePessimisticFixpoint();
10974 return;
10975 }
10976 Value *Stripped = getAssociatedValue().stripPointerCasts();
10977 if (isa<Constant>(Val: Stripped) && !isa<ConstantExpr>(Val: Stripped)) {
10978 addValue(A, State&: getState(), V&: *Stripped, CtxI: getCtxI(), S: AA::AnyScope,
10979 AnchorScope: getAnchorScope());
10980 indicateOptimisticFixpoint();
10981 return;
10982 }
10983 AAPotentialValues::initialize(A);
10984 }
10985
10986 /// See AbstractAttribute::getAsStr().
10987 const std::string getAsStr(Attributor *A) const override {
10988 std::string Str;
10989 llvm::raw_string_ostream OS(Str);
10990 OS << getState();
10991 return Str;
10992 }
10993
10994 template <typename AAType>
10995 static std::optional<Value *> askOtherAA(Attributor &A,
10996 const AbstractAttribute &AA,
10997 const IRPosition &IRP, Type &Ty) {
10998 if (isa<Constant>(Val: IRP.getAssociatedValue()))
10999 return &IRP.getAssociatedValue();
11000 std::optional<Constant *> C = askForAssumedConstant<AAType>(A, AA, IRP, Ty);
11001 if (!C)
11002 return std::nullopt;
11003 if (*C)
11004 if (auto *CC = AA::getWithType(V&: **C, Ty))
11005 return CC;
11006 return nullptr;
11007 }
11008
11009 virtual void addValue(Attributor &A, StateType &State, Value &V,
11010 const Instruction *CtxI, AA::ValueScope S,
11011 Function *AnchorScope) const {
11012
11013 IRPosition ValIRP = IRPosition::value(V);
11014 if (auto *CB = dyn_cast_or_null<CallBase>(Val: CtxI)) {
11015 for (const auto &U : CB->args()) {
11016 if (U.get() != &V)
11017 continue;
11018 ValIRP = IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U));
11019 break;
11020 }
11021 }
11022
11023 Value *VPtr = &V;
11024 if (ValIRP.getAssociatedType()->isIntegerTy()) {
11025 Type &Ty = *getAssociatedType();
11026 std::optional<Value *> SimpleV =
11027 askOtherAA<AAValueConstantRange>(A, AA: *this, IRP: ValIRP, Ty);
11028 if (SimpleV.has_value() && !*SimpleV) {
11029 auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
11030 QueryingAA: *this, IRP: ValIRP, DepClass: DepClassTy::OPTIONAL);
11031 if (PotentialConstantsAA && PotentialConstantsAA->isValidState()) {
11032 for (const auto &It : PotentialConstantsAA->getAssumedSet())
11033 State.unionAssumed(C: {{*ConstantInt::get(Ty: &Ty, V: It), nullptr}, S});
11034 if (PotentialConstantsAA->undefIsContained())
11035 State.unionAssumed(C: {{*UndefValue::get(T: &Ty), nullptr}, S});
11036 return;
11037 }
11038 }
11039 if (!SimpleV.has_value())
11040 return;
11041
11042 if (*SimpleV)
11043 VPtr = *SimpleV;
11044 }
11045
11046 if (isa<ConstantInt>(Val: VPtr))
11047 CtxI = nullptr;
11048 if (!AA::isValidInScope(V: *VPtr, Scope: AnchorScope))
11049 S = AA::ValueScope(S | AA::Interprocedural);
11050
11051 State.unionAssumed(C: {{*VPtr, CtxI}, S});
11052 }
11053
11054 /// Helper struct to tie a value+context pair together with the scope for
11055 /// which this is the simplified version.
11056 struct ItemInfo {
11057 AA::ValueAndContext I;
11058 AA::ValueScope S;
11059
11060 bool operator==(const ItemInfo &II) const {
11061 return II.I == I && II.S == S;
11062 };
11063 bool operator<(const ItemInfo &II) const {
11064 return std::tie(args: I, args: S) < std::tie(args: II.I, args: II.S);
11065 };
11066 };
11067
11068 bool recurseForValue(Attributor &A, const IRPosition &IRP, AA::ValueScope S) {
11069 SmallMapVector<AA::ValueAndContext, int, 8> ValueScopeMap;
11070 for (auto CS : {AA::Intraprocedural, AA::Interprocedural}) {
11071 if (!(CS & S))
11072 continue;
11073
11074 bool UsedAssumedInformation = false;
11075 SmallVector<AA::ValueAndContext> Values;
11076 if (!A.getAssumedSimplifiedValues(IRP, AA: this, Values, S: CS,
11077 UsedAssumedInformation))
11078 return false;
11079
11080 for (auto &It : Values)
11081 ValueScopeMap[It] += CS;
11082 }
11083 for (auto &It : ValueScopeMap)
11084 addValue(A, State&: getState(), V&: *It.first.getValue(), CtxI: It.first.getCtxI(),
11085 S: AA::ValueScope(It.second), AnchorScope: getAnchorScope());
11086
11087 return true;
11088 }
11089
11090 void giveUpOnIntraprocedural(Attributor &A) {
11091 auto NewS = StateType::getBestState(PVS: getState());
11092 for (const auto &It : getAssumedSet()) {
11093 if (It.second == AA::Intraprocedural)
11094 continue;
11095 addValue(A, State&: NewS, V&: *It.first.getValue(), CtxI: It.first.getCtxI(),
11096 S: AA::Interprocedural, AnchorScope: getAnchorScope());
11097 }
11098 assert(!undefIsContained() && "Undef should be an explicit value!");
11099 addValue(A, State&: NewS, V&: getAssociatedValue(), CtxI: getCtxI(), S: AA::Intraprocedural,
11100 AnchorScope: getAnchorScope());
11101 getState() = NewS;
11102 }
11103
11104 /// See AbstractState::indicatePessimisticFixpoint(...).
11105 ChangeStatus indicatePessimisticFixpoint() override {
11106 getState() = StateType::getBestState(PVS: getState());
11107 getState().unionAssumed(C: {{getAssociatedValue(), getCtxI()}, AA::AnyScope});
11108 AAPotentialValues::indicateOptimisticFixpoint();
11109 return ChangeStatus::CHANGED;
11110 }
11111
11112 /// See AbstractAttribute::updateImpl(...).
11113 ChangeStatus updateImpl(Attributor &A) override {
11114 return indicatePessimisticFixpoint();
11115 }
11116
11117 /// See AbstractAttribute::manifest(...).
11118 ChangeStatus manifest(Attributor &A) override {
11119 SmallVector<AA::ValueAndContext> Values;
11120 for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {
11121 Values.clear();
11122 if (!getAssumedSimplifiedValues(A, Values, S))
11123 continue;
11124 Value &OldV = getAssociatedValue();
11125 if (isa<UndefValue>(Val: OldV))
11126 continue;
11127 Value *NewV = getSingleValue(A, AA: *this, IRP: getIRPosition(), Values);
11128 if (!NewV || NewV == &OldV)
11129 continue;
11130 if (getCtxI() &&
11131 !AA::isValidAtPosition(VAC: {*NewV, *getCtxI()}, InfoCache&: A.getInfoCache()))
11132 continue;
11133 if (A.changeAfterManifest(IRP: getIRPosition(), NV&: *NewV))
11134 return ChangeStatus::CHANGED;
11135 }
11136 return ChangeStatus::UNCHANGED;
11137 }
11138
11139 bool getAssumedSimplifiedValues(
11140 Attributor &A, SmallVectorImpl<AA::ValueAndContext> &Values,
11141 AA::ValueScope S, bool RecurseForSelectAndPHI = false) const override {
11142 if (!isValidState())
11143 return false;
11144 bool UsedAssumedInformation = false;
11145 for (const auto &It : getAssumedSet())
11146 if (It.second & S) {
11147 if (RecurseForSelectAndPHI && (isa<PHINode>(Val: It.first.getValue()) ||
11148 isa<SelectInst>(Val: It.first.getValue()))) {
11149 if (A.getAssumedSimplifiedValues(
11150 IRP: IRPosition::inst(I: *cast<Instruction>(Val: It.first.getValue())),
11151 AA: this, Values, S, UsedAssumedInformation))
11152 continue;
11153 }
11154 Values.push_back(Elt: It.first);
11155 }
11156 assert(!undefIsContained() && "Undef should be an explicit value!");
11157 return true;
11158 }
11159};
11160
11161struct AAPotentialValuesFloating : AAPotentialValuesImpl {
11162 AAPotentialValuesFloating(const IRPosition &IRP, Attributor &A)
11163 : AAPotentialValuesImpl(IRP, A) {}
11164
11165 /// See AbstractAttribute::updateImpl(...).
11166 ChangeStatus updateImpl(Attributor &A) override {
11167 auto AssumedBefore = getAssumed();
11168
11169 genericValueTraversal(A, InitialV: &getAssociatedValue());
11170
11171 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11172 : ChangeStatus::CHANGED;
11173 }
11174
11175 /// Helper struct to remember which AAIsDead instances we actually used.
11176 struct LivenessInfo {
11177 const AAIsDead *LivenessAA = nullptr;
11178 bool AnyDead = false;
11179 };
11180
11181 /// Check if \p Cmp is a comparison we can simplify.
11182 ///
11183 /// We handle multiple cases, one in which at least one operand is an
11184 /// (assumed) nullptr. If so, try to simplify it using AANonNull on the other
11185 /// operand. Return true if successful, in that case Worklist will be updated.
11186 bool handleCmp(Attributor &A, Value &Cmp, Value *LHS, Value *RHS,
11187 CmpInst::Predicate Pred, ItemInfo II,
11188 SmallVectorImpl<ItemInfo> &Worklist) {
11189
11190 // Simplify the operands first.
11191 bool UsedAssumedInformation = false;
11192 SmallVector<AA::ValueAndContext> LHSValues, RHSValues;
11193 auto GetSimplifiedValues = [&](Value &V,
11194 SmallVector<AA::ValueAndContext> &Values) {
11195 if (!A.getAssumedSimplifiedValues(
11196 IRP: IRPosition::value(V, CBContext: getCallBaseContext()), AA: this, Values,
11197 S: AA::Intraprocedural, UsedAssumedInformation)) {
11198 Values.clear();
11199 Values.push_back(Elt: AA::ValueAndContext{V, II.I.getCtxI()});
11200 }
11201 return Values.empty();
11202 };
11203 if (GetSimplifiedValues(*LHS, LHSValues))
11204 return true;
11205 if (GetSimplifiedValues(*RHS, RHSValues))
11206 return true;
11207
11208 LLVMContext &Ctx = LHS->getContext();
11209
11210 InformationCache &InfoCache = A.getInfoCache();
11211 Instruction *CmpI = dyn_cast<Instruction>(Val: &Cmp);
11212 Function *F = CmpI ? CmpI->getFunction() : nullptr;
11213 const auto *DT =
11214 F ? InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F)
11215 : nullptr;
11216 const auto *TLI =
11217 F ? A.getInfoCache().getTargetLibraryInfoForFunction(F: *F) : nullptr;
11218 auto *AC =
11219 F ? InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F)
11220 : nullptr;
11221
11222 const DataLayout &DL = A.getDataLayout();
11223 SimplifyQuery Q(DL, TLI, DT, AC, CmpI);
11224
11225 auto CheckPair = [&](Value &LHSV, Value &RHSV) {
11226 if (isa<UndefValue>(Val: LHSV) || isa<UndefValue>(Val: RHSV)) {
11227 addValue(A, State&: getState(), V&: *UndefValue::get(T: Cmp.getType()),
11228 /* CtxI */ nullptr, S: II.S, AnchorScope: getAnchorScope());
11229 return true;
11230 }
11231
11232 // Handle the trivial case first in which we don't even need to think
11233 // about null or non-null.
11234 if (&LHSV == &RHSV &&
11235 (CmpInst::isTrueWhenEqual(predicate: Pred) || CmpInst::isFalseWhenEqual(predicate: Pred))) {
11236 Constant *NewV = ConstantInt::get(Ty: Type::getInt1Ty(C&: Ctx),
11237 V: CmpInst::isTrueWhenEqual(predicate: Pred));
11238 addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11239 AnchorScope: getAnchorScope());
11240 return true;
11241 }
11242
11243 auto *TypedLHS = AA::getWithType(V&: LHSV, Ty&: *LHS->getType());
11244 auto *TypedRHS = AA::getWithType(V&: RHSV, Ty&: *RHS->getType());
11245 if (TypedLHS && TypedRHS) {
11246 Value *NewV = simplifyCmpInst(Predicate: Pred, LHS: TypedLHS, RHS: TypedRHS, Q);
11247 if (NewV && NewV != &Cmp) {
11248 addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11249 AnchorScope: getAnchorScope());
11250 return true;
11251 }
11252 }
11253
11254 // From now on we only handle equalities (==, !=).
11255 if (!CmpInst::isEquality(pred: Pred))
11256 return false;
11257
11258 bool LHSIsNull = isa<ConstantPointerNull>(Val: LHSV);
11259 bool RHSIsNull = isa<ConstantPointerNull>(Val: RHSV);
11260 if (!LHSIsNull && !RHSIsNull)
11261 return false;
11262
11263 // Left is the nullptr ==/!= non-nullptr case. We'll use AANonNull on the
11264 // non-nullptr operand and if we assume it's non-null we can conclude the
11265 // result of the comparison.
11266 assert((LHSIsNull || RHSIsNull) &&
11267 "Expected nullptr versus non-nullptr comparison at this point");
11268
11269 // The index is the operand that we assume is not null.
11270 unsigned PtrIdx = LHSIsNull;
11271 bool IsKnownNonNull;
11272 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
11273 A, QueryingAA: this, IRP: IRPosition::value(V: *(PtrIdx ? &RHSV : &LHSV)),
11274 DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNonNull);
11275 if (!IsAssumedNonNull)
11276 return false;
11277
11278 // The new value depends on the predicate, true for != and false for ==.
11279 Constant *NewV =
11280 ConstantInt::get(Ty: Type::getInt1Ty(C&: Ctx), V: Pred == CmpInst::ICMP_NE);
11281 addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11282 AnchorScope: getAnchorScope());
11283 return true;
11284 };
11285
11286 for (auto &LHSValue : LHSValues)
11287 for (auto &RHSValue : RHSValues)
11288 if (!CheckPair(*LHSValue.getValue(), *RHSValue.getValue()))
11289 return false;
11290 return true;
11291 }
11292
11293 bool handleSelectInst(Attributor &A, SelectInst &SI, ItemInfo II,
11294 SmallVectorImpl<ItemInfo> &Worklist) {
11295 const Instruction *CtxI = II.I.getCtxI();
11296 bool UsedAssumedInformation = false;
11297
11298 std::optional<Constant *> C =
11299 A.getAssumedConstant(V: *SI.getCondition(), AA: *this, UsedAssumedInformation);
11300 bool NoValueYet = !C.has_value();
11301 if (NoValueYet || isa_and_nonnull<UndefValue>(Val: *C))
11302 return true;
11303 if (auto *CI = dyn_cast_or_null<ConstantInt>(Val: *C)) {
11304 if (CI->isZero())
11305 Worklist.push_back(Elt: {.I: {*SI.getFalseValue(), CtxI}, .S: II.S});
11306 else
11307 Worklist.push_back(Elt: {.I: {*SI.getTrueValue(), CtxI}, .S: II.S});
11308 } else if (&SI == &getAssociatedValue()) {
11309 // We could not simplify the condition, assume both values.
11310 Worklist.push_back(Elt: {.I: {*SI.getTrueValue(), CtxI}, .S: II.S});
11311 Worklist.push_back(Elt: {.I: {*SI.getFalseValue(), CtxI}, .S: II.S});
11312 } else {
11313 std::optional<Value *> SimpleV = A.getAssumedSimplified(
11314 IRP: IRPosition::inst(I: SI), AA: *this, UsedAssumedInformation, S: II.S);
11315 if (!SimpleV.has_value())
11316 return true;
11317 if (*SimpleV) {
11318 addValue(A, State&: getState(), V&: **SimpleV, CtxI, S: II.S, AnchorScope: getAnchorScope());
11319 return true;
11320 }
11321 return false;
11322 }
11323 return true;
11324 }
11325
11326 bool handleLoadInst(Attributor &A, LoadInst &LI, ItemInfo II,
11327 SmallVectorImpl<ItemInfo> &Worklist) {
11328 SmallSetVector<Value *, 4> PotentialCopies;
11329 SmallSetVector<Instruction *, 4> PotentialValueOrigins;
11330 bool UsedAssumedInformation = false;
11331 if (!AA::getPotentiallyLoadedValues(A, LI, PotentialValues&: PotentialCopies,
11332 PotentialValueOrigins, QueryingAA: *this,
11333 UsedAssumedInformation,
11334 /* OnlyExact */ true)) {
11335 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Failed to get potentially "
11336 "loaded values for load instruction "
11337 << LI << "\n");
11338 return false;
11339 }
11340
11341 // Do not simplify loads that are only used in llvm.assume if we cannot also
11342 // remove all stores that may feed into the load. The reason is that the
11343 // assume is probably worth something as long as the stores are around.
11344 InformationCache &InfoCache = A.getInfoCache();
11345 if (InfoCache.isOnlyUsedByAssume(I: LI)) {
11346 if (!llvm::all_of(Range&: PotentialValueOrigins, P: [&](Instruction *I) {
11347 if (!I || isa<AssumeInst>(Val: I))
11348 return true;
11349 if (auto *SI = dyn_cast<StoreInst>(Val: I))
11350 return A.isAssumedDead(U: SI->getOperandUse(i: 0), QueryingAA: this,
11351 /* LivenessAA */ FnLivenessAA: nullptr,
11352 UsedAssumedInformation,
11353 /* CheckBBLivenessOnly */ false);
11354 return A.isAssumedDead(I: *I, QueryingAA: this, /* LivenessAA */ nullptr,
11355 UsedAssumedInformation,
11356 /* CheckBBLivenessOnly */ false);
11357 })) {
11358 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Load is onl used by assumes "
11359 "and we cannot delete all the stores: "
11360 << LI << "\n");
11361 return false;
11362 }
11363 }
11364
11365 // Values have to be dynamically unique or we loose the fact that a
11366 // single llvm::Value might represent two runtime values (e.g.,
11367 // stack locations in different recursive calls).
11368 const Instruction *CtxI = II.I.getCtxI();
11369 bool ScopeIsLocal = (II.S & AA::Intraprocedural);
11370 bool AllLocal = ScopeIsLocal;
11371 bool DynamicallyUnique = llvm::all_of(Range&: PotentialCopies, P: [&](Value *PC) {
11372 AllLocal &= AA::isValidInScope(V: *PC, Scope: getAnchorScope());
11373 return AA::isDynamicallyUnique(A, QueryingAA: *this, V: *PC);
11374 });
11375 if (!DynamicallyUnique) {
11376 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Not all potentially loaded "
11377 "values are dynamically unique: "
11378 << LI << "\n");
11379 return false;
11380 }
11381
11382 for (auto *PotentialCopy : PotentialCopies) {
11383 if (AllLocal) {
11384 Worklist.push_back(Elt: {.I: {*PotentialCopy, CtxI}, .S: II.S});
11385 } else {
11386 Worklist.push_back(Elt: {.I: {*PotentialCopy, CtxI}, .S: AA::Interprocedural});
11387 }
11388 }
11389 if (!AllLocal && ScopeIsLocal)
11390 addValue(A, State&: getState(), V&: LI, CtxI, S: AA::Intraprocedural, AnchorScope: getAnchorScope());
11391 return true;
11392 }
11393
11394 bool handlePHINode(
11395 Attributor &A, PHINode &PHI, ItemInfo II,
11396 SmallVectorImpl<ItemInfo> &Worklist,
11397 SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
11398 auto GetLivenessInfo = [&](const Function &F) -> LivenessInfo & {
11399 LivenessInfo &LI = LivenessAAs[&F];
11400 if (!LI.LivenessAA)
11401 LI.LivenessAA = A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: IRPosition::function(F),
11402 DepClass: DepClassTy::NONE);
11403 return LI;
11404 };
11405
11406 if (&PHI == &getAssociatedValue()) {
11407 LivenessInfo &LI = GetLivenessInfo(*PHI.getFunction());
11408 const auto *CI =
11409 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
11410 F: *PHI.getFunction());
11411
11412 CycleRef C;
11413 bool CyclePHI = mayBeInCycle(CI, I: &PHI, /* HeaderOnly */ true, CPtr: &C);
11414 for (unsigned u = 0, e = PHI.getNumIncomingValues(); u < e; u++) {
11415 BasicBlock *IncomingBB = PHI.getIncomingBlock(i: u);
11416 if (LI.LivenessAA &&
11417 LI.LivenessAA->isEdgeDead(From: IncomingBB, To: PHI.getParent())) {
11418 LI.AnyDead = true;
11419 continue;
11420 }
11421 Value *V = PHI.getIncomingValue(i: u);
11422 if (V == &PHI)
11423 continue;
11424
11425 // If the incoming value is not the PHI but an instruction in the same
11426 // cycle we might have multiple versions of it flying around.
11427 if (CyclePHI && isa<Instruction>(Val: V) &&
11428 (!C || CI->contains(C, Block: cast<Instruction>(Val: V)->getParent())))
11429 return false;
11430
11431 Worklist.push_back(Elt: {.I: {*V, IncomingBB->getTerminator()}, .S: II.S});
11432 }
11433 return true;
11434 }
11435
11436 bool UsedAssumedInformation = false;
11437 std::optional<Value *> SimpleV = A.getAssumedSimplified(
11438 IRP: IRPosition::inst(I: PHI), AA: *this, UsedAssumedInformation, S: II.S);
11439 if (!SimpleV.has_value())
11440 return true;
11441 if (!(*SimpleV))
11442 return false;
11443 addValue(A, State&: getState(), V&: **SimpleV, CtxI: &PHI, S: II.S, AnchorScope: getAnchorScope());
11444 return true;
11445 }
11446
11447 /// Use the generic, non-optimistic InstSimplfy functionality if we managed to
11448 /// simplify any operand of the instruction \p I. Return true if successful,
11449 /// in that case Worklist will be updated.
11450 bool handleGenericInst(Attributor &A, Instruction &I, ItemInfo II,
11451 SmallVectorImpl<ItemInfo> &Worklist) {
11452 bool SomeSimplified = false;
11453 bool UsedAssumedInformation = false;
11454
11455 SmallVector<Value *, 8> NewOps(I.getNumOperands());
11456 int Idx = 0;
11457 for (Value *Op : I.operands()) {
11458 const auto &SimplifiedOp = A.getAssumedSimplified(
11459 IRP: IRPosition::value(V: *Op, CBContext: getCallBaseContext()), AA: *this,
11460 UsedAssumedInformation, S: AA::Intraprocedural);
11461 // If we are not sure about any operand we are not sure about the entire
11462 // instruction, we'll wait.
11463 if (!SimplifiedOp.has_value())
11464 return true;
11465
11466 if (*SimplifiedOp)
11467 NewOps[Idx] = *SimplifiedOp;
11468 else
11469 NewOps[Idx] = Op;
11470
11471 SomeSimplified |= (NewOps[Idx] != Op);
11472 ++Idx;
11473 }
11474
11475 // We won't bother with the InstSimplify interface if we didn't simplify any
11476 // operand ourselves.
11477 if (!SomeSimplified)
11478 return false;
11479
11480 InformationCache &InfoCache = A.getInfoCache();
11481 Function *F = I.getFunction();
11482 const auto *DT =
11483 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
11484 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
11485 auto *AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
11486
11487 const DataLayout &DL = I.getDataLayout();
11488 SimplifyQuery Q(DL, TLI, DT, AC, &I);
11489 Value *NewV = simplifyInstructionWithOperands(I: &I, NewOps, Q);
11490 if (!NewV || NewV == &I)
11491 return false;
11492
11493 LLVM_DEBUG(dbgs() << "Generic inst " << I << " assumed simplified to "
11494 << *NewV << "\n");
11495 Worklist.push_back(Elt: {.I: {*NewV, II.I.getCtxI()}, .S: II.S});
11496 return true;
11497 }
11498
11499 bool simplifyInstruction(
11500 Attributor &A, Instruction &I, ItemInfo II,
11501 SmallVectorImpl<ItemInfo> &Worklist,
11502 SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
11503 if (auto *CI = dyn_cast<CmpInst>(Val: &I))
11504 return handleCmp(A, Cmp&: *CI, LHS: CI->getOperand(i_nocapture: 0), RHS: CI->getOperand(i_nocapture: 1),
11505 Pred: CI->getPredicate(), II, Worklist);
11506
11507 switch (I.getOpcode()) {
11508 case Instruction::Select:
11509 return handleSelectInst(A, SI&: cast<SelectInst>(Val&: I), II, Worklist);
11510 case Instruction::PHI:
11511 return handlePHINode(A, PHI&: cast<PHINode>(Val&: I), II, Worklist, LivenessAAs);
11512 case Instruction::Load:
11513 return handleLoadInst(A, LI&: cast<LoadInst>(Val&: I), II, Worklist);
11514 default:
11515 return handleGenericInst(A, I, II, Worklist);
11516 };
11517 return false;
11518 }
11519
11520 void genericValueTraversal(Attributor &A, Value *InitialV) {
11521 SmallMapVector<const Function *, LivenessInfo, 4> LivenessAAs;
11522
11523 SmallSet<ItemInfo, 16> Visited;
11524 SmallVector<ItemInfo, 16> Worklist;
11525 Worklist.push_back(Elt: {.I: {*InitialV, getCtxI()}, .S: AA::AnyScope});
11526
11527 int Iteration = 0;
11528 do {
11529 ItemInfo II = Worklist.pop_back_val();
11530 Value *V = II.I.getValue();
11531 assert(V);
11532 const Instruction *CtxI = II.I.getCtxI();
11533 AA::ValueScope S = II.S;
11534
11535 // Check if we should process the current value. To prevent endless
11536 // recursion keep a record of the values we followed!
11537 if (!Visited.insert(V: II).second)
11538 continue;
11539
11540 // Make sure we limit the compile time for complex expressions.
11541 if (Iteration++ >= MaxPotentialValuesIterations) {
11542 LLVM_DEBUG(dbgs() << "Generic value traversal reached iteration limit: "
11543 << Iteration << "!\n");
11544 addValue(A, State&: getState(), V&: *V, CtxI, S, AnchorScope: getAnchorScope());
11545 continue;
11546 }
11547
11548 // Explicitly look through calls with a "returned" attribute if we do
11549 // not have a pointer as stripPointerCasts only works on them.
11550 Value *NewV = nullptr;
11551 if (V->getType()->isPointerTy()) {
11552 NewV = AA::getWithType(V&: *V->stripPointerCasts(), Ty&: *V->getType());
11553 } else {
11554 if (auto *CB = dyn_cast<CallBase>(Val: V))
11555 if (auto *Callee =
11556 dyn_cast_if_present<Function>(Val: CB->getCalledOperand())) {
11557 for (Argument &Arg : Callee->args())
11558 if (Arg.hasReturnedAttr()) {
11559 NewV = CB->getArgOperand(i: Arg.getArgNo());
11560 break;
11561 }
11562 }
11563 }
11564 if (NewV && NewV != V) {
11565 Worklist.push_back(Elt: {.I: {*NewV, CtxI}, .S: S});
11566 continue;
11567 }
11568
11569 if (auto *I = dyn_cast<Instruction>(Val: V)) {
11570 if (simplifyInstruction(A, I&: *I, II, Worklist, LivenessAAs))
11571 continue;
11572 }
11573
11574 if (V != InitialV || isa<Argument>(Val: V))
11575 if (recurseForValue(A, IRP: IRPosition::value(V: *V), S: II.S))
11576 continue;
11577
11578 // If we haven't stripped anything we give up.
11579 if (V == InitialV && CtxI == getCtxI()) {
11580 indicatePessimisticFixpoint();
11581 return;
11582 }
11583
11584 addValue(A, State&: getState(), V&: *V, CtxI, S, AnchorScope: getAnchorScope());
11585 } while (!Worklist.empty());
11586
11587 // If we actually used liveness information so we have to record a
11588 // dependence.
11589 for (auto &It : LivenessAAs)
11590 if (It.second.AnyDead)
11591 A.recordDependence(FromAA: *It.second.LivenessAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
11592 }
11593
11594 /// See AbstractAttribute::trackStatistics()
11595 void trackStatistics() const override {
11596 STATS_DECLTRACK_FLOATING_ATTR(potential_values)
11597 }
11598};
11599
11600struct AAPotentialValuesArgument final : AAPotentialValuesImpl {
11601 using Base = AAPotentialValuesImpl;
11602 AAPotentialValuesArgument(const IRPosition &IRP, Attributor &A)
11603 : Base(IRP, A) {}
11604
11605 /// See AbstractAttribute::initialize(..).
11606 void initialize(Attributor &A) override {
11607 auto &Arg = cast<Argument>(Val&: getAssociatedValue());
11608 if (Arg.hasPointeeInMemoryValueAttr())
11609 indicatePessimisticFixpoint();
11610 }
11611
11612 /// See AbstractAttribute::updateImpl(...).
11613 ChangeStatus updateImpl(Attributor &A) override {
11614 auto AssumedBefore = getAssumed();
11615
11616 unsigned ArgNo = getCalleeArgNo();
11617
11618 bool UsedAssumedInformation = false;
11619 SmallVector<AA::ValueAndContext> Values;
11620 auto CallSitePred = [&](AbstractCallSite ACS) {
11621 const auto CSArgIRP = IRPosition::callsite_argument(ACS, ArgNo);
11622 if (CSArgIRP.getPositionKind() == IRP_INVALID)
11623 return false;
11624
11625 if (!A.getAssumedSimplifiedValues(IRP: CSArgIRP, AA: this, Values,
11626 S: AA::Interprocedural,
11627 UsedAssumedInformation))
11628 return false;
11629
11630 return isValidState();
11631 };
11632
11633 if (!A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this,
11634 /* RequireAllCallSites */ true,
11635 UsedAssumedInformation))
11636 return indicatePessimisticFixpoint();
11637
11638 Function *Fn = getAssociatedFunction();
11639 bool AnyNonLocal = false;
11640 for (auto &It : Values) {
11641 if (isa<Constant>(Val: It.getValue())) {
11642 addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::AnyScope,
11643 AnchorScope: getAnchorScope());
11644 continue;
11645 }
11646 if (!AA::isDynamicallyUnique(A, QueryingAA: *this, V: *It.getValue()))
11647 return indicatePessimisticFixpoint();
11648
11649 if (auto *Arg = dyn_cast<Argument>(Val: It.getValue()))
11650 if (Arg->getParent() == Fn) {
11651 addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::AnyScope,
11652 AnchorScope: getAnchorScope());
11653 continue;
11654 }
11655 addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::Interprocedural,
11656 AnchorScope: getAnchorScope());
11657 AnyNonLocal = true;
11658 }
11659 assert(!undefIsContained() && "Undef should be an explicit value!");
11660 if (AnyNonLocal)
11661 giveUpOnIntraprocedural(A);
11662
11663 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11664 : ChangeStatus::CHANGED;
11665 }
11666
11667 /// See AbstractAttribute::trackStatistics()
11668 void trackStatistics() const override {
11669 STATS_DECLTRACK_ARG_ATTR(potential_values)
11670 }
11671};
11672
11673struct AAPotentialValuesReturned : public AAPotentialValuesFloating {
11674 using Base = AAPotentialValuesFloating;
11675 AAPotentialValuesReturned(const IRPosition &IRP, Attributor &A)
11676 : Base(IRP, A) {}
11677
11678 /// See AbstractAttribute::initialize(..).
11679 void initialize(Attributor &A) override {
11680 Function *F = getAssociatedFunction();
11681 if (!F || F->isDeclaration() || F->getReturnType()->isVoidTy()) {
11682 indicatePessimisticFixpoint();
11683 return;
11684 }
11685
11686 for (Argument &Arg : F->args())
11687 if (Arg.hasReturnedAttr()) {
11688 addValue(A, State&: getState(), V&: Arg, CtxI: nullptr, S: AA::AnyScope, AnchorScope: F);
11689 ReturnedArg = &Arg;
11690 break;
11691 }
11692 if (!A.isFunctionIPOAmendable(F: *F) ||
11693 A.hasSimplificationCallback(IRP: getIRPosition())) {
11694 if (!ReturnedArg)
11695 indicatePessimisticFixpoint();
11696 else
11697 indicateOptimisticFixpoint();
11698 }
11699 }
11700
11701 /// See AbstractAttribute::updateImpl(...).
11702 ChangeStatus updateImpl(Attributor &A) override {
11703 auto AssumedBefore = getAssumed();
11704 bool UsedAssumedInformation = false;
11705
11706 SmallVector<AA::ValueAndContext> Values;
11707 Function *AnchorScope = getAnchorScope();
11708 auto HandleReturnedValue = [&](Value &V, Instruction *CtxI,
11709 bool AddValues) {
11710 for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {
11711 Values.clear();
11712 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V), AA: this, Values, S,
11713 UsedAssumedInformation,
11714 /* RecurseForSelectAndPHI */ true))
11715 return false;
11716 if (!AddValues)
11717 continue;
11718
11719 bool AllInterAreIntra = false;
11720 if (S == AA::Interprocedural)
11721 AllInterAreIntra =
11722 llvm::all_of(Range&: Values, P: [&](const AA::ValueAndContext &VAC) {
11723 return AA::isValidInScope(V: *VAC.getValue(), Scope: AnchorScope);
11724 });
11725
11726 for (const AA::ValueAndContext &VAC : Values) {
11727 addValue(A, State&: getState(), V&: *VAC.getValue(),
11728 CtxI: VAC.getCtxI() ? VAC.getCtxI() : CtxI,
11729 S: AllInterAreIntra ? AA::AnyScope : S, AnchorScope);
11730 }
11731 if (AllInterAreIntra)
11732 break;
11733 }
11734 return true;
11735 };
11736
11737 if (ReturnedArg) {
11738 HandleReturnedValue(*ReturnedArg, nullptr, true);
11739 } else {
11740 auto RetInstPred = [&](Instruction &RetI) {
11741 bool AddValues = true;
11742 if (isa<PHINode>(Val: RetI.getOperand(i: 0)) ||
11743 isa<SelectInst>(Val: RetI.getOperand(i: 0))) {
11744 addValue(A, State&: getState(), V&: *RetI.getOperand(i: 0), CtxI: &RetI, S: AA::AnyScope,
11745 AnchorScope);
11746 AddValues = false;
11747 }
11748 return HandleReturnedValue(*RetI.getOperand(i: 0), &RetI, AddValues);
11749 };
11750
11751 if (!A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
11752 UsedAssumedInformation,
11753 /* CheckBBLivenessOnly */ true))
11754 return indicatePessimisticFixpoint();
11755 }
11756
11757 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11758 : ChangeStatus::CHANGED;
11759 }
11760
11761 ChangeStatus manifest(Attributor &A) override {
11762 if (ReturnedArg)
11763 return ChangeStatus::UNCHANGED;
11764 SmallVector<AA::ValueAndContext> Values;
11765 if (!getAssumedSimplifiedValues(A, Values, S: AA::ValueScope::Intraprocedural,
11766 /* RecurseForSelectAndPHI */ true))
11767 return ChangeStatus::UNCHANGED;
11768 Value *NewVal = getSingleValue(A, AA: *this, IRP: getIRPosition(), Values);
11769 if (!NewVal)
11770 return ChangeStatus::UNCHANGED;
11771
11772 ChangeStatus Changed = ChangeStatus::UNCHANGED;
11773 if (auto *Arg = dyn_cast<Argument>(Val: NewVal)) {
11774 STATS_DECLTRACK(UniqueReturnValue, FunctionReturn,
11775 "Number of function with unique return");
11776 Changed |= A.manifestAttrs(
11777 IRP: IRPosition::argument(Arg: *Arg),
11778 DeducedAttrs: {Attribute::get(Context&: Arg->getContext(), Kind: Attribute::Returned)});
11779 STATS_DECLTRACK_ARG_ATTR(returned);
11780 }
11781
11782 auto RetInstPred = [&](Instruction &RetI) {
11783 Value *RetOp = RetI.getOperand(i: 0);
11784 if (isa<UndefValue>(Val: RetOp) || RetOp == NewVal)
11785 return true;
11786 if (AA::isValidAtPosition(VAC: {*NewVal, RetI}, InfoCache&: A.getInfoCache()))
11787 if (A.changeUseAfterManifest(U&: RetI.getOperandUse(i: 0), NV&: *NewVal))
11788 Changed = ChangeStatus::CHANGED;
11789 return true;
11790 };
11791 bool UsedAssumedInformation = false;
11792 (void)A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
11793 UsedAssumedInformation,
11794 /* CheckBBLivenessOnly */ true);
11795 return Changed;
11796 }
11797
11798 ChangeStatus indicatePessimisticFixpoint() override {
11799 return AAPotentialValues::indicatePessimisticFixpoint();
11800 }
11801
11802 /// See AbstractAttribute::trackStatistics()
11803 void trackStatistics() const override{
11804 STATS_DECLTRACK_FNRET_ATTR(potential_values)}
11805
11806 /// The argumented with an existing `returned` attribute.
11807 Argument *ReturnedArg = nullptr;
11808};
11809
11810struct AAPotentialValuesFunction : AAPotentialValuesImpl {
11811 AAPotentialValuesFunction(const IRPosition &IRP, Attributor &A)
11812 : AAPotentialValuesImpl(IRP, A) {}
11813
11814 /// See AbstractAttribute::updateImpl(...).
11815 ChangeStatus updateImpl(Attributor &A) override {
11816 llvm_unreachable("AAPotentialValues(Function|CallSite)::updateImpl will "
11817 "not be called");
11818 }
11819
11820 /// See AbstractAttribute::trackStatistics()
11821 void trackStatistics() const override {
11822 STATS_DECLTRACK_FN_ATTR(potential_values)
11823 }
11824};
11825
11826struct AAPotentialValuesCallSite : AAPotentialValuesFunction {
11827 AAPotentialValuesCallSite(const IRPosition &IRP, Attributor &A)
11828 : AAPotentialValuesFunction(IRP, A) {}
11829
11830 /// See AbstractAttribute::trackStatistics()
11831 void trackStatistics() const override {
11832 STATS_DECLTRACK_CS_ATTR(potential_values)
11833 }
11834};
11835
11836struct AAPotentialValuesCallSiteReturned : AAPotentialValuesImpl {
11837 AAPotentialValuesCallSiteReturned(const IRPosition &IRP, Attributor &A)
11838 : AAPotentialValuesImpl(IRP, A) {}
11839
11840 /// See AbstractAttribute::updateImpl(...).
11841 ChangeStatus updateImpl(Attributor &A) override {
11842 auto AssumedBefore = getAssumed();
11843
11844 Function *Callee = getAssociatedFunction();
11845 if (!Callee)
11846 return indicatePessimisticFixpoint();
11847
11848 bool UsedAssumedInformation = false;
11849 auto *CB = cast<CallBase>(Val: getCtxI());
11850 if (CB->isMustTailCall() &&
11851 !A.isAssumedDead(IRP: IRPosition::inst(I: *CB), QueryingAA: this, FnLivenessAA: nullptr,
11852 UsedAssumedInformation))
11853 return indicatePessimisticFixpoint();
11854
11855 Function *Caller = CB->getCaller();
11856
11857 auto AddScope = [&](AA::ValueScope S) {
11858 SmallVector<AA::ValueAndContext> Values;
11859 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::returned(F: *Callee), AA: this,
11860 Values, S, UsedAssumedInformation))
11861 return false;
11862
11863 for (auto &It : Values) {
11864 Value *V = It.getValue();
11865 std::optional<Value *> CallerV = A.translateArgumentToCallSiteContent(
11866 V, CB&: *CB, AA: *this, UsedAssumedInformation);
11867 if (!CallerV.has_value()) {
11868 // Nothing to do as long as no value was determined.
11869 continue;
11870 }
11871 V = *CallerV ? *CallerV : V;
11872 if (*CallerV && AA::isDynamicallyUnique(A, QueryingAA: *this, V: *V)) {
11873 if (recurseForValue(A, IRP: IRPosition::value(V: *V), S))
11874 continue;
11875 }
11876 if (S == AA::Intraprocedural && !AA::isValidInScope(V: *V, Scope: Caller)) {
11877 giveUpOnIntraprocedural(A);
11878 return true;
11879 }
11880 addValue(A, State&: getState(), V&: *V, CtxI: CB, S, AnchorScope: getAnchorScope());
11881 }
11882 return true;
11883 };
11884 if (!AddScope(AA::Intraprocedural))
11885 return indicatePessimisticFixpoint();
11886 if (!AddScope(AA::Interprocedural))
11887 return indicatePessimisticFixpoint();
11888 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11889 : ChangeStatus::CHANGED;
11890 }
11891
11892 ChangeStatus indicatePessimisticFixpoint() override {
11893 return AAPotentialValues::indicatePessimisticFixpoint();
11894 }
11895
11896 /// See AbstractAttribute::trackStatistics()
11897 void trackStatistics() const override {
11898 STATS_DECLTRACK_CSRET_ATTR(potential_values)
11899 }
11900};
11901
11902struct AAPotentialValuesCallSiteArgument : AAPotentialValuesFloating {
11903 AAPotentialValuesCallSiteArgument(const IRPosition &IRP, Attributor &A)
11904 : AAPotentialValuesFloating(IRP, A) {}
11905
11906 /// See AbstractAttribute::trackStatistics()
11907 void trackStatistics() const override {
11908 STATS_DECLTRACK_CSARG_ATTR(potential_values)
11909 }
11910};
11911} // namespace
11912
11913/// ---------------------- Assumption Propagation ------------------------------
11914namespace {
11915struct AAAssumptionInfoImpl : public AAAssumptionInfo {
11916 AAAssumptionInfoImpl(const IRPosition &IRP, Attributor &A,
11917 const DenseSet<StringRef> &Known)
11918 : AAAssumptionInfo(IRP, A, Known) {}
11919
11920 /// See AbstractAttribute::manifest(...).
11921 ChangeStatus manifest(Attributor &A) override {
11922 // Don't manifest a universal set if it somehow made it here.
11923 if (getKnown().isUniversal())
11924 return ChangeStatus::UNCHANGED;
11925
11926 const IRPosition &IRP = getIRPosition();
11927 SmallVector<StringRef, 0> Set(getAssumed().getSet().begin(),
11928 getAssumed().getSet().end());
11929 llvm::sort(C&: Set);
11930 return A.manifestAttrs(IRP,
11931 DeducedAttrs: Attribute::get(Context&: IRP.getAnchorValue().getContext(),
11932 Kind: AssumptionAttrKey,
11933 Val: llvm::join(R&: Set, Separator: ",")),
11934 /*ForceReplace=*/true);
11935 }
11936
11937 bool hasAssumption(const StringRef Assumption) const override {
11938 return isValidState() && setContains(Assumption);
11939 }
11940
11941 /// See AbstractAttribute::getAsStr()
11942 const std::string getAsStr(Attributor *A) const override {
11943 const SetContents &Known = getKnown();
11944 const SetContents &Assumed = getAssumed();
11945
11946 SmallVector<StringRef, 0> Set(Known.getSet().begin(), Known.getSet().end());
11947 llvm::sort(C&: Set);
11948 const std::string KnownStr = llvm::join(R&: Set, Separator: ",");
11949
11950 std::string AssumedStr = "Universal";
11951 if (!Assumed.isUniversal()) {
11952 Set.assign(in_start: Assumed.getSet().begin(), in_end: Assumed.getSet().end());
11953 AssumedStr = llvm::join(R&: Set, Separator: ",");
11954 }
11955 return "Known [" + KnownStr + "]," + " Assumed [" + AssumedStr + "]";
11956 }
11957};
11958
11959/// Propagates assumption information from parent functions to all of their
11960/// successors. An assumption can be propagated if the containing function
11961/// dominates the called function.
11962///
11963/// We start with a "known" set of assumptions already valid for the associated
11964/// function and an "assumed" set that initially contains all possible
11965/// assumptions. The assumed set is inter-procedurally updated by narrowing its
11966/// contents as concrete values are known. The concrete values are seeded by the
11967/// first nodes that are either entries into the call graph, or contains no
11968/// assumptions. Each node is updated as the intersection of the assumed state
11969/// with all of its predecessors.
11970struct AAAssumptionInfoFunction final : AAAssumptionInfoImpl {
11971 AAAssumptionInfoFunction(const IRPosition &IRP, Attributor &A)
11972 : AAAssumptionInfoImpl(IRP, A,
11973 getAssumptions(F: *IRP.getAssociatedFunction())) {}
11974
11975 /// See AbstractAttribute::updateImpl(...).
11976 ChangeStatus updateImpl(Attributor &A) override {
11977 bool Changed = false;
11978
11979 auto CallSitePred = [&](AbstractCallSite ACS) {
11980 const auto *AssumptionAA = A.getAAFor<AAAssumptionInfo>(
11981 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *ACS.getInstruction()),
11982 DepClass: DepClassTy::REQUIRED);
11983 if (!AssumptionAA)
11984 return false;
11985 // Get the set of assumptions shared by all of this function's callers.
11986 Changed |= getIntersection(RHS: AssumptionAA->getAssumed());
11987 return !getAssumed().empty() || !getKnown().empty();
11988 };
11989
11990 bool UsedAssumedInformation = false;
11991 // Get the intersection of all assumptions held by this node's predecessors.
11992 // If we don't know all the call sites then this is either an entry into the
11993 // call graph or an empty node. This node is known to only contain its own
11994 // assumptions and can be propagated to its successors.
11995 if (!A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this, RequireAllCallSites: true,
11996 UsedAssumedInformation))
11997 return indicatePessimisticFixpoint();
11998
11999 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
12000 }
12001
12002 void trackStatistics() const override {}
12003};
12004
12005/// Assumption Info defined for call sites.
12006struct AAAssumptionInfoCallSite final : AAAssumptionInfoImpl {
12007
12008 AAAssumptionInfoCallSite(const IRPosition &IRP, Attributor &A)
12009 : AAAssumptionInfoImpl(IRP, A, getInitialAssumptions(IRP)) {}
12010
12011 /// See AbstractAttribute::initialize(...).
12012 void initialize(Attributor &A) override {
12013 const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
12014 A.getAAFor<AAAssumptionInfo>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
12015 }
12016
12017 /// See AbstractAttribute::updateImpl(...).
12018 ChangeStatus updateImpl(Attributor &A) override {
12019 const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
12020 auto *AssumptionAA =
12021 A.getAAFor<AAAssumptionInfo>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
12022 if (!AssumptionAA)
12023 return indicatePessimisticFixpoint();
12024 bool Changed = getIntersection(RHS: AssumptionAA->getAssumed());
12025 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
12026 }
12027
12028 /// See AbstractAttribute::trackStatistics()
12029 void trackStatistics() const override {}
12030
12031private:
12032 /// Helper to initialized the known set as all the assumptions this call and
12033 /// the callee contain.
12034 DenseSet<StringRef> getInitialAssumptions(const IRPosition &IRP) {
12035 const CallBase &CB = cast<CallBase>(Val&: IRP.getAssociatedValue());
12036 auto Assumptions = getAssumptions(CB);
12037 if (const Function *F = CB.getCaller())
12038 set_union(S1&: Assumptions, S2: getAssumptions(F: *F));
12039 if (Function *F = IRP.getAssociatedFunction())
12040 set_union(S1&: Assumptions, S2: getAssumptions(F: *F));
12041 return Assumptions;
12042 }
12043};
12044} // namespace
12045
12046AACallGraphNode *AACallEdgeIterator::operator*() const {
12047 return static_cast<AACallGraphNode *>(const_cast<AACallEdges *>(
12048 A.getOrCreateAAFor<AACallEdges>(IRP: IRPosition::function(F: **I))));
12049}
12050
12051void AttributorCallGraph::print() { llvm::WriteGraph(O&: outs(), G: this); }
12052
12053/// ------------------------ UnderlyingObjects ---------------------------------
12054
12055namespace {
12056struct AAUnderlyingObjectsImpl
12057 : StateWrapper<BooleanState, AAUnderlyingObjects> {
12058 using BaseTy = StateWrapper<BooleanState, AAUnderlyingObjects>;
12059 AAUnderlyingObjectsImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
12060
12061 /// See AbstractAttribute::getAsStr().
12062 const std::string getAsStr(Attributor *A) const override {
12063 if (!isValidState())
12064 return "<invalid>";
12065 std::string Str;
12066 llvm::raw_string_ostream OS(Str);
12067 OS << "underlying objects: inter " << InterAssumedUnderlyingObjects.size()
12068 << " objects, intra " << IntraAssumedUnderlyingObjects.size()
12069 << " objects.\n";
12070 if (!InterAssumedUnderlyingObjects.empty()) {
12071 OS << "inter objects:\n";
12072 for (auto *Obj : InterAssumedUnderlyingObjects)
12073 OS << *Obj << '\n';
12074 }
12075 if (!IntraAssumedUnderlyingObjects.empty()) {
12076 OS << "intra objects:\n";
12077 for (auto *Obj : IntraAssumedUnderlyingObjects)
12078 OS << *Obj << '\n';
12079 }
12080 return Str;
12081 }
12082
12083 /// See AbstractAttribute::trackStatistics()
12084 void trackStatistics() const override {}
12085
12086 /// See AbstractAttribute::updateImpl(...).
12087 ChangeStatus updateImpl(Attributor &A) override {
12088 auto &Ptr = getAssociatedValue();
12089
12090 bool UsedAssumedInformation = false;
12091 auto DoUpdate = [&](SmallSetVector<Value *, 8> &UnderlyingObjects,
12092 AA::ValueScope Scope) {
12093 SmallPtrSet<Value *, 8> SeenObjects;
12094 SmallVector<AA::ValueAndContext> Values;
12095
12096 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: Ptr), AA: *this, Values,
12097 S: Scope, UsedAssumedInformation))
12098 return UnderlyingObjects.insert(X: &Ptr);
12099
12100 bool Changed = false;
12101
12102 for (unsigned I = 0; I < Values.size(); ++I) {
12103 auto &VAC = Values[I];
12104 auto *Obj = VAC.getValue();
12105 Value *UO = getUnderlyingObject(V: Obj);
12106 if (!SeenObjects.insert(Ptr: UO ? UO : Obj).second)
12107 continue;
12108 if (UO && UO != Obj) {
12109 if (isa<AllocaInst>(Val: UO) || isa<GlobalValue>(Val: UO)) {
12110 Changed |= UnderlyingObjects.insert(X: UO);
12111 continue;
12112 }
12113
12114 const auto *OtherAA = A.getAAFor<AAUnderlyingObjects>(
12115 QueryingAA: *this, IRP: IRPosition::value(V: *UO), DepClass: DepClassTy::OPTIONAL);
12116 auto Pred = [&](Value &V) {
12117 if (&V == UO)
12118 Changed |= UnderlyingObjects.insert(X: UO);
12119 else
12120 Values.emplace_back(Args&: V, Args: nullptr);
12121 return true;
12122 };
12123
12124 if (!OtherAA || !OtherAA->forallUnderlyingObjects(Pred, Scope))
12125 llvm_unreachable(
12126 "The forall call should not return false at this position");
12127 UsedAssumedInformation |= !OtherAA->getState().isAtFixpoint();
12128 continue;
12129 }
12130
12131 if (isa<SelectInst>(Val: Obj)) {
12132 Changed |= handleIndirect(A, V&: *Obj, UnderlyingObjects, Scope,
12133 UsedAssumedInformation);
12134 continue;
12135 }
12136 if (auto *PHI = dyn_cast<PHINode>(Val: Obj)) {
12137 // Explicitly look through PHIs as we do not care about dynamically
12138 // uniqueness.
12139 for (unsigned u = 0, e = PHI->getNumIncomingValues(); u < e; u++) {
12140 Changed |=
12141 handleIndirect(A, V&: *PHI->getIncomingValue(i: u), UnderlyingObjects,
12142 Scope, UsedAssumedInformation);
12143 }
12144 continue;
12145 }
12146
12147 Changed |= UnderlyingObjects.insert(X: Obj);
12148 }
12149
12150 return Changed;
12151 };
12152
12153 bool Changed = false;
12154 Changed |= DoUpdate(IntraAssumedUnderlyingObjects, AA::Intraprocedural);
12155 Changed |= DoUpdate(InterAssumedUnderlyingObjects, AA::Interprocedural);
12156 if (!UsedAssumedInformation)
12157 indicateOptimisticFixpoint();
12158 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
12159 }
12160
12161 bool forallUnderlyingObjects(
12162 function_ref<bool(Value &)> Pred,
12163 AA::ValueScope Scope = AA::Interprocedural) const override {
12164 if (!isValidState())
12165 return Pred(getAssociatedValue());
12166
12167 auto &AssumedUnderlyingObjects = Scope == AA::Intraprocedural
12168 ? IntraAssumedUnderlyingObjects
12169 : InterAssumedUnderlyingObjects;
12170 for (Value *Obj : AssumedUnderlyingObjects)
12171 if (!Pred(*Obj))
12172 return false;
12173
12174 return true;
12175 }
12176
12177private:
12178 /// Handle the case where the value is not the actual underlying value, such
12179 /// as a phi node or a select instruction.
12180 bool handleIndirect(Attributor &A, Value &V,
12181 SmallSetVector<Value *, 8> &UnderlyingObjects,
12182 AA::ValueScope Scope, bool &UsedAssumedInformation) {
12183 bool Changed = false;
12184 const auto *AA = A.getAAFor<AAUnderlyingObjects>(
12185 QueryingAA: *this, IRP: IRPosition::value(V), DepClass: DepClassTy::OPTIONAL);
12186 auto Pred = [&](Value &V) {
12187 Changed |= UnderlyingObjects.insert(X: &V);
12188 return true;
12189 };
12190 if (!AA || !AA->forallUnderlyingObjects(Pred, Scope))
12191 llvm_unreachable(
12192 "The forall call should not return false at this position");
12193 UsedAssumedInformation |= !AA->getState().isAtFixpoint();
12194 return Changed;
12195 }
12196
12197 /// All the underlying objects collected so far via intra procedural scope.
12198 SmallSetVector<Value *, 8> IntraAssumedUnderlyingObjects;
12199 /// All the underlying objects collected so far via inter procedural scope.
12200 SmallSetVector<Value *, 8> InterAssumedUnderlyingObjects;
12201};
12202
12203struct AAUnderlyingObjectsFloating final : AAUnderlyingObjectsImpl {
12204 AAUnderlyingObjectsFloating(const IRPosition &IRP, Attributor &A)
12205 : AAUnderlyingObjectsImpl(IRP, A) {}
12206};
12207
12208struct AAUnderlyingObjectsArgument final : AAUnderlyingObjectsImpl {
12209 AAUnderlyingObjectsArgument(const IRPosition &IRP, Attributor &A)
12210 : AAUnderlyingObjectsImpl(IRP, A) {}
12211};
12212
12213struct AAUnderlyingObjectsCallSite final : AAUnderlyingObjectsImpl {
12214 AAUnderlyingObjectsCallSite(const IRPosition &IRP, Attributor &A)
12215 : AAUnderlyingObjectsImpl(IRP, A) {}
12216};
12217
12218struct AAUnderlyingObjectsCallSiteArgument final : AAUnderlyingObjectsImpl {
12219 AAUnderlyingObjectsCallSiteArgument(const IRPosition &IRP, Attributor &A)
12220 : AAUnderlyingObjectsImpl(IRP, A) {}
12221};
12222
12223struct AAUnderlyingObjectsReturned final : AAUnderlyingObjectsImpl {
12224 AAUnderlyingObjectsReturned(const IRPosition &IRP, Attributor &A)
12225 : AAUnderlyingObjectsImpl(IRP, A) {}
12226};
12227
12228struct AAUnderlyingObjectsCallSiteReturned final : AAUnderlyingObjectsImpl {
12229 AAUnderlyingObjectsCallSiteReturned(const IRPosition &IRP, Attributor &A)
12230 : AAUnderlyingObjectsImpl(IRP, A) {}
12231};
12232
12233struct AAUnderlyingObjectsFunction final : AAUnderlyingObjectsImpl {
12234 AAUnderlyingObjectsFunction(const IRPosition &IRP, Attributor &A)
12235 : AAUnderlyingObjectsImpl(IRP, A) {}
12236};
12237} // namespace
12238
12239/// ------------------------ Global Value Info -------------------------------
12240namespace {
12241struct AAGlobalValueInfoFloating : public AAGlobalValueInfo {
12242 AAGlobalValueInfoFloating(const IRPosition &IRP, Attributor &A)
12243 : AAGlobalValueInfo(IRP, A) {}
12244
12245 /// See AbstractAttribute::initialize(...).
12246 void initialize(Attributor &A) override {}
12247
12248 bool checkUse(Attributor &A, const Use &U, bool &Follow,
12249 SmallVectorImpl<const Value *> &Worklist) {
12250 Instruction *UInst = dyn_cast<Instruction>(Val: U.getUser());
12251 if (!UInst) {
12252 // Outside a closed world, code outside the module can read an
12253 // externally visible global, so the value escapes through it.
12254 if (auto *GV = dyn_cast<GlobalValue>(Val: U.getUser()))
12255 if (!GV->hasLocalLinkage() && !A.isClosedWorldModule())
12256 return false;
12257 Follow = true;
12258 return true;
12259 }
12260
12261 LLVM_DEBUG(dbgs() << "[AAGlobalValueInfo] Check use: " << *U.get() << " in "
12262 << *UInst << "\n");
12263
12264 if (auto *Cmp = dyn_cast<ICmpInst>(Val: U.getUser())) {
12265 int Idx = &Cmp->getOperandUse(i: 0) == &U;
12266 if (isa<Constant>(Val: Cmp->getOperand(i_nocapture: Idx)))
12267 return true;
12268 return U == &getAnchorValue();
12269 }
12270
12271 // Explicitly catch return instructions.
12272 if (isa<ReturnInst>(Val: UInst)) {
12273 auto CallSitePred = [&](AbstractCallSite ACS) {
12274 Worklist.push_back(Elt: ACS.getInstruction());
12275 return true;
12276 };
12277 bool UsedAssumedInformation = false;
12278 // TODO: We should traverse the uses or add a "non-call-site" CB.
12279 if (!A.checkForAllCallSites(Pred: CallSitePred, Fn: *UInst->getFunction(),
12280 /*RequireAllCallSites=*/true, QueryingAA: this,
12281 UsedAssumedInformation))
12282 return false;
12283 return true;
12284 }
12285
12286 // For now we only use special logic for call sites. However, the tracker
12287 // itself knows about a lot of other non-capturing cases already.
12288 auto *CB = dyn_cast<CallBase>(Val: UInst);
12289 if (!CB)
12290 return false;
12291 // Direct calls are OK uses.
12292 if (CB->isCallee(U: &U))
12293 return true;
12294 // Non-argument uses are scary.
12295 if (!CB->isArgOperand(U: &U))
12296 return false;
12297 // TODO: Iterate callees.
12298 auto *Fn = dyn_cast<Function>(Val: CB->getCalledOperand());
12299 if (!Fn || !A.isFunctionIPOAmendable(F: *Fn))
12300 return false;
12301
12302 unsigned ArgNo = CB->getArgOperandNo(U: &U);
12303 Worklist.push_back(Elt: Fn->getArg(i: ArgNo));
12304 return true;
12305 }
12306
12307 ChangeStatus updateImpl(Attributor &A) override {
12308 unsigned NumUsesBefore = Uses.size();
12309
12310 SmallPtrSet<const Value *, 8> Visited;
12311 SmallVector<const Value *> Worklist;
12312 Worklist.push_back(Elt: &getAnchorValue());
12313
12314 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
12315 Uses.insert(Ptr: &U);
12316 // TODO(captures): Make this more precise.
12317 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
12318 if (CI.isPassthrough()) {
12319 Follow = true;
12320 return true;
12321 }
12322 return checkUse(A, U, Follow, Worklist);
12323 };
12324 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
12325 Uses.insert(Ptr: &OldU);
12326 return true;
12327 };
12328
12329 while (!Worklist.empty()) {
12330 const Value *V = Worklist.pop_back_val();
12331 if (!Visited.insert(Ptr: V).second)
12332 continue;
12333 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: *V,
12334 /* CheckBBLivenessOnly */ true,
12335 LivenessDepClass: DepClassTy::OPTIONAL,
12336 /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
12337 return indicatePessimisticFixpoint();
12338 }
12339 }
12340
12341 return Uses.size() == NumUsesBefore ? ChangeStatus::UNCHANGED
12342 : ChangeStatus::CHANGED;
12343 }
12344
12345 bool isPotentialUse(const Use &U) const override {
12346 return !isValidState() || Uses.contains(Ptr: &U);
12347 }
12348
12349 /// See AbstractAttribute::manifest(...).
12350 ChangeStatus manifest(Attributor &A) override {
12351 return ChangeStatus::UNCHANGED;
12352 }
12353
12354 /// See AbstractAttribute::getAsStr().
12355 const std::string getAsStr(Attributor *A) const override {
12356 return "[" + std::to_string(val: Uses.size()) + " uses]";
12357 }
12358
12359 void trackStatistics() const override {
12360 STATS_DECLTRACK_FLOATING_ATTR(GlobalValuesTracked);
12361 }
12362
12363private:
12364 /// Set of (transitive) uses of this GlobalValue.
12365 SmallPtrSet<const Use *, 8> Uses;
12366};
12367} // namespace
12368
12369/// ------------------------ Indirect Call Info -------------------------------
12370namespace {
12371struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo {
12372 AAIndirectCallInfoCallSite(const IRPosition &IRP, Attributor &A)
12373 : AAIndirectCallInfo(IRP, A) {}
12374
12375 /// See AbstractAttribute::initialize(...).
12376 void initialize(Attributor &A) override {
12377 auto *MD = getCtxI()->getMetadata(KindID: LLVMContext::MD_callees);
12378 if (!MD && !A.isClosedWorldModule())
12379 return;
12380
12381 if (MD) {
12382 for (const auto &Op : MD->operands())
12383 if (Function *Callee = mdconst::dyn_extract_or_null<Function>(MD: Op))
12384 PotentialCallees.insert(X: Callee);
12385 } else if (A.isClosedWorldModule()) {
12386 ArrayRef<Function *> IndirectlyCallableFunctions =
12387 A.getInfoCache().getIndirectlyCallableFunctions(A);
12388 PotentialCallees.insert_range(R&: IndirectlyCallableFunctions);
12389 }
12390
12391 if (PotentialCallees.empty())
12392 indicateOptimisticFixpoint();
12393 }
12394
12395 ChangeStatus updateImpl(Attributor &A) override {
12396 CallBase *CB = cast<CallBase>(Val: getCtxI());
12397 const Use &CalleeUse = CB->getCalledOperandUse();
12398 Value *FP = CB->getCalledOperand();
12399
12400 SmallSetVector<Function *, 4> AssumedCalleesNow;
12401 bool AllCalleesKnownNow = AllCalleesKnown;
12402
12403 auto CheckPotentialCalleeUse = [&](Function &PotentialCallee,
12404 bool &UsedAssumedInformation) {
12405 const auto *GIAA = A.getAAFor<AAGlobalValueInfo>(
12406 QueryingAA: *this, IRP: IRPosition::value(V: PotentialCallee), DepClass: DepClassTy::OPTIONAL);
12407 if (!GIAA || GIAA->isPotentialUse(U: CalleeUse))
12408 return true;
12409 UsedAssumedInformation = !GIAA->isAtFixpoint();
12410 return false;
12411 };
12412
12413 auto AddPotentialCallees = [&]() {
12414 for (auto *PotentialCallee : PotentialCallees) {
12415 bool UsedAssumedInformation = false;
12416 if (CheckPotentialCalleeUse(*PotentialCallee, UsedAssumedInformation))
12417 AssumedCalleesNow.insert(X: PotentialCallee);
12418 }
12419 };
12420
12421 // Use simplification to find potential callees, if !callees was present,
12422 // fallback to that set if necessary.
12423 bool UsedAssumedInformation = false;
12424 SmallVector<AA::ValueAndContext> Values;
12425 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *FP), AA: this, Values,
12426 S: AA::ValueScope::AnyScope,
12427 UsedAssumedInformation)) {
12428 if (PotentialCallees.empty())
12429 return indicatePessimisticFixpoint();
12430 AddPotentialCallees();
12431 }
12432
12433 // Try to find a reason for \p Fn not to be a potential callee. If none was
12434 // found, add it to the assumed callees set.
12435 auto CheckPotentialCallee = [&](Function &Fn) {
12436 if (!PotentialCallees.empty() && !PotentialCallees.count(key: &Fn))
12437 return false;
12438
12439 auto &CachedResult = FilterResults[&Fn];
12440 if (CachedResult.has_value())
12441 return CachedResult.value();
12442
12443 bool UsedAssumedInformation = false;
12444 if (!CheckPotentialCalleeUse(Fn, UsedAssumedInformation)) {
12445 if (!UsedAssumedInformation)
12446 CachedResult = false;
12447 return false;
12448 }
12449
12450 int NumFnArgs = Fn.arg_size();
12451 int NumCBArgs = CB->arg_size();
12452
12453 // Check if any excess argument (which we fill up with poison) is known to
12454 // be UB on undef.
12455 for (int I = NumCBArgs; I < NumFnArgs; ++I) {
12456 bool IsKnown = false;
12457 if (AA::hasAssumedIRAttr<Attribute::NoUndef>(
12458 A, QueryingAA: this, IRP: IRPosition::argument(Arg: *Fn.getArg(i: I)),
12459 DepClass: DepClassTy::OPTIONAL, IsKnown)) {
12460 if (IsKnown)
12461 CachedResult = false;
12462 return false;
12463 }
12464 }
12465
12466 CachedResult = true;
12467 return true;
12468 };
12469
12470 // Check simplification result, prune known UB callees, also restrict it to
12471 // the !callees set, if present.
12472 for (auto &VAC : Values) {
12473 if (isa<UndefValue>(Val: VAC.getValue()))
12474 continue;
12475 if (isa<ConstantPointerNull>(Val: VAC.getValue()) &&
12476 VAC.getValue()->getType()->getPointerAddressSpace() == 0)
12477 continue;
12478 // TODO: Check for known UB, e.g., poison + noundef.
12479 if (auto *VACFn = dyn_cast<Function>(Val: VAC.getValue())) {
12480 if (CheckPotentialCallee(*VACFn))
12481 AssumedCalleesNow.insert(X: VACFn);
12482 continue;
12483 }
12484 if (!PotentialCallees.empty()) {
12485 AddPotentialCallees();
12486 break;
12487 }
12488 AllCalleesKnownNow = false;
12489 }
12490
12491 if (AssumedCalleesNow == AssumedCallees &&
12492 AllCalleesKnown == AllCalleesKnownNow)
12493 return ChangeStatus::UNCHANGED;
12494
12495 std::swap(LHS&: AssumedCallees, RHS&: AssumedCalleesNow);
12496 AllCalleesKnown = AllCalleesKnownNow;
12497 return ChangeStatus::CHANGED;
12498 }
12499
12500 /// See AbstractAttribute::manifest(...).
12501 ChangeStatus manifest(Attributor &A) override {
12502 // If we can't specialize at all, give up now.
12503 if (!AllCalleesKnown && AssumedCallees.empty())
12504 return ChangeStatus::UNCHANGED;
12505
12506 CallBase *CB = cast<CallBase>(Val: getCtxI());
12507 bool UsedAssumedInformation = false;
12508 if (A.isAssumedDead(I: *CB, QueryingAA: this, /*LivenessAA=*/nullptr,
12509 UsedAssumedInformation))
12510 return ChangeStatus::UNCHANGED;
12511
12512 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12513 unsigned ProgramAS = CB->getDataLayout().getProgramAddressSpace();
12514 Value *FP = CB->getCalledOperand();
12515 if (FP->getType()->getPointerAddressSpace() != ProgramAS)
12516 FP = new AddrSpaceCastInst(
12517 FP, PointerType::get(C&: FP->getContext(), AddressSpace: ProgramAS),
12518 FP->getName() + ".as" + Twine(ProgramAS), CB->getIterator());
12519
12520 bool CBIsVoid = CB->getType()->isVoidTy();
12521 BasicBlock::iterator IP = CB->getIterator();
12522 FunctionType *CSFT = CB->getFunctionType();
12523 SmallVector<Value *> CSArgs(CB->args());
12524
12525 // If we know all callees and there are none, the call site is (effectively)
12526 // dead (or UB).
12527 if (AssumedCallees.empty()) {
12528 assert(AllCalleesKnown &&
12529 "Expected all callees to be known if there are none.");
12530 A.changeToUnreachableAfterManifest(I: CB);
12531 return ChangeStatus::CHANGED;
12532 }
12533
12534 // Special handling for the single callee case.
12535 if (AllCalleesKnown && AssumedCallees.size() == 1) {
12536 auto *NewCallee = AssumedCallees.front();
12537 if (isLegalToPromote(CB: *CB, Callee: NewCallee)) {
12538 promoteCall(CB&: *CB, Callee: NewCallee, RetBitCast: nullptr);
12539 NumIndirectCallsPromoted++;
12540 return ChangeStatus::CHANGED;
12541 }
12542 Instruction *NewCall =
12543 CallInst::Create(Func: FunctionCallee(CSFT, NewCallee), Args: CSArgs,
12544 NameStr: CB->getName(), InsertBefore: CB->getIterator());
12545 if (!CBIsVoid)
12546 A.changeAfterManifest(IRP: IRPosition::callsite_returned(CB: *CB), NV&: *NewCall);
12547 A.deleteAfterManifest(I&: *CB);
12548 return ChangeStatus::CHANGED;
12549 }
12550
12551 // For each potential value we create a conditional
12552 //
12553 // ```
12554 // if (ptr == value) value(args);
12555 // else ...
12556 // ```
12557 //
12558 bool SpecializedForAnyCallees = false;
12559 bool SpecializedForAllCallees = AllCalleesKnown;
12560 ICmpInst *LastCmp = nullptr;
12561 SmallVector<Function *, 8> SkippedAssumedCallees;
12562 SmallVector<std::pair<CallInst *, Instruction *>> NewCalls;
12563 for (Function *NewCallee : AssumedCallees) {
12564 if (!A.shouldSpecializeCallSiteForCallee(AA: *this, CB&: *CB, Callee&: *NewCallee,
12565 NumAssumedCallees: AssumedCallees.size())) {
12566 SkippedAssumedCallees.push_back(Elt: NewCallee);
12567 SpecializedForAllCallees = false;
12568 continue;
12569 }
12570 SpecializedForAnyCallees = true;
12571
12572 LastCmp = new ICmpInst(IP, llvm::CmpInst::ICMP_EQ, FP, NewCallee);
12573 Instruction *ThenTI =
12574 SplitBlockAndInsertIfThen(Cond: LastCmp, SplitBefore: IP, /* Unreachable */ false);
12575 BasicBlock *CBBB = CB->getParent();
12576 A.registerManifestAddedBasicBlock(BB&: *ThenTI->getParent());
12577 A.registerManifestAddedBasicBlock(BB&: *IP->getParent());
12578 auto *SplitTI = cast<CondBrInst>(Val: LastCmp->getNextNode());
12579 BasicBlock *ElseBB;
12580 if (&*IP == CB) {
12581 ElseBB = BasicBlock::Create(Context&: ThenTI->getContext(), Name: "",
12582 Parent: ThenTI->getFunction(), InsertBefore: CBBB);
12583 A.registerManifestAddedBasicBlock(BB&: *ElseBB);
12584 IP = UncondBrInst::Create(Target: CBBB, InsertBefore: ElseBB)->getIterator();
12585 SplitTI->replaceUsesOfWith(From: CBBB, To: ElseBB);
12586 } else {
12587 ElseBB = IP->getParent();
12588 ThenTI->replaceUsesOfWith(From: ElseBB, To: CBBB);
12589 }
12590 CastInst *RetBC = nullptr;
12591 CallInst *NewCall = nullptr;
12592 if (isLegalToPromote(CB: *CB, Callee: NewCallee)) {
12593 auto *CBClone = cast<CallBase>(Val: CB->clone());
12594 CBClone->insertBefore(InsertPos: ThenTI->getIterator());
12595 NewCall = &cast<CallInst>(Val&: promoteCall(CB&: *CBClone, Callee: NewCallee, RetBitCast: &RetBC));
12596 NumIndirectCallsPromoted++;
12597 } else {
12598 NewCall = CallInst::Create(Func: FunctionCallee(CSFT, NewCallee), Args: CSArgs,
12599 NameStr: CB->getName(), InsertBefore: ThenTI->getIterator());
12600 }
12601 NewCalls.push_back(Elt: {NewCall, RetBC});
12602 }
12603
12604 auto AttachCalleeMetadata = [&](CallBase &IndirectCB) {
12605 if (!AllCalleesKnown)
12606 return ChangeStatus::UNCHANGED;
12607 MDBuilder MDB(IndirectCB.getContext());
12608 MDNode *Callees = MDB.createCallees(Callees: SkippedAssumedCallees);
12609 IndirectCB.setMetadata(KindID: LLVMContext::MD_callees, Node: Callees);
12610 return ChangeStatus::CHANGED;
12611 };
12612
12613 if (!SpecializedForAnyCallees)
12614 return AttachCalleeMetadata(*CB);
12615
12616 // Check if we need the fallback indirect call still.
12617 if (SpecializedForAllCallees) {
12618 LastCmp->replaceAllUsesWith(V: ConstantInt::getTrue(Context&: LastCmp->getContext()));
12619 LastCmp->eraseFromParent();
12620 new UnreachableInst(IP->getContext(), IP);
12621 IP->eraseFromParent();
12622 } else {
12623 auto *CBClone = cast<CallInst>(Val: CB->clone());
12624 CBClone->setName(CB->getName());
12625 CBClone->insertBefore(BB&: *IP->getParent(), InsertPos: IP);
12626 NewCalls.push_back(Elt: {CBClone, nullptr});
12627 AttachCalleeMetadata(*CBClone);
12628 }
12629
12630 // Check if we need a PHI to merge the results.
12631 if (!CBIsVoid) {
12632 auto *PHI = PHINode::Create(Ty: CB->getType(), NumReservedValues: NewCalls.size(),
12633 NameStr: CB->getName() + ".phi",
12634 InsertBefore: CB->getParent()->getFirstInsertionPt());
12635 for (auto &It : NewCalls) {
12636 CallBase *NewCall = It.first;
12637 Instruction *CallRet = It.second ? It.second : It.first;
12638 if (CallRet->getType() == CB->getType())
12639 PHI->addIncoming(V: CallRet, BB: CallRet->getParent());
12640 else if (NewCall->getType()->isVoidTy())
12641 PHI->addIncoming(V: PoisonValue::get(T: CB->getType()),
12642 BB: NewCall->getParent());
12643 else
12644 llvm_unreachable("Call return should match or be void!");
12645 }
12646 A.changeAfterManifest(IRP: IRPosition::callsite_returned(CB: *CB), NV&: *PHI);
12647 }
12648
12649 A.deleteAfterManifest(I&: *CB);
12650 Changed = ChangeStatus::CHANGED;
12651
12652 return Changed;
12653 }
12654
12655 /// See AbstractAttribute::getAsStr().
12656 const std::string getAsStr(Attributor *A) const override {
12657 return std::string(AllCalleesKnown ? "eliminate" : "specialize") +
12658 " indirect call site with " + std::to_string(val: AssumedCallees.size()) +
12659 " functions";
12660 }
12661
12662 void trackStatistics() const override {
12663 if (AllCalleesKnown) {
12664 STATS_DECLTRACK(
12665 Eliminated, CallSites,
12666 "Number of indirect call sites eliminated via specialization")
12667 } else {
12668 STATS_DECLTRACK(Specialized, CallSites,
12669 "Number of indirect call sites specialized")
12670 }
12671 }
12672
12673 bool foreachCallee(function_ref<bool(Function *)> CB) const override {
12674 return isValidState() && AllCalleesKnown && all_of(Range: AssumedCallees, P: CB);
12675 }
12676
12677private:
12678 /// Map to remember filter results.
12679 DenseMap<Function *, std::optional<bool>> FilterResults;
12680
12681 /// If the !callee metadata was present, this set will contain all potential
12682 /// callees (superset).
12683 SmallSetVector<Function *, 4> PotentialCallees;
12684
12685 /// This set contains all currently assumed calllees, which might grow over
12686 /// time.
12687 SmallSetVector<Function *, 4> AssumedCallees;
12688
12689 /// Flag to indicate if all possible callees are in the AssumedCallees set or
12690 /// if there could be others.
12691 bool AllCalleesKnown = true;
12692};
12693} // namespace
12694
12695/// --------------------- Invariant Load Pointer -------------------------------
12696namespace {
12697
12698struct AAInvariantLoadPointerImpl
12699 : public StateWrapper<BitIntegerState<uint8_t, 15>,
12700 AAInvariantLoadPointer> {
12701
12702 enum {
12703 // pointer does not alias within the bounds of the function
12704 IS_NOALIAS = 1 << 0,
12705 // pointer is not involved in any effectful instructions within the bounds
12706 // of the function
12707 IS_NOEFFECT = 1 << 1,
12708 // loads are invariant within the bounds of the function
12709 IS_LOCALLY_INVARIANT = 1 << 2,
12710 // memory lifetime is constrained within the bounds of the function
12711 IS_LOCALLY_CONSTRAINED = 1 << 3,
12712
12713 IS_BEST_STATE = IS_NOALIAS | IS_NOEFFECT | IS_LOCALLY_INVARIANT |
12714 IS_LOCALLY_CONSTRAINED,
12715 };
12716 static_assert(getBestState() == IS_BEST_STATE, "Unexpected best state");
12717
12718 using Base =
12719 StateWrapper<BitIntegerState<uint8_t, 15>, AAInvariantLoadPointer>;
12720
12721 // the BitIntegerState is optimistic about IS_NOALIAS and IS_NOEFFECT, but
12722 // pessimistic about IS_KNOWN_INVARIANT
12723 AAInvariantLoadPointerImpl(const IRPosition &IRP, Attributor &A)
12724 : Base(IRP) {}
12725
12726 bool isKnownInvariant() const final {
12727 return isKnownLocallyInvariant() && isKnown(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12728 }
12729
12730 bool isKnownLocallyInvariant() const final {
12731 if (isKnown(BitsEncoding: IS_LOCALLY_INVARIANT))
12732 return true;
12733 return isKnown(BitsEncoding: IS_NOALIAS | IS_NOEFFECT);
12734 }
12735
12736 bool isAssumedInvariant() const final {
12737 return isAssumedLocallyInvariant() && isAssumed(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12738 }
12739
12740 bool isAssumedLocallyInvariant() const final {
12741 if (isAssumed(BitsEncoding: IS_LOCALLY_INVARIANT))
12742 return true;
12743 return isAssumed(BitsEncoding: IS_NOALIAS | IS_NOEFFECT);
12744 }
12745
12746 ChangeStatus updateImpl(Attributor &A) override {
12747 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12748
12749 Changed |= updateNoAlias(A);
12750 if (requiresNoAlias() && !isAssumed(BitsEncoding: IS_NOALIAS))
12751 return indicatePessimisticFixpoint();
12752
12753 Changed |= updateNoEffect(A);
12754
12755 Changed |= updateLocalInvariance(A);
12756
12757 return Changed;
12758 }
12759
12760 ChangeStatus manifest(Attributor &A) override {
12761 if (!isKnownInvariant())
12762 return ChangeStatus::UNCHANGED;
12763
12764 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12765 const Value *Ptr = &getAssociatedValue();
12766 const auto TagInvariantLoads = [&](const Use &U, bool &) {
12767 if (U.get() != Ptr)
12768 return true;
12769 auto *I = dyn_cast<Instruction>(Val: U.getUser());
12770 if (!I)
12771 return true;
12772
12773 // Ensure that we are only changing uses from the corresponding callgraph
12774 // SSC in the case that the AA isn't run on the entire module
12775 if (!A.isRunOn(Fn: I->getFunction()))
12776 return true;
12777
12778 if (I->hasMetadata(KindID: LLVMContext::MD_invariant_load))
12779 return true;
12780
12781 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
12782 LI->setMetadata(KindID: LLVMContext::MD_invariant_load,
12783 Node: MDNode::get(Context&: LI->getContext(), MDs: {}));
12784 Changed = ChangeStatus::CHANGED;
12785 }
12786 return true;
12787 };
12788
12789 (void)A.checkForAllUses(Pred: TagInvariantLoads, QueryingAA: *this, V: *Ptr);
12790 return Changed;
12791 }
12792
12793 /// See AbstractAttribute::getAsStr().
12794 const std::string getAsStr(Attributor *) const override {
12795 if (isKnownInvariant())
12796 return "load-invariant pointer";
12797 return "non-invariant pointer";
12798 }
12799
12800 /// See AbstractAttribute::trackStatistics().
12801 void trackStatistics() const override {}
12802
12803private:
12804 /// Indicate that noalias is required for the pointer to be invariant.
12805 bool requiresNoAlias() const {
12806 switch (getPositionKind()) {
12807 default:
12808 // Conservatively default to require noalias.
12809 return true;
12810 case IRP_FLOAT:
12811 case IRP_RETURNED:
12812 case IRP_CALL_SITE:
12813 return false;
12814 case IRP_CALL_SITE_RETURNED: {
12815 const auto &CB = cast<CallBase>(Val&: getAnchorValue());
12816 return !isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
12817 Call: &CB, /*MustPreserveOffset=*/false);
12818 }
12819 case IRP_ARGUMENT: {
12820 const Function *F = getAssociatedFunction();
12821 assert(F && "no associated function for argument");
12822 return !isCallableCC(CC: F->getCallingConv());
12823 }
12824 }
12825 }
12826
12827 bool isExternal() const {
12828 const Function *F = getAssociatedFunction();
12829 if (!F)
12830 return true;
12831 return isCallableCC(CC: F->getCallingConv()) &&
12832 getPositionKind() != IRP_CALL_SITE_RETURNED;
12833 }
12834
12835 ChangeStatus updateNoAlias(Attributor &A) {
12836 if (isKnown(BitsEncoding: IS_NOALIAS) || !isAssumed(BitsEncoding: IS_NOALIAS))
12837 return ChangeStatus::UNCHANGED;
12838
12839 // Try to use AANoAlias.
12840 if (const auto *ANoAlias = A.getOrCreateAAFor<AANoAlias>(
12841 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED)) {
12842 if (ANoAlias->isKnownNoAlias()) {
12843 addKnownBits(Bits: IS_NOALIAS);
12844 return ChangeStatus::CHANGED;
12845 }
12846
12847 if (!ANoAlias->isAssumedNoAlias()) {
12848 removeAssumedBits(BitsEncoding: IS_NOALIAS);
12849 return ChangeStatus::CHANGED;
12850 }
12851
12852 return ChangeStatus::UNCHANGED;
12853 }
12854
12855 // Try to infer noalias from argument attribute, since it is applicable for
12856 // the duration of the function.
12857 if (const Argument *Arg = getAssociatedArgument()) {
12858 if (Arg->hasNoAliasAttr()) {
12859 addKnownBits(Bits: IS_NOALIAS);
12860 return ChangeStatus::UNCHANGED;
12861 }
12862
12863 // Noalias information is not provided, and cannot be inferred,
12864 // so we conservatively assume the pointer aliases.
12865 removeAssumedBits(BitsEncoding: IS_NOALIAS);
12866 return ChangeStatus::CHANGED;
12867 }
12868
12869 return ChangeStatus::UNCHANGED;
12870 }
12871
12872 ChangeStatus updateNoEffect(Attributor &A) {
12873 if (isKnown(BitsEncoding: IS_NOEFFECT) || !isAssumed(BitsEncoding: IS_NOEFFECT))
12874 return ChangeStatus::UNCHANGED;
12875
12876 if (!getAssociatedFunction())
12877 return indicatePessimisticFixpoint();
12878
12879 if (isa<AllocaInst>(Val: &getAssociatedValue()))
12880 return indicatePessimisticFixpoint();
12881
12882 const auto HasNoEffectLoads = [&](const Use &U, bool &) {
12883 const auto *LI = dyn_cast<LoadInst>(Val: U.getUser());
12884 return !LI || !LI->mayHaveSideEffects();
12885 };
12886 if (!A.checkForAllUses(Pred: HasNoEffectLoads, QueryingAA: *this, V: getAssociatedValue()))
12887 return indicatePessimisticFixpoint();
12888
12889 if (const auto *AMemoryBehavior = A.getOrCreateAAFor<AAMemoryBehavior>(
12890 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED)) {
12891 // For non-instructions, try to use AAMemoryBehavior to infer the readonly
12892 // attribute
12893 if (!AMemoryBehavior->isAssumedReadOnly())
12894 return indicatePessimisticFixpoint();
12895
12896 if (AMemoryBehavior->isKnownReadOnly()) {
12897 addKnownBits(Bits: IS_NOEFFECT);
12898 return ChangeStatus::UNCHANGED;
12899 }
12900
12901 return ChangeStatus::UNCHANGED;
12902 }
12903
12904 if (const Argument *Arg = getAssociatedArgument()) {
12905 if (Arg->onlyReadsMemory()) {
12906 addKnownBits(Bits: IS_NOEFFECT);
12907 return ChangeStatus::UNCHANGED;
12908 }
12909
12910 // Readonly information is not provided, and cannot be inferred from
12911 // AAMemoryBehavior.
12912 return indicatePessimisticFixpoint();
12913 }
12914
12915 return ChangeStatus::UNCHANGED;
12916 }
12917
12918 ChangeStatus updateLocalInvariance(Attributor &A) {
12919 if (isKnown(BitsEncoding: IS_LOCALLY_INVARIANT) || !isAssumed(BitsEncoding: IS_LOCALLY_INVARIANT))
12920 return ChangeStatus::UNCHANGED;
12921
12922 // try to infer invariance from underlying objects
12923 const auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(
12924 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED);
12925 if (!AUO)
12926 return ChangeStatus::UNCHANGED;
12927
12928 bool UsedAssumedInformation = false;
12929 const auto IsLocallyInvariantLoadIfPointer = [&](const Value &V) {
12930 if (!V.getType()->isPointerTy())
12931 return true;
12932 const auto *IsInvariantLoadPointer =
12933 A.getOrCreateAAFor<AAInvariantLoadPointer>(IRP: IRPosition::value(V), QueryingAA: this,
12934 DepClass: DepClassTy::REQUIRED);
12935 // Conservatively fail if invariance cannot be inferred.
12936 if (!IsInvariantLoadPointer)
12937 return false;
12938
12939 if (IsInvariantLoadPointer->isKnownLocallyInvariant())
12940 return true;
12941 if (!IsInvariantLoadPointer->isAssumedLocallyInvariant())
12942 return false;
12943
12944 UsedAssumedInformation = true;
12945 return true;
12946 };
12947 if (!AUO->forallUnderlyingObjects(Pred: IsLocallyInvariantLoadIfPointer))
12948 return indicatePessimisticFixpoint();
12949
12950 if (const auto *CB = dyn_cast<CallBase>(Val: &getAnchorValue())) {
12951 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
12952 Call: CB, /*MustPreserveOffset=*/false)) {
12953 for (const Value *Arg : CB->args()) {
12954 if (!IsLocallyInvariantLoadIfPointer(*Arg))
12955 return indicatePessimisticFixpoint();
12956 }
12957 }
12958 }
12959
12960 if (!UsedAssumedInformation) {
12961 // Pointer is known and not just assumed to be locally invariant.
12962 addKnownBits(Bits: IS_LOCALLY_INVARIANT);
12963 return ChangeStatus::CHANGED;
12964 }
12965
12966 return ChangeStatus::UNCHANGED;
12967 }
12968};
12969
12970struct AAInvariantLoadPointerFloating final : AAInvariantLoadPointerImpl {
12971 AAInvariantLoadPointerFloating(const IRPosition &IRP, Attributor &A)
12972 : AAInvariantLoadPointerImpl(IRP, A) {}
12973};
12974
12975struct AAInvariantLoadPointerReturned final : AAInvariantLoadPointerImpl {
12976 AAInvariantLoadPointerReturned(const IRPosition &IRP, Attributor &A)
12977 : AAInvariantLoadPointerImpl(IRP, A) {}
12978
12979 void initialize(Attributor &) override {
12980 removeAssumedBits(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12981 }
12982};
12983
12984struct AAInvariantLoadPointerCallSiteReturned final
12985 : AAInvariantLoadPointerImpl {
12986 AAInvariantLoadPointerCallSiteReturned(const IRPosition &IRP, Attributor &A)
12987 : AAInvariantLoadPointerImpl(IRP, A) {}
12988
12989 void initialize(Attributor &A) override {
12990 const Function *F = getAssociatedFunction();
12991 assert(F && "no associated function for return from call");
12992
12993 if (!F->isDeclaration() && !F->isIntrinsic())
12994 return AAInvariantLoadPointerImpl::initialize(A);
12995
12996 const auto &CB = cast<CallBase>(Val&: getAnchorValue());
12997 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
12998 Call: &CB, /*MustPreserveOffset=*/false))
12999 return AAInvariantLoadPointerImpl::initialize(A);
13000
13001 if (F->onlyReadsMemory() && F->hasNoSync())
13002 return AAInvariantLoadPointerImpl::initialize(A);
13003
13004 // At this point, the function is opaque, so we conservatively assume
13005 // non-invariance.
13006 indicatePessimisticFixpoint();
13007 }
13008};
13009
13010struct AAInvariantLoadPointerArgument final : AAInvariantLoadPointerImpl {
13011 AAInvariantLoadPointerArgument(const IRPosition &IRP, Attributor &A)
13012 : AAInvariantLoadPointerImpl(IRP, A) {}
13013
13014 void initialize(Attributor &) override {
13015 const Function *F = getAssociatedFunction();
13016 assert(F && "no associated function for argument");
13017
13018 if (!isCallableCC(CC: F->getCallingConv())) {
13019 addKnownBits(Bits: IS_LOCALLY_CONSTRAINED);
13020 return;
13021 }
13022
13023 if (!F->hasLocalLinkage())
13024 removeAssumedBits(BitsEncoding: IS_LOCALLY_CONSTRAINED);
13025 }
13026};
13027
13028struct AAInvariantLoadPointerCallSiteArgument final
13029 : AAInvariantLoadPointerImpl {
13030 AAInvariantLoadPointerCallSiteArgument(const IRPosition &IRP, Attributor &A)
13031 : AAInvariantLoadPointerImpl(IRP, A) {}
13032};
13033} // namespace
13034
13035/// ------------------------ Address Space ------------------------------------
13036namespace {
13037
13038template <typename InstType>
13039static bool makeChange(Attributor &A, InstType *MemInst, const Use &U,
13040 Value *OriginalValue, PointerType *NewPtrTy,
13041 bool UseOriginalValue) {
13042 if (U.getOperandNo() != InstType::getPointerOperandIndex())
13043 return false;
13044
13045 if (MemInst->isVolatile()) {
13046 auto *TTI = A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(
13047 *MemInst->getFunction());
13048 unsigned NewAS = NewPtrTy->getPointerAddressSpace();
13049 if (!TTI || !TTI->hasVolatileVariant(MemInst, NewAS))
13050 return false;
13051 }
13052
13053 if (UseOriginalValue) {
13054 A.changeUseAfterManifest(U&: const_cast<Use &>(U), NV&: *OriginalValue);
13055 return true;
13056 }
13057
13058 Instruction *CastInst = new AddrSpaceCastInst(OriginalValue, NewPtrTy);
13059 CastInst->insertBefore(MemInst->getIterator());
13060 A.changeUseAfterManifest(U&: const_cast<Use &>(U), NV&: *CastInst);
13061 return true;
13062}
13063
13064struct AAAddressSpaceImpl : public AAAddressSpace {
13065 AAAddressSpaceImpl(const IRPosition &IRP, Attributor &A)
13066 : AAAddressSpace(IRP, A) {}
13067
13068 uint32_t getAddressSpace() const override {
13069 assert(isValidState() && "the AA is invalid");
13070 return AssumedAddressSpace;
13071 }
13072
13073 /// See AbstractAttribute::initialize(...).
13074 void initialize(Attributor &A) override {
13075 assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
13076 "Associated value is not a pointer");
13077
13078 if (!A.getInfoCache().getFlatAddressSpace().has_value()) {
13079 indicatePessimisticFixpoint();
13080 return;
13081 }
13082
13083 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13084 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13085 if (AS != FlatAS) {
13086 [[maybe_unused]] bool R = takeAddressSpace(AS);
13087 assert(R && "The take should happen");
13088 indicateOptimisticFixpoint();
13089 }
13090 }
13091
13092 ChangeStatus updateImpl(Attributor &A) override {
13093 uint32_t OldAddressSpace = AssumedAddressSpace;
13094 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13095
13096 auto CheckAddressSpace = [&](Value &Obj) {
13097 // Ignore undef.
13098 if (isa<UndefValue>(Val: &Obj))
13099 return true;
13100
13101 // If the object already has a non-flat address space, we simply take it.
13102 unsigned ObjAS = Obj.getType()->getPointerAddressSpace();
13103 if (ObjAS != FlatAS)
13104 return takeAddressSpace(AS: ObjAS);
13105
13106 // At this point, we know Obj is in the flat address space. For a final
13107 // attempt, we want to use getAssumedAddrSpace, but first we must get the
13108 // associated function, if possible.
13109 Function *F = nullptr;
13110 if (auto *Arg = dyn_cast<Argument>(Val: &Obj))
13111 F = Arg->getParent();
13112 else if (auto *I = dyn_cast<Instruction>(Val: &Obj))
13113 F = I->getFunction();
13114
13115 // Use getAssumedAddrSpace if the associated function exists.
13116 if (F) {
13117 auto *TTI =
13118 A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(F: *F);
13119 unsigned AssumedAS = TTI->getAssumedAddrSpace(V: &Obj);
13120 if (AssumedAS != ~0U)
13121 return takeAddressSpace(AS: AssumedAS);
13122 }
13123
13124 // Now we can't do anything else but to take the flat AS.
13125 return takeAddressSpace(AS: FlatAS);
13126 };
13127
13128 auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(IRP: getIRPosition(), QueryingAA: this,
13129 DepClass: DepClassTy::REQUIRED);
13130 if (!AUO->forallUnderlyingObjects(Pred: CheckAddressSpace))
13131 return indicatePessimisticFixpoint();
13132
13133 return OldAddressSpace == AssumedAddressSpace ? ChangeStatus::UNCHANGED
13134 : ChangeStatus::CHANGED;
13135 }
13136
13137 /// See AbstractAttribute::manifest(...).
13138 ChangeStatus manifest(Attributor &A) override {
13139 unsigned NewAS = getAddressSpace();
13140
13141 if (NewAS == InvalidAddressSpace ||
13142 NewAS == getAssociatedType()->getPointerAddressSpace())
13143 return ChangeStatus::UNCHANGED;
13144
13145 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13146
13147 Value *AssociatedValue = &getAssociatedValue();
13148 Value *OriginalValue = peelAddrspacecast(V: AssociatedValue, FlatAS);
13149
13150 PointerType *NewPtrTy =
13151 PointerType::get(C&: getAssociatedType()->getContext(), AddressSpace: NewAS);
13152 bool UseOriginalValue =
13153 OriginalValue->getType()->getPointerAddressSpace() == NewAS;
13154
13155 bool Changed = false;
13156
13157 auto Pred = [&](const Use &U, bool &) {
13158 if (U.get() != AssociatedValue)
13159 return true;
13160 auto *Inst = dyn_cast<Instruction>(Val: U.getUser());
13161 if (!Inst)
13162 return true;
13163 // This is a WA to make sure we only change uses from the corresponding
13164 // CGSCC if the AA is run on CGSCC instead of the entire module.
13165 if (!A.isRunOn(Fn: Inst->getFunction()))
13166 return true;
13167 if (auto *LI = dyn_cast<LoadInst>(Val: Inst)) {
13168 Changed |=
13169 makeChange(A, MemInst: LI, U, OriginalValue, NewPtrTy, UseOriginalValue);
13170 } else if (auto *SI = dyn_cast<StoreInst>(Val: Inst)) {
13171 Changed |=
13172 makeChange(A, MemInst: SI, U, OriginalValue, NewPtrTy, UseOriginalValue);
13173 } else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: Inst)) {
13174 Changed |=
13175 makeChange(A, MemInst: RMW, U, OriginalValue, NewPtrTy, UseOriginalValue);
13176 } else if (auto *CmpX = dyn_cast<AtomicCmpXchgInst>(Val: Inst)) {
13177 Changed |=
13178 makeChange(A, MemInst: CmpX, U, OriginalValue, NewPtrTy, UseOriginalValue);
13179 }
13180 return true;
13181 };
13182
13183 // It doesn't matter if we can't check all uses as we can simply
13184 // conservatively ignore those that can not be visited.
13185 (void)A.checkForAllUses(Pred, QueryingAA: *this, V: getAssociatedValue(),
13186 /* CheckBBLivenessOnly */ true);
13187
13188 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
13189 }
13190
13191 /// See AbstractAttribute::getAsStr().
13192 const std::string getAsStr(Attributor *A) const override {
13193 if (!isValidState())
13194 return "addrspace(<invalid>)";
13195 return "addrspace(" +
13196 (AssumedAddressSpace == InvalidAddressSpace
13197 ? "none"
13198 : std::to_string(val: AssumedAddressSpace)) +
13199 ")";
13200 }
13201
13202private:
13203 uint32_t AssumedAddressSpace = InvalidAddressSpace;
13204
13205 bool takeAddressSpace(uint32_t AS) {
13206 if (AssumedAddressSpace == InvalidAddressSpace) {
13207 AssumedAddressSpace = AS;
13208 return true;
13209 }
13210 return AssumedAddressSpace == AS;
13211 }
13212
13213 static Value *peelAddrspacecast(Value *V, unsigned FlatAS) {
13214 if (auto *I = dyn_cast<AddrSpaceCastInst>(Val: V)) {
13215 assert(I->getSrcAddressSpace() != FlatAS &&
13216 "there should not be flat AS -> non-flat AS");
13217 return I->getPointerOperand();
13218 }
13219 if (auto *C = dyn_cast<ConstantExpr>(Val: V))
13220 if (C->getOpcode() == Instruction::AddrSpaceCast) {
13221 assert(C->getOperand(0)->getType()->getPointerAddressSpace() !=
13222 FlatAS &&
13223 "there should not be flat AS -> non-flat AS X");
13224 return C->getOperand(i_nocapture: 0);
13225 }
13226 return V;
13227 }
13228};
13229
13230struct AAAddressSpaceFloating final : AAAddressSpaceImpl {
13231 AAAddressSpaceFloating(const IRPosition &IRP, Attributor &A)
13232 : AAAddressSpaceImpl(IRP, A) {}
13233
13234 void trackStatistics() const override {
13235 STATS_DECLTRACK_FLOATING_ATTR(addrspace);
13236 }
13237};
13238
13239struct AAAddressSpaceReturned final : AAAddressSpaceImpl {
13240 AAAddressSpaceReturned(const IRPosition &IRP, Attributor &A)
13241 : AAAddressSpaceImpl(IRP, A) {}
13242
13243 /// See AbstractAttribute::initialize(...).
13244 void initialize(Attributor &A) override {
13245 // TODO: we don't rewrite function argument for now because it will need to
13246 // rewrite the function signature and all call sites.
13247 (void)indicatePessimisticFixpoint();
13248 }
13249
13250 void trackStatistics() const override {
13251 STATS_DECLTRACK_FNRET_ATTR(addrspace);
13252 }
13253};
13254
13255struct AAAddressSpaceCallSiteReturned final : AAAddressSpaceImpl {
13256 AAAddressSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
13257 : AAAddressSpaceImpl(IRP, A) {}
13258
13259 void trackStatistics() const override {
13260 STATS_DECLTRACK_CSRET_ATTR(addrspace);
13261 }
13262};
13263
13264struct AAAddressSpaceArgument final : AAAddressSpaceImpl {
13265 AAAddressSpaceArgument(const IRPosition &IRP, Attributor &A)
13266 : AAAddressSpaceImpl(IRP, A) {}
13267
13268 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(addrspace); }
13269};
13270
13271struct AAAddressSpaceCallSiteArgument final : AAAddressSpaceImpl {
13272 AAAddressSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
13273 : AAAddressSpaceImpl(IRP, A) {}
13274
13275 /// See AbstractAttribute::initialize(...).
13276 void initialize(Attributor &A) override {
13277 // TODO: we don't rewrite call site argument for now because it will need to
13278 // rewrite the function signature of the callee.
13279 (void)indicatePessimisticFixpoint();
13280 }
13281
13282 void trackStatistics() const override {
13283 STATS_DECLTRACK_CSARG_ATTR(addrspace);
13284 }
13285};
13286} // namespace
13287
13288/// ------------------------ No Alias Address Space ---------------------------
13289// This attribute assumes flat address space can alias all other address space
13290
13291// TODO: this is similar to AAAddressSpace, most of the code should be merged.
13292// But merging it created failing cased on gateway test that cannot be
13293// reproduced locally. So should open a separated PR to handle the merge of
13294// AANoAliasAddrSpace and AAAddressSpace attribute
13295
13296namespace {
13297struct AANoAliasAddrSpaceImpl : public AANoAliasAddrSpace {
13298 AANoAliasAddrSpaceImpl(const IRPosition &IRP, Attributor &A)
13299 : AANoAliasAddrSpace(IRP, A) {}
13300
13301 void initialize(Attributor &A) override {
13302 assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
13303 "Associated value is not a pointer");
13304
13305 resetASRanges(A);
13306
13307 std::optional<unsigned> FlatAS = A.getInfoCache().getFlatAddressSpace();
13308 if (!FlatAS.has_value()) {
13309 indicatePessimisticFixpoint();
13310 return;
13311 }
13312
13313 removeAS(AS: *FlatAS);
13314
13315 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13316 if (AS != *FlatAS) {
13317 removeAS(AS);
13318 indicateOptimisticFixpoint();
13319 }
13320 }
13321
13322 ChangeStatus updateImpl(Attributor &A) override {
13323 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13324 uint32_t OldAssumed = getAssumed();
13325
13326 auto CheckAddressSpace = [&](Value &Obj) {
13327 if (isa<PoisonValue>(Val: &Obj))
13328 return true;
13329
13330 unsigned AS = Obj.getType()->getPointerAddressSpace();
13331 if (AS == FlatAS)
13332 return false;
13333
13334 removeAS(AS: Obj.getType()->getPointerAddressSpace());
13335 return true;
13336 };
13337
13338 const AAUnderlyingObjects *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(
13339 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED);
13340 if (!AUO->forallUnderlyingObjects(Pred: CheckAddressSpace))
13341 return indicatePessimisticFixpoint();
13342
13343 return OldAssumed == getAssumed() ? ChangeStatus::UNCHANGED
13344 : ChangeStatus::CHANGED;
13345 }
13346
13347 /// See AbstractAttribute::manifest(...).
13348 ChangeStatus manifest(Attributor &A) override {
13349 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13350
13351 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13352 if (AS != FlatAS || Map.empty())
13353 return ChangeStatus::UNCHANGED;
13354
13355 LLVMContext &Ctx = getAssociatedValue().getContext();
13356 MDNode *NoAliasASNode = nullptr;
13357 MDBuilder MDB(Ctx);
13358 // Has to use iterator to get the range info.
13359 for (RangeMap::const_iterator I = Map.begin(); I != Map.end(); I++) {
13360 if (!I.value())
13361 continue;
13362 unsigned Upper = I.stop();
13363 unsigned Lower = I.start();
13364 if (!NoAliasASNode) {
13365 NoAliasASNode = MDB.createRange(Lo: APInt(32, Lower), Hi: APInt(32, Upper + 1));
13366 continue;
13367 }
13368 MDNode *ASRange = MDB.createRange(Lo: APInt(32, Lower), Hi: APInt(32, Upper + 1));
13369 NoAliasASNode = MDNode::getMostGenericRange(A: NoAliasASNode, B: ASRange);
13370 }
13371
13372 Value *AssociatedValue = &getAssociatedValue();
13373 bool Changed = false;
13374
13375 auto AddNoAliasAttr = [&](const Use &U, bool &) {
13376 if (U.get() != AssociatedValue)
13377 return true;
13378 Instruction *Inst = dyn_cast<Instruction>(Val: U.getUser());
13379 if (!Inst || Inst->hasMetadata(KindID: LLVMContext::MD_noalias_addrspace))
13380 return true;
13381 if (!isa<LoadInst>(Val: Inst) && !isa<StoreInst>(Val: Inst) &&
13382 !isa<AtomicCmpXchgInst>(Val: Inst) && !isa<AtomicRMWInst>(Val: Inst))
13383 return true;
13384 if (!A.isRunOn(Fn: Inst->getFunction()))
13385 return true;
13386 Inst->setMetadata(KindID: LLVMContext::MD_noalias_addrspace, Node: NoAliasASNode);
13387 Changed = true;
13388 return true;
13389 };
13390 (void)A.checkForAllUses(Pred: AddNoAliasAttr, QueryingAA: *this, V: *AssociatedValue,
13391 /*CheckBBLivenessOnly=*/true);
13392 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
13393 }
13394
13395 /// See AbstractAttribute::getAsStr().
13396 const std::string getAsStr(Attributor *A) const override {
13397 if (!isValidState())
13398 return "<invalid>";
13399 std::string Str;
13400 raw_string_ostream OS(Str);
13401 OS << "CanNotBeAddrSpace(";
13402 for (RangeMap::const_iterator I = Map.begin(); I != Map.end(); I++) {
13403 unsigned Upper = I.stop();
13404 unsigned Lower = I.start();
13405 OS << ' ' << '[' << Upper << ',' << Lower + 1 << ')';
13406 }
13407 OS << " )";
13408 return OS.str();
13409 }
13410
13411private:
13412 void removeAS(unsigned AS) {
13413 RangeMap::iterator I = Map.find(x: AS);
13414
13415 if (I != Map.end()) {
13416 unsigned Upper = I.stop();
13417 unsigned Lower = I.start();
13418 I.erase();
13419 if (Upper == Lower)
13420 return;
13421 if (AS != ~((unsigned)0) && AS + 1 <= Upper)
13422 Map.insert(a: AS + 1, b: Upper, /*what ever this variable name is=*/y: true);
13423 if (AS != 0 && Lower <= AS - 1)
13424 Map.insert(a: Lower, b: AS - 1, y: true);
13425 }
13426 }
13427
13428 void resetASRanges(Attributor &A) {
13429 Map.clear();
13430 Map.insert(a: 0, b: A.getInfoCache().getMaxAddrSpace(), y: true);
13431 }
13432};
13433
13434struct AANoAliasAddrSpaceFloating final : AANoAliasAddrSpaceImpl {
13435 AANoAliasAddrSpaceFloating(const IRPosition &IRP, Attributor &A)
13436 : AANoAliasAddrSpaceImpl(IRP, A) {}
13437
13438 void trackStatistics() const override {
13439 STATS_DECLTRACK_FLOATING_ATTR(noaliasaddrspace);
13440 }
13441};
13442
13443struct AANoAliasAddrSpaceReturned final : AANoAliasAddrSpaceImpl {
13444 AANoAliasAddrSpaceReturned(const IRPosition &IRP, Attributor &A)
13445 : AANoAliasAddrSpaceImpl(IRP, A) {}
13446
13447 void trackStatistics() const override {
13448 STATS_DECLTRACK_FNRET_ATTR(noaliasaddrspace);
13449 }
13450};
13451
13452struct AANoAliasAddrSpaceCallSiteReturned final : AANoAliasAddrSpaceImpl {
13453 AANoAliasAddrSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
13454 : AANoAliasAddrSpaceImpl(IRP, A) {}
13455
13456 void trackStatistics() const override {
13457 STATS_DECLTRACK_CSRET_ATTR(noaliasaddrspace);
13458 }
13459};
13460
13461struct AANoAliasAddrSpaceArgument final : AANoAliasAddrSpaceImpl {
13462 AANoAliasAddrSpaceArgument(const IRPosition &IRP, Attributor &A)
13463 : AANoAliasAddrSpaceImpl(IRP, A) {}
13464
13465 void trackStatistics() const override {
13466 STATS_DECLTRACK_ARG_ATTR(noaliasaddrspace);
13467 }
13468};
13469
13470struct AANoAliasAddrSpaceCallSiteArgument final : AANoAliasAddrSpaceImpl {
13471 AANoAliasAddrSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
13472 : AANoAliasAddrSpaceImpl(IRP, A) {}
13473
13474 void trackStatistics() const override {
13475 STATS_DECLTRACK_CSARG_ATTR(noaliasaddrspace);
13476 }
13477};
13478} // namespace
13479/// ----------- Allocation Info ----------
13480namespace {
13481struct AAAllocationInfoImpl : public AAAllocationInfo {
13482 AAAllocationInfoImpl(const IRPosition &IRP, Attributor &A)
13483 : AAAllocationInfo(IRP, A) {}
13484
13485 std::optional<TypeSize> getAllocatedSize() const override {
13486 assert(isValidState() && "the AA is invalid");
13487 return AssumedAllocatedSize;
13488 }
13489
13490 std::optional<TypeSize> findInitialAllocationSize(Instruction *I,
13491 const DataLayout &DL) {
13492
13493 // TODO: implement case for malloc like instructions
13494 switch (I->getOpcode()) {
13495 case Instruction::Alloca: {
13496 AllocaInst *AI = cast<AllocaInst>(Val: I);
13497 return AI->getAllocationSize(DL);
13498 }
13499 default:
13500 return std::nullopt;
13501 }
13502 }
13503
13504 ChangeStatus updateImpl(Attributor &A) override {
13505
13506 const IRPosition &IRP = getIRPosition();
13507 Instruction *I = IRP.getCtxI();
13508
13509 // TODO: update check for malloc like calls
13510 if (!isa<AllocaInst>(Val: I))
13511 return indicatePessimisticFixpoint();
13512
13513 bool IsKnownNoCapture;
13514 if (!AA::hasAssumedIRAttr<Attribute::Captures>(
13515 A, QueryingAA: this, IRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture))
13516 return indicatePessimisticFixpoint();
13517
13518 const AAPointerInfo *PI =
13519 A.getOrCreateAAFor<AAPointerInfo>(IRP, QueryingAA: *this, DepClass: DepClassTy::REQUIRED);
13520
13521 if (!PI)
13522 return indicatePessimisticFixpoint();
13523
13524 if (!PI->getState().isValidState() || PI->reachesReturn())
13525 return indicatePessimisticFixpoint();
13526
13527 const DataLayout &DL = A.getDataLayout();
13528 const auto AllocationSize = findInitialAllocationSize(I, DL);
13529
13530 // If allocation size is nullopt, we give up.
13531 if (!AllocationSize)
13532 return indicatePessimisticFixpoint();
13533
13534 // For zero sized allocations, we give up.
13535 // Since we can't reduce further
13536 if (*AllocationSize == 0)
13537 return indicatePessimisticFixpoint();
13538
13539 int64_t BinSize = PI->numOffsetBins();
13540
13541 // TODO: implement for multiple bins
13542 if (BinSize > 1)
13543 return indicatePessimisticFixpoint();
13544
13545 if (BinSize == 0) {
13546 auto NewAllocationSize = std::make_optional<TypeSize>(args: 0, args: false);
13547 if (!changeAllocationSize(Size: NewAllocationSize))
13548 return ChangeStatus::UNCHANGED;
13549 return ChangeStatus::CHANGED;
13550 }
13551
13552 // TODO: refactor this to be part of multiple bin case
13553 const auto &It = PI->begin();
13554
13555 // TODO: handle if Offset is not zero
13556 if (It->first.Offset != 0)
13557 return indicatePessimisticFixpoint();
13558
13559 uint64_t SizeOfBin = It->first.Offset + It->first.Size;
13560
13561 if (SizeOfBin >= *AllocationSize)
13562 return indicatePessimisticFixpoint();
13563
13564 auto NewAllocationSize = std::make_optional<TypeSize>(args: SizeOfBin * 8, args: false);
13565
13566 if (!changeAllocationSize(Size: NewAllocationSize))
13567 return ChangeStatus::UNCHANGED;
13568
13569 return ChangeStatus::CHANGED;
13570 }
13571
13572 /// See AbstractAttribute::manifest(...).
13573 ChangeStatus manifest(Attributor &A) override {
13574
13575 assert(isValidState() &&
13576 "Manifest should only be called if the state is valid.");
13577
13578 Instruction *I = getIRPosition().getCtxI();
13579
13580 auto FixedAllocatedSizeInBits = getAllocatedSize()->getFixedValue();
13581
13582 unsigned long NumBytesToAllocate = (FixedAllocatedSizeInBits + 7) / 8;
13583
13584 switch (I->getOpcode()) {
13585 // TODO: add case for malloc like calls
13586 case Instruction::Alloca: {
13587
13588 AllocaInst *AI = cast<AllocaInst>(Val: I);
13589
13590 Type *CharType = Type::getInt8Ty(C&: I->getContext());
13591
13592 auto *NumBytesToValue =
13593 ConstantInt::get(Context&: I->getContext(), V: APInt(32, NumBytesToAllocate));
13594
13595 BasicBlock::iterator insertPt = AI->getIterator();
13596 insertPt = std::next(x: insertPt);
13597 AllocaInst *NewAllocaInst =
13598 new AllocaInst(CharType, AI->getAddressSpace(), NumBytesToValue,
13599 AI->getAlign(), AI->getName(), insertPt);
13600
13601 if (A.changeAfterManifest(IRP: IRPosition::inst(I: *AI), NV&: *NewAllocaInst))
13602 return ChangeStatus::CHANGED;
13603
13604 break;
13605 }
13606 default:
13607 break;
13608 }
13609
13610 return ChangeStatus::UNCHANGED;
13611 }
13612
13613 /// See AbstractAttribute::getAsStr().
13614 const std::string getAsStr(Attributor *A) const override {
13615 if (!isValidState())
13616 return "allocationinfo(<invalid>)";
13617 return "allocationinfo(" +
13618 (AssumedAllocatedSize == HasNoAllocationSize
13619 ? "none"
13620 : std::to_string(val: AssumedAllocatedSize->getFixedValue())) +
13621 ")";
13622 }
13623
13624private:
13625 std::optional<TypeSize> AssumedAllocatedSize = HasNoAllocationSize;
13626
13627 // Maintain the computed allocation size of the object.
13628 // Returns (bool) weather the size of the allocation was modified or not.
13629 bool changeAllocationSize(std::optional<TypeSize> Size) {
13630 if (AssumedAllocatedSize == HasNoAllocationSize ||
13631 AssumedAllocatedSize != Size) {
13632 AssumedAllocatedSize = Size;
13633 return true;
13634 }
13635 return false;
13636 }
13637};
13638
13639struct AAAllocationInfoFloating : AAAllocationInfoImpl {
13640 AAAllocationInfoFloating(const IRPosition &IRP, Attributor &A)
13641 : AAAllocationInfoImpl(IRP, A) {}
13642
13643 void trackStatistics() const override {
13644 STATS_DECLTRACK_FLOATING_ATTR(allocationinfo);
13645 }
13646};
13647
13648struct AAAllocationInfoReturned : AAAllocationInfoImpl {
13649 AAAllocationInfoReturned(const IRPosition &IRP, Attributor &A)
13650 : AAAllocationInfoImpl(IRP, A) {}
13651
13652 /// See AbstractAttribute::initialize(...).
13653 void initialize(Attributor &A) override {
13654 // TODO: we don't rewrite function argument for now because it will need to
13655 // rewrite the function signature and all call sites
13656 (void)indicatePessimisticFixpoint();
13657 }
13658
13659 void trackStatistics() const override {
13660 STATS_DECLTRACK_FNRET_ATTR(allocationinfo);
13661 }
13662};
13663
13664struct AAAllocationInfoCallSiteReturned : AAAllocationInfoImpl {
13665 AAAllocationInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
13666 : AAAllocationInfoImpl(IRP, A) {}
13667
13668 void trackStatistics() const override {
13669 STATS_DECLTRACK_CSRET_ATTR(allocationinfo);
13670 }
13671};
13672
13673struct AAAllocationInfoArgument : AAAllocationInfoImpl {
13674 AAAllocationInfoArgument(const IRPosition &IRP, Attributor &A)
13675 : AAAllocationInfoImpl(IRP, A) {}
13676
13677 void trackStatistics() const override {
13678 STATS_DECLTRACK_ARG_ATTR(allocationinfo);
13679 }
13680};
13681
13682struct AAAllocationInfoCallSiteArgument : AAAllocationInfoImpl {
13683 AAAllocationInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
13684 : AAAllocationInfoImpl(IRP, A) {}
13685
13686 /// See AbstractAttribute::initialize(...).
13687 void initialize(Attributor &A) override {
13688
13689 (void)indicatePessimisticFixpoint();
13690 }
13691
13692 void trackStatistics() const override {
13693 STATS_DECLTRACK_CSARG_ATTR(allocationinfo);
13694 }
13695};
13696} // namespace
13697
13698const char AANoUnwind::ID = 0;
13699const char AANoSync::ID = 0;
13700const char AANoFree::ID = 0;
13701const char AANonNull::ID = 0;
13702const char AAMustProgress::ID = 0;
13703const char AANoRecurse::ID = 0;
13704const char AANonConvergent::ID = 0;
13705const char AAWillReturn::ID = 0;
13706const char AAUndefinedBehavior::ID = 0;
13707const char AANoAlias::ID = 0;
13708const char AAIntraFnReachability::ID = 0;
13709const char AANoReturn::ID = 0;
13710const char AAIsDead::ID = 0;
13711const char AADereferenceable::ID = 0;
13712const char AAAlign::ID = 0;
13713const char AAInstanceInfo::ID = 0;
13714const char AANoCapture::ID = 0;
13715const char AAValueSimplify::ID = 0;
13716const char AAHeapToStack::ID = 0;
13717const char AAPrivatizablePtr::ID = 0;
13718const char AAMemoryBehavior::ID = 0;
13719const char AAMemoryLocation::ID = 0;
13720const char AAValueConstantRange::ID = 0;
13721const char AAPotentialConstantValues::ID = 0;
13722const char AAPotentialValues::ID = 0;
13723const char AANoUndef::ID = 0;
13724const char AANoFPClass::ID = 0;
13725const char AACallEdges::ID = 0;
13726const char AAInterFnReachability::ID = 0;
13727const char AAPointerInfo::ID = 0;
13728const char AAAssumptionInfo::ID = 0;
13729const char AAUnderlyingObjects::ID = 0;
13730const char AAInvariantLoadPointer::ID = 0;
13731const char AAAddressSpace::ID = 0;
13732const char AANoAliasAddrSpace::ID = 0;
13733const char AAAllocationInfo::ID = 0;
13734const char AAIndirectCallInfo::ID = 0;
13735const char AAGlobalValueInfo::ID = 0;
13736const char AADenormalFPMath::ID = 0;
13737
13738// Macro magic to create the static generator function for attributes that
13739// follow the naming scheme.
13740
13741#define SWITCH_PK_INV(CLASS, PK, POS_NAME) \
13742 case IRPosition::PK: \
13743 llvm_unreachable("Cannot create " #CLASS " for a " POS_NAME " position!");
13744
13745#define SWITCH_PK_CREATE(CLASS, IRP, PK, SUFFIX) \
13746 case IRPosition::PK: \
13747 AA = new (A.Allocator) CLASS##SUFFIX(IRP, A); \
13748 ++NumAAs; \
13749 break;
13750
13751#define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13752 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13753 CLASS *AA = nullptr; \
13754 switch (IRP.getPositionKind()) { \
13755 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13756 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
13757 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
13758 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13759 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
13760 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
13761 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13762 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13763 } \
13764 return *AA; \
13765 }
13766
13767#define CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13768 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13769 CLASS *AA = nullptr; \
13770 switch (IRP.getPositionKind()) { \
13771 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13772 SWITCH_PK_INV(CLASS, IRP_FUNCTION, "function") \
13773 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
13774 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13775 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13776 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
13777 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13778 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13779 } \
13780 return *AA; \
13781 }
13782
13783#define CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(POS, SUFFIX, CLASS) \
13784 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13785 CLASS *AA = nullptr; \
13786 switch (IRP.getPositionKind()) { \
13787 SWITCH_PK_CREATE(CLASS, IRP, POS, SUFFIX) \
13788 default: \
13789 llvm_unreachable("Cannot create " #CLASS " for position otherthan " #POS \
13790 " position!"); \
13791 } \
13792 return *AA; \
13793 }
13794
13795#define CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13796 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13797 CLASS *AA = nullptr; \
13798 switch (IRP.getPositionKind()) { \
13799 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13800 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13801 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13802 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13803 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13804 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
13805 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13806 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13807 } \
13808 return *AA; \
13809 }
13810
13811#define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13812 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13813 CLASS *AA = nullptr; \
13814 switch (IRP.getPositionKind()) { \
13815 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13816 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
13817 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
13818 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13819 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
13820 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
13821 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
13822 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13823 } \
13824 return *AA; \
13825 }
13826
13827#define CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13828 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13829 CLASS *AA = nullptr; \
13830 switch (IRP.getPositionKind()) { \
13831 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13832 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13833 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13834 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13835 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13836 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13837 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13838 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13839 } \
13840 return *AA; \
13841 }
13842
13843CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUnwind)
13844CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoSync)
13845CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoRecurse)
13846CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAWillReturn)
13847CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoReturn)
13848CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryLocation)
13849CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AACallEdges)
13850CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAssumptionInfo)
13851CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMustProgress)
13852
13853CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonNull)
13854CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAlias)
13855CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPrivatizablePtr)
13856CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADereferenceable)
13857CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAlign)
13858CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInstanceInfo)
13859CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoCapture)
13860CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueConstantRange)
13861CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialConstantValues)
13862CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialValues)
13863CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUndef)
13864CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFPClass)
13865CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPointerInfo)
13866CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInvariantLoadPointer)
13867CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAddressSpace)
13868CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAliasAddrSpace)
13869CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAllocationInfo)
13870
13871CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueSimplify)
13872CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIsDead)
13873CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFree)
13874CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUnderlyingObjects)
13875
13876CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_CALL_SITE, CallSite,
13877 AAIndirectCallInfo)
13878CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_FLOAT, Floating,
13879 AAGlobalValueInfo)
13880
13881CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAHeapToStack)
13882CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUndefinedBehavior)
13883CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonConvergent)
13884CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIntraFnReachability)
13885CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInterFnReachability)
13886CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADenormalFPMath)
13887
13888CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryBehavior)
13889
13890#undef CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION
13891#undef CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION
13892#undef CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION
13893#undef CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION
13894#undef CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION
13895#undef CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION
13896#undef SWITCH_PK_CREATE
13897#undef SWITCH_PK_INV
13898