1//===- AttributorAttributes.cpp - Attributes for Attributor deduction -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// See the Attributor.h file comment and the class descriptions in that file for
10// more information.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/IPO/Attributor.h"
15
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/DenseMapInfo.h"
19#include "llvm/ADT/MapVector.h"
20#include "llvm/ADT/SCCIterator.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/ADT/SetOperations.h"
23#include "llvm/ADT/SetVector.h"
24#include "llvm/ADT/SmallPtrSet.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/ADT/StringExtras.h"
28#include "llvm/Analysis/AliasAnalysis.h"
29#include "llvm/Analysis/AssumeBundleQueries.h"
30#include "llvm/Analysis/AssumptionCache.h"
31#include "llvm/Analysis/CaptureTracking.h"
32#include "llvm/Analysis/CycleAnalysis.h"
33#include "llvm/Analysis/InstructionSimplify.h"
34#include "llvm/Analysis/LazyValueInfo.h"
35#include "llvm/Analysis/MemoryBuiltins.h"
36#include "llvm/Analysis/ScalarEvolution.h"
37#include "llvm/Analysis/TargetTransformInfo.h"
38#include "llvm/Analysis/ValueTracking.h"
39#include "llvm/IR/Argument.h"
40#include "llvm/IR/Assumptions.h"
41#include "llvm/IR/Attributes.h"
42#include "llvm/IR/BasicBlock.h"
43#include "llvm/IR/Constant.h"
44#include "llvm/IR/Constants.h"
45#include "llvm/IR/DataLayout.h"
46#include "llvm/IR/DerivedTypes.h"
47#include "llvm/IR/GlobalValue.h"
48#include "llvm/IR/IRBuilder.h"
49#include "llvm/IR/InlineAsm.h"
50#include "llvm/IR/InstrTypes.h"
51#include "llvm/IR/Instruction.h"
52#include "llvm/IR/Instructions.h"
53#include "llvm/IR/IntrinsicInst.h"
54#include "llvm/IR/IntrinsicsAMDGPU.h"
55#include "llvm/IR/IntrinsicsNVPTX.h"
56#include "llvm/IR/LLVMContext.h"
57#include "llvm/IR/MDBuilder.h"
58#include "llvm/IR/NoFolder.h"
59#include "llvm/IR/Value.h"
60#include "llvm/IR/ValueHandle.h"
61#include "llvm/Support/Alignment.h"
62#include "llvm/Support/Casting.h"
63#include "llvm/Support/CommandLine.h"
64#include "llvm/Support/ErrorHandling.h"
65#include "llvm/Support/GraphWriter.h"
66#include "llvm/Support/InterleavedRange.h"
67#include "llvm/Support/KnownFPClass.h"
68#include "llvm/Support/MathExtras.h"
69#include "llvm/Support/TypeSize.h"
70#include "llvm/Support/raw_ostream.h"
71#include "llvm/Transforms/Utils/BasicBlockUtils.h"
72#include "llvm/Transforms/Utils/CallPromotionUtils.h"
73#include "llvm/Transforms/Utils/Local.h"
74#include "llvm/Transforms/Utils/ValueMapper.h"
75#include <cassert>
76#include <numeric>
77#include <optional>
78#include <string>
79
80using namespace llvm;
81
82#define DEBUG_TYPE "attributor"
83
84static cl::opt<bool> ManifestInternal(
85 "attributor-manifest-internal", cl::Hidden,
86 cl::desc("Manifest Attributor internal string attributes."),
87 cl::init(Val: false));
88
89static cl::opt<int> MaxHeapToStackSize("max-heap-to-stack-size", cl::init(Val: 128),
90 cl::Hidden);
91
92template <>
93unsigned llvm::PotentialConstantIntValuesState::MaxPotentialValues = 0;
94
95template <> unsigned llvm::PotentialLLVMValuesState::MaxPotentialValues = -1;
96
97static cl::opt<unsigned, true> MaxPotentialValues(
98 "attributor-max-potential-values", cl::Hidden,
99 cl::desc("Maximum number of potential values to be "
100 "tracked for each position."),
101 cl::location(L&: llvm::PotentialConstantIntValuesState::MaxPotentialValues),
102 cl::init(Val: 7));
103
104static cl::opt<int> MaxPotentialValuesIterations(
105 "attributor-max-potential-values-iterations", cl::Hidden,
106 cl::desc(
107 "Maximum number of iterations we keep dismantling potential values."),
108 cl::init(Val: 64));
109
110STATISTIC(NumAAs, "Number of abstract attributes created");
111STATISTIC(NumIndirectCallsPromoted, "Number of indirect calls promoted");
112
113// Some helper macros to deal with statistics tracking.
114//
115// Usage:
116// For simple IR attribute tracking overload trackStatistics in the abstract
117// attribute and choose the right STATS_DECLTRACK_********* macro,
118// e.g.,:
119// void trackStatistics() const override {
120// STATS_DECLTRACK_ARG_ATTR(returned)
121// }
122// If there is a single "increment" side one can use the macro
123// STATS_DECLTRACK with a custom message. If there are multiple increment
124// sides, STATS_DECL and STATS_TRACK can also be used separately.
125//
126#define BUILD_STAT_MSG_IR_ATTR(TYPE, NAME) \
127 ("Number of " #TYPE " marked '" #NAME "'")
128#define BUILD_STAT_NAME(NAME, TYPE) NumIR##TYPE##_##NAME
129#define STATS_DECL_(NAME, MSG) STATISTIC(NAME, MSG);
130#define STATS_DECL(NAME, TYPE, MSG) \
131 STATS_DECL_(BUILD_STAT_NAME(NAME, TYPE), MSG);
132#define STATS_TRACK(NAME, TYPE) ++(BUILD_STAT_NAME(NAME, TYPE));
133#define STATS_DECLTRACK(NAME, TYPE, MSG) \
134 {STATS_DECL(NAME, TYPE, MSG) STATS_TRACK(NAME, TYPE)}
135#define STATS_DECLTRACK_ARG_ATTR(NAME) \
136 STATS_DECLTRACK(NAME, Arguments, BUILD_STAT_MSG_IR_ATTR(arguments, NAME))
137#define STATS_DECLTRACK_CSARG_ATTR(NAME) \
138 STATS_DECLTRACK(NAME, CSArguments, \
139 BUILD_STAT_MSG_IR_ATTR(call site arguments, NAME))
140#define STATS_DECLTRACK_FN_ATTR(NAME) \
141 STATS_DECLTRACK(NAME, Function, BUILD_STAT_MSG_IR_ATTR(functions, NAME))
142#define STATS_DECLTRACK_CS_ATTR(NAME) \
143 STATS_DECLTRACK(NAME, CS, BUILD_STAT_MSG_IR_ATTR(call site, NAME))
144#define STATS_DECLTRACK_FNRET_ATTR(NAME) \
145 STATS_DECLTRACK(NAME, FunctionReturn, \
146 BUILD_STAT_MSG_IR_ATTR(function returns, NAME))
147#define STATS_DECLTRACK_CSRET_ATTR(NAME) \
148 STATS_DECLTRACK(NAME, CSReturn, \
149 BUILD_STAT_MSG_IR_ATTR(call site returns, NAME))
150#define STATS_DECLTRACK_FLOATING_ATTR(NAME) \
151 STATS_DECLTRACK(NAME, Floating, \
152 ("Number of floating values known to be '" #NAME "'"))
153
154// Specialization of the operator<< for abstract attributes subclasses. This
155// disambiguates situations where multiple operators are applicable.
156namespace llvm {
157#define PIPE_OPERATOR(CLASS) \
158 raw_ostream &operator<<(raw_ostream &OS, const CLASS &AA) { \
159 return OS << static_cast<const AbstractAttribute &>(AA); \
160 }
161
162PIPE_OPERATOR(AAIsDead)
163PIPE_OPERATOR(AANoUnwind)
164PIPE_OPERATOR(AANoSync)
165PIPE_OPERATOR(AANoRecurse)
166PIPE_OPERATOR(AANonConvergent)
167PIPE_OPERATOR(AAWillReturn)
168PIPE_OPERATOR(AANoReturn)
169PIPE_OPERATOR(AANonNull)
170PIPE_OPERATOR(AAMustProgress)
171PIPE_OPERATOR(AANoAlias)
172PIPE_OPERATOR(AADereferenceable)
173PIPE_OPERATOR(AAAlign)
174PIPE_OPERATOR(AAInstanceInfo)
175PIPE_OPERATOR(AANoCapture)
176PIPE_OPERATOR(AAValueSimplify)
177PIPE_OPERATOR(AANoFree)
178PIPE_OPERATOR(AAHeapToStack)
179PIPE_OPERATOR(AAIntraFnReachability)
180PIPE_OPERATOR(AAMemoryBehavior)
181PIPE_OPERATOR(AAMemoryLocation)
182PIPE_OPERATOR(AAValueConstantRange)
183PIPE_OPERATOR(AAPrivatizablePtr)
184PIPE_OPERATOR(AAUndefinedBehavior)
185PIPE_OPERATOR(AAPotentialConstantValues)
186PIPE_OPERATOR(AAPotentialValues)
187PIPE_OPERATOR(AANoUndef)
188PIPE_OPERATOR(AANoFPClass)
189PIPE_OPERATOR(AACallEdges)
190PIPE_OPERATOR(AAInterFnReachability)
191PIPE_OPERATOR(AAPointerInfo)
192PIPE_OPERATOR(AAAssumptionInfo)
193PIPE_OPERATOR(AAUnderlyingObjects)
194PIPE_OPERATOR(AAInvariantLoadPointer)
195PIPE_OPERATOR(AAAddressSpace)
196PIPE_OPERATOR(AANoAliasAddrSpace)
197PIPE_OPERATOR(AAAllocationInfo)
198PIPE_OPERATOR(AAIndirectCallInfo)
199PIPE_OPERATOR(AAGlobalValueInfo)
200PIPE_OPERATOR(AADenormalFPMath)
201
202#undef PIPE_OPERATOR
203
204template <>
205ChangeStatus clampStateAndIndicateChange<DerefState>(DerefState &S,
206 const DerefState &R) {
207 ChangeStatus CS0 =
208 clampStateAndIndicateChange(S&: S.DerefBytesState, R: R.DerefBytesState);
209 ChangeStatus CS1 = clampStateAndIndicateChange(S&: S.GlobalState, R: R.GlobalState);
210 return CS0 | CS1;
211}
212
213} // namespace llvm
214
215static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I,
216 bool HeaderOnly, Cycle **CPtr = nullptr) {
217 if (!CI)
218 return true;
219 auto *BB = I->getParent();
220 auto *C = CI->getCycle(Block: BB);
221 if (!C)
222 return false;
223 if (CPtr)
224 *CPtr = C;
225 return !HeaderOnly || BB == C->getHeader();
226}
227
228/// Checks if a type could have padding bytes.
229static bool isDenselyPacked(Type *Ty, const DataLayout &DL) {
230 // There is no size information, so be conservative.
231 if (!Ty->isSized())
232 return false;
233
234 // If the alloc size is not equal to the storage size, then there are padding
235 // bytes. For x86_fp80 on x86-64, size: 80 alloc size: 128.
236 if (DL.getTypeSizeInBits(Ty) != DL.getTypeAllocSizeInBits(Ty))
237 return false;
238
239 // FIXME: This isn't the right way to check for padding in vectors with
240 // non-byte-size elements.
241 if (VectorType *SeqTy = dyn_cast<VectorType>(Val: Ty))
242 return isDenselyPacked(Ty: SeqTy->getElementType(), DL);
243
244 // For array types, check for padding within members.
245 if (ArrayType *SeqTy = dyn_cast<ArrayType>(Val: Ty))
246 return isDenselyPacked(Ty: SeqTy->getElementType(), DL);
247
248 if (!isa<StructType>(Val: Ty))
249 return true;
250
251 // Check for padding within and between elements of a struct.
252 StructType *StructTy = cast<StructType>(Val: Ty);
253 const StructLayout *Layout = DL.getStructLayout(Ty: StructTy);
254 uint64_t StartPos = 0;
255 for (unsigned I = 0, E = StructTy->getNumElements(); I < E; ++I) {
256 Type *ElTy = StructTy->getElementType(N: I);
257 if (!isDenselyPacked(Ty: ElTy, DL))
258 return false;
259 if (StartPos != Layout->getElementOffsetInBits(Idx: I))
260 return false;
261 StartPos += DL.getTypeAllocSizeInBits(Ty: ElTy);
262 }
263
264 return true;
265}
266
267/// Get pointer operand of memory accessing instruction. If \p I is
268/// not a memory accessing instruction, return nullptr. If \p AllowVolatile,
269/// is set to false and the instruction is volatile, return nullptr.
270static const Value *getPointerOperand(const Instruction *I,
271 bool AllowVolatile) {
272 if (!AllowVolatile && I->isVolatile())
273 return nullptr;
274
275 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
276 return LI->getPointerOperand();
277 }
278
279 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
280 return SI->getPointerOperand();
281 }
282
283 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
284 return CXI->getPointerOperand();
285 }
286
287 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: I)) {
288 return RMWI->getPointerOperand();
289 }
290
291 return nullptr;
292}
293
294/// Helper function to create a pointer based on \p Ptr, and advanced by \p
295/// Offset bytes.
296static Value *constructPointer(Value *Ptr, int64_t Offset,
297 IRBuilder<NoFolder> &IRB) {
298 LLVM_DEBUG(dbgs() << "Construct pointer: " << *Ptr << " + " << Offset
299 << "-bytes\n");
300
301 if (Offset)
302 Ptr = IRB.CreatePtrAdd(Ptr, Offset: IRB.getInt64(C: Offset),
303 Name: Ptr->getName() + ".b" + Twine(Offset));
304 return Ptr;
305}
306
307static const Value *
308stripAndAccumulateOffsets(Attributor &A, const AbstractAttribute &QueryingAA,
309 const Value *Val, const DataLayout &DL, APInt &Offset,
310 bool GetMinOffset, bool AllowNonInbounds,
311 bool UseAssumed = false) {
312
313 auto AttributorAnalysis = [&](Value &V, APInt &ROffset) -> bool {
314 const IRPosition &Pos = IRPosition::value(V);
315 // Only track dependence if we are going to use the assumed info.
316 const AAValueConstantRange *ValueConstantRangeAA =
317 A.getAAFor<AAValueConstantRange>(QueryingAA, IRP: Pos,
318 DepClass: UseAssumed ? DepClassTy::OPTIONAL
319 : DepClassTy::NONE);
320 if (!ValueConstantRangeAA)
321 return false;
322 ConstantRange Range = UseAssumed ? ValueConstantRangeAA->getAssumed()
323 : ValueConstantRangeAA->getKnown();
324 if (Range.isFullSet())
325 return false;
326
327 // We can only use the lower part of the range because the upper part can
328 // be higher than what the value can really be.
329 if (GetMinOffset)
330 ROffset = Range.getSignedMin();
331 else
332 ROffset = Range.getSignedMax();
333 return true;
334 };
335
336 return Val->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds,
337 /* AllowInvariant */ AllowInvariantGroup: true,
338 ExternalAnalysis: AttributorAnalysis);
339}
340
341static const Value *
342getMinimalBaseOfPointer(Attributor &A, const AbstractAttribute &QueryingAA,
343 const Value *Ptr, int64_t &BytesOffset,
344 const DataLayout &DL, bool AllowNonInbounds = false) {
345 APInt OffsetAPInt(DL.getIndexTypeSizeInBits(Ty: Ptr->getType()), 0);
346 const Value *Base =
347 stripAndAccumulateOffsets(A, QueryingAA, Val: Ptr, DL, Offset&: OffsetAPInt,
348 /* GetMinOffset */ true, AllowNonInbounds);
349
350 BytesOffset = OffsetAPInt.getSExtValue();
351 return Base;
352}
353
354/// Clamp the information known for all returned values of a function
355/// (identified by \p QueryingAA) into \p S.
356template <typename AAType, typename StateType = typename AAType::StateType,
357 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
358 bool RecurseForSelectAndPHI = true>
359static void clampReturnedValueStates(
360 Attributor &A, const AAType &QueryingAA, StateType &S,
361 const IRPosition::CallBaseContext *CBContext = nullptr) {
362 LLVM_DEBUG(dbgs() << "[Attributor] Clamp return value states for "
363 << QueryingAA << " into " << S << "\n");
364
365 assert((QueryingAA.getIRPosition().getPositionKind() ==
366 IRPosition::IRP_RETURNED ||
367 QueryingAA.getIRPosition().getPositionKind() ==
368 IRPosition::IRP_CALL_SITE_RETURNED) &&
369 "Can only clamp returned value states for a function returned or call "
370 "site returned position!");
371
372 // Use an optional state as there might not be any return values and we want
373 // to join (IntegerState::operator&) the state of all there are.
374 std::optional<StateType> T;
375
376 // Callback for each possibly returned value.
377 auto CheckReturnValue = [&](Value &RV) -> bool {
378 const IRPosition &RVPos = IRPosition::value(V: RV, CBContext);
379 // If possible, use the hasAssumedIRAttr interface.
380 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
381 bool IsKnown;
382 return AA::hasAssumedIRAttr<IRAttributeKind>(
383 A, &QueryingAA, RVPos, DepClassTy::REQUIRED, IsKnown);
384 }
385
386 const AAType *AA =
387 A.getAAFor<AAType>(QueryingAA, RVPos, DepClassTy::REQUIRED);
388 if (!AA)
389 return false;
390 LLVM_DEBUG(dbgs() << "[Attributor] RV: " << RV
391 << " AA: " << AA->getAsStr(&A) << " @ " << RVPos << "\n");
392 const StateType &AAS = AA->getState();
393 if (!T)
394 T = StateType::getBestState(AAS);
395 *T &= AAS;
396 LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " RV State: " << T
397 << "\n");
398 return T->isValidState();
399 };
400
401 if (!A.checkForAllReturnedValues(Pred: CheckReturnValue, QueryingAA,
402 S: AA::ValueScope::Intraprocedural,
403 RecurseForSelectAndPHI))
404 S.indicatePessimisticFixpoint();
405 else if (T)
406 S ^= *T;
407}
408
409namespace {
410/// Helper class for generic deduction: return value -> returned position.
411template <typename AAType, typename BaseType,
412 typename StateType = typename BaseType::StateType,
413 bool PropagateCallBaseContext = false,
414 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
415 bool RecurseForSelectAndPHI = true>
416struct AAReturnedFromReturnedValues : public BaseType {
417 AAReturnedFromReturnedValues(const IRPosition &IRP, Attributor &A)
418 : BaseType(IRP, A) {}
419
420 /// See AbstractAttribute::updateImpl(...).
421 ChangeStatus updateImpl(Attributor &A) override {
422 StateType S(StateType::getBestState(this->getState()));
423 clampReturnedValueStates<AAType, StateType, IRAttributeKind,
424 RecurseForSelectAndPHI>(
425 A, *this, S,
426 PropagateCallBaseContext ? this->getCallBaseContext() : nullptr);
427 // TODO: If we know we visited all returned values, thus no are assumed
428 // dead, we can take the known information from the state T.
429 return clampStateAndIndicateChange<StateType>(this->getState(), S);
430 }
431};
432
433/// Clamp the information known at all call sites for a given argument
434/// (identified by \p QueryingAA) into \p S.
435template <typename AAType, typename StateType = typename AAType::StateType,
436 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
437static void clampCallSiteArgumentStates(Attributor &A, const AAType &QueryingAA,
438 StateType &S) {
439 LLVM_DEBUG(dbgs() << "[Attributor] Clamp call site argument states for "
440 << QueryingAA << " into " << S << "\n");
441
442 assert(QueryingAA.getIRPosition().getPositionKind() ==
443 IRPosition::IRP_ARGUMENT &&
444 "Can only clamp call site argument states for an argument position!");
445
446 // Use an optional state as there might not be any return values and we want
447 // to join (IntegerState::operator&) the state of all there are.
448 std::optional<StateType> T;
449
450 // The argument number which is also the call site argument number.
451 unsigned ArgNo = QueryingAA.getIRPosition().getCallSiteArgNo();
452
453 auto CallSiteCheck = [&](AbstractCallSite ACS) {
454 const IRPosition &ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
455 // Check if a coresponding argument was found or if it is on not associated
456 // (which can happen for callback calls).
457 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
458 return false;
459
460 // If possible, use the hasAssumedIRAttr interface.
461 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
462 bool IsKnown;
463 return AA::hasAssumedIRAttr<IRAttributeKind>(
464 A, &QueryingAA, ACSArgPos, DepClassTy::REQUIRED, IsKnown);
465 }
466
467 const AAType *AA =
468 A.getAAFor<AAType>(QueryingAA, ACSArgPos, DepClassTy::REQUIRED);
469 if (!AA)
470 return false;
471 LLVM_DEBUG(dbgs() << "[Attributor] ACS: " << *ACS.getInstruction()
472 << " AA: " << AA->getAsStr(&A) << " @" << ACSArgPos
473 << "\n");
474 const StateType &AAS = AA->getState();
475 if (!T)
476 T = StateType::getBestState(AAS);
477 *T &= AAS;
478 LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " CSA State: " << T
479 << "\n");
480 return T->isValidState();
481 };
482
483 bool UsedAssumedInformation = false;
484 if (!A.checkForAllCallSites(CallSiteCheck, QueryingAA, true,
485 UsedAssumedInformation))
486 S.indicatePessimisticFixpoint();
487 else if (T)
488 S ^= *T;
489}
490
491/// This function is the bridge between argument position and the call base
492/// context.
493template <typename AAType, typename BaseType,
494 typename StateType = typename AAType::StateType,
495 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
496bool getArgumentStateFromCallBaseContext(Attributor &A,
497 BaseType &QueryingAttribute,
498 IRPosition &Pos, StateType &State) {
499 assert((Pos.getPositionKind() == IRPosition::IRP_ARGUMENT) &&
500 "Expected an 'argument' position !");
501 const CallBase *CBContext = Pos.getCallBaseContext();
502 if (!CBContext)
503 return false;
504
505 int ArgNo = Pos.getCallSiteArgNo();
506 assert(ArgNo >= 0 && "Invalid Arg No!");
507 const IRPosition CBArgPos = IRPosition::callsite_argument(CB: *CBContext, ArgNo);
508
509 // If possible, use the hasAssumedIRAttr interface.
510 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
511 bool IsKnown;
512 return AA::hasAssumedIRAttr<IRAttributeKind>(
513 A, &QueryingAttribute, CBArgPos, DepClassTy::REQUIRED, IsKnown);
514 }
515
516 const auto *AA =
517 A.getAAFor<AAType>(QueryingAttribute, CBArgPos, DepClassTy::REQUIRED);
518 if (!AA)
519 return false;
520 const StateType &CBArgumentState =
521 static_cast<const StateType &>(AA->getState());
522
523 LLVM_DEBUG(dbgs() << "[Attributor] Briding Call site context to argument"
524 << "Position:" << Pos << "CB Arg state:" << CBArgumentState
525 << "\n");
526
527 // NOTE: If we want to do call site grouping it should happen here.
528 State ^= CBArgumentState;
529 return true;
530}
531
532/// Helper class for generic deduction: call site argument -> argument position.
533template <typename AAType, typename BaseType,
534 typename StateType = typename AAType::StateType,
535 bool BridgeCallBaseContext = false,
536 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
537struct AAArgumentFromCallSiteArguments : public BaseType {
538 AAArgumentFromCallSiteArguments(const IRPosition &IRP, Attributor &A)
539 : BaseType(IRP, A) {}
540
541 /// See AbstractAttribute::updateImpl(...).
542 ChangeStatus updateImpl(Attributor &A) override {
543 StateType S = StateType::getBestState(this->getState());
544
545 if (BridgeCallBaseContext) {
546 bool Success =
547 getArgumentStateFromCallBaseContext<AAType, BaseType, StateType,
548 IRAttributeKind>(
549 A, *this, this->getIRPosition(), S);
550 if (Success)
551 return clampStateAndIndicateChange<StateType>(this->getState(), S);
552 }
553 clampCallSiteArgumentStates<AAType, StateType, IRAttributeKind>(A, *this,
554 S);
555
556 // TODO: If we know we visited all incoming values, thus no are assumed
557 // dead, we can take the known information from the state T.
558 return clampStateAndIndicateChange<StateType>(this->getState(), S);
559 }
560};
561
562/// Helper class for generic replication: function returned -> cs returned.
563template <typename AAType, typename BaseType,
564 typename StateType = typename BaseType::StateType,
565 bool IntroduceCallBaseContext = false,
566 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
567struct AACalleeToCallSite : public BaseType {
568 AACalleeToCallSite(const IRPosition &IRP, Attributor &A) : BaseType(IRP, A) {}
569
570 /// See AbstractAttribute::updateImpl(...).
571 ChangeStatus updateImpl(Attributor &A) override {
572 auto IRPKind = this->getIRPosition().getPositionKind();
573 assert((IRPKind == IRPosition::IRP_CALL_SITE_RETURNED ||
574 IRPKind == IRPosition::IRP_CALL_SITE) &&
575 "Can only wrap function returned positions for call site "
576 "returned positions!");
577 auto &S = this->getState();
578
579 CallBase &CB = cast<CallBase>(this->getAnchorValue());
580 if (IntroduceCallBaseContext)
581 LLVM_DEBUG(dbgs() << "[Attributor] Introducing call base context:" << CB
582 << "\n");
583
584 ChangeStatus Changed = ChangeStatus::UNCHANGED;
585 auto CalleePred = [&](ArrayRef<const Function *> Callees) {
586 for (const Function *Callee : Callees) {
587 IRPosition FnPos =
588 IRPKind == llvm::IRPosition::IRP_CALL_SITE_RETURNED
589 ? IRPosition::returned(F: *Callee,
590 CBContext: IntroduceCallBaseContext ? &CB : nullptr)
591 : IRPosition::function(
592 F: *Callee, CBContext: IntroduceCallBaseContext ? &CB : nullptr);
593 // If possible, use the hasAssumedIRAttr interface.
594 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
595 bool IsKnown;
596 if (!AA::hasAssumedIRAttr<IRAttributeKind>(
597 A, this, FnPos, DepClassTy::REQUIRED, IsKnown))
598 return false;
599 continue;
600 }
601
602 const AAType *AA =
603 A.getAAFor<AAType>(*this, FnPos, DepClassTy::REQUIRED);
604 if (!AA)
605 return false;
606 Changed |= clampStateAndIndicateChange(S, AA->getState());
607 if (S.isAtFixpoint())
608 return S.isValidState();
609 }
610 return true;
611 };
612 if (!A.checkForAllCallees(Pred: CalleePred, QueryingAA: *this, CB))
613 return S.indicatePessimisticFixpoint();
614 return Changed;
615 }
616};
617
618/// Helper function to accumulate uses.
619template <class AAType, typename StateType = typename AAType::StateType>
620static void followUsesInContext(AAType &AA, Attributor &A,
621 MustBeExecutedContextExplorer &Explorer,
622 const Instruction *CtxI,
623 SetVector<const Use *> &Uses,
624 StateType &State) {
625 auto EIt = Explorer.begin(PP: CtxI), EEnd = Explorer.end(CtxI);
626 for (unsigned u = 0; u < Uses.size(); ++u) {
627 const Use *U = Uses[u];
628 if (const Instruction *UserI = dyn_cast<Instruction>(Val: U->getUser())) {
629 bool Found = Explorer.findInContextOf(I: UserI, EIt, EEnd);
630 if (Found && AA.followUseInMBEC(A, U, UserI, State))
631 Uses.insert_range(R: llvm::make_pointer_range(Range: UserI->uses()));
632 }
633 }
634}
635
636/// Use the must-be-executed-context around \p I to add information into \p S.
637/// The AAType class is required to have `followUseInMBEC` method with the
638/// following signature and behaviour:
639///
640/// bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I)
641/// U - Underlying use.
642/// I - The user of the \p U.
643/// Returns true if the value should be tracked transitively.
644///
645template <class AAType, typename StateType = typename AAType::StateType>
646static void followUsesInMBEC(AAType &AA, Attributor &A, StateType &S,
647 Instruction &CtxI) {
648 const Value &Val = AA.getIRPosition().getAssociatedValue();
649 if (isa<ConstantData>(Val))
650 return;
651
652 MustBeExecutedContextExplorer *Explorer =
653 A.getInfoCache().getMustBeExecutedContextExplorer();
654 if (!Explorer)
655 return;
656
657 // Container for (transitive) uses of the associated value.
658 SetVector<const Use *> Uses;
659 for (const Use &U : Val.uses())
660 Uses.insert(X: &U);
661
662 followUsesInContext<AAType>(AA, A, *Explorer, &CtxI, Uses, S);
663
664 if (S.isAtFixpoint())
665 return;
666
667 SmallVector<const CondBrInst *, 4> BrInsts;
668 auto Pred = [&](const Instruction *I) {
669 if (const CondBrInst *Br = dyn_cast<CondBrInst>(Val: I))
670 BrInsts.push_back(Elt: Br);
671 return true;
672 };
673
674 // Here, accumulate conditional branch instructions in the context. We
675 // explore the child paths and collect the known states. The disjunction of
676 // those states can be merged to its own state. Let ParentState_i be a state
677 // to indicate the known information for an i-th branch instruction in the
678 // context. ChildStates are created for its successors respectively.
679 //
680 // ParentS_1 = ChildS_{1, 1} /\ ChildS_{1, 2} /\ ... /\ ChildS_{1, n_1}
681 // ParentS_2 = ChildS_{2, 1} /\ ChildS_{2, 2} /\ ... /\ ChildS_{2, n_2}
682 // ...
683 // ParentS_m = ChildS_{m, 1} /\ ChildS_{m, 2} /\ ... /\ ChildS_{m, n_m}
684 //
685 // Known State |= ParentS_1 \/ ParentS_2 \/... \/ ParentS_m
686 //
687 // FIXME: Currently, recursive branches are not handled. For example, we
688 // can't deduce that ptr must be dereferenced in below function.
689 //
690 // void f(int a, int c, int *ptr) {
691 // if(a)
692 // if (b) {
693 // *ptr = 0;
694 // } else {
695 // *ptr = 1;
696 // }
697 // else {
698 // if (b) {
699 // *ptr = 0;
700 // } else {
701 // *ptr = 1;
702 // }
703 // }
704 // }
705
706 Explorer->checkForAllContext(PP: &CtxI, Pred);
707 for (const CondBrInst *Br : BrInsts) {
708 StateType ParentState;
709
710 // The known state of the parent state is a conjunction of children's
711 // known states so it is initialized with a best state.
712 ParentState.indicateOptimisticFixpoint();
713
714 for (const BasicBlock *BB : Br->successors()) {
715 StateType ChildState;
716
717 size_t BeforeSize = Uses.size();
718 followUsesInContext(AA, A, *Explorer, &BB->front(), Uses, ChildState);
719
720 // Erase uses which only appear in the child.
721 for (auto It = Uses.begin() + BeforeSize; It != Uses.end();)
722 It = Uses.erase(I: It);
723
724 ParentState &= ChildState;
725 }
726
727 // Use only known state.
728 S += ParentState;
729 }
730}
731} // namespace
732
733/// ------------------------ PointerInfo ---------------------------------------
734
735namespace llvm {
736namespace AA {
737namespace PointerInfo {
738
739struct State;
740
741} // namespace PointerInfo
742} // namespace AA
743
744/// Helper for AA::PointerInfo::Access DenseMap/Set usage.
745template <>
746struct DenseMapInfo<AAPointerInfo::Access> : DenseMapInfo<Instruction *> {
747 using Access = AAPointerInfo::Access;
748 static inline Access getEmptyKey();
749 static inline Access getTombstoneKey();
750 static unsigned getHashValue(const Access &A);
751 static bool isEqual(const Access &LHS, const Access &RHS);
752};
753
754/// Helper that allows RangeTy as a key in a DenseMap.
755template <> struct DenseMapInfo<AA::RangeTy> {
756 static inline AA::RangeTy getEmptyKey() {
757 auto EmptyKey = DenseMapInfo<int64_t>::getEmptyKey();
758 return AA::RangeTy{EmptyKey, EmptyKey};
759 }
760
761 static inline AA::RangeTy getTombstoneKey() {
762 auto TombstoneKey = DenseMapInfo<int64_t>::getTombstoneKey();
763 return AA::RangeTy{TombstoneKey, TombstoneKey};
764 }
765
766 static unsigned getHashValue(const AA::RangeTy &Range) {
767 return detail::combineHashValue(
768 a: DenseMapInfo<int64_t>::getHashValue(Val: Range.Offset),
769 b: DenseMapInfo<int64_t>::getHashValue(Val: Range.Size));
770 }
771
772 static bool isEqual(const AA::RangeTy &A, const AA::RangeTy B) {
773 return A == B;
774 }
775};
776
777/// Helper for AA::PointerInfo::Access DenseMap/Set usage ignoring everythign
778/// but the instruction
779struct AccessAsInstructionInfo : DenseMapInfo<Instruction *> {
780 using Base = DenseMapInfo<Instruction *>;
781 using Access = AAPointerInfo::Access;
782 static inline Access getEmptyKey();
783 static inline Access getTombstoneKey();
784 static unsigned getHashValue(const Access &A);
785 static bool isEqual(const Access &LHS, const Access &RHS);
786};
787
788} // namespace llvm
789
790/// A type to track pointer/struct usage and accesses for AAPointerInfo.
791struct AA::PointerInfo::State : public AbstractState {
792 /// Return the best possible representable state.
793 static State getBestState(const State &SIS) { return State(); }
794
795 /// Return the worst possible representable state.
796 static State getWorstState(const State &SIS) {
797 State R;
798 R.indicatePessimisticFixpoint();
799 return R;
800 }
801
802 State() = default;
803 State(State &&SIS) = default;
804
805 const State &getAssumed() const { return *this; }
806
807 /// See AbstractState::isValidState().
808 bool isValidState() const override { return BS.isValidState(); }
809
810 /// See AbstractState::isAtFixpoint().
811 bool isAtFixpoint() const override { return BS.isAtFixpoint(); }
812
813 /// See AbstractState::indicateOptimisticFixpoint().
814 ChangeStatus indicateOptimisticFixpoint() override {
815 BS.indicateOptimisticFixpoint();
816 return ChangeStatus::UNCHANGED;
817 }
818
819 /// See AbstractState::indicatePessimisticFixpoint().
820 ChangeStatus indicatePessimisticFixpoint() override {
821 BS.indicatePessimisticFixpoint();
822 return ChangeStatus::CHANGED;
823 }
824
825 State &operator=(const State &R) {
826 if (this == &R)
827 return *this;
828 BS = R.BS;
829 AccessList = R.AccessList;
830 OffsetBins = R.OffsetBins;
831 RemoteIMap = R.RemoteIMap;
832 ReturnedOffsets = R.ReturnedOffsets;
833 return *this;
834 }
835
836 State &operator=(State &&R) {
837 if (this == &R)
838 return *this;
839 std::swap(a&: BS, b&: R.BS);
840 std::swap(LHS&: AccessList, RHS&: R.AccessList);
841 std::swap(a&: OffsetBins, b&: R.OffsetBins);
842 std::swap(a&: RemoteIMap, b&: R.RemoteIMap);
843 std::swap(a&: ReturnedOffsets, b&: R.ReturnedOffsets);
844 return *this;
845 }
846
847 /// Add a new Access to the state at offset \p Offset and with size \p Size.
848 /// The access is associated with \p I, writes \p Content (if anything), and
849 /// is of kind \p Kind. If an Access already exists for the same \p I and same
850 /// \p RemoteI, the two are combined, potentially losing information about
851 /// offset and size. The resulting access must now be moved from its original
852 /// OffsetBin to the bin for its new offset.
853 ///
854 /// \Returns CHANGED, if the state changed, UNCHANGED otherwise.
855 ChangeStatus addAccess(Attributor &A, const AAPointerInfo::RangeList &Ranges,
856 Instruction &I, std::optional<Value *> Content,
857 AAPointerInfo::AccessKind Kind, Type *Ty,
858 Instruction *RemoteI = nullptr);
859
860 AAPointerInfo::const_bin_iterator begin() const { return OffsetBins.begin(); }
861 AAPointerInfo::const_bin_iterator end() const { return OffsetBins.end(); }
862 int64_t numOffsetBins() const { return OffsetBins.size(); }
863
864 const AAPointerInfo::Access &getAccess(unsigned Index) const {
865 return AccessList[Index];
866 }
867
868protected:
869 // Every memory instruction results in an Access object. We maintain a list of
870 // all Access objects that we own, along with the following maps:
871 //
872 // - OffsetBins: RangeTy -> { Access }
873 // - RemoteIMap: RemoteI x LocalI -> Access
874 //
875 // A RemoteI is any instruction that accesses memory. RemoteI is different
876 // from LocalI if and only if LocalI is a call; then RemoteI is some
877 // instruction in the callgraph starting from LocalI. Multiple paths in the
878 // callgraph from LocalI to RemoteI may produce multiple accesses, but these
879 // are all combined into a single Access object. This may result in loss of
880 // information in RangeTy in the Access object.
881 SmallVector<AAPointerInfo::Access> AccessList;
882 AAPointerInfo::OffsetBinsTy OffsetBins;
883 DenseMap<const Instruction *, SmallVector<unsigned>> RemoteIMap;
884
885 /// Flag to determine if the underlying pointer is reaching a return statement
886 /// in the associated function or not. Returns in other functions cause
887 /// invalidation.
888 AAPointerInfo::OffsetInfo ReturnedOffsets;
889
890 /// See AAPointerInfo::forallInterferingAccesses.
891 template <typename F>
892 bool forallInterferingAccesses(AA::RangeTy Range, F CB) const {
893 if (!isValidState() || !ReturnedOffsets.isUnassigned())
894 return false;
895
896 for (const auto &It : OffsetBins) {
897 AA::RangeTy ItRange = It.getFirst();
898 if (!Range.mayOverlap(Range: ItRange))
899 continue;
900 bool IsExact = Range == ItRange && !Range.offsetOrSizeAreUnknown();
901 for (auto Index : It.getSecond()) {
902 auto &Access = AccessList[Index];
903 if (!CB(Access, IsExact))
904 return false;
905 }
906 }
907 return true;
908 }
909
910 /// See AAPointerInfo::forallInterferingAccesses.
911 template <typename F>
912 bool forallInterferingAccesses(Instruction &I, F CB,
913 AA::RangeTy &Range) const {
914 if (!isValidState() || !ReturnedOffsets.isUnassigned())
915 return false;
916
917 auto LocalList = RemoteIMap.find(Val: &I);
918 if (LocalList == RemoteIMap.end()) {
919 return true;
920 }
921
922 for (unsigned Index : LocalList->getSecond()) {
923 for (auto &R : AccessList[Index]) {
924 Range &= R;
925 if (Range.offsetAndSizeAreUnknown())
926 break;
927 }
928 }
929 return forallInterferingAccesses(Range, CB);
930 }
931
932private:
933 /// State to track fixpoint and validity.
934 BooleanState BS;
935};
936
937ChangeStatus AA::PointerInfo::State::addAccess(
938 Attributor &A, const AAPointerInfo::RangeList &Ranges, Instruction &I,
939 std::optional<Value *> Content, AAPointerInfo::AccessKind Kind, Type *Ty,
940 Instruction *RemoteI) {
941 RemoteI = RemoteI ? RemoteI : &I;
942
943 // Check if we have an access for this instruction, if not, simply add it.
944 auto &LocalList = RemoteIMap[RemoteI];
945 bool AccExists = false;
946 unsigned AccIndex = AccessList.size();
947 for (auto Index : LocalList) {
948 auto &A = AccessList[Index];
949 if (A.getLocalInst() == &I) {
950 AccExists = true;
951 AccIndex = Index;
952 break;
953 }
954 }
955
956 auto AddToBins = [&](const AAPointerInfo::RangeList &ToAdd) {
957 LLVM_DEBUG(if (ToAdd.size()) dbgs()
958 << "[AAPointerInfo] Inserting access in new offset bins\n";);
959
960 for (auto Key : ToAdd) {
961 LLVM_DEBUG(dbgs() << " key " << Key << "\n");
962 OffsetBins[Key].insert(V: AccIndex);
963 }
964 };
965
966 if (!AccExists) {
967 AccessList.emplace_back(Args: &I, Args&: RemoteI, Args: Ranges, Args&: Content, Args&: Kind, Args&: Ty);
968 assert((AccessList.size() == AccIndex + 1) &&
969 "New Access should have been at AccIndex");
970 LocalList.push_back(Elt: AccIndex);
971 AddToBins(AccessList[AccIndex].getRanges());
972 return ChangeStatus::CHANGED;
973 }
974
975 // Combine the new Access with the existing Access, and then update the
976 // mapping in the offset bins.
977 AAPointerInfo::Access Acc(&I, RemoteI, Ranges, Content, Kind, Ty);
978 auto &Current = AccessList[AccIndex];
979 auto Before = Current;
980 Current &= Acc;
981 if (Current == Before)
982 return ChangeStatus::UNCHANGED;
983
984 auto &ExistingRanges = Before.getRanges();
985 auto &NewRanges = Current.getRanges();
986
987 // Ranges that are in the old access but not the new access need to be removed
988 // from the offset bins.
989 AAPointerInfo::RangeList ToRemove;
990 AAPointerInfo::RangeList::set_difference(L: ExistingRanges, R: NewRanges, D&: ToRemove);
991 LLVM_DEBUG(if (ToRemove.size()) dbgs()
992 << "[AAPointerInfo] Removing access from old offset bins\n";);
993
994 for (auto Key : ToRemove) {
995 LLVM_DEBUG(dbgs() << " key " << Key << "\n");
996 assert(OffsetBins.count(Key) && "Existing Access must be in some bin.");
997 auto &Bin = OffsetBins[Key];
998 assert(Bin.count(AccIndex) &&
999 "Expected bin to actually contain the Access.");
1000 Bin.erase(V: AccIndex);
1001 }
1002
1003 // Ranges that are in the new access but not the old access need to be added
1004 // to the offset bins.
1005 AAPointerInfo::RangeList ToAdd;
1006 AAPointerInfo::RangeList::set_difference(L: NewRanges, R: ExistingRanges, D&: ToAdd);
1007 AddToBins(ToAdd);
1008 return ChangeStatus::CHANGED;
1009}
1010
1011namespace {
1012
1013#ifndef NDEBUG
1014static raw_ostream &operator<<(raw_ostream &OS,
1015 const AAPointerInfo::OffsetInfo &OI) {
1016 OS << llvm::interleaved_array(OI);
1017 return OS;
1018}
1019#endif // NDEBUG
1020
1021struct AAPointerInfoImpl
1022 : public StateWrapper<AA::PointerInfo::State, AAPointerInfo> {
1023 using BaseTy = StateWrapper<AA::PointerInfo::State, AAPointerInfo>;
1024 AAPointerInfoImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
1025
1026 /// See AbstractAttribute::getAsStr().
1027 const std::string getAsStr(Attributor *A) const override {
1028 return std::string("PointerInfo ") +
1029 (isValidState() ? (std::string("#") +
1030 std::to_string(val: OffsetBins.size()) + " bins")
1031 : "<invalid>") +
1032 (reachesReturn()
1033 ? (" (returned:" +
1034 join(R: map_range(C: ReturnedOffsets,
1035 F: [](int64_t O) { return std::to_string(val: O); }),
1036 Separator: ", ") +
1037 ")")
1038 : "");
1039 }
1040
1041 /// See AbstractAttribute::manifest(...).
1042 ChangeStatus manifest(Attributor &A) override {
1043 return AAPointerInfo::manifest(A);
1044 }
1045
1046 const_bin_iterator begin() const override { return State::begin(); }
1047 const_bin_iterator end() const override { return State::end(); }
1048 int64_t numOffsetBins() const override { return State::numOffsetBins(); }
1049 bool reachesReturn() const override {
1050 return !ReturnedOffsets.isUnassigned();
1051 }
1052 void addReturnedOffsetsTo(OffsetInfo &OI) const override {
1053 if (ReturnedOffsets.isUnknown()) {
1054 OI.setUnknown();
1055 return;
1056 }
1057
1058 OffsetInfo MergedOI;
1059 for (auto Offset : ReturnedOffsets) {
1060 OffsetInfo TmpOI = OI;
1061 TmpOI.addToAll(Inc: Offset);
1062 MergedOI.merge(R: TmpOI);
1063 }
1064 OI = std::move(MergedOI);
1065 }
1066
1067 ChangeStatus setReachesReturn(const OffsetInfo &ReachedReturnedOffsets) {
1068 if (ReturnedOffsets.isUnknown())
1069 return ChangeStatus::UNCHANGED;
1070 if (ReachedReturnedOffsets.isUnknown()) {
1071 ReturnedOffsets.setUnknown();
1072 return ChangeStatus::CHANGED;
1073 }
1074 if (ReturnedOffsets.merge(R: ReachedReturnedOffsets))
1075 return ChangeStatus::CHANGED;
1076 return ChangeStatus::UNCHANGED;
1077 }
1078
1079 bool forallInterferingAccesses(
1080 AA::RangeTy Range,
1081 function_ref<bool(const AAPointerInfo::Access &, bool)> CB)
1082 const override {
1083 return State::forallInterferingAccesses(Range, CB);
1084 }
1085
1086 bool forallInterferingAccesses(
1087 Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I,
1088 bool FindInterferingWrites, bool FindInterferingReads,
1089 function_ref<bool(const Access &, bool)> UserCB, bool &HasBeenWrittenTo,
1090 AA::RangeTy &Range,
1091 function_ref<bool(const Access &)> SkipCB) const override {
1092 HasBeenWrittenTo = false;
1093
1094 SmallPtrSet<const Access *, 8> DominatingWrites;
1095 SmallVector<std::pair<const Access *, bool>, 8> InterferingAccesses;
1096
1097 Function &Scope = *I.getFunction();
1098 bool IsKnownNoSync;
1099 bool IsAssumedNoSync = AA::hasAssumedIRAttr<Attribute::NoSync>(
1100 A, QueryingAA: &QueryingAA, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL,
1101 IsKnown&: IsKnownNoSync);
1102 const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
1103 IRP: IRPosition::function(F: Scope), QueryingAA: &QueryingAA, DepClass: DepClassTy::NONE);
1104 bool AllInSameNoSyncFn = IsAssumedNoSync;
1105 bool InstIsExecutedByInitialThreadOnly =
1106 ExecDomainAA && ExecDomainAA->isExecutedByInitialThreadOnly(I);
1107
1108 // If the function is not ending in aligned barriers, we need the stores to
1109 // be in aligned barriers. The load being in one is not sufficient since the
1110 // store might be executed by a thread that disappears after, causing the
1111 // aligned barrier guarding the load to unblock and the load to read a value
1112 // that has no CFG path to the load.
1113 bool InstIsExecutedInAlignedRegion =
1114 FindInterferingReads && ExecDomainAA &&
1115 ExecDomainAA->isExecutedInAlignedRegion(A, I);
1116
1117 if (InstIsExecutedInAlignedRegion || InstIsExecutedByInitialThreadOnly)
1118 A.recordDependence(FromAA: *ExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1119
1120 InformationCache &InfoCache = A.getInfoCache();
1121 bool IsThreadLocalObj =
1122 AA::isAssumedThreadLocalObject(A, Obj&: getAssociatedValue(), QueryingAA: *this);
1123
1124 // Helper to determine if we need to consider threading, which we cannot
1125 // right now. However, if the function is (assumed) nosync or the thread
1126 // executing all instructions is the main thread only we can ignore
1127 // threading. Also, thread-local objects do not require threading reasoning.
1128 // Finally, we can ignore threading if either access is executed in an
1129 // aligned region.
1130 auto CanIgnoreThreadingForInst = [&](const Instruction &I) -> bool {
1131 if (IsThreadLocalObj || AllInSameNoSyncFn)
1132 return true;
1133 const auto *FnExecDomainAA =
1134 I.getFunction() == &Scope
1135 ? ExecDomainAA
1136 : A.lookupAAFor<AAExecutionDomain>(
1137 IRP: IRPosition::function(F: *I.getFunction()), QueryingAA: &QueryingAA,
1138 DepClass: DepClassTy::NONE);
1139 if (!FnExecDomainAA)
1140 return false;
1141 if (InstIsExecutedInAlignedRegion ||
1142 (FindInterferingWrites &&
1143 FnExecDomainAA->isExecutedInAlignedRegion(A, I))) {
1144 A.recordDependence(FromAA: *FnExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1145 return true;
1146 }
1147 if (InstIsExecutedByInitialThreadOnly &&
1148 FnExecDomainAA->isExecutedByInitialThreadOnly(I)) {
1149 A.recordDependence(FromAA: *FnExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1150 return true;
1151 }
1152 return false;
1153 };
1154
1155 // Helper to determine if the access is executed by the same thread as the
1156 // given instruction, for now it is sufficient to avoid any potential
1157 // threading effects as we cannot deal with them anyway.
1158 auto CanIgnoreThreading = [&](const Access &Acc) -> bool {
1159 return CanIgnoreThreadingForInst(*Acc.getRemoteInst()) ||
1160 (Acc.getRemoteInst() != Acc.getLocalInst() &&
1161 CanIgnoreThreadingForInst(*Acc.getLocalInst()));
1162 };
1163
1164 // TODO: Use inter-procedural reachability and dominance.
1165 bool IsKnownNoRecurse;
1166 AA::hasAssumedIRAttr<Attribute::NoRecurse>(
1167 A, QueryingAA: this, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL,
1168 IsKnown&: IsKnownNoRecurse);
1169
1170 // TODO: Use reaching kernels from AAKernelInfo (or move it to
1171 // AAExecutionDomain) such that we allow scopes other than kernels as long
1172 // as the reaching kernels are disjoint.
1173 bool InstInKernel = A.getInfoCache().isKernel(F: Scope);
1174 bool ObjHasKernelLifetime = false;
1175 const bool UseDominanceReasoning =
1176 FindInterferingWrites && IsKnownNoRecurse;
1177 const DominatorTree *DT =
1178 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: Scope);
1179
1180 // Helper to check if a value has "kernel lifetime", that is it will not
1181 // outlive a GPU kernel. This is true for shared, constant, and local
1182 // globals on AMD and NVIDIA GPUs.
1183 auto HasKernelLifetime = [&](Value *V, Module &M) {
1184 if (!AA::isGPU(M))
1185 return false;
1186 switch (AA::GPUAddressSpace(V->getType()->getPointerAddressSpace())) {
1187 case AA::GPUAddressSpace::Shared:
1188 case AA::GPUAddressSpace::Constant:
1189 case AA::GPUAddressSpace::Local:
1190 return true;
1191 default:
1192 return false;
1193 };
1194 };
1195
1196 // The IsLiveInCalleeCB will be used by the AA::isPotentiallyReachable query
1197 // to determine if we should look at reachability from the callee. For
1198 // certain pointers we know the lifetime and we do not have to step into the
1199 // callee to determine reachability as the pointer would be dead in the
1200 // callee. See the conditional initialization below.
1201 std::function<bool(const Function &)> IsLiveInCalleeCB;
1202
1203 if (auto *AI = dyn_cast<AllocaInst>(Val: &getAssociatedValue())) {
1204 // If the alloca containing function is not recursive the alloca
1205 // must be dead in the callee.
1206 const Function *AIFn = AI->getFunction();
1207 ObjHasKernelLifetime = A.getInfoCache().isKernel(F: *AIFn);
1208 bool IsKnownNoRecurse;
1209 if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
1210 A, QueryingAA: this, IRP: IRPosition::function(F: *AIFn), DepClass: DepClassTy::OPTIONAL,
1211 IsKnown&: IsKnownNoRecurse)) {
1212 IsLiveInCalleeCB = [AIFn](const Function &Fn) { return AIFn != &Fn; };
1213 }
1214 } else if (auto *GV = dyn_cast<GlobalValue>(Val: &getAssociatedValue())) {
1215 // If the global has kernel lifetime we can stop if we reach a kernel
1216 // as it is "dead" in the (unknown) callees.
1217 ObjHasKernelLifetime = HasKernelLifetime(GV, *GV->getParent());
1218 if (ObjHasKernelLifetime)
1219 IsLiveInCalleeCB = [&A](const Function &Fn) {
1220 return !A.getInfoCache().isKernel(F: Fn);
1221 };
1222 }
1223
1224 // Set of accesses/instructions that will overwrite the result and are
1225 // therefore blockers in the reachability traversal.
1226 AA::InstExclusionSetTy ExclusionSet;
1227
1228 auto AccessCB = [&](const Access &Acc, bool Exact) {
1229 Function *AccScope = Acc.getRemoteInst()->getFunction();
1230 bool AccInSameScope = AccScope == &Scope;
1231
1232 // If the object has kernel lifetime we can ignore accesses only reachable
1233 // by other kernels. For now we only skip accesses *in* other kernels.
1234 if (InstInKernel && ObjHasKernelLifetime && !AccInSameScope &&
1235 A.getInfoCache().isKernel(F: *AccScope))
1236 return true;
1237
1238 if (Exact && Acc.isMustAccess() && Acc.getRemoteInst() != &I) {
1239 if (Acc.isWrite() || (isa<LoadInst>(Val: I) && Acc.isWriteOrAssumption()))
1240 ExclusionSet.insert(Ptr: Acc.getRemoteInst());
1241 }
1242
1243 if ((!FindInterferingWrites || !Acc.isWriteOrAssumption()) &&
1244 (!FindInterferingReads || !Acc.isRead()))
1245 return true;
1246
1247 bool Dominates = FindInterferingWrites && DT && Exact &&
1248 Acc.isMustAccess() && AccInSameScope &&
1249 DT->dominates(Def: Acc.getRemoteInst(), User: &I);
1250 if (Dominates)
1251 DominatingWrites.insert(Ptr: &Acc);
1252
1253 // Track if all interesting accesses are in the same `nosync` function as
1254 // the given instruction.
1255 AllInSameNoSyncFn &= Acc.getRemoteInst()->getFunction() == &Scope;
1256
1257 InterferingAccesses.push_back(Elt: {&Acc, Exact});
1258 return true;
1259 };
1260 if (!State::forallInterferingAccesses(I, CB: AccessCB, Range))
1261 return false;
1262
1263 HasBeenWrittenTo = !DominatingWrites.empty();
1264
1265 // Dominating writes form a chain, find the least/lowest member.
1266 Instruction *LeastDominatingWriteInst = nullptr;
1267 for (const Access *Acc : DominatingWrites) {
1268 if (!LeastDominatingWriteInst) {
1269 LeastDominatingWriteInst = Acc->getRemoteInst();
1270 } else if (DT->dominates(Def: LeastDominatingWriteInst,
1271 User: Acc->getRemoteInst())) {
1272 LeastDominatingWriteInst = Acc->getRemoteInst();
1273 }
1274 }
1275
1276 // Helper to determine if we can skip a specific write access.
1277 auto CanSkipAccess = [&](const Access &Acc, bool Exact) {
1278 if (SkipCB && SkipCB(Acc))
1279 return true;
1280 if (!CanIgnoreThreading(Acc))
1281 return false;
1282
1283 // Check read (RAW) dependences and write (WAR) dependences as necessary.
1284 // If we successfully excluded all effects we are interested in, the
1285 // access can be skipped.
1286 bool ReadChecked = !FindInterferingReads;
1287 bool WriteChecked = !FindInterferingWrites;
1288
1289 // If the instruction cannot reach the access, the former does not
1290 // interfere with what the access reads.
1291 if (!ReadChecked) {
1292 if (!AA::isPotentiallyReachable(A, FromI: I, ToI: *Acc.getRemoteInst(), QueryingAA,
1293 ExclusionSet: &ExclusionSet, GoBackwardsCB: IsLiveInCalleeCB))
1294 ReadChecked = true;
1295 }
1296 // If the instruction cannot be reach from the access, the latter does not
1297 // interfere with what the instruction reads.
1298 if (!WriteChecked) {
1299 if (!AA::isPotentiallyReachable(A, FromI: *Acc.getRemoteInst(), ToI: I, QueryingAA,
1300 ExclusionSet: &ExclusionSet, GoBackwardsCB: IsLiveInCalleeCB))
1301 WriteChecked = true;
1302 }
1303
1304 // If we still might be affected by the write of the access but there are
1305 // dominating writes in the function of the instruction
1306 // (HasBeenWrittenTo), we can try to reason that the access is overwritten
1307 // by them. This would have happend above if they are all in the same
1308 // function, so we only check the inter-procedural case. Effectively, we
1309 // want to show that there is no call after the dominting write that might
1310 // reach the access, and when it returns reach the instruction with the
1311 // updated value. To this end, we iterate all call sites, check if they
1312 // might reach the instruction without going through another access
1313 // (ExclusionSet) and at the same time might reach the access. However,
1314 // that is all part of AAInterFnReachability.
1315 if (!WriteChecked && HasBeenWrittenTo &&
1316 Acc.getRemoteInst()->getFunction() != &Scope) {
1317
1318 const auto *FnReachabilityAA = A.getAAFor<AAInterFnReachability>(
1319 QueryingAA, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL);
1320 if (FnReachabilityAA) {
1321 // Without going backwards in the call tree, can we reach the access
1322 // from the least dominating write. Do not allow to pass the
1323 // instruction itself either.
1324 bool Inserted = ExclusionSet.insert(Ptr: &I).second;
1325
1326 if (!FnReachabilityAA->instructionCanReach(
1327 A, Inst: *LeastDominatingWriteInst,
1328 Fn: *Acc.getRemoteInst()->getFunction(), ExclusionSet: &ExclusionSet))
1329 WriteChecked = true;
1330
1331 if (Inserted)
1332 ExclusionSet.erase(Ptr: &I);
1333 }
1334 }
1335
1336 if (ReadChecked && WriteChecked)
1337 return true;
1338
1339 if (!DT || !UseDominanceReasoning)
1340 return false;
1341 if (!DominatingWrites.count(Ptr: &Acc))
1342 return false;
1343 return LeastDominatingWriteInst != Acc.getRemoteInst();
1344 };
1345
1346 // Run the user callback on all accesses we cannot skip and return if
1347 // that succeeded for all or not.
1348 for (auto &It : InterferingAccesses) {
1349 if ((!AllInSameNoSyncFn && !IsThreadLocalObj && !ExecDomainAA) ||
1350 !CanSkipAccess(*It.first, It.second)) {
1351 if (!UserCB(*It.first, It.second))
1352 return false;
1353 }
1354 }
1355 return true;
1356 }
1357
1358 ChangeStatus translateAndAddStateFromCallee(Attributor &A,
1359 const AAPointerInfo &OtherAA,
1360 CallBase &CB) {
1361 using namespace AA::PointerInfo;
1362 if (!OtherAA.getState().isValidState() || !isValidState())
1363 return indicatePessimisticFixpoint();
1364
1365 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1366 const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1367 bool IsByval = OtherAAImpl.getAssociatedArgument()->hasByValAttr();
1368 Changed |= setReachesReturn(OtherAAImpl.ReturnedOffsets);
1369
1370 // Combine the accesses bin by bin.
1371 const auto &State = OtherAAImpl.getState();
1372 for (const auto &It : State) {
1373 for (auto Index : It.getSecond()) {
1374 const auto &RAcc = State.getAccess(Index);
1375 if (IsByval && !RAcc.isRead())
1376 continue;
1377 bool UsedAssumedInformation = false;
1378 AccessKind AK = RAcc.getKind();
1379 auto Content = A.translateArgumentToCallSiteContent(
1380 V: RAcc.getContent(), CB, AA: *this, UsedAssumedInformation);
1381 AK = AccessKind(AK & (IsByval ? AccessKind::AK_R : AccessKind::AK_RW));
1382 AK = AccessKind(AK | (RAcc.isMayAccess() ? AK_MAY : AK_MUST));
1383
1384 Changed |= addAccess(A, Ranges: RAcc.getRanges(), I&: CB, Content, Kind: AK,
1385 Ty: RAcc.getType(), RemoteI: RAcc.getRemoteInst());
1386 }
1387 }
1388 return Changed;
1389 }
1390
1391 ChangeStatus translateAndAddState(Attributor &A, const AAPointerInfo &OtherAA,
1392 const OffsetInfo &Offsets, CallBase &CB,
1393 bool IsMustAcc) {
1394 using namespace AA::PointerInfo;
1395 if (!OtherAA.getState().isValidState() || !isValidState())
1396 return indicatePessimisticFixpoint();
1397
1398 const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1399
1400 // Combine the accesses bin by bin.
1401 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1402 const auto &State = OtherAAImpl.getState();
1403 for (const auto &It : State) {
1404 for (auto Index : It.getSecond()) {
1405 const auto &RAcc = State.getAccess(Index);
1406 if (!IsMustAcc && RAcc.isAssumption())
1407 continue;
1408 for (auto Offset : Offsets) {
1409 auto NewRanges = Offset == AA::RangeTy::Unknown
1410 ? AA::RangeTy::getUnknown()
1411 : RAcc.getRanges();
1412 if (!NewRanges.isUnknown()) {
1413 NewRanges.addToAllOffsets(Inc: Offset);
1414 }
1415 AccessKind AK = RAcc.getKind();
1416 if (!IsMustAcc)
1417 AK = AccessKind((AK & ~AK_MUST) | AK_MAY);
1418 Changed |= addAccess(A, Ranges: NewRanges, I&: CB, Content: RAcc.getContent(), Kind: AK,
1419 Ty: RAcc.getType(), RemoteI: RAcc.getRemoteInst());
1420 }
1421 }
1422 }
1423 return Changed;
1424 }
1425
1426 /// Statistic tracking for all AAPointerInfo implementations.
1427 /// See AbstractAttribute::trackStatistics().
1428 void trackPointerInfoStatistics(const IRPosition &IRP) const {}
1429
1430 /// Dump the state into \p O.
1431 void dumpState(raw_ostream &O) {
1432 for (auto &It : OffsetBins) {
1433 O << "[" << It.first.Offset << "-" << It.first.Offset + It.first.Size
1434 << "] : " << It.getSecond().size() << "\n";
1435 for (auto AccIndex : It.getSecond()) {
1436 auto &Acc = AccessList[AccIndex];
1437 O << " - " << Acc.getKind() << " - " << *Acc.getLocalInst() << "\n";
1438 if (Acc.getLocalInst() != Acc.getRemoteInst())
1439 O << " --> " << *Acc.getRemoteInst()
1440 << "\n";
1441 if (!Acc.isWrittenValueYetUndetermined()) {
1442 if (isa_and_nonnull<Function>(Val: Acc.getWrittenValue()))
1443 O << " - c: func " << Acc.getWrittenValue()->getName()
1444 << "\n";
1445 else if (Acc.getWrittenValue())
1446 O << " - c: " << *Acc.getWrittenValue() << "\n";
1447 else
1448 O << " - c: <unknown>\n";
1449 }
1450 }
1451 }
1452 }
1453};
1454
1455struct AAPointerInfoFloating : public AAPointerInfoImpl {
1456 using AccessKind = AAPointerInfo::AccessKind;
1457 AAPointerInfoFloating(const IRPosition &IRP, Attributor &A)
1458 : AAPointerInfoImpl(IRP, A) {}
1459
1460 /// Deal with an access and signal if it was handled successfully.
1461 bool handleAccess(Attributor &A, Instruction &I,
1462 std::optional<Value *> Content, AccessKind Kind,
1463 OffsetInfo::VecTy &Offsets, ChangeStatus &Changed,
1464 Type &Ty) {
1465 using namespace AA::PointerInfo;
1466 auto Size = AA::RangeTy::Unknown;
1467 const DataLayout &DL = A.getDataLayout();
1468 TypeSize AccessSize = DL.getTypeStoreSize(Ty: &Ty);
1469 if (!AccessSize.isScalable())
1470 Size = AccessSize.getFixedValue();
1471
1472 // Make a strictly ascending list of offsets as required by addAccess()
1473 SmallVector<int64_t> OffsetsSorted(Offsets.begin(), Offsets.end());
1474 llvm::sort(C&: OffsetsSorted);
1475
1476 VectorType *VT = dyn_cast<VectorType>(Val: &Ty);
1477 if (!VT || VT->getElementCount().isScalable() ||
1478 !Content.value_or(u: nullptr) || !isa<Constant>(Val: *Content) ||
1479 (*Content)->getType() != VT ||
1480 DL.getTypeStoreSize(Ty: VT->getElementType()).isScalable()) {
1481 Changed =
1482 Changed | addAccess(A, Ranges: {OffsetsSorted, Size}, I, Content, Kind, Ty: &Ty);
1483 } else {
1484 // Handle vector stores with constant content element-wise.
1485 // TODO: We could look for the elements or create instructions
1486 // representing them.
1487 // TODO: We need to push the Content into the range abstraction
1488 // (AA::RangeTy) to allow different content values for different
1489 // ranges. ranges. Hence, support vectors storing different values.
1490 Type *ElementType = VT->getElementType();
1491 int64_t ElementSize = DL.getTypeStoreSize(Ty: ElementType).getFixedValue();
1492 auto *ConstContent = cast<Constant>(Val: *Content);
1493 Type *Int32Ty = Type::getInt32Ty(C&: ElementType->getContext());
1494 SmallVector<int64_t> ElementOffsets(Offsets.begin(), Offsets.end());
1495
1496 for (int i = 0, e = VT->getElementCount().getFixedValue(); i != e; ++i) {
1497 Value *ElementContent = ConstantExpr::getExtractElement(
1498 Vec: ConstContent, Idx: ConstantInt::get(Ty: Int32Ty, V: i));
1499
1500 // Add the element access.
1501 Changed = Changed | addAccess(A, Ranges: {ElementOffsets, ElementSize}, I,
1502 Content: ElementContent, Kind, Ty: ElementType);
1503
1504 // Advance the offsets for the next element.
1505 for (auto &ElementOffset : ElementOffsets)
1506 ElementOffset += ElementSize;
1507 }
1508 }
1509 return true;
1510 };
1511
1512 /// See AbstractAttribute::updateImpl(...).
1513 ChangeStatus updateImpl(Attributor &A) override;
1514
1515 /// If the indices to \p GEP can be traced to constants, incorporate all
1516 /// of these into \p UsrOI.
1517 ///
1518 /// \return true iff \p UsrOI is updated.
1519 bool collectConstantsForGEP(Attributor &A, const DataLayout &DL,
1520 OffsetInfo &UsrOI, const OffsetInfo &PtrOI,
1521 const GEPOperator *GEP);
1522
1523 /// See AbstractAttribute::trackStatistics()
1524 void trackStatistics() const override {
1525 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1526 }
1527};
1528
1529bool AAPointerInfoFloating::collectConstantsForGEP(Attributor &A,
1530 const DataLayout &DL,
1531 OffsetInfo &UsrOI,
1532 const OffsetInfo &PtrOI,
1533 const GEPOperator *GEP) {
1534 unsigned BitWidth = DL.getIndexTypeSizeInBits(Ty: GEP->getType());
1535 SmallMapVector<Value *, APInt, 4> VariableOffsets;
1536 APInt ConstantOffset(BitWidth, 0);
1537
1538 assert(!UsrOI.isUnknown() && !PtrOI.isUnknown() &&
1539 "Don't look for constant values if the offset has already been "
1540 "determined to be unknown.");
1541
1542 if (!GEP->collectOffset(DL, BitWidth, VariableOffsets, ConstantOffset)) {
1543 UsrOI.setUnknown();
1544 return true;
1545 }
1546
1547 LLVM_DEBUG(dbgs() << "[AAPointerInfo] GEP offset is "
1548 << (VariableOffsets.empty() ? "" : "not") << " constant "
1549 << *GEP << "\n");
1550
1551 auto Union = PtrOI;
1552 Union.addToAll(Inc: ConstantOffset.getSExtValue());
1553
1554 // Each VI in VariableOffsets has a set of potential constant values. Every
1555 // combination of elements, picked one each from these sets, is separately
1556 // added to the original set of offsets, thus resulting in more offsets.
1557 for (const auto &VI : VariableOffsets) {
1558 auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
1559 QueryingAA: *this, IRP: IRPosition::value(V: *VI.first), DepClass: DepClassTy::OPTIONAL);
1560 if (!PotentialConstantsAA || !PotentialConstantsAA->isValidState()) {
1561 UsrOI.setUnknown();
1562 return true;
1563 }
1564
1565 // UndefValue is treated as a zero, which leaves Union as is.
1566 if (PotentialConstantsAA->undefIsContained())
1567 continue;
1568
1569 // We need at least one constant in every set to compute an actual offset.
1570 // Otherwise, we end up pessimizing AAPointerInfo by respecting offsets that
1571 // don't actually exist. In other words, the absence of constant values
1572 // implies that the operation can be assumed dead for now.
1573 auto &AssumedSet = PotentialConstantsAA->getAssumedSet();
1574 if (AssumedSet.empty())
1575 return false;
1576
1577 OffsetInfo Product;
1578 for (const auto &ConstOffset : AssumedSet) {
1579 auto CopyPerOffset = Union;
1580 CopyPerOffset.addToAll(Inc: ConstOffset.getSExtValue() *
1581 VI.second.getZExtValue());
1582 Product.merge(R: CopyPerOffset);
1583 }
1584 Union = Product;
1585 }
1586
1587 UsrOI = std::move(Union);
1588 return true;
1589}
1590
1591ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
1592 using namespace AA::PointerInfo;
1593 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1594 const DataLayout &DL = A.getDataLayout();
1595 Value &AssociatedValue = getAssociatedValue();
1596
1597 DenseMap<Value *, OffsetInfo> OffsetInfoMap;
1598 OffsetInfoMap[&AssociatedValue].insert(Offset: 0);
1599
1600 auto HandlePassthroughUser = [&](Value *Usr, Value *CurPtr, bool &Follow) {
1601 // One does not simply walk into a map and assign a reference to a possibly
1602 // new location. That can cause an invalidation before the assignment
1603 // happens, like so:
1604 //
1605 // OffsetInfoMap[Usr] = OffsetInfoMap[CurPtr]; /* bad idea! */
1606 //
1607 // The RHS is a reference that may be invalidated by an insertion caused by
1608 // the LHS. So we ensure that the side-effect of the LHS happens first.
1609
1610 assert(OffsetInfoMap.contains(CurPtr) &&
1611 "CurPtr does not exist in the map!");
1612
1613 auto &UsrOI = OffsetInfoMap[Usr];
1614 auto &PtrOI = OffsetInfoMap[CurPtr];
1615 assert(!PtrOI.isUnassigned() &&
1616 "Cannot pass through if the input Ptr was not visited!");
1617 UsrOI.merge(R: PtrOI);
1618 Follow = true;
1619 return true;
1620 };
1621
1622 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
1623 Value *CurPtr = U.get();
1624 User *Usr = U.getUser();
1625 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Analyze " << *CurPtr << " in " << *Usr
1626 << "\n");
1627 assert(OffsetInfoMap.count(CurPtr) &&
1628 "The current pointer offset should have been seeded!");
1629 assert(!OffsetInfoMap[CurPtr].isUnassigned() &&
1630 "Current pointer should be assigned");
1631
1632 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: Usr)) {
1633 if (CE->isCast())
1634 return HandlePassthroughUser(Usr, CurPtr, Follow);
1635 if (!isa<GEPOperator>(Val: CE)) {
1636 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled constant user " << *CE
1637 << "\n");
1638 return false;
1639 }
1640 }
1641 if (auto *GEP = dyn_cast<GEPOperator>(Val: Usr)) {
1642 // Note the order here, the Usr access might change the map, CurPtr is
1643 // already in it though.
1644 auto &UsrOI = OffsetInfoMap[Usr];
1645 auto &PtrOI = OffsetInfoMap[CurPtr];
1646
1647 if (UsrOI.isUnknown())
1648 return true;
1649
1650 if (PtrOI.isUnknown()) {
1651 Follow = true;
1652 UsrOI.setUnknown();
1653 return true;
1654 }
1655
1656 Follow = collectConstantsForGEP(A, DL, UsrOI, PtrOI, GEP);
1657 return true;
1658 }
1659 if (isa<PtrToIntInst>(Val: Usr))
1660 return false;
1661 if (isa<CastInst>(Val: Usr) || isa<SelectInst>(Val: Usr))
1662 return HandlePassthroughUser(Usr, CurPtr, Follow);
1663 // Returns are allowed if they are in the associated functions. Users can
1664 // then check the call site return. Returns from other functions can't be
1665 // tracked and are cause for invalidation.
1666 if (auto *RI = dyn_cast<ReturnInst>(Val: Usr)) {
1667 if (RI->getFunction() == getAssociatedFunction()) {
1668 auto &PtrOI = OffsetInfoMap[CurPtr];
1669 Changed |= setReachesReturn(PtrOI);
1670 return true;
1671 }
1672 return false;
1673 }
1674
1675 // For PHIs we need to take care of the recurrence explicitly as the value
1676 // might change while we iterate through a loop. For now, we give up if
1677 // the PHI is not invariant.
1678 if (auto *PHI = dyn_cast<PHINode>(Val: Usr)) {
1679 // Note the order here, the Usr access might change the map, CurPtr is
1680 // already in it though.
1681 auto [PhiIt, IsFirstPHIUser] = OffsetInfoMap.try_emplace(Key: PHI);
1682 auto &UsrOI = PhiIt->second;
1683 auto &PtrOI = OffsetInfoMap[CurPtr];
1684
1685 // Check if the PHI operand has already an unknown offset as we can't
1686 // improve on that anymore.
1687 if (PtrOI.isUnknown()) {
1688 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand offset unknown "
1689 << *CurPtr << " in " << *PHI << "\n");
1690 Follow = !UsrOI.isUnknown();
1691 UsrOI.setUnknown();
1692 return true;
1693 }
1694
1695 // Check if the PHI is invariant (so far).
1696 if (UsrOI == PtrOI) {
1697 assert(!PtrOI.isUnassigned() &&
1698 "Cannot assign if the current Ptr was not visited!");
1699 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant (so far)");
1700 return true;
1701 }
1702
1703 // Check if the PHI operand can be traced back to AssociatedValue.
1704 APInt Offset(
1705 DL.getIndexSizeInBits(AS: CurPtr->getType()->getPointerAddressSpace()),
1706 0);
1707 Value *CurPtrBase = CurPtr->stripAndAccumulateConstantOffsets(
1708 DL, Offset, /* AllowNonInbounds */ true);
1709 auto It = OffsetInfoMap.find(Val: CurPtrBase);
1710 if (It == OffsetInfoMap.end()) {
1711 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand is too complex "
1712 << *CurPtr << " in " << *PHI
1713 << " (base: " << *CurPtrBase << ")\n");
1714 UsrOI.setUnknown();
1715 Follow = true;
1716 return true;
1717 }
1718
1719 // Check if the PHI operand is not dependent on the PHI itself. Every
1720 // recurrence is a cyclic net of PHIs in the data flow, and has an
1721 // equivalent Cycle in the control flow. One of those PHIs must be in the
1722 // header of that control flow Cycle. This is independent of the choice of
1723 // Cycles reported by CycleInfo. It is sufficient to check the PHIs in
1724 // every Cycle header; if such a node is marked unknown, this will
1725 // eventually propagate through the whole net of PHIs in the recurrence.
1726 const auto *CI =
1727 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
1728 F: *PHI->getFunction());
1729 if (mayBeInCycle(CI, I: cast<Instruction>(Val: Usr), /* HeaderOnly */ true)) {
1730 auto BaseOI = It->getSecond();
1731 BaseOI.addToAll(Inc: Offset.getZExtValue());
1732 if (IsFirstPHIUser || BaseOI == UsrOI) {
1733 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant " << *CurPtr
1734 << " in " << *Usr << "\n");
1735 return HandlePassthroughUser(Usr, CurPtr, Follow);
1736 }
1737
1738 LLVM_DEBUG(
1739 dbgs() << "[AAPointerInfo] PHI operand pointer offset mismatch "
1740 << *CurPtr << " in " << *PHI << "\n");
1741 UsrOI.setUnknown();
1742 Follow = true;
1743 return true;
1744 }
1745
1746 UsrOI.merge(R: PtrOI);
1747 Follow = true;
1748 return true;
1749 }
1750
1751 if (auto *LoadI = dyn_cast<LoadInst>(Val: Usr)) {
1752 // If the access is to a pointer that may or may not be the associated
1753 // value, e.g. due to a PHI, we cannot assume it will be read.
1754 AccessKind AK = AccessKind::AK_R;
1755 if (getUnderlyingObject(V: CurPtr) == &AssociatedValue)
1756 AK = AccessKind(AK | AccessKind::AK_MUST);
1757 else
1758 AK = AccessKind(AK | AccessKind::AK_MAY);
1759 if (!handleAccess(A, I&: *LoadI, /* Content */ nullptr, Kind: AK,
1760 Offsets&: OffsetInfoMap[CurPtr].Offsets, Changed,
1761 Ty&: *LoadI->getType()))
1762 return false;
1763
1764 auto IsAssumption = [](Instruction &I) {
1765 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
1766 return II->isAssumeLikeIntrinsic();
1767 return false;
1768 };
1769
1770 auto IsImpactedInRange = [&](Instruction *FromI, Instruction *ToI) {
1771 // Check if the assumption and the load are executed together without
1772 // memory modification.
1773 do {
1774 if (FromI->mayWriteToMemory() && !IsAssumption(*FromI))
1775 return true;
1776 FromI = FromI->getNextNode();
1777 } while (FromI && FromI != ToI);
1778 return false;
1779 };
1780
1781 BasicBlock *BB = LoadI->getParent();
1782 auto IsValidAssume = [&](IntrinsicInst &IntrI) {
1783 if (IntrI.getIntrinsicID() != Intrinsic::assume)
1784 return false;
1785 BasicBlock *IntrBB = IntrI.getParent();
1786 if (IntrI.getParent() == BB) {
1787 if (IsImpactedInRange(LoadI->getNextNode(), &IntrI))
1788 return false;
1789 } else {
1790 auto PredIt = pred_begin(BB: IntrBB);
1791 if (PredIt == pred_end(BB: IntrBB))
1792 return false;
1793 if ((*PredIt) != BB)
1794 return false;
1795 if (++PredIt != pred_end(BB: IntrBB))
1796 return false;
1797 for (auto *SuccBB : successors(BB)) {
1798 if (SuccBB == IntrBB)
1799 continue;
1800 if (isa<UnreachableInst>(Val: SuccBB->getTerminator()))
1801 continue;
1802 return false;
1803 }
1804 if (IsImpactedInRange(LoadI->getNextNode(), BB->getTerminator()))
1805 return false;
1806 if (IsImpactedInRange(&IntrBB->front(), &IntrI))
1807 return false;
1808 }
1809 return true;
1810 };
1811
1812 std::pair<Value *, IntrinsicInst *> Assumption;
1813 for (const Use &LoadU : LoadI->uses()) {
1814 if (auto *CmpI = dyn_cast<CmpInst>(Val: LoadU.getUser())) {
1815 if (!CmpI->isEquality() || !CmpI->isTrueWhenEqual())
1816 continue;
1817 for (const Use &CmpU : CmpI->uses()) {
1818 if (auto *IntrI = dyn_cast<IntrinsicInst>(Val: CmpU.getUser())) {
1819 if (!IsValidAssume(*IntrI))
1820 continue;
1821 int Idx = CmpI->getOperandUse(i: 0) == LoadU;
1822 Assumption = {CmpI->getOperand(i_nocapture: Idx), IntrI};
1823 break;
1824 }
1825 }
1826 }
1827 if (Assumption.first)
1828 break;
1829 }
1830
1831 // Check if we found an assumption associated with this load.
1832 if (!Assumption.first || !Assumption.second)
1833 return true;
1834
1835 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Assumption found "
1836 << *Assumption.second << ": " << *LoadI
1837 << " == " << *Assumption.first << "\n");
1838 bool UsedAssumedInformation = false;
1839 std::optional<Value *> Content = nullptr;
1840 if (Assumption.first)
1841 Content =
1842 A.getAssumedSimplified(V: *Assumption.first, AA: *this,
1843 UsedAssumedInformation, S: AA::Interprocedural);
1844 return handleAccess(
1845 A, I&: *Assumption.second, Content, Kind: AccessKind::AK_ASSUMPTION,
1846 Offsets&: OffsetInfoMap[CurPtr].Offsets, Changed, Ty&: *LoadI->getType());
1847 }
1848
1849 auto HandleStoreLike = [&](Instruction &I, Value *ValueOp, Type &ValueTy,
1850 ArrayRef<Value *> OtherOps, AccessKind AK) {
1851 for (auto *OtherOp : OtherOps) {
1852 if (OtherOp == CurPtr) {
1853 LLVM_DEBUG(
1854 dbgs()
1855 << "[AAPointerInfo] Escaping use in store like instruction " << I
1856 << "\n");
1857 return false;
1858 }
1859 }
1860
1861 // If the access is to a pointer that may or may not be the associated
1862 // value, e.g. due to a PHI, we cannot assume it will be written.
1863 if (getUnderlyingObject(V: CurPtr) == &AssociatedValue)
1864 AK = AccessKind(AK | AccessKind::AK_MUST);
1865 else
1866 AK = AccessKind(AK | AccessKind::AK_MAY);
1867 bool UsedAssumedInformation = false;
1868 std::optional<Value *> Content = nullptr;
1869 if (ValueOp)
1870 Content = A.getAssumedSimplified(
1871 V: *ValueOp, AA: *this, UsedAssumedInformation, S: AA::Interprocedural);
1872 return handleAccess(A, I, Content, Kind: AK, Offsets&: OffsetInfoMap[CurPtr].Offsets,
1873 Changed, Ty&: ValueTy);
1874 };
1875
1876 if (auto *StoreI = dyn_cast<StoreInst>(Val: Usr))
1877 return HandleStoreLike(*StoreI, StoreI->getValueOperand(),
1878 *StoreI->getValueOperand()->getType(),
1879 {StoreI->getValueOperand()}, AccessKind::AK_W);
1880 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: Usr))
1881 return HandleStoreLike(*RMWI, nullptr, *RMWI->getValOperand()->getType(),
1882 {RMWI->getValOperand()}, AccessKind::AK_RW);
1883 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: Usr))
1884 return HandleStoreLike(
1885 *CXI, nullptr, *CXI->getNewValOperand()->getType(),
1886 {CXI->getCompareOperand(), CXI->getNewValOperand()},
1887 AccessKind::AK_RW);
1888
1889 if (auto *CB = dyn_cast<CallBase>(Val: Usr)) {
1890 if (CB->isLifetimeStartOrEnd())
1891 return true;
1892 const auto *TLI =
1893 A.getInfoCache().getTargetLibraryInfoForFunction(F: *CB->getFunction());
1894 if (getFreedOperand(CB, TLI) == U)
1895 return true;
1896 if (CB->isArgOperand(U: &U)) {
1897 unsigned ArgNo = CB->getArgOperandNo(U: &U);
1898 const auto *CSArgPI = A.getAAFor<AAPointerInfo>(
1899 QueryingAA: *this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
1900 DepClass: DepClassTy::REQUIRED);
1901 if (!CSArgPI)
1902 return false;
1903 bool IsArgMustAcc = (getUnderlyingObject(V: CurPtr) == &AssociatedValue);
1904 Changed = translateAndAddState(A, OtherAA: *CSArgPI, Offsets: OffsetInfoMap[CurPtr], CB&: *CB,
1905 IsMustAcc: IsArgMustAcc) |
1906 Changed;
1907 if (!CSArgPI->reachesReturn())
1908 return isValidState();
1909
1910 Function *Callee = CB->getCalledFunction();
1911 if (!Callee || Callee->arg_size() <= ArgNo)
1912 return false;
1913 bool UsedAssumedInformation = false;
1914 auto ReturnedValue = A.getAssumedSimplified(
1915 IRP: IRPosition::returned(F: *Callee), AA: *this, UsedAssumedInformation,
1916 S: AA::ValueScope::Intraprocedural);
1917 auto *ReturnedArg =
1918 dyn_cast_or_null<Argument>(Val: ReturnedValue.value_or(u: nullptr));
1919 auto *Arg = Callee->getArg(i: ArgNo);
1920 if (ReturnedArg && Arg != ReturnedArg)
1921 return true;
1922 bool IsRetMustAcc = IsArgMustAcc && (ReturnedArg == Arg);
1923 const auto *CSRetPI = A.getAAFor<AAPointerInfo>(
1924 QueryingAA: *this, IRP: IRPosition::callsite_returned(CB: *CB), DepClass: DepClassTy::REQUIRED);
1925 if (!CSRetPI)
1926 return false;
1927 OffsetInfo OI = OffsetInfoMap[CurPtr];
1928 CSArgPI->addReturnedOffsetsTo(OI);
1929 Changed =
1930 translateAndAddState(A, OtherAA: *CSRetPI, Offsets: OI, CB&: *CB, IsMustAcc: IsRetMustAcc) | Changed;
1931 return isValidState();
1932 }
1933 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Call user not handled " << *CB
1934 << "\n");
1935 return false;
1936 }
1937
1938 LLVM_DEBUG(dbgs() << "[AAPointerInfo] User not handled " << *Usr << "\n");
1939 return false;
1940 };
1941 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
1942 assert(OffsetInfoMap.count(OldU) && "Old use should be known already!");
1943 assert(!OffsetInfoMap[OldU].isUnassigned() && "Old use should be assinged");
1944 if (OffsetInfoMap.count(Val: NewU)) {
1945 LLVM_DEBUG({
1946 if (!(OffsetInfoMap[NewU] == OffsetInfoMap[OldU])) {
1947 dbgs() << "[AAPointerInfo] Equivalent use callback failed: "
1948 << OffsetInfoMap[NewU] << " vs " << OffsetInfoMap[OldU]
1949 << "\n";
1950 }
1951 });
1952 return OffsetInfoMap[NewU] == OffsetInfoMap[OldU];
1953 }
1954 bool Unused;
1955 return HandlePassthroughUser(NewU.get(), OldU.get(), Unused);
1956 };
1957 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: AssociatedValue,
1958 /* CheckBBLivenessOnly */ true, LivenessDepClass: DepClassTy::OPTIONAL,
1959 /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
1960 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Check for all uses failed, abort!\n");
1961 return indicatePessimisticFixpoint();
1962 }
1963
1964 LLVM_DEBUG({
1965 dbgs() << "Accesses by bin after update:\n";
1966 dumpState(dbgs());
1967 });
1968
1969 return Changed;
1970}
1971
1972struct AAPointerInfoReturned final : AAPointerInfoImpl {
1973 AAPointerInfoReturned(const IRPosition &IRP, Attributor &A)
1974 : AAPointerInfoImpl(IRP, A) {}
1975
1976 /// See AbstractAttribute::updateImpl(...).
1977 ChangeStatus updateImpl(Attributor &A) override {
1978 return indicatePessimisticFixpoint();
1979 }
1980
1981 /// See AbstractAttribute::trackStatistics()
1982 void trackStatistics() const override {
1983 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1984 }
1985};
1986
1987struct AAPointerInfoArgument final : AAPointerInfoFloating {
1988 AAPointerInfoArgument(const IRPosition &IRP, Attributor &A)
1989 : AAPointerInfoFloating(IRP, A) {}
1990
1991 /// See AbstractAttribute::trackStatistics()
1992 void trackStatistics() const override {
1993 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1994 }
1995};
1996
1997struct AAPointerInfoCallSiteArgument final : AAPointerInfoFloating {
1998 AAPointerInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
1999 : AAPointerInfoFloating(IRP, A) {}
2000
2001 /// See AbstractAttribute::updateImpl(...).
2002 ChangeStatus updateImpl(Attributor &A) override {
2003 using namespace AA::PointerInfo;
2004 // We handle memory intrinsics explicitly, at least the first (=
2005 // destination) and second (=source) arguments as we know how they are
2006 // accessed.
2007 if (auto *MI = dyn_cast_or_null<MemIntrinsic>(Val: getCtxI())) {
2008 int64_t LengthVal = AA::RangeTy::Unknown;
2009 if (auto Length = MI->getLengthInBytes())
2010 LengthVal = Length->getSExtValue();
2011 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
2012 ChangeStatus Changed = ChangeStatus::UNCHANGED;
2013 if (ArgNo > 1) {
2014 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled memory intrinsic "
2015 << *MI << "\n");
2016 return indicatePessimisticFixpoint();
2017 } else {
2018 auto Kind =
2019 ArgNo == 0 ? AccessKind::AK_MUST_WRITE : AccessKind::AK_MUST_READ;
2020 Changed =
2021 Changed | addAccess(A, Ranges: {0, LengthVal}, I&: *MI, Content: nullptr, Kind, Ty: nullptr);
2022 }
2023 LLVM_DEBUG({
2024 dbgs() << "Accesses by bin after update:\n";
2025 dumpState(dbgs());
2026 });
2027
2028 return Changed;
2029 }
2030
2031 // TODO: Once we have call site specific value information we can provide
2032 // call site specific liveness information and then it makes
2033 // sense to specialize attributes for call sites arguments instead of
2034 // redirecting requests to the callee argument.
2035 Argument *Arg = getAssociatedArgument();
2036 if (Arg) {
2037 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
2038 auto *ArgAA =
2039 A.getAAFor<AAPointerInfo>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
2040 if (ArgAA && ArgAA->getState().isValidState())
2041 return translateAndAddStateFromCallee(A, OtherAA: *ArgAA,
2042 CB&: *cast<CallBase>(Val: getCtxI()));
2043 if (!Arg->getParent()->isDeclaration())
2044 return indicatePessimisticFixpoint();
2045 }
2046
2047 bool IsKnownNoCapture;
2048 if (!AA::hasAssumedIRAttr<Attribute::Captures>(
2049 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture))
2050 return indicatePessimisticFixpoint();
2051
2052 bool IsKnown = false;
2053 if (AA::isAssumedReadNone(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
2054 return ChangeStatus::UNCHANGED;
2055 bool ReadOnly = AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown);
2056 auto Kind =
2057 ReadOnly ? AccessKind::AK_MAY_READ : AccessKind::AK_MAY_READ_WRITE;
2058 return addAccess(A, Ranges: AA::RangeTy::getUnknown(), I&: *getCtxI(), Content: nullptr, Kind,
2059 Ty: nullptr);
2060 }
2061
2062 /// See AbstractAttribute::trackStatistics()
2063 void trackStatistics() const override {
2064 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
2065 }
2066};
2067
2068struct AAPointerInfoCallSiteReturned final : AAPointerInfoFloating {
2069 AAPointerInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
2070 : AAPointerInfoFloating(IRP, A) {}
2071
2072 /// See AbstractAttribute::trackStatistics()
2073 void trackStatistics() const override {
2074 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
2075 }
2076};
2077} // namespace
2078
2079/// -----------------------NoUnwind Function Attribute--------------------------
2080
2081namespace {
2082struct AANoUnwindImpl : AANoUnwind {
2083 AANoUnwindImpl(const IRPosition &IRP, Attributor &A) : AANoUnwind(IRP, A) {}
2084
2085 /// See AbstractAttribute::initialize(...).
2086 void initialize(Attributor &A) override {
2087 bool IsKnown;
2088 assert(!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
2089 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2090 (void)IsKnown;
2091 }
2092
2093 const std::string getAsStr(Attributor *A) const override {
2094 return getAssumed() ? "nounwind" : "may-unwind";
2095 }
2096
2097 /// See AbstractAttribute::updateImpl(...).
2098 ChangeStatus updateImpl(Attributor &A) override {
2099 auto Opcodes = {
2100 (unsigned)Instruction::Invoke, (unsigned)Instruction::CallBr,
2101 (unsigned)Instruction::Call, (unsigned)Instruction::CleanupRet,
2102 (unsigned)Instruction::CatchSwitch, (unsigned)Instruction::Resume};
2103
2104 auto CheckForNoUnwind = [&](Instruction &I) {
2105 if (!I.mayThrow(/* IncludePhaseOneUnwind */ true))
2106 return true;
2107
2108 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
2109 bool IsKnownNoUnwind;
2110 return AA::hasAssumedIRAttr<Attribute::NoUnwind>(
2111 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::REQUIRED,
2112 IsKnown&: IsKnownNoUnwind);
2113 }
2114 return false;
2115 };
2116
2117 bool UsedAssumedInformation = false;
2118 if (!A.checkForAllInstructions(Pred: CheckForNoUnwind, QueryingAA: *this, Opcodes,
2119 UsedAssumedInformation))
2120 return indicatePessimisticFixpoint();
2121
2122 return ChangeStatus::UNCHANGED;
2123 }
2124};
2125
2126struct AANoUnwindFunction final : public AANoUnwindImpl {
2127 AANoUnwindFunction(const IRPosition &IRP, Attributor &A)
2128 : AANoUnwindImpl(IRP, A) {}
2129
2130 /// See AbstractAttribute::trackStatistics()
2131 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nounwind) }
2132};
2133
2134/// NoUnwind attribute deduction for a call sites.
2135struct AANoUnwindCallSite final
2136 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl> {
2137 AANoUnwindCallSite(const IRPosition &IRP, Attributor &A)
2138 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl>(IRP, A) {}
2139
2140 /// See AbstractAttribute::trackStatistics()
2141 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nounwind); }
2142};
2143} // namespace
2144
2145/// ------------------------ NoSync Function Attribute -------------------------
2146
2147bool AANoSync::isAlignedBarrier(const CallBase &CB, bool ExecutedAligned) {
2148 switch (CB.getIntrinsicID()) {
2149 case Intrinsic::nvvm_barrier_cta_sync_aligned_all:
2150 case Intrinsic::nvvm_barrier_cta_sync_aligned_count:
2151 case Intrinsic::nvvm_barrier_cta_red_and_aligned_all:
2152 case Intrinsic::nvvm_barrier_cta_red_and_aligned_count:
2153 case Intrinsic::nvvm_barrier_cta_red_or_aligned_all:
2154 case Intrinsic::nvvm_barrier_cta_red_or_aligned_count:
2155 case Intrinsic::nvvm_barrier_cta_red_popc_aligned_all:
2156 case Intrinsic::nvvm_barrier_cta_red_popc_aligned_count:
2157 return true;
2158 case Intrinsic::amdgcn_s_barrier:
2159 if (ExecutedAligned)
2160 return true;
2161 break;
2162 default:
2163 break;
2164 }
2165 return hasAssumption(CB, AssumptionStr: KnownAssumptionString("ompx_aligned_barrier"));
2166}
2167
2168bool AANoSync::isNonRelaxedAtomic(const Instruction *I) {
2169 if (!I->isAtomic())
2170 return false;
2171
2172 if (auto *FI = dyn_cast<FenceInst>(Val: I))
2173 // All legal orderings for fence are stronger than monotonic.
2174 return FI->getSyncScopeID() != SyncScope::SingleThread;
2175 if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
2176 // Unordered is not a legal ordering for cmpxchg.
2177 return (AI->getSuccessOrdering() != AtomicOrdering::Monotonic ||
2178 AI->getFailureOrdering() != AtomicOrdering::Monotonic);
2179 }
2180
2181 AtomicOrdering Ordering;
2182 switch (I->getOpcode()) {
2183 case Instruction::AtomicRMW:
2184 Ordering = cast<AtomicRMWInst>(Val: I)->getOrdering();
2185 break;
2186 case Instruction::Store:
2187 Ordering = cast<StoreInst>(Val: I)->getOrdering();
2188 break;
2189 case Instruction::Load:
2190 Ordering = cast<LoadInst>(Val: I)->getOrdering();
2191 break;
2192 default:
2193 llvm_unreachable(
2194 "New atomic operations need to be known in the attributor.");
2195 }
2196
2197 return (Ordering != AtomicOrdering::Unordered &&
2198 Ordering != AtomicOrdering::Monotonic);
2199}
2200
2201/// Return true if this intrinsic is nosync. This is only used for intrinsics
2202/// which would be nosync except that they have a volatile flag. All other
2203/// intrinsics are simply annotated with the nosync attribute in Intrinsics.td.
2204bool AANoSync::isNoSyncIntrinsic(const Instruction *I) {
2205 if (auto *MI = dyn_cast<MemIntrinsic>(Val: I))
2206 return !MI->isVolatile();
2207 return false;
2208}
2209
2210namespace {
2211struct AANoSyncImpl : AANoSync {
2212 AANoSyncImpl(const IRPosition &IRP, Attributor &A) : AANoSync(IRP, A) {}
2213
2214 /// See AbstractAttribute::initialize(...).
2215 void initialize(Attributor &A) override {
2216 bool IsKnown;
2217 assert(!AA::hasAssumedIRAttr<Attribute::NoSync>(A, nullptr, getIRPosition(),
2218 DepClassTy::NONE, IsKnown));
2219 (void)IsKnown;
2220 }
2221
2222 const std::string getAsStr(Attributor *A) const override {
2223 return getAssumed() ? "nosync" : "may-sync";
2224 }
2225
2226 /// See AbstractAttribute::updateImpl(...).
2227 ChangeStatus updateImpl(Attributor &A) override;
2228};
2229
2230ChangeStatus AANoSyncImpl::updateImpl(Attributor &A) {
2231
2232 auto CheckRWInstForNoSync = [&](Instruction &I) {
2233 return AA::isNoSyncInst(A, I, QueryingAA: *this);
2234 };
2235
2236 auto CheckForNoSync = [&](Instruction &I) {
2237 // At this point we handled all read/write effects and they are all
2238 // nosync, so they can be skipped.
2239 if (I.mayReadOrWriteMemory())
2240 return true;
2241
2242 bool IsKnown;
2243 CallBase &CB = cast<CallBase>(Val&: I);
2244 if (AA::hasAssumedIRAttr<Attribute::NoSync>(
2245 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::OPTIONAL,
2246 IsKnown))
2247 return true;
2248
2249 // non-convergent and readnone imply nosync.
2250 return !CB.isConvergent();
2251 };
2252
2253 bool UsedAssumedInformation = false;
2254 if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInstForNoSync, QueryingAA&: *this,
2255 UsedAssumedInformation) ||
2256 !A.checkForAllCallLikeInstructions(Pred: CheckForNoSync, QueryingAA: *this,
2257 UsedAssumedInformation))
2258 return indicatePessimisticFixpoint();
2259
2260 return ChangeStatus::UNCHANGED;
2261}
2262
2263struct AANoSyncFunction final : public AANoSyncImpl {
2264 AANoSyncFunction(const IRPosition &IRP, Attributor &A)
2265 : AANoSyncImpl(IRP, A) {}
2266
2267 /// See AbstractAttribute::trackStatistics()
2268 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nosync) }
2269};
2270
2271/// NoSync attribute deduction for a call sites.
2272struct AANoSyncCallSite final : AACalleeToCallSite<AANoSync, AANoSyncImpl> {
2273 AANoSyncCallSite(const IRPosition &IRP, Attributor &A)
2274 : AACalleeToCallSite<AANoSync, AANoSyncImpl>(IRP, A) {}
2275
2276 /// See AbstractAttribute::trackStatistics()
2277 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nosync); }
2278};
2279} // namespace
2280
2281/// ------------------------ No-Free Attributes ----------------------------
2282
2283namespace {
2284struct AANoFreeImpl : public AANoFree {
2285 AANoFreeImpl(const IRPosition &IRP, Attributor &A) : AANoFree(IRP, A) {}
2286
2287 /// See AbstractAttribute::initialize(...).
2288 void initialize(Attributor &A) override {
2289 bool IsKnown;
2290 assert(!AA::hasAssumedIRAttr<Attribute::NoFree>(A, nullptr, getIRPosition(),
2291 DepClassTy::NONE, IsKnown));
2292 (void)IsKnown;
2293 }
2294
2295 /// See AbstractAttribute::updateImpl(...).
2296 ChangeStatus updateImpl(Attributor &A) override {
2297 auto CheckForNoFree = [&](Instruction &I) {
2298 bool IsKnown;
2299 return AA::hasAssumedIRAttr<Attribute::NoFree>(
2300 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB: cast<CallBase>(Val&: I)),
2301 DepClass: DepClassTy::REQUIRED, IsKnown);
2302 };
2303
2304 bool UsedAssumedInformation = false;
2305 if (!A.checkForAllCallLikeInstructions(Pred: CheckForNoFree, QueryingAA: *this,
2306 UsedAssumedInformation))
2307 return indicatePessimisticFixpoint();
2308 return ChangeStatus::UNCHANGED;
2309 }
2310
2311 /// See AbstractAttribute::getAsStr().
2312 const std::string getAsStr(Attributor *A) const override {
2313 return getAssumed() ? "nofree" : "may-free";
2314 }
2315};
2316
2317struct AANoFreeFunction final : public AANoFreeImpl {
2318 AANoFreeFunction(const IRPosition &IRP, Attributor &A)
2319 : AANoFreeImpl(IRP, A) {}
2320
2321 /// See AbstractAttribute::trackStatistics()
2322 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nofree) }
2323};
2324
2325/// NoFree attribute deduction for a call sites.
2326struct AANoFreeCallSite final : AACalleeToCallSite<AANoFree, AANoFreeImpl> {
2327 AANoFreeCallSite(const IRPosition &IRP, Attributor &A)
2328 : AACalleeToCallSite<AANoFree, AANoFreeImpl>(IRP, A) {}
2329
2330 /// See AbstractAttribute::trackStatistics()
2331 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nofree); }
2332};
2333
2334/// NoFree attribute for floating values.
2335struct AANoFreeFloating : AANoFreeImpl {
2336 AANoFreeFloating(const IRPosition &IRP, Attributor &A)
2337 : AANoFreeImpl(IRP, A) {}
2338
2339 /// See AbstractAttribute::trackStatistics()
2340 void trackStatistics() const override{STATS_DECLTRACK_FLOATING_ATTR(nofree)}
2341
2342 /// See Abstract Attribute::updateImpl(...).
2343 ChangeStatus updateImpl(Attributor &A) override {
2344 const IRPosition &IRP = getIRPosition();
2345
2346 bool IsKnown;
2347 if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, QueryingAA: this,
2348 IRP: IRPosition::function_scope(IRP),
2349 DepClass: DepClassTy::OPTIONAL, IsKnown))
2350 return ChangeStatus::UNCHANGED;
2351
2352 Value &AssociatedValue = getIRPosition().getAssociatedValue();
2353 auto Pred = [&](const Use &U, bool &Follow) -> bool {
2354 Instruction *UserI = cast<Instruction>(Val: U.getUser());
2355 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
2356 if (CB->isBundleOperand(U: &U))
2357 return false;
2358 if (!CB->isArgOperand(U: &U))
2359 return true;
2360 unsigned ArgNo = CB->getArgOperandNo(U: &U);
2361
2362 bool IsKnown;
2363 return AA::hasAssumedIRAttr<Attribute::NoFree>(
2364 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
2365 DepClass: DepClassTy::REQUIRED, IsKnown);
2366 }
2367
2368 if (isa<GetElementPtrInst>(Val: UserI) || isa<PHINode>(Val: UserI) ||
2369 isa<SelectInst>(Val: UserI)) {
2370 Follow = true;
2371 return true;
2372 }
2373 if (isa<StoreInst>(Val: UserI) || isa<LoadInst>(Val: UserI))
2374 return true;
2375
2376 if (isa<ReturnInst>(Val: UserI) && getIRPosition().isArgumentPosition())
2377 return true;
2378
2379 // Unknown user.
2380 return false;
2381 };
2382 if (!A.checkForAllUses(Pred, QueryingAA: *this, V: AssociatedValue))
2383 return indicatePessimisticFixpoint();
2384
2385 return ChangeStatus::UNCHANGED;
2386 }
2387};
2388
2389/// NoFree attribute for a call site argument.
2390struct AANoFreeArgument final : AANoFreeFloating {
2391 AANoFreeArgument(const IRPosition &IRP, Attributor &A)
2392 : AANoFreeFloating(IRP, A) {}
2393
2394 /// See AbstractAttribute::trackStatistics()
2395 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofree) }
2396};
2397
2398/// NoFree attribute for call site arguments.
2399struct AANoFreeCallSiteArgument final : AANoFreeFloating {
2400 AANoFreeCallSiteArgument(const IRPosition &IRP, Attributor &A)
2401 : AANoFreeFloating(IRP, A) {}
2402
2403 /// See AbstractAttribute::updateImpl(...).
2404 ChangeStatus updateImpl(Attributor &A) override {
2405 // TODO: Once we have call site specific value information we can provide
2406 // call site specific liveness information and then it makes
2407 // sense to specialize attributes for call sites arguments instead of
2408 // redirecting requests to the callee argument.
2409 Argument *Arg = getAssociatedArgument();
2410 if (!Arg)
2411 return indicatePessimisticFixpoint();
2412 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
2413 bool IsKnown;
2414 if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, QueryingAA: this, IRP: ArgPos,
2415 DepClass: DepClassTy::REQUIRED, IsKnown))
2416 return ChangeStatus::UNCHANGED;
2417 return indicatePessimisticFixpoint();
2418 }
2419
2420 /// See AbstractAttribute::trackStatistics()
2421 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nofree) };
2422};
2423
2424/// NoFree attribute for function return value.
2425struct AANoFreeReturned final : AANoFreeFloating {
2426 AANoFreeReturned(const IRPosition &IRP, Attributor &A)
2427 : AANoFreeFloating(IRP, A) {
2428 llvm_unreachable("NoFree is not applicable to function returns!");
2429 }
2430
2431 /// See AbstractAttribute::initialize(...).
2432 void initialize(Attributor &A) override {
2433 llvm_unreachable("NoFree is not applicable to function returns!");
2434 }
2435
2436 /// See AbstractAttribute::updateImpl(...).
2437 ChangeStatus updateImpl(Attributor &A) override {
2438 llvm_unreachable("NoFree is not applicable to function returns!");
2439 }
2440
2441 /// See AbstractAttribute::trackStatistics()
2442 void trackStatistics() const override {}
2443};
2444
2445/// NoFree attribute deduction for a call site return value.
2446struct AANoFreeCallSiteReturned final : AANoFreeFloating {
2447 AANoFreeCallSiteReturned(const IRPosition &IRP, Attributor &A)
2448 : AANoFreeFloating(IRP, A) {}
2449
2450 ChangeStatus manifest(Attributor &A) override {
2451 return ChangeStatus::UNCHANGED;
2452 }
2453 /// See AbstractAttribute::trackStatistics()
2454 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nofree) }
2455};
2456} // namespace
2457
2458/// ------------------------ NonNull Argument Attribute ------------------------
2459
2460bool AANonNull::isImpliedByIR(Attributor &A, const IRPosition &IRP,
2461 Attribute::AttrKind ImpliedAttributeKind,
2462 bool IgnoreSubsumingPositions) {
2463 SmallVector<Attribute::AttrKind, 2> AttrKinds;
2464 AttrKinds.push_back(Elt: Attribute::NonNull);
2465 if (!NullPointerIsDefined(F: IRP.getAnchorScope(),
2466 AS: IRP.getAssociatedType()->getPointerAddressSpace()))
2467 AttrKinds.push_back(Elt: Attribute::Dereferenceable);
2468 if (A.hasAttr(IRP, AKs: AttrKinds, IgnoreSubsumingPositions, ImpliedAttributeKind: Attribute::NonNull))
2469 return true;
2470
2471 DominatorTree *DT = nullptr;
2472 AssumptionCache *AC = nullptr;
2473 InformationCache &InfoCache = A.getInfoCache();
2474 if (const Function *Fn = IRP.getAnchorScope()) {
2475 if (!Fn->isDeclaration()) {
2476 DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *Fn);
2477 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *Fn);
2478 }
2479 }
2480
2481 SmallVector<AA::ValueAndContext> Worklist;
2482 if (IRP.getPositionKind() != IRP_RETURNED) {
2483 Worklist.push_back(Elt: {IRP.getAssociatedValue(), IRP.getCtxI()});
2484 } else {
2485 bool UsedAssumedInformation = false;
2486 if (!A.checkForAllInstructions(
2487 Pred: [&](Instruction &I) {
2488 Worklist.push_back(Elt: {*cast<ReturnInst>(Val&: I).getReturnValue(), &I});
2489 return true;
2490 },
2491 Fn: IRP.getAssociatedFunction(), QueryingAA: nullptr, Opcodes: {Instruction::Ret},
2492 UsedAssumedInformation, CheckBBLivenessOnly: false, /*CheckPotentiallyDead=*/true))
2493 return false;
2494 }
2495
2496 if (llvm::any_of(Range&: Worklist, P: [&](AA::ValueAndContext VAC) {
2497 return !isKnownNonZero(
2498 V: VAC.getValue(),
2499 Q: SimplifyQuery(A.getDataLayout(), DT, AC, VAC.getCtxI()));
2500 }))
2501 return false;
2502
2503 A.manifestAttrs(IRP, DeducedAttrs: {Attribute::get(Context&: IRP.getAnchorValue().getContext(),
2504 Kind: Attribute::NonNull)});
2505 return true;
2506}
2507
2508namespace {
2509static int64_t getKnownNonNullAndDerefBytesForUse(
2510 Attributor &A, const AbstractAttribute &QueryingAA, Value &AssociatedValue,
2511 const Use *U, const Instruction *I, bool &IsNonNull, bool &TrackUse) {
2512 TrackUse = false;
2513
2514 const Value *UseV = U->get();
2515 if (!UseV->getType()->isPointerTy())
2516 return 0;
2517
2518 // We need to follow common pointer manipulation uses to the accesses they
2519 // feed into. We can try to be smart to avoid looking through things we do not
2520 // like for now, e.g., non-inbounds GEPs.
2521 if (isa<CastInst>(Val: I)) {
2522 TrackUse = true;
2523 return 0;
2524 }
2525
2526 if (isa<GetElementPtrInst>(Val: I)) {
2527 TrackUse = true;
2528 return 0;
2529 }
2530
2531 Type *PtrTy = UseV->getType();
2532 const Function *F = I->getFunction();
2533 bool NullPointerIsDefined =
2534 F ? llvm::NullPointerIsDefined(F, AS: PtrTy->getPointerAddressSpace()) : true;
2535 const DataLayout &DL = A.getInfoCache().getDL();
2536 if (const auto *CB = dyn_cast<CallBase>(Val: I)) {
2537 if (CB->isBundleOperand(U)) {
2538 if (RetainedKnowledge RK = getKnowledgeFromUse(
2539 U, AttrKinds: {Attribute::NonNull, Attribute::Dereferenceable})) {
2540 IsNonNull |=
2541 (RK.AttrKind == Attribute::NonNull || !NullPointerIsDefined);
2542 return RK.ArgValue;
2543 }
2544 return 0;
2545 }
2546
2547 if (CB->isCallee(U)) {
2548 IsNonNull |= !NullPointerIsDefined;
2549 return 0;
2550 }
2551
2552 unsigned ArgNo = CB->getArgOperandNo(U);
2553 IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
2554 // As long as we only use known information there is no need to track
2555 // dependences here.
2556 bool IsKnownNonNull;
2557 AA::hasAssumedIRAttr<Attribute::NonNull>(A, QueryingAA: &QueryingAA, IRP,
2558 DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
2559 IsNonNull |= IsKnownNonNull;
2560 auto *DerefAA =
2561 A.getAAFor<AADereferenceable>(QueryingAA, IRP, DepClass: DepClassTy::NONE);
2562 return DerefAA ? DerefAA->getKnownDereferenceableBytes() : 0;
2563 }
2564
2565 std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(Inst: I);
2566 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() ||
2567 Loc->Size.isScalable() || I->isVolatile())
2568 return 0;
2569
2570 int64_t Offset;
2571 const Value *Base =
2572 getMinimalBaseOfPointer(A, QueryingAA, Ptr: Loc->Ptr, BytesOffset&: Offset, DL);
2573 if (Base && Base == &AssociatedValue) {
2574 int64_t DerefBytes = Loc->Size.getValue() + Offset;
2575 IsNonNull |= !NullPointerIsDefined;
2576 return std::max(a: int64_t(0), b: DerefBytes);
2577 }
2578
2579 /// Corner case when an offset is 0.
2580 Base = GetPointerBaseWithConstantOffset(Ptr: Loc->Ptr, Offset, DL,
2581 /*AllowNonInbounds*/ true);
2582 if (Base && Base == &AssociatedValue && Offset == 0) {
2583 int64_t DerefBytes = Loc->Size.getValue();
2584 IsNonNull |= !NullPointerIsDefined;
2585 return std::max(a: int64_t(0), b: DerefBytes);
2586 }
2587
2588 return 0;
2589}
2590
2591struct AANonNullImpl : AANonNull {
2592 AANonNullImpl(const IRPosition &IRP, Attributor &A) : AANonNull(IRP, A) {}
2593
2594 /// See AbstractAttribute::initialize(...).
2595 void initialize(Attributor &A) override {
2596 Value &V = *getAssociatedValue().stripPointerCasts();
2597 if (isa<ConstantPointerNull>(Val: V)) {
2598 indicatePessimisticFixpoint();
2599 return;
2600 }
2601
2602 if (Instruction *CtxI = getCtxI())
2603 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
2604 }
2605
2606 /// See followUsesInMBEC
2607 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
2608 AANonNull::StateType &State) {
2609 bool IsNonNull = false;
2610 bool TrackUse = false;
2611 getKnownNonNullAndDerefBytesForUse(A, QueryingAA: *this, AssociatedValue&: getAssociatedValue(), U, I,
2612 IsNonNull, TrackUse);
2613 State.setKnown(IsNonNull);
2614 return TrackUse;
2615 }
2616
2617 /// See AbstractAttribute::getAsStr().
2618 const std::string getAsStr(Attributor *A) const override {
2619 return getAssumed() ? "nonnull" : "may-null";
2620 }
2621};
2622
2623/// NonNull attribute for a floating value.
2624struct AANonNullFloating : public AANonNullImpl {
2625 AANonNullFloating(const IRPosition &IRP, Attributor &A)
2626 : AANonNullImpl(IRP, A) {}
2627
2628 /// See AbstractAttribute::updateImpl(...).
2629 ChangeStatus updateImpl(Attributor &A) override {
2630 auto CheckIRP = [&](const IRPosition &IRP) {
2631 bool IsKnownNonNull;
2632 return AA::hasAssumedIRAttr<Attribute::NonNull>(
2633 A, QueryingAA: *this, IRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNonNull);
2634 };
2635
2636 bool Stripped;
2637 bool UsedAssumedInformation = false;
2638 Value *AssociatedValue = &getAssociatedValue();
2639 SmallVector<AA::ValueAndContext> Values;
2640 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
2641 S: AA::AnyScope, UsedAssumedInformation))
2642 Stripped = false;
2643 else
2644 Stripped =
2645 Values.size() != 1 || Values.front().getValue() != AssociatedValue;
2646
2647 if (!Stripped) {
2648 bool IsKnown;
2649 if (auto *PHI = dyn_cast<PHINode>(Val: AssociatedValue))
2650 if (llvm::all_of(Range: PHI->incoming_values(), P: [&](Value *Op) {
2651 return AA::hasAssumedIRAttr<Attribute::NonNull>(
2652 A, QueryingAA: this, IRP: IRPosition::value(V: *Op), DepClass: DepClassTy::OPTIONAL,
2653 IsKnown);
2654 }))
2655 return ChangeStatus::UNCHANGED;
2656 if (auto *Select = dyn_cast<SelectInst>(Val: AssociatedValue))
2657 if (AA::hasAssumedIRAttr<Attribute::NonNull>(
2658 A, QueryingAA: this, IRP: IRPosition::value(V: *Select->getFalseValue()),
2659 DepClass: DepClassTy::OPTIONAL, IsKnown) &&
2660 AA::hasAssumedIRAttr<Attribute::NonNull>(
2661 A, QueryingAA: this, IRP: IRPosition::value(V: *Select->getTrueValue()),
2662 DepClass: DepClassTy::OPTIONAL, IsKnown))
2663 return ChangeStatus::UNCHANGED;
2664
2665 // If we haven't stripped anything we might still be able to use a
2666 // different AA, but only if the IRP changes. Effectively when we
2667 // interpret this not as a call site value but as a floating/argument
2668 // value.
2669 const IRPosition AVIRP = IRPosition::value(V: *AssociatedValue);
2670 if (AVIRP == getIRPosition() || !CheckIRP(AVIRP))
2671 return indicatePessimisticFixpoint();
2672 return ChangeStatus::UNCHANGED;
2673 }
2674
2675 for (const auto &VAC : Values)
2676 if (!CheckIRP(IRPosition::value(V: *VAC.getValue())))
2677 return indicatePessimisticFixpoint();
2678
2679 return ChangeStatus::UNCHANGED;
2680 }
2681
2682 /// See AbstractAttribute::trackStatistics()
2683 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2684};
2685
2686/// NonNull attribute for function return value.
2687struct AANonNullReturned final
2688 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2689 false, AANonNull::IRAttributeKind, false> {
2690 AANonNullReturned(const IRPosition &IRP, Attributor &A)
2691 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2692 false, Attribute::NonNull, false>(IRP, A) {
2693 }
2694
2695 /// See AbstractAttribute::getAsStr().
2696 const std::string getAsStr(Attributor *A) const override {
2697 return getAssumed() ? "nonnull" : "may-null";
2698 }
2699
2700 /// See AbstractAttribute::trackStatistics()
2701 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2702};
2703
2704/// NonNull attribute for function argument.
2705struct AANonNullArgument final
2706 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl> {
2707 AANonNullArgument(const IRPosition &IRP, Attributor &A)
2708 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl>(IRP, A) {}
2709
2710 /// See AbstractAttribute::trackStatistics()
2711 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nonnull) }
2712};
2713
2714struct AANonNullCallSiteArgument final : AANonNullFloating {
2715 AANonNullCallSiteArgument(const IRPosition &IRP, Attributor &A)
2716 : AANonNullFloating(IRP, A) {}
2717
2718 /// See AbstractAttribute::trackStatistics()
2719 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nonnull) }
2720};
2721
2722/// NonNull attribute for a call site return position.
2723struct AANonNullCallSiteReturned final
2724 : AACalleeToCallSite<AANonNull, AANonNullImpl> {
2725 AANonNullCallSiteReturned(const IRPosition &IRP, Attributor &A)
2726 : AACalleeToCallSite<AANonNull, AANonNullImpl>(IRP, A) {}
2727
2728 /// See AbstractAttribute::trackStatistics()
2729 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nonnull) }
2730};
2731} // namespace
2732
2733/// ------------------------ Must-Progress Attributes --------------------------
2734namespace {
2735struct AAMustProgressImpl : public AAMustProgress {
2736 AAMustProgressImpl(const IRPosition &IRP, Attributor &A)
2737 : AAMustProgress(IRP, A) {}
2738
2739 /// See AbstractAttribute::initialize(...).
2740 void initialize(Attributor &A) override {
2741 bool IsKnown;
2742 assert(!AA::hasAssumedIRAttr<Attribute::MustProgress>(
2743 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2744 (void)IsKnown;
2745 }
2746
2747 /// See AbstractAttribute::getAsStr()
2748 const std::string getAsStr(Attributor *A) const override {
2749 return getAssumed() ? "mustprogress" : "may-not-progress";
2750 }
2751};
2752
2753struct AAMustProgressFunction final : AAMustProgressImpl {
2754 AAMustProgressFunction(const IRPosition &IRP, Attributor &A)
2755 : AAMustProgressImpl(IRP, A) {}
2756
2757 /// See AbstractAttribute::updateImpl(...).
2758 ChangeStatus updateImpl(Attributor &A) override {
2759 bool IsKnown;
2760 if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
2761 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown)) {
2762 if (IsKnown)
2763 return indicateOptimisticFixpoint();
2764 return ChangeStatus::UNCHANGED;
2765 }
2766
2767 auto CheckForMustProgress = [&](AbstractCallSite ACS) {
2768 IRPosition IPos = IRPosition::callsite_function(CB: *ACS.getInstruction());
2769 bool IsKnownMustProgress;
2770 return AA::hasAssumedIRAttr<Attribute::MustProgress>(
2771 A, QueryingAA: this, IRP: IPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownMustProgress,
2772 /* IgnoreSubsumingPositions */ true);
2773 };
2774
2775 bool AllCallSitesKnown = true;
2776 if (!A.checkForAllCallSites(Pred: CheckForMustProgress, QueryingAA: *this,
2777 /* RequireAllCallSites */ true,
2778 UsedAssumedInformation&: AllCallSitesKnown))
2779 return indicatePessimisticFixpoint();
2780
2781 return ChangeStatus::UNCHANGED;
2782 }
2783
2784 /// See AbstractAttribute::trackStatistics()
2785 void trackStatistics() const override {
2786 STATS_DECLTRACK_FN_ATTR(mustprogress)
2787 }
2788};
2789
2790/// MustProgress attribute deduction for a call sites.
2791struct AAMustProgressCallSite final : AAMustProgressImpl {
2792 AAMustProgressCallSite(const IRPosition &IRP, Attributor &A)
2793 : AAMustProgressImpl(IRP, A) {}
2794
2795 /// See AbstractAttribute::updateImpl(...).
2796 ChangeStatus updateImpl(Attributor &A) override {
2797 // TODO: Once we have call site specific value information we can provide
2798 // call site specific liveness information and then it makes
2799 // sense to specialize attributes for call sites arguments instead of
2800 // redirecting requests to the callee argument.
2801 const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
2802 bool IsKnownMustProgress;
2803 if (!AA::hasAssumedIRAttr<Attribute::MustProgress>(
2804 A, QueryingAA: this, IRP: FnPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownMustProgress))
2805 return indicatePessimisticFixpoint();
2806 return ChangeStatus::UNCHANGED;
2807 }
2808
2809 /// See AbstractAttribute::trackStatistics()
2810 void trackStatistics() const override {
2811 STATS_DECLTRACK_CS_ATTR(mustprogress);
2812 }
2813};
2814} // namespace
2815
2816/// ------------------------ No-Recurse Attributes ----------------------------
2817
2818namespace {
2819struct AANoRecurseImpl : public AANoRecurse {
2820 AANoRecurseImpl(const IRPosition &IRP, Attributor &A) : AANoRecurse(IRP, A) {}
2821
2822 /// See AbstractAttribute::initialize(...).
2823 void initialize(Attributor &A) override {
2824 bool IsKnown;
2825 assert(!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
2826 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2827 (void)IsKnown;
2828 }
2829
2830 /// See AbstractAttribute::getAsStr()
2831 const std::string getAsStr(Attributor *A) const override {
2832 return getAssumed() ? "norecurse" : "may-recurse";
2833 }
2834};
2835
2836struct AANoRecurseFunction final : AANoRecurseImpl {
2837 AANoRecurseFunction(const IRPosition &IRP, Attributor &A)
2838 : AANoRecurseImpl(IRP, A) {}
2839
2840 /// See AbstractAttribute::updateImpl(...).
2841 ChangeStatus updateImpl(Attributor &A) override {
2842
2843 // If all live call sites are known to be no-recurse, we are as well.
2844 auto CallSitePred = [&](AbstractCallSite ACS) {
2845 bool IsKnownNoRecurse;
2846 if (!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
2847 A, QueryingAA: this,
2848 IRP: IRPosition::function(F: *ACS.getInstruction()->getFunction()),
2849 DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoRecurse))
2850 return false;
2851 return IsKnownNoRecurse;
2852 };
2853 bool UsedAssumedInformation = false;
2854 if (A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this, RequireAllCallSites: true,
2855 UsedAssumedInformation)) {
2856 // If we know all call sites and all are known no-recurse, we are done.
2857 // If all known call sites, which might not be all that exist, are known
2858 // to be no-recurse, we are not done but we can continue to assume
2859 // no-recurse. If one of the call sites we have not visited will become
2860 // live, another update is triggered.
2861 if (!UsedAssumedInformation)
2862 indicateOptimisticFixpoint();
2863 return ChangeStatus::UNCHANGED;
2864 }
2865
2866 const AAInterFnReachability *EdgeReachability =
2867 A.getAAFor<AAInterFnReachability>(QueryingAA: *this, IRP: getIRPosition(),
2868 DepClass: DepClassTy::REQUIRED);
2869 if (EdgeReachability && EdgeReachability->canReach(A, Fn: *getAnchorScope()))
2870 return indicatePessimisticFixpoint();
2871 return ChangeStatus::UNCHANGED;
2872 }
2873
2874 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(norecurse) }
2875};
2876
2877/// NoRecurse attribute deduction for a call sites.
2878struct AANoRecurseCallSite final
2879 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl> {
2880 AANoRecurseCallSite(const IRPosition &IRP, Attributor &A)
2881 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl>(IRP, A) {}
2882
2883 /// See AbstractAttribute::trackStatistics()
2884 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(norecurse); }
2885};
2886} // namespace
2887
2888/// ------------------------ No-Convergent Attribute --------------------------
2889
2890namespace {
2891struct AANonConvergentImpl : public AANonConvergent {
2892 AANonConvergentImpl(const IRPosition &IRP, Attributor &A)
2893 : AANonConvergent(IRP, A) {}
2894
2895 /// See AbstractAttribute::getAsStr()
2896 const std::string getAsStr(Attributor *A) const override {
2897 return getAssumed() ? "non-convergent" : "may-be-convergent";
2898 }
2899};
2900
2901struct AANonConvergentFunction final : AANonConvergentImpl {
2902 AANonConvergentFunction(const IRPosition &IRP, Attributor &A)
2903 : AANonConvergentImpl(IRP, A) {}
2904
2905 /// See AbstractAttribute::updateImpl(...).
2906 ChangeStatus updateImpl(Attributor &A) override {
2907 // If all function calls are known to not be convergent, we are not
2908 // convergent.
2909 auto CalleeIsNotConvergent = [&](Instruction &Inst) {
2910 CallBase &CB = cast<CallBase>(Val&: Inst);
2911 auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand());
2912 if (!Callee || Callee->isIntrinsic()) {
2913 return false;
2914 }
2915 if (Callee->isDeclaration()) {
2916 return !Callee->hasFnAttribute(Kind: Attribute::Convergent);
2917 }
2918 const auto *ConvergentAA = A.getAAFor<AANonConvergent>(
2919 QueryingAA: *this, IRP: IRPosition::function(F: *Callee), DepClass: DepClassTy::REQUIRED);
2920 return ConvergentAA && ConvergentAA->isAssumedNotConvergent();
2921 };
2922
2923 bool UsedAssumedInformation = false;
2924 if (!A.checkForAllCallLikeInstructions(Pred: CalleeIsNotConvergent, QueryingAA: *this,
2925 UsedAssumedInformation)) {
2926 return indicatePessimisticFixpoint();
2927 }
2928 return ChangeStatus::UNCHANGED;
2929 }
2930
2931 ChangeStatus manifest(Attributor &A) override {
2932 if (isKnownNotConvergent() &&
2933 A.hasAttr(IRP: getIRPosition(), AKs: Attribute::Convergent)) {
2934 A.removeAttrs(IRP: getIRPosition(), AttrKinds: {Attribute::Convergent});
2935 return ChangeStatus::CHANGED;
2936 }
2937 return ChangeStatus::UNCHANGED;
2938 }
2939
2940 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(convergent) }
2941};
2942} // namespace
2943
2944/// -------------------- Undefined-Behavior Attributes ------------------------
2945
2946namespace {
2947struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior {
2948 AAUndefinedBehaviorImpl(const IRPosition &IRP, Attributor &A)
2949 : AAUndefinedBehavior(IRP, A) {}
2950
2951 /// See AbstractAttribute::updateImpl(...).
2952 // through a pointer (i.e. also branches etc.)
2953 ChangeStatus updateImpl(Attributor &A) override {
2954 const size_t UBPrevSize = KnownUBInsts.size();
2955 const size_t NoUBPrevSize = AssumedNoUBInsts.size();
2956
2957 auto InspectMemAccessInstForUB = [&](Instruction &I) {
2958 // Lang ref now states volatile store is not UB, let's skip them.
2959 if (I.isVolatile() && I.mayWriteToMemory())
2960 return true;
2961
2962 // Skip instructions that are already saved.
2963 if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Ptr: &I))
2964 return true;
2965
2966 // If we reach here, we know we have an instruction
2967 // that accesses memory through a pointer operand,
2968 // for which getPointerOperand() should give it to us.
2969 Value *PtrOp =
2970 const_cast<Value *>(getPointerOperand(I: &I, /* AllowVolatile */ true));
2971 assert(PtrOp &&
2972 "Expected pointer operand of memory accessing instruction");
2973
2974 // Either we stopped and the appropriate action was taken,
2975 // or we got back a simplified value to continue.
2976 std::optional<Value *> SimplifiedPtrOp =
2977 stopOnUndefOrAssumed(A, V: PtrOp, I: &I);
2978 if (!SimplifiedPtrOp || !*SimplifiedPtrOp)
2979 return true;
2980 const Value *PtrOpVal = *SimplifiedPtrOp;
2981
2982 // A memory access through a pointer is considered UB
2983 // only if the pointer has constant null value.
2984 // TODO: Expand it to not only check constant values.
2985 if (!isa<ConstantPointerNull>(Val: PtrOpVal)) {
2986 AssumedNoUBInsts.insert(Ptr: &I);
2987 return true;
2988 }
2989 const Type *PtrTy = PtrOpVal->getType();
2990
2991 // Because we only consider instructions inside functions,
2992 // assume that a parent function exists.
2993 const Function *F = I.getFunction();
2994
2995 // A memory access using constant null pointer is only considered UB
2996 // if null pointer is _not_ defined for the target platform.
2997 if (llvm::NullPointerIsDefined(F, AS: PtrTy->getPointerAddressSpace()))
2998 AssumedNoUBInsts.insert(Ptr: &I);
2999 else
3000 KnownUBInsts.insert(Ptr: &I);
3001 return true;
3002 };
3003
3004 auto InspectBrInstForUB = [&](Instruction &I) {
3005 // A conditional branch instruction is considered UB if it has `undef`
3006 // condition.
3007
3008 // Skip instructions that are already saved.
3009 if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Ptr: &I))
3010 return true;
3011
3012 // We know we have a branch instruction.
3013 auto *BrInst = cast<CondBrInst>(Val: &I);
3014
3015 // Either we stopped and the appropriate action was taken,
3016 // or we got back a simplified value to continue.
3017 std::optional<Value *> SimplifiedCond =
3018 stopOnUndefOrAssumed(A, V: BrInst->getCondition(), I: BrInst);
3019 if (!SimplifiedCond || !*SimplifiedCond)
3020 return true;
3021 AssumedNoUBInsts.insert(Ptr: &I);
3022 return true;
3023 };
3024
3025 auto InspectCallSiteForUB = [&](Instruction &I) {
3026 // Check whether a callsite always cause UB or not
3027
3028 // Skip instructions that are already saved.
3029 if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Ptr: &I))
3030 return true;
3031
3032 // Check nonnull and noundef argument attribute violation for each
3033 // callsite.
3034 CallBase &CB = cast<CallBase>(Val&: I);
3035 auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand());
3036 if (!Callee)
3037 return true;
3038 for (unsigned idx = 0; idx < CB.arg_size(); idx++) {
3039 // If current argument is known to be simplified to null pointer and the
3040 // corresponding argument position is known to have nonnull attribute,
3041 // the argument is poison. Furthermore, if the argument is poison and
3042 // the position is known to have noundef attriubte, this callsite is
3043 // considered UB.
3044 if (idx >= Callee->arg_size())
3045 break;
3046 Value *ArgVal = CB.getArgOperand(i: idx);
3047 if (!ArgVal)
3048 continue;
3049 // Here, we handle three cases.
3050 // (1) Not having a value means it is dead. (we can replace the value
3051 // with undef)
3052 // (2) Simplified to undef. The argument violate noundef attriubte.
3053 // (3) Simplified to null pointer where known to be nonnull.
3054 // The argument is a poison value and violate noundef attribute.
3055 IRPosition CalleeArgumentIRP = IRPosition::callsite_argument(CB, ArgNo: idx);
3056 bool IsKnownNoUndef;
3057 AA::hasAssumedIRAttr<Attribute::NoUndef>(
3058 A, QueryingAA: this, IRP: CalleeArgumentIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoUndef);
3059 if (!IsKnownNoUndef)
3060 continue;
3061 bool UsedAssumedInformation = false;
3062 std::optional<Value *> SimplifiedVal =
3063 A.getAssumedSimplified(IRP: IRPosition::value(V: *ArgVal), AA: *this,
3064 UsedAssumedInformation, S: AA::Interprocedural);
3065 if (UsedAssumedInformation)
3066 continue;
3067 if (SimplifiedVal && !*SimplifiedVal)
3068 return true;
3069 if (!SimplifiedVal || isa<UndefValue>(Val: **SimplifiedVal)) {
3070 KnownUBInsts.insert(Ptr: &I);
3071 continue;
3072 }
3073 if (!ArgVal->getType()->isPointerTy() ||
3074 !isa<ConstantPointerNull>(Val: **SimplifiedVal))
3075 continue;
3076 bool IsKnownNonNull;
3077 AA::hasAssumedIRAttr<Attribute::NonNull>(
3078 A, QueryingAA: this, IRP: CalleeArgumentIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
3079 if (IsKnownNonNull)
3080 KnownUBInsts.insert(Ptr: &I);
3081 }
3082 return true;
3083 };
3084
3085 auto InspectReturnInstForUB = [&](Instruction &I) {
3086 auto &RI = cast<ReturnInst>(Val&: I);
3087 // Either we stopped and the appropriate action was taken,
3088 // or we got back a simplified return value to continue.
3089 std::optional<Value *> SimplifiedRetValue =
3090 stopOnUndefOrAssumed(A, V: RI.getReturnValue(), I: &I);
3091 if (!SimplifiedRetValue || !*SimplifiedRetValue)
3092 return true;
3093
3094 // Check if a return instruction always cause UB or not
3095 // Note: It is guaranteed that the returned position of the anchor
3096 // scope has noundef attribute when this is called.
3097 // We also ensure the return position is not "assumed dead"
3098 // because the returned value was then potentially simplified to
3099 // `undef` in AAReturnedValues without removing the `noundef`
3100 // attribute yet.
3101
3102 // When the returned position has noundef attriubte, UB occurs in the
3103 // following cases.
3104 // (1) Returned value is known to be undef.
3105 // (2) The value is known to be a null pointer and the returned
3106 // position has nonnull attribute (because the returned value is
3107 // poison).
3108 if (isa<ConstantPointerNull>(Val: *SimplifiedRetValue)) {
3109 bool IsKnownNonNull;
3110 AA::hasAssumedIRAttr<Attribute::NonNull>(
3111 A, QueryingAA: this, IRP: IRPosition::returned(F: *getAnchorScope()), DepClass: DepClassTy::NONE,
3112 IsKnown&: IsKnownNonNull);
3113 if (IsKnownNonNull)
3114 KnownUBInsts.insert(Ptr: &I);
3115 }
3116
3117 return true;
3118 };
3119
3120 bool UsedAssumedInformation = false;
3121 A.checkForAllInstructions(Pred: InspectMemAccessInstForUB, QueryingAA: *this,
3122 Opcodes: {Instruction::Load, Instruction::Store,
3123 Instruction::AtomicCmpXchg,
3124 Instruction::AtomicRMW},
3125 UsedAssumedInformation,
3126 /* CheckBBLivenessOnly */ true);
3127 A.checkForAllInstructions(Pred: InspectBrInstForUB, QueryingAA: *this, Opcodes: {Instruction::CondBr},
3128 UsedAssumedInformation,
3129 /* CheckBBLivenessOnly */ true);
3130 A.checkForAllCallLikeInstructions(Pred: InspectCallSiteForUB, QueryingAA: *this,
3131 UsedAssumedInformation);
3132
3133 // If the returned position of the anchor scope has noundef attriubte, check
3134 // all returned instructions.
3135 if (!getAnchorScope()->getReturnType()->isVoidTy()) {
3136 const IRPosition &ReturnIRP = IRPosition::returned(F: *getAnchorScope());
3137 if (!A.isAssumedDead(IRP: ReturnIRP, QueryingAA: this, FnLivenessAA: nullptr, UsedAssumedInformation)) {
3138 bool IsKnownNoUndef;
3139 AA::hasAssumedIRAttr<Attribute::NoUndef>(
3140 A, QueryingAA: this, IRP: ReturnIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoUndef);
3141 if (IsKnownNoUndef)
3142 A.checkForAllInstructions(Pred: InspectReturnInstForUB, QueryingAA: *this,
3143 Opcodes: {Instruction::Ret}, UsedAssumedInformation,
3144 /* CheckBBLivenessOnly */ true);
3145 }
3146 }
3147
3148 if (NoUBPrevSize != AssumedNoUBInsts.size() ||
3149 UBPrevSize != KnownUBInsts.size())
3150 return ChangeStatus::CHANGED;
3151 return ChangeStatus::UNCHANGED;
3152 }
3153
3154 bool isKnownToCauseUB(Instruction *I) const override {
3155 return KnownUBInsts.count(Ptr: I);
3156 }
3157
3158 bool isAssumedToCauseUB(Instruction *I) const override {
3159 // In simple words, if an instruction is not in the assumed to _not_
3160 // cause UB, then it is assumed UB (that includes those
3161 // in the KnownUBInsts set). The rest is boilerplate
3162 // is to ensure that it is one of the instructions we test
3163 // for UB.
3164
3165 switch (I->getOpcode()) {
3166 case Instruction::Load:
3167 case Instruction::Store:
3168 case Instruction::AtomicCmpXchg:
3169 case Instruction::AtomicRMW:
3170 case Instruction::CondBr:
3171 return !AssumedNoUBInsts.count(Ptr: I);
3172 default:
3173 return false;
3174 }
3175 return false;
3176 }
3177
3178 ChangeStatus manifest(Attributor &A) override {
3179 if (KnownUBInsts.empty())
3180 return ChangeStatus::UNCHANGED;
3181 for (Instruction *I : KnownUBInsts)
3182 A.changeToUnreachableAfterManifest(I);
3183 return ChangeStatus::CHANGED;
3184 }
3185
3186 /// See AbstractAttribute::getAsStr()
3187 const std::string getAsStr(Attributor *A) const override {
3188 return getAssumed() ? "undefined-behavior" : "no-ub";
3189 }
3190
3191 /// Note: The correctness of this analysis depends on the fact that the
3192 /// following 2 sets will stop changing after some point.
3193 /// "Change" here means that their size changes.
3194 /// The size of each set is monotonically increasing
3195 /// (we only add items to them) and it is upper bounded by the number of
3196 /// instructions in the processed function (we can never save more
3197 /// elements in either set than this number). Hence, at some point,
3198 /// they will stop increasing.
3199 /// Consequently, at some point, both sets will have stopped
3200 /// changing, effectively making the analysis reach a fixpoint.
3201
3202 /// Note: These 2 sets are disjoint and an instruction can be considered
3203 /// one of 3 things:
3204 /// 1) Known to cause UB (AAUndefinedBehavior could prove it) and put it in
3205 /// the KnownUBInsts set.
3206 /// 2) Assumed to cause UB (in every updateImpl, AAUndefinedBehavior
3207 /// has a reason to assume it).
3208 /// 3) Assumed to not cause UB. very other instruction - AAUndefinedBehavior
3209 /// could not find a reason to assume or prove that it can cause UB,
3210 /// hence it assumes it doesn't. We have a set for these instructions
3211 /// so that we don't reprocess them in every update.
3212 /// Note however that instructions in this set may cause UB.
3213
3214protected:
3215 /// A set of all live instructions _known_ to cause UB.
3216 SmallPtrSet<Instruction *, 8> KnownUBInsts;
3217
3218private:
3219 /// A set of all the (live) instructions that are assumed to _not_ cause UB.
3220 SmallPtrSet<Instruction *, 8> AssumedNoUBInsts;
3221
3222 // Should be called on updates in which if we're processing an instruction
3223 // \p I that depends on a value \p V, one of the following has to happen:
3224 // - If the value is assumed, then stop.
3225 // - If the value is known but undef, then consider it UB.
3226 // - Otherwise, do specific processing with the simplified value.
3227 // We return std::nullopt in the first 2 cases to signify that an appropriate
3228 // action was taken and the caller should stop.
3229 // Otherwise, we return the simplified value that the caller should
3230 // use for specific processing.
3231 std::optional<Value *> stopOnUndefOrAssumed(Attributor &A, Value *V,
3232 Instruction *I) {
3233 bool UsedAssumedInformation = false;
3234 std::optional<Value *> SimplifiedV =
3235 A.getAssumedSimplified(IRP: IRPosition::value(V: *V), AA: *this,
3236 UsedAssumedInformation, S: AA::Interprocedural);
3237 if (!UsedAssumedInformation) {
3238 // Don't depend on assumed values.
3239 if (!SimplifiedV) {
3240 // If it is known (which we tested above) but it doesn't have a value,
3241 // then we can assume `undef` and hence the instruction is UB.
3242 KnownUBInsts.insert(Ptr: I);
3243 return std::nullopt;
3244 }
3245 if (!*SimplifiedV)
3246 return nullptr;
3247 V = *SimplifiedV;
3248 }
3249 if (isa<UndefValue>(Val: V)) {
3250 KnownUBInsts.insert(Ptr: I);
3251 return std::nullopt;
3252 }
3253 return V;
3254 }
3255};
3256
3257struct AAUndefinedBehaviorFunction final : AAUndefinedBehaviorImpl {
3258 AAUndefinedBehaviorFunction(const IRPosition &IRP, Attributor &A)
3259 : AAUndefinedBehaviorImpl(IRP, A) {}
3260
3261 /// See AbstractAttribute::trackStatistics()
3262 void trackStatistics() const override {
3263 STATS_DECL(UndefinedBehaviorInstruction, Instruction,
3264 "Number of instructions known to have UB");
3265 BUILD_STAT_NAME(UndefinedBehaviorInstruction, Instruction) +=
3266 KnownUBInsts.size();
3267 }
3268};
3269} // namespace
3270
3271/// ------------------------ Will-Return Attributes ----------------------------
3272
3273namespace {
3274// Helper function that checks whether a function has any cycle which we don't
3275// know if it is bounded or not.
3276// Loops with maximum trip count are considered bounded, any other cycle not.
3277static bool mayContainUnboundedCycle(Function &F, Attributor &A) {
3278 ScalarEvolution *SE =
3279 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(F);
3280 LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F);
3281 // If either SCEV or LoopInfo is not available for the function then we assume
3282 // any cycle to be unbounded cycle.
3283 // We use scc_iterator which uses Tarjan algorithm to find all the maximal
3284 // SCCs.To detect if there's a cycle, we only need to find the maximal ones.
3285 if (!SE || !LI) {
3286 for (scc_iterator<Function *> SCCI = scc_begin(G: &F); !SCCI.isAtEnd(); ++SCCI)
3287 if (SCCI.hasCycle())
3288 return true;
3289 return false;
3290 }
3291
3292 // If there's irreducible control, the function may contain non-loop cycles.
3293 if (mayContainIrreducibleControl(F, LI))
3294 return true;
3295
3296 // Any loop that does not have a max trip count is considered unbounded cycle.
3297 for (auto *L : LI->getLoopsInPreorder()) {
3298 if (!SE->getSmallConstantMaxTripCount(L))
3299 return true;
3300 }
3301 return false;
3302}
3303
3304struct AAWillReturnImpl : public AAWillReturn {
3305 AAWillReturnImpl(const IRPosition &IRP, Attributor &A)
3306 : AAWillReturn(IRP, A) {}
3307
3308 /// See AbstractAttribute::initialize(...).
3309 void initialize(Attributor &A) override {
3310 bool IsKnown;
3311 assert(!AA::hasAssumedIRAttr<Attribute::WillReturn>(
3312 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
3313 (void)IsKnown;
3314 }
3315
3316 /// Check for `mustprogress` and `readonly` as they imply `willreturn`.
3317 bool isImpliedByMustprogressAndReadonly(Attributor &A, bool KnownOnly) {
3318 if (!A.hasAttr(IRP: getIRPosition(), AKs: {Attribute::MustProgress}))
3319 return false;
3320
3321 bool IsKnown;
3322 if (AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
3323 return IsKnown || !KnownOnly;
3324 return false;
3325 }
3326
3327 /// See AbstractAttribute::updateImpl(...).
3328 ChangeStatus updateImpl(Attributor &A) override {
3329 if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3330 return ChangeStatus::UNCHANGED;
3331
3332 auto CheckForWillReturn = [&](Instruction &I) {
3333 IRPosition IPos = IRPosition::callsite_function(CB: cast<CallBase>(Val&: I));
3334 bool IsKnown;
3335 if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
3336 A, QueryingAA: this, IRP: IPos, DepClass: DepClassTy::REQUIRED, IsKnown)) {
3337 if (IsKnown)
3338 return true;
3339 } else {
3340 return false;
3341 }
3342 bool IsKnownNoRecurse;
3343 return AA::hasAssumedIRAttr<Attribute::NoRecurse>(
3344 A, QueryingAA: this, IRP: IPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoRecurse);
3345 };
3346
3347 bool UsedAssumedInformation = false;
3348 if (!A.checkForAllCallLikeInstructions(Pred: CheckForWillReturn, QueryingAA: *this,
3349 UsedAssumedInformation))
3350 return indicatePessimisticFixpoint();
3351
3352 return ChangeStatus::UNCHANGED;
3353 }
3354
3355 /// See AbstractAttribute::getAsStr()
3356 const std::string getAsStr(Attributor *A) const override {
3357 return getAssumed() ? "willreturn" : "may-noreturn";
3358 }
3359};
3360
3361struct AAWillReturnFunction final : AAWillReturnImpl {
3362 AAWillReturnFunction(const IRPosition &IRP, Attributor &A)
3363 : AAWillReturnImpl(IRP, A) {}
3364
3365 /// See AbstractAttribute::initialize(...).
3366 void initialize(Attributor &A) override {
3367 AAWillReturnImpl::initialize(A);
3368
3369 Function *F = getAnchorScope();
3370 assert(F && "Did expect an anchor function");
3371 if (F->isDeclaration() || mayContainUnboundedCycle(F&: *F, A))
3372 indicatePessimisticFixpoint();
3373 }
3374
3375 /// See AbstractAttribute::trackStatistics()
3376 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(willreturn) }
3377};
3378
3379/// WillReturn attribute deduction for a call sites.
3380struct AAWillReturnCallSite final
3381 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl> {
3382 AAWillReturnCallSite(const IRPosition &IRP, Attributor &A)
3383 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl>(IRP, A) {}
3384
3385 /// See AbstractAttribute::updateImpl(...).
3386 ChangeStatus updateImpl(Attributor &A) override {
3387 if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3388 return ChangeStatus::UNCHANGED;
3389
3390 return AACalleeToCallSite::updateImpl(A);
3391 }
3392
3393 /// See AbstractAttribute::trackStatistics()
3394 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(willreturn); }
3395};
3396} // namespace
3397
3398/// -------------------AAIntraFnReachability Attribute--------------------------
3399
3400/// All information associated with a reachability query. This boilerplate code
3401/// is used by both AAIntraFnReachability and AAInterFnReachability, with
3402/// different \p ToTy values.
3403template <typename ToTy> struct ReachabilityQueryInfo {
3404 enum class Reachable {
3405 No,
3406 Yes,
3407 };
3408
3409 /// Start here,
3410 const Instruction *From = nullptr;
3411 /// reach this place,
3412 const ToTy *To = nullptr;
3413 /// without going through any of these instructions,
3414 const AA::InstExclusionSetTy *ExclusionSet = nullptr;
3415 /// and remember if it worked:
3416 Reachable Result = Reachable::No;
3417
3418 /// Precomputed hash for this RQI.
3419 unsigned Hash = 0;
3420
3421 unsigned computeHashValue() const {
3422 assert(Hash == 0 && "Computed hash twice!");
3423 using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;
3424 using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;
3425 return const_cast<ReachabilityQueryInfo<ToTy> *>(this)->Hash =
3426 detail::combineHashValue(a: PairDMI ::getHashValue({From, To}),
3427 b: InstSetDMI::getHashValue(BES: ExclusionSet));
3428 }
3429
3430 ReachabilityQueryInfo(const Instruction *From, const ToTy *To)
3431 : From(From), To(To) {}
3432
3433 /// Constructor replacement to ensure unique and stable sets are used for the
3434 /// cache.
3435 ReachabilityQueryInfo(Attributor &A, const Instruction &From, const ToTy &To,
3436 const AA::InstExclusionSetTy *ES, bool MakeUnique)
3437 : From(&From), To(&To), ExclusionSet(ES) {
3438
3439 if (!ES || ES->empty()) {
3440 ExclusionSet = nullptr;
3441 } else if (MakeUnique) {
3442 ExclusionSet = A.getInfoCache().getOrCreateUniqueBlockExecutionSet(BES: ES);
3443 }
3444 }
3445
3446 ReachabilityQueryInfo(const ReachabilityQueryInfo &RQI)
3447 : From(RQI.From), To(RQI.To), ExclusionSet(RQI.ExclusionSet) {}
3448};
3449
3450namespace llvm {
3451template <typename ToTy> struct DenseMapInfo<ReachabilityQueryInfo<ToTy> *> {
3452 using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;
3453 using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;
3454
3455 static ReachabilityQueryInfo<ToTy> EmptyKey;
3456 static ReachabilityQueryInfo<ToTy> TombstoneKey;
3457
3458 static inline ReachabilityQueryInfo<ToTy> *getEmptyKey() { return &EmptyKey; }
3459 static inline ReachabilityQueryInfo<ToTy> *getTombstoneKey() {
3460 return &TombstoneKey;
3461 }
3462 static unsigned getHashValue(const ReachabilityQueryInfo<ToTy> *RQI) {
3463 return RQI->Hash ? RQI->Hash : RQI->computeHashValue();
3464 }
3465 static bool isEqual(const ReachabilityQueryInfo<ToTy> *LHS,
3466 const ReachabilityQueryInfo<ToTy> *RHS) {
3467 if (!PairDMI::isEqual({LHS->From, LHS->To}, {RHS->From, RHS->To}))
3468 return false;
3469 return InstSetDMI::isEqual(LHS: LHS->ExclusionSet, RHS: RHS->ExclusionSet);
3470 }
3471};
3472
3473#define DefineKeys(ToTy) \
3474 template <> \
3475 ReachabilityQueryInfo<ToTy> \
3476 DenseMapInfo<ReachabilityQueryInfo<ToTy> *>::EmptyKey = \
3477 ReachabilityQueryInfo<ToTy>( \
3478 DenseMapInfo<const Instruction *>::getEmptyKey(), \
3479 DenseMapInfo<const ToTy *>::getEmptyKey()); \
3480 template <> \
3481 ReachabilityQueryInfo<ToTy> \
3482 DenseMapInfo<ReachabilityQueryInfo<ToTy> *>::TombstoneKey = \
3483 ReachabilityQueryInfo<ToTy>( \
3484 DenseMapInfo<const Instruction *>::getTombstoneKey(), \
3485 DenseMapInfo<const ToTy *>::getTombstoneKey());
3486
3487DefineKeys(Instruction) DefineKeys(Function)
3488#undef DefineKeys
3489
3490} // namespace llvm
3491
3492namespace {
3493
3494template <typename BaseTy, typename ToTy>
3495struct CachedReachabilityAA : public BaseTy {
3496 using RQITy = ReachabilityQueryInfo<ToTy>;
3497
3498 CachedReachabilityAA(const IRPosition &IRP, Attributor &A) : BaseTy(IRP, A) {}
3499
3500 /// See AbstractAttribute::isQueryAA.
3501 bool isQueryAA() const override { return true; }
3502
3503 /// See AbstractAttribute::updateImpl(...).
3504 ChangeStatus updateImpl(Attributor &A) override {
3505 ChangeStatus Changed = ChangeStatus::UNCHANGED;
3506 for (unsigned u = 0, e = QueryVector.size(); u < e; ++u) {
3507 RQITy *RQI = QueryVector[u];
3508 if (RQI->Result == RQITy::Reachable::No &&
3509 isReachableImpl(A, RQI&: *RQI, /*IsTemporaryRQI=*/false))
3510 Changed = ChangeStatus::CHANGED;
3511 }
3512 return Changed;
3513 }
3514
3515 virtual bool isReachableImpl(Attributor &A, RQITy &RQI,
3516 bool IsTemporaryRQI) = 0;
3517
3518 bool rememberResult(Attributor &A, typename RQITy::Reachable Result,
3519 RQITy &RQI, bool UsedExclusionSet, bool IsTemporaryRQI) {
3520 RQI.Result = Result;
3521
3522 // Remove the temporary RQI from the cache.
3523 if (IsTemporaryRQI)
3524 QueryCache.erase(&RQI);
3525
3526 // Insert a plain RQI (w/o exclusion set) if that makes sense. Two options:
3527 // 1) If it is reachable, it doesn't matter if we have an exclusion set for
3528 // this query. 2) We did not use the exclusion set, potentially because
3529 // there is none.
3530 if (Result == RQITy::Reachable::Yes || !UsedExclusionSet) {
3531 RQITy PlainRQI(RQI.From, RQI.To);
3532 if (!QueryCache.count(&PlainRQI)) {
3533 RQITy *RQIPtr = new (A.Allocator) RQITy(RQI.From, RQI.To);
3534 RQIPtr->Result = Result;
3535 QueryVector.push_back(RQIPtr);
3536 QueryCache.insert(RQIPtr);
3537 }
3538 }
3539
3540 // Check if we need to insert a new permanent RQI with the exclusion set.
3541 if (IsTemporaryRQI && Result != RQITy::Reachable::Yes && UsedExclusionSet) {
3542 assert((!RQI.ExclusionSet || !RQI.ExclusionSet->empty()) &&
3543 "Did not expect empty set!");
3544 RQITy *RQIPtr = new (A.Allocator)
3545 RQITy(A, *RQI.From, *RQI.To, RQI.ExclusionSet, true);
3546 assert(RQIPtr->Result == RQITy::Reachable::No && "Already reachable?");
3547 RQIPtr->Result = Result;
3548 assert(!QueryCache.count(RQIPtr));
3549 QueryVector.push_back(RQIPtr);
3550 QueryCache.insert(RQIPtr);
3551 }
3552
3553 if (Result == RQITy::Reachable::No && IsTemporaryRQI)
3554 A.registerForUpdate(AA&: *this);
3555 return Result == RQITy::Reachable::Yes;
3556 }
3557
3558 const std::string getAsStr(Attributor *A) const override {
3559 // TODO: Return the number of reachable queries.
3560 return "#queries(" + std::to_string(QueryVector.size()) + ")";
3561 }
3562
3563 bool checkQueryCache(Attributor &A, RQITy &StackRQI,
3564 typename RQITy::Reachable &Result) {
3565 if (!this->getState().isValidState()) {
3566 Result = RQITy::Reachable::Yes;
3567 return true;
3568 }
3569
3570 // If we have an exclusion set we might be able to find our answer by
3571 // ignoring it first.
3572 if (StackRQI.ExclusionSet) {
3573 RQITy PlainRQI(StackRQI.From, StackRQI.To);
3574 auto It = QueryCache.find(&PlainRQI);
3575 if (It != QueryCache.end() && (*It)->Result == RQITy::Reachable::No) {
3576 Result = RQITy::Reachable::No;
3577 return true;
3578 }
3579 }
3580
3581 auto It = QueryCache.find(&StackRQI);
3582 if (It != QueryCache.end()) {
3583 Result = (*It)->Result;
3584 return true;
3585 }
3586
3587 // Insert a temporary for recursive queries. We will replace it with a
3588 // permanent entry later.
3589 QueryCache.insert(&StackRQI);
3590 return false;
3591 }
3592
3593private:
3594 SmallVector<RQITy *> QueryVector;
3595 DenseSet<RQITy *> QueryCache;
3596};
3597
3598struct AAIntraFnReachabilityFunction final
3599 : public CachedReachabilityAA<AAIntraFnReachability, Instruction> {
3600 using Base = CachedReachabilityAA<AAIntraFnReachability, Instruction>;
3601 AAIntraFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
3602 : Base(IRP, A) {
3603 DT = A.getInfoCache().getAnalysisResultForFunction<DominatorTreeAnalysis>(
3604 F: *IRP.getAssociatedFunction());
3605 }
3606
3607 bool isAssumedReachable(
3608 Attributor &A, const Instruction &From, const Instruction &To,
3609 const AA::InstExclusionSetTy *ExclusionSet) const override {
3610 auto *NonConstThis = const_cast<AAIntraFnReachabilityFunction *>(this);
3611 if (&From == &To)
3612 return true;
3613
3614 RQITy StackRQI(A, From, To, ExclusionSet, false);
3615 RQITy::Reachable Result;
3616 if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
3617 return NonConstThis->isReachableImpl(A, RQI&: StackRQI,
3618 /*IsTemporaryRQI=*/true);
3619 return Result == RQITy::Reachable::Yes;
3620 }
3621
3622 ChangeStatus updateImpl(Attributor &A) override {
3623 // We only depend on liveness. DeadEdges is all we care about, check if any
3624 // of them changed.
3625 auto *LivenessAA =
3626 A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL);
3627 if (LivenessAA &&
3628 llvm::all_of(Range&: DeadEdges,
3629 P: [&](const auto &DeadEdge) {
3630 return LivenessAA->isEdgeDead(From: DeadEdge.first,
3631 To: DeadEdge.second);
3632 }) &&
3633 llvm::all_of(Range&: DeadBlocks, P: [&](const BasicBlock *BB) {
3634 return LivenessAA->isAssumedDead(BB);
3635 })) {
3636 return ChangeStatus::UNCHANGED;
3637 }
3638 DeadEdges.clear();
3639 DeadBlocks.clear();
3640 return Base::updateImpl(A);
3641 }
3642
3643 bool isReachableImpl(Attributor &A, RQITy &RQI,
3644 bool IsTemporaryRQI) override {
3645 const Instruction *Origin = RQI.From;
3646 bool UsedExclusionSet = false;
3647
3648 auto WillReachInBlock = [&](const Instruction &From, const Instruction &To,
3649 const AA::InstExclusionSetTy *ExclusionSet) {
3650 const Instruction *IP = &From;
3651 while (IP && IP != &To) {
3652 if (ExclusionSet && IP != Origin && ExclusionSet->count(Ptr: IP)) {
3653 UsedExclusionSet = true;
3654 break;
3655 }
3656 IP = IP->getNextNode();
3657 }
3658 return IP == &To;
3659 };
3660
3661 const BasicBlock *FromBB = RQI.From->getParent();
3662 const BasicBlock *ToBB = RQI.To->getParent();
3663 assert(FromBB->getParent() == ToBB->getParent() &&
3664 "Not an intra-procedural query!");
3665
3666 // Check intra-block reachability, however, other reaching paths are still
3667 // possible.
3668 if (FromBB == ToBB &&
3669 WillReachInBlock(*RQI.From, *RQI.To, RQI.ExclusionSet))
3670 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3671 IsTemporaryRQI);
3672
3673 // Check if reaching the ToBB block is sufficient or if even that would not
3674 // ensure reaching the target. In the latter case we are done.
3675 if (!WillReachInBlock(ToBB->front(), *RQI.To, RQI.ExclusionSet))
3676 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3677 IsTemporaryRQI);
3678
3679 const Function *Fn = FromBB->getParent();
3680 SmallPtrSet<const BasicBlock *, 16> ExclusionBlocks;
3681 if (RQI.ExclusionSet)
3682 for (auto *I : *RQI.ExclusionSet)
3683 if (I->getFunction() == Fn)
3684 ExclusionBlocks.insert(Ptr: I->getParent());
3685
3686 // Check if we make it out of the FromBB block at all.
3687 if (ExclusionBlocks.count(Ptr: FromBB) &&
3688 !WillReachInBlock(*RQI.From, *FromBB->getTerminator(),
3689 RQI.ExclusionSet))
3690 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet: true, IsTemporaryRQI);
3691
3692 auto *LivenessAA =
3693 A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL);
3694 if (LivenessAA && LivenessAA->isAssumedDead(BB: ToBB)) {
3695 DeadBlocks.insert(V: ToBB);
3696 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3697 IsTemporaryRQI);
3698 }
3699
3700 SmallPtrSet<const BasicBlock *, 16> Visited;
3701 SmallVector<const BasicBlock *, 16> Worklist;
3702 Worklist.push_back(Elt: FromBB);
3703
3704 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> LocalDeadEdges;
3705 while (!Worklist.empty()) {
3706 const BasicBlock *BB = Worklist.pop_back_val();
3707 if (!Visited.insert(Ptr: BB).second)
3708 continue;
3709 for (const BasicBlock *SuccBB : successors(BB)) {
3710 if (LivenessAA && LivenessAA->isEdgeDead(From: BB, To: SuccBB)) {
3711 LocalDeadEdges.insert(V: {BB, SuccBB});
3712 continue;
3713 }
3714 // We checked before if we just need to reach the ToBB block.
3715 if (SuccBB == ToBB)
3716 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3717 IsTemporaryRQI);
3718 if (DT && ExclusionBlocks.empty() && DT->dominates(A: BB, B: ToBB))
3719 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3720 IsTemporaryRQI);
3721
3722 if (ExclusionBlocks.count(Ptr: SuccBB)) {
3723 UsedExclusionSet = true;
3724 continue;
3725 }
3726 Worklist.push_back(Elt: SuccBB);
3727 }
3728 }
3729
3730 DeadEdges.insert_range(R&: LocalDeadEdges);
3731 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3732 IsTemporaryRQI);
3733 }
3734
3735 /// See AbstractAttribute::trackStatistics()
3736 void trackStatistics() const override {}
3737
3738private:
3739 // Set of assumed dead blocks we used in the last query. If any changes we
3740 // update the state.
3741 DenseSet<const BasicBlock *> DeadBlocks;
3742
3743 // Set of assumed dead edges we used in the last query. If any changes we
3744 // update the state.
3745 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> DeadEdges;
3746
3747 /// The dominator tree of the function to short-circuit reasoning.
3748 const DominatorTree *DT = nullptr;
3749};
3750} // namespace
3751
3752/// ------------------------ NoAlias Argument Attribute ------------------------
3753
3754bool AANoAlias::isImpliedByIR(Attributor &A, const IRPosition &IRP,
3755 Attribute::AttrKind ImpliedAttributeKind,
3756 bool IgnoreSubsumingPositions) {
3757 assert(ImpliedAttributeKind == Attribute::NoAlias &&
3758 "Unexpected attribute kind");
3759 Value *Val = &IRP.getAssociatedValue();
3760 if (IRP.getPositionKind() != IRP_CALL_SITE_ARGUMENT) {
3761 if (isa<AllocaInst>(Val))
3762 return true;
3763 } else {
3764 IgnoreSubsumingPositions = true;
3765 }
3766
3767 if (isa<UndefValue>(Val))
3768 return true;
3769
3770 if (isa<ConstantPointerNull>(Val) &&
3771 !NullPointerIsDefined(F: IRP.getAnchorScope(),
3772 AS: Val->getType()->getPointerAddressSpace()))
3773 return true;
3774
3775 if (A.hasAttr(IRP, AKs: {Attribute::ByVal, Attribute::NoAlias},
3776 IgnoreSubsumingPositions, ImpliedAttributeKind: Attribute::NoAlias))
3777 return true;
3778
3779 return false;
3780}
3781
3782namespace {
3783struct AANoAliasImpl : AANoAlias {
3784 AANoAliasImpl(const IRPosition &IRP, Attributor &A) : AANoAlias(IRP, A) {
3785 assert(getAssociatedType()->isPointerTy() &&
3786 "Noalias is a pointer attribute");
3787 }
3788
3789 const std::string getAsStr(Attributor *A) const override {
3790 return getAssumed() ? "noalias" : "may-alias";
3791 }
3792};
3793
3794/// NoAlias attribute for a floating value.
3795struct AANoAliasFloating final : AANoAliasImpl {
3796 AANoAliasFloating(const IRPosition &IRP, Attributor &A)
3797 : AANoAliasImpl(IRP, A) {}
3798
3799 /// See AbstractAttribute::updateImpl(...).
3800 ChangeStatus updateImpl(Attributor &A) override {
3801 // TODO: Implement this.
3802 return indicatePessimisticFixpoint();
3803 }
3804
3805 /// See AbstractAttribute::trackStatistics()
3806 void trackStatistics() const override {
3807 STATS_DECLTRACK_FLOATING_ATTR(noalias)
3808 }
3809};
3810
3811/// NoAlias attribute for an argument.
3812struct AANoAliasArgument final
3813 : AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl> {
3814 using Base = AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl>;
3815 AANoAliasArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
3816
3817 /// See AbstractAttribute::update(...).
3818 ChangeStatus updateImpl(Attributor &A) override {
3819 // We have to make sure no-alias on the argument does not break
3820 // synchronization when this is a callback argument, see also [1] below.
3821 // If synchronization cannot be affected, we delegate to the base updateImpl
3822 // function, otherwise we give up for now.
3823
3824 // If the function is no-sync, no-alias cannot break synchronization.
3825 bool IsKnownNoSycn;
3826 if (AA::hasAssumedIRAttr<Attribute::NoSync>(
3827 A, QueryingAA: this, IRP: IRPosition::function_scope(IRP: getIRPosition()),
3828 DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoSycn))
3829 return Base::updateImpl(A);
3830
3831 // If the argument is read-only, no-alias cannot break synchronization.
3832 bool IsKnown;
3833 if (AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
3834 return Base::updateImpl(A);
3835
3836 // If the argument is never passed through callbacks, no-alias cannot break
3837 // synchronization.
3838 bool UsedAssumedInformation = false;
3839 if (A.checkForAllCallSites(
3840 Pred: [](AbstractCallSite ACS) { return !ACS.isCallbackCall(); }, QueryingAA: *this,
3841 RequireAllCallSites: true, UsedAssumedInformation))
3842 return Base::updateImpl(A);
3843
3844 // TODO: add no-alias but make sure it doesn't break synchronization by
3845 // introducing fake uses. See:
3846 // [1] Compiler Optimizations for OpenMP, J. Doerfert and H. Finkel,
3847 // International Workshop on OpenMP 2018,
3848 // http://compilers.cs.uni-saarland.de/people/doerfert/par_opt18.pdf
3849
3850 return indicatePessimisticFixpoint();
3851 }
3852
3853 /// See AbstractAttribute::trackStatistics()
3854 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noalias) }
3855};
3856
3857struct AANoAliasCallSiteArgument final : AANoAliasImpl {
3858 AANoAliasCallSiteArgument(const IRPosition &IRP, Attributor &A)
3859 : AANoAliasImpl(IRP, A) {}
3860
3861 /// Determine if the underlying value may alias with the call site argument
3862 /// \p OtherArgNo of \p ICS (= the underlying call site).
3863 bool mayAliasWithArgument(Attributor &A, AAResults *&AAR,
3864 const AAMemoryBehavior &MemBehaviorAA,
3865 const CallBase &CB, unsigned OtherArgNo) {
3866 // We do not need to worry about aliasing with the underlying IRP.
3867 if (this->getCalleeArgNo() == (int)OtherArgNo)
3868 return false;
3869
3870 // If it is not a pointer or pointer vector we do not alias.
3871 const Value *ArgOp = CB.getArgOperand(i: OtherArgNo);
3872 if (!ArgOp->getType()->isPtrOrPtrVectorTy())
3873 return false;
3874
3875 auto *CBArgMemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
3876 QueryingAA: *this, IRP: IRPosition::callsite_argument(CB, ArgNo: OtherArgNo), DepClass: DepClassTy::NONE);
3877
3878 // If the argument is readnone, there is no read-write aliasing.
3879 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadNone()) {
3880 A.recordDependence(FromAA: *CBArgMemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3881 return false;
3882 }
3883
3884 // If the argument is readonly and the underlying value is readonly, there
3885 // is no read-write aliasing.
3886 bool IsReadOnly = MemBehaviorAA.isAssumedReadOnly();
3887 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadOnly() &&
3888 IsReadOnly) {
3889 A.recordDependence(FromAA: MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3890 A.recordDependence(FromAA: *CBArgMemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3891 return false;
3892 }
3893
3894 // We have to utilize actual alias analysis queries so we need the object.
3895 if (!AAR)
3896 AAR = A.getInfoCache().getAnalysisResultForFunction<AAManager>(
3897 F: *getAnchorScope());
3898
3899 // Try to rule it out at the call site.
3900 bool IsAliasing = !AAR || !AAR->isNoAlias(V1: &getAssociatedValue(), V2: ArgOp);
3901 LLVM_DEBUG(dbgs() << "[NoAliasCSArg] Check alias between "
3902 "callsite arguments: "
3903 << getAssociatedValue() << " " << *ArgOp << " => "
3904 << (IsAliasing ? "" : "no-") << "alias \n");
3905
3906 return IsAliasing;
3907 }
3908
3909 bool isKnownNoAliasDueToNoAliasPreservation(
3910 Attributor &A, AAResults *&AAR, const AAMemoryBehavior &MemBehaviorAA) {
3911 // We can deduce "noalias" if the following conditions hold.
3912 // (i) Associated value is assumed to be noalias in the definition.
3913 // (ii) Associated value is assumed to be no-capture in all the uses
3914 // possibly executed before this callsite.
3915 // (iii) There is no other pointer argument which could alias with the
3916 // value.
3917
3918 const IRPosition &VIRP = IRPosition::value(V: getAssociatedValue());
3919 const Function *ScopeFn = VIRP.getAnchorScope();
3920 // Check whether the value is captured in the scope using AANoCapture.
3921 // Look at CFG and check only uses possibly executed before this
3922 // callsite.
3923 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
3924 Instruction *UserI = cast<Instruction>(Val: U.getUser());
3925
3926 // If UserI is the curr instruction and there is a single potential use of
3927 // the value in UserI we allow the use.
3928 // TODO: We should inspect the operands and allow those that cannot alias
3929 // with the value.
3930 if (UserI == getCtxI() && UserI->getNumOperands() == 1)
3931 return true;
3932
3933 if (ScopeFn) {
3934 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
3935 if (CB->isArgOperand(U: &U)) {
3936
3937 unsigned ArgNo = CB->getArgOperandNo(U: &U);
3938
3939 bool IsKnownNoCapture;
3940 if (AA::hasAssumedIRAttr<Attribute::Captures>(
3941 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
3942 DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture))
3943 return true;
3944 }
3945 }
3946
3947 if (!AA::isPotentiallyReachable(
3948 A, FromI: *UserI, ToI: *getCtxI(), QueryingAA: *this, /* ExclusionSet */ nullptr,
3949 GoBackwardsCB: [ScopeFn](const Function &Fn) { return &Fn != ScopeFn; }))
3950 return true;
3951 }
3952
3953 // TODO: We should track the capturing uses in AANoCapture but the problem
3954 // is CGSCC runs. For those we would need to "allow" AANoCapture for
3955 // a value in the module slice.
3956 // TODO(captures): Make this more precise.
3957 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
3958 if (capturesNothing(CC: CI))
3959 return true;
3960 if (CI.isPassthrough()) {
3961 Follow = true;
3962 return true;
3963 }
3964 LLVM_DEBUG(dbgs() << "[AANoAliasCSArg] Unknown user: " << *UserI << "\n");
3965 return false;
3966 };
3967
3968 bool IsKnownNoCapture;
3969 const AANoCapture *NoCaptureAA = nullptr;
3970 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
3971 A, QueryingAA: this, IRP: VIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false, AAPtr: &NoCaptureAA);
3972 if (!IsAssumedNoCapture &&
3973 (!NoCaptureAA || !NoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
3974 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: getAssociatedValue())) {
3975 LLVM_DEBUG(
3976 dbgs() << "[AANoAliasCSArg] " << getAssociatedValue()
3977 << " cannot be noalias as it is potentially captured\n");
3978 return false;
3979 }
3980 }
3981 if (NoCaptureAA)
3982 A.recordDependence(FromAA: *NoCaptureAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3983
3984 // Check there is no other pointer argument which could alias with the
3985 // value passed at this call site.
3986 // TODO: AbstractCallSite
3987 const auto &CB = cast<CallBase>(Val&: getAnchorValue());
3988 for (unsigned OtherArgNo = 0; OtherArgNo < CB.arg_size(); OtherArgNo++)
3989 if (mayAliasWithArgument(A, AAR, MemBehaviorAA, CB, OtherArgNo))
3990 return false;
3991
3992 return true;
3993 }
3994
3995 /// See AbstractAttribute::updateImpl(...).
3996 ChangeStatus updateImpl(Attributor &A) override {
3997 // If the argument is readnone we are done as there are no accesses via the
3998 // argument.
3999 auto *MemBehaviorAA =
4000 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::NONE);
4001 if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
4002 A.recordDependence(FromAA: *MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
4003 return ChangeStatus::UNCHANGED;
4004 }
4005
4006 bool IsKnownNoAlias;
4007 const IRPosition &VIRP = IRPosition::value(V: getAssociatedValue());
4008 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
4009 A, QueryingAA: this, IRP: VIRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoAlias)) {
4010 LLVM_DEBUG(dbgs() << "[AANoAlias] " << getAssociatedValue()
4011 << " is not no-alias at the definition\n");
4012 return indicatePessimisticFixpoint();
4013 }
4014
4015 AAResults *AAR = nullptr;
4016 if (MemBehaviorAA &&
4017 isKnownNoAliasDueToNoAliasPreservation(A, AAR, MemBehaviorAA: *MemBehaviorAA)) {
4018 LLVM_DEBUG(
4019 dbgs() << "[AANoAlias] No-Alias deduced via no-alias preservation\n");
4020 return ChangeStatus::UNCHANGED;
4021 }
4022
4023 return indicatePessimisticFixpoint();
4024 }
4025
4026 /// See AbstractAttribute::trackStatistics()
4027 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noalias) }
4028};
4029
4030/// NoAlias attribute for function return value.
4031struct AANoAliasReturned final : AANoAliasImpl {
4032 AANoAliasReturned(const IRPosition &IRP, Attributor &A)
4033 : AANoAliasImpl(IRP, A) {}
4034
4035 /// See AbstractAttribute::updateImpl(...).
4036 ChangeStatus updateImpl(Attributor &A) override {
4037
4038 auto CheckReturnValue = [&](Value &RV) -> bool {
4039 if (Constant *C = dyn_cast<Constant>(Val: &RV))
4040 if (C->isNullValue() || isa<UndefValue>(Val: C))
4041 return true;
4042
4043 /// For now, we can only deduce noalias if we have call sites.
4044 /// FIXME: add more support.
4045 if (!isa<CallBase>(Val: &RV))
4046 return false;
4047
4048 const IRPosition &RVPos = IRPosition::value(V: RV);
4049 bool IsKnownNoAlias;
4050 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
4051 A, QueryingAA: this, IRP: RVPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoAlias))
4052 return false;
4053
4054 bool IsKnownNoCapture;
4055 const AANoCapture *NoCaptureAA = nullptr;
4056 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
4057 A, QueryingAA: this, IRP: RVPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
4058 AAPtr: &NoCaptureAA);
4059 return IsAssumedNoCapture ||
4060 (NoCaptureAA && NoCaptureAA->isAssumedNoCaptureMaybeReturned());
4061 };
4062
4063 if (!A.checkForAllReturnedValues(Pred: CheckReturnValue, QueryingAA: *this))
4064 return indicatePessimisticFixpoint();
4065
4066 return ChangeStatus::UNCHANGED;
4067 }
4068
4069 /// See AbstractAttribute::trackStatistics()
4070 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noalias) }
4071};
4072
4073/// NoAlias attribute deduction for a call site return value.
4074struct AANoAliasCallSiteReturned final
4075 : AACalleeToCallSite<AANoAlias, AANoAliasImpl> {
4076 AANoAliasCallSiteReturned(const IRPosition &IRP, Attributor &A)
4077 : AACalleeToCallSite<AANoAlias, AANoAliasImpl>(IRP, A) {}
4078
4079 /// See AbstractAttribute::trackStatistics()
4080 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noalias); }
4081};
4082} // namespace
4083
4084/// -------------------AAIsDead Function Attribute-----------------------
4085
4086namespace {
4087struct AAIsDeadValueImpl : public AAIsDead {
4088 AAIsDeadValueImpl(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4089
4090 /// See AAIsDead::isAssumedDead().
4091 bool isAssumedDead() const override { return isAssumed(BitsEncoding: IS_DEAD); }
4092
4093 /// See AAIsDead::isKnownDead().
4094 bool isKnownDead() const override { return isKnown(BitsEncoding: IS_DEAD); }
4095
4096 /// See AAIsDead::isAssumedDead(BasicBlock *).
4097 bool isAssumedDead(const BasicBlock *BB) const override { return false; }
4098
4099 /// See AAIsDead::isKnownDead(BasicBlock *).
4100 bool isKnownDead(const BasicBlock *BB) const override { return false; }
4101
4102 /// See AAIsDead::isAssumedDead(Instruction *I).
4103 bool isAssumedDead(const Instruction *I) const override {
4104 return I == getCtxI() && isAssumedDead();
4105 }
4106
4107 /// See AAIsDead::isKnownDead(Instruction *I).
4108 bool isKnownDead(const Instruction *I) const override {
4109 return isAssumedDead(I) && isKnownDead();
4110 }
4111
4112 /// See AbstractAttribute::getAsStr().
4113 const std::string getAsStr(Attributor *A) const override {
4114 return isAssumedDead() ? "assumed-dead" : "assumed-live";
4115 }
4116
4117 /// Check if all uses are assumed dead.
4118 bool areAllUsesAssumedDead(Attributor &A, Value &V) {
4119 // Callers might not check the type, void has no uses.
4120 if (V.getType()->isVoidTy() || V.use_empty())
4121 return true;
4122
4123 // If we replace a value with a constant there are no uses left afterwards.
4124 if (!isa<Constant>(Val: V)) {
4125 if (auto *I = dyn_cast<Instruction>(Val: &V))
4126 if (!A.isRunOn(Fn&: *I->getFunction()))
4127 return false;
4128 bool UsedAssumedInformation = false;
4129 std::optional<Constant *> C =
4130 A.getAssumedConstant(V, AA: *this, UsedAssumedInformation);
4131 if (!C || *C)
4132 return true;
4133 }
4134
4135 auto UsePred = [&](const Use &U, bool &Follow) { return false; };
4136 // Explicitly set the dependence class to required because we want a long
4137 // chain of N dependent instructions to be considered live as soon as one is
4138 // without going through N update cycles. This is not required for
4139 // correctness.
4140 return A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V, /* CheckBBLivenessOnly */ false,
4141 LivenessDepClass: DepClassTy::REQUIRED,
4142 /* IgnoreDroppableUses */ false);
4143 }
4144
4145 /// Determine if \p I is assumed to be side-effect free.
4146 bool isAssumedSideEffectFree(Attributor &A, Instruction *I) {
4147 if (!I || wouldInstructionBeTriviallyDead(I))
4148 return true;
4149
4150 auto *CB = dyn_cast<CallBase>(Val: I);
4151 if (!CB || isa<IntrinsicInst>(Val: CB))
4152 return false;
4153
4154 const IRPosition &CallIRP = IRPosition::callsite_function(CB: *CB);
4155
4156 bool IsKnownNoUnwind;
4157 if (!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
4158 A, QueryingAA: this, IRP: CallIRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoUnwind))
4159 return false;
4160
4161 bool IsKnown;
4162 return AA::isAssumedReadOnly(A, IRP: CallIRP, QueryingAA: *this, IsKnown);
4163 }
4164};
4165
4166struct AAIsDeadFloating : public AAIsDeadValueImpl {
4167 AAIsDeadFloating(const IRPosition &IRP, Attributor &A)
4168 : AAIsDeadValueImpl(IRP, A) {}
4169
4170 /// See AbstractAttribute::initialize(...).
4171 void initialize(Attributor &A) override {
4172 AAIsDeadValueImpl::initialize(A);
4173
4174 if (isa<UndefValue>(Val: getAssociatedValue())) {
4175 indicatePessimisticFixpoint();
4176 return;
4177 }
4178
4179 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4180 if (!isAssumedSideEffectFree(A, I)) {
4181 if (!isa_and_nonnull<StoreInst>(Val: I) && !isa_and_nonnull<FenceInst>(Val: I))
4182 indicatePessimisticFixpoint();
4183 else
4184 removeAssumedBits(BitsEncoding: HAS_NO_EFFECT);
4185 }
4186 }
4187
4188 bool isDeadFence(Attributor &A, FenceInst &FI) {
4189 const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
4190 IRP: IRPosition::function(F: *FI.getFunction()), QueryingAA: *this, DepClass: DepClassTy::NONE);
4191 if (!ExecDomainAA || !ExecDomainAA->isNoOpFence(FI))
4192 return false;
4193 A.recordDependence(FromAA: *ExecDomainAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
4194 return true;
4195 }
4196
4197 bool isDeadStore(Attributor &A, StoreInst &SI,
4198 SmallSetVector<Instruction *, 8> *AssumeOnlyInst = nullptr) {
4199 // Lang ref now states volatile store is not UB/dead, let's skip them.
4200 if (SI.isVolatile())
4201 return false;
4202
4203 // If we are collecting assumes to be deleted we are in the manifest stage.
4204 // It's problematic to collect the potential copies again now so we use the
4205 // cached ones.
4206 bool UsedAssumedInformation = false;
4207 if (!AssumeOnlyInst) {
4208 PotentialCopies.clear();
4209 if (!AA::getPotentialCopiesOfStoredValue(A, SI, PotentialCopies, QueryingAA: *this,
4210 UsedAssumedInformation)) {
4211 LLVM_DEBUG(
4212 dbgs()
4213 << "[AAIsDead] Could not determine potential copies of store!\n");
4214 return false;
4215 }
4216 }
4217 LLVM_DEBUG(dbgs() << "[AAIsDead] Store has " << PotentialCopies.size()
4218 << " potential copies.\n");
4219
4220 InformationCache &InfoCache = A.getInfoCache();
4221 return llvm::all_of(Range&: PotentialCopies, P: [&](Value *V) {
4222 if (A.isAssumedDead(IRP: IRPosition::value(V: *V), QueryingAA: this, FnLivenessAA: nullptr,
4223 UsedAssumedInformation))
4224 return true;
4225 if (auto *LI = dyn_cast<LoadInst>(Val: V)) {
4226 if (llvm::all_of(Range: LI->uses(), P: [&](const Use &U) {
4227 auto &UserI = cast<Instruction>(Val&: *U.getUser());
4228 if (InfoCache.isOnlyUsedByAssume(I: UserI)) {
4229 if (AssumeOnlyInst)
4230 AssumeOnlyInst->insert(X: &UserI);
4231 return true;
4232 }
4233 return A.isAssumedDead(U, QueryingAA: this, FnLivenessAA: nullptr, UsedAssumedInformation);
4234 })) {
4235 return true;
4236 }
4237 }
4238 LLVM_DEBUG(dbgs() << "[AAIsDead] Potential copy " << *V
4239 << " is assumed live!\n");
4240 return false;
4241 });
4242 }
4243
4244 /// See AbstractAttribute::getAsStr().
4245 const std::string getAsStr(Attributor *A) const override {
4246 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4247 if (isa_and_nonnull<StoreInst>(Val: I))
4248 if (isValidState())
4249 return "assumed-dead-store";
4250 if (isa_and_nonnull<FenceInst>(Val: I))
4251 if (isValidState())
4252 return "assumed-dead-fence";
4253 return AAIsDeadValueImpl::getAsStr(A);
4254 }
4255
4256 /// See AbstractAttribute::updateImpl(...).
4257 ChangeStatus updateImpl(Attributor &A) override {
4258 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4259 if (auto *SI = dyn_cast_or_null<StoreInst>(Val: I)) {
4260 if (!isDeadStore(A, SI&: *SI))
4261 return indicatePessimisticFixpoint();
4262 } else if (auto *FI = dyn_cast_or_null<FenceInst>(Val: I)) {
4263 if (!isDeadFence(A, FI&: *FI))
4264 return indicatePessimisticFixpoint();
4265 } else {
4266 if (!isAssumedSideEffectFree(A, I))
4267 return indicatePessimisticFixpoint();
4268 if (!areAllUsesAssumedDead(A, V&: getAssociatedValue()))
4269 return indicatePessimisticFixpoint();
4270 }
4271 return ChangeStatus::UNCHANGED;
4272 }
4273
4274 bool isRemovableStore() const override {
4275 return isAssumed(BitsEncoding: IS_REMOVABLE) && isa<StoreInst>(Val: &getAssociatedValue());
4276 }
4277
4278 /// See AbstractAttribute::manifest(...).
4279 ChangeStatus manifest(Attributor &A) override {
4280 Value &V = getAssociatedValue();
4281 if (auto *I = dyn_cast<Instruction>(Val: &V)) {
4282 // If we get here we basically know the users are all dead. We check if
4283 // isAssumedSideEffectFree returns true here again because it might not be
4284 // the case and only the users are dead but the instruction (=call) is
4285 // still needed.
4286 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
4287 SmallSetVector<Instruction *, 8> AssumeOnlyInst;
4288 bool IsDead = isDeadStore(A, SI&: *SI, AssumeOnlyInst: &AssumeOnlyInst);
4289 (void)IsDead;
4290 assert(IsDead && "Store was assumed to be dead!");
4291 A.deleteAfterManifest(I&: *I);
4292 for (size_t i = 0; i < AssumeOnlyInst.size(); ++i) {
4293 Instruction *AOI = AssumeOnlyInst[i];
4294 for (auto *Usr : AOI->users())
4295 AssumeOnlyInst.insert(X: cast<Instruction>(Val: Usr));
4296 A.deleteAfterManifest(I&: *AOI);
4297 }
4298 return ChangeStatus::CHANGED;
4299 }
4300 if (auto *FI = dyn_cast<FenceInst>(Val: I)) {
4301 assert(isDeadFence(A, *FI));
4302 A.deleteAfterManifest(I&: *FI);
4303 return ChangeStatus::CHANGED;
4304 }
4305 if (isAssumedSideEffectFree(A, I) && !isa<InvokeInst>(Val: I)) {
4306 A.deleteAfterManifest(I&: *I);
4307 return ChangeStatus::CHANGED;
4308 }
4309 }
4310 return ChangeStatus::UNCHANGED;
4311 }
4312
4313 /// See AbstractAttribute::trackStatistics()
4314 void trackStatistics() const override {
4315 STATS_DECLTRACK_FLOATING_ATTR(IsDead)
4316 }
4317
4318private:
4319 // The potential copies of a dead store, used for deletion during manifest.
4320 SmallSetVector<Value *, 4> PotentialCopies;
4321};
4322
4323struct AAIsDeadArgument : public AAIsDeadFloating {
4324 AAIsDeadArgument(const IRPosition &IRP, Attributor &A)
4325 : AAIsDeadFloating(IRP, A) {}
4326
4327 /// See AbstractAttribute::manifest(...).
4328 ChangeStatus manifest(Attributor &A) override {
4329 Argument &Arg = *getAssociatedArgument();
4330 if (A.isValidFunctionSignatureRewrite(Arg, /* ReplacementTypes */ {}))
4331 if (A.registerFunctionSignatureRewrite(
4332 Arg, /* ReplacementTypes */ {},
4333 CalleeRepairCB: Attributor::ArgumentReplacementInfo::CalleeRepairCBTy{},
4334 ACSRepairCB: Attributor::ArgumentReplacementInfo::ACSRepairCBTy{})) {
4335 return ChangeStatus::CHANGED;
4336 }
4337 return ChangeStatus::UNCHANGED;
4338 }
4339
4340 /// See AbstractAttribute::trackStatistics()
4341 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(IsDead) }
4342};
4343
4344struct AAIsDeadCallSiteArgument : public AAIsDeadValueImpl {
4345 AAIsDeadCallSiteArgument(const IRPosition &IRP, Attributor &A)
4346 : AAIsDeadValueImpl(IRP, A) {}
4347
4348 /// See AbstractAttribute::initialize(...).
4349 void initialize(Attributor &A) override {
4350 AAIsDeadValueImpl::initialize(A);
4351 if (isa<UndefValue>(Val: getAssociatedValue()))
4352 indicatePessimisticFixpoint();
4353 }
4354
4355 /// See AbstractAttribute::updateImpl(...).
4356 ChangeStatus updateImpl(Attributor &A) override {
4357 // TODO: Once we have call site specific value information we can provide
4358 // call site specific liveness information and then it makes
4359 // sense to specialize attributes for call sites arguments instead of
4360 // redirecting requests to the callee argument.
4361 Argument *Arg = getAssociatedArgument();
4362 if (!Arg)
4363 return indicatePessimisticFixpoint();
4364 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
4365 auto *ArgAA = A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
4366 if (!ArgAA)
4367 return indicatePessimisticFixpoint();
4368 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
4369 }
4370
4371 /// See AbstractAttribute::manifest(...).
4372 ChangeStatus manifest(Attributor &A) override {
4373 CallBase &CB = cast<CallBase>(Val&: getAnchorValue());
4374 Use &U = CB.getArgOperandUse(i: getCallSiteArgNo());
4375 assert(!isa<UndefValue>(U.get()) &&
4376 "Expected undef values to be filtered out!");
4377 UndefValue &UV = *UndefValue::get(T: U->getType());
4378 if (A.changeUseAfterManifest(U, NV&: UV))
4379 return ChangeStatus::CHANGED;
4380 return ChangeStatus::UNCHANGED;
4381 }
4382
4383 /// See AbstractAttribute::trackStatistics()
4384 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(IsDead) }
4385};
4386
4387struct AAIsDeadCallSiteReturned : public AAIsDeadFloating {
4388 AAIsDeadCallSiteReturned(const IRPosition &IRP, Attributor &A)
4389 : AAIsDeadFloating(IRP, A) {}
4390
4391 /// See AAIsDead::isAssumedDead().
4392 bool isAssumedDead() const override {
4393 return AAIsDeadFloating::isAssumedDead() && IsAssumedSideEffectFree;
4394 }
4395
4396 /// See AbstractAttribute::initialize(...).
4397 void initialize(Attributor &A) override {
4398 AAIsDeadFloating::initialize(A);
4399 if (isa<UndefValue>(Val: getAssociatedValue())) {
4400 indicatePessimisticFixpoint();
4401 return;
4402 }
4403
4404 // We track this separately as a secondary state.
4405 IsAssumedSideEffectFree = isAssumedSideEffectFree(A, I: getCtxI());
4406 }
4407
4408 /// See AbstractAttribute::updateImpl(...).
4409 ChangeStatus updateImpl(Attributor &A) override {
4410 ChangeStatus Changed = ChangeStatus::UNCHANGED;
4411 if (IsAssumedSideEffectFree && !isAssumedSideEffectFree(A, I: getCtxI())) {
4412 IsAssumedSideEffectFree = false;
4413 Changed = ChangeStatus::CHANGED;
4414 }
4415 if (!areAllUsesAssumedDead(A, V&: getAssociatedValue()))
4416 return indicatePessimisticFixpoint();
4417 return Changed;
4418 }
4419
4420 /// See AbstractAttribute::trackStatistics()
4421 void trackStatistics() const override {
4422 if (IsAssumedSideEffectFree)
4423 STATS_DECLTRACK_CSRET_ATTR(IsDead)
4424 else
4425 STATS_DECLTRACK_CSRET_ATTR(UnusedResult)
4426 }
4427
4428 /// See AbstractAttribute::getAsStr().
4429 const std::string getAsStr(Attributor *A) const override {
4430 return isAssumedDead()
4431 ? "assumed-dead"
4432 : (getAssumed() ? "assumed-dead-users" : "assumed-live");
4433 }
4434
4435private:
4436 bool IsAssumedSideEffectFree = true;
4437};
4438
4439struct AAIsDeadReturned : public AAIsDeadValueImpl {
4440 AAIsDeadReturned(const IRPosition &IRP, Attributor &A)
4441 : AAIsDeadValueImpl(IRP, A) {}
4442
4443 /// See AbstractAttribute::updateImpl(...).
4444 ChangeStatus updateImpl(Attributor &A) override {
4445
4446 bool UsedAssumedInformation = false;
4447 A.checkForAllInstructions(Pred: [](Instruction &) { return true; }, QueryingAA: *this,
4448 Opcodes: {Instruction::Ret}, UsedAssumedInformation);
4449
4450 auto PredForCallSite = [&](AbstractCallSite ACS) {
4451 if (ACS.isCallbackCall() || !ACS.getInstruction())
4452 return false;
4453 return areAllUsesAssumedDead(A, V&: *ACS.getInstruction());
4454 };
4455
4456 if (!A.checkForAllCallSites(Pred: PredForCallSite, QueryingAA: *this, RequireAllCallSites: true,
4457 UsedAssumedInformation))
4458 return indicatePessimisticFixpoint();
4459
4460 return ChangeStatus::UNCHANGED;
4461 }
4462
4463 /// See AbstractAttribute::manifest(...).
4464 ChangeStatus manifest(Attributor &A) override {
4465 // TODO: Rewrite the signature to return void?
4466 bool AnyChange = false;
4467 UndefValue &UV = *UndefValue::get(T: getAssociatedFunction()->getReturnType());
4468 auto RetInstPred = [&](Instruction &I) {
4469 ReturnInst &RI = cast<ReturnInst>(Val&: I);
4470 if (!isa<UndefValue>(Val: RI.getReturnValue()))
4471 AnyChange |= A.changeUseAfterManifest(U&: RI.getOperandUse(i: 0), NV&: UV);
4472 return true;
4473 };
4474 bool UsedAssumedInformation = false;
4475 A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
4476 UsedAssumedInformation);
4477 return AnyChange ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
4478 }
4479
4480 /// See AbstractAttribute::trackStatistics()
4481 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(IsDead) }
4482};
4483
4484struct AAIsDeadFunction : public AAIsDead {
4485 AAIsDeadFunction(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4486
4487 /// See AbstractAttribute::initialize(...).
4488 void initialize(Attributor &A) override {
4489 Function *F = getAnchorScope();
4490 assert(F && "Did expect an anchor function");
4491 if (!isAssumedDeadInternalFunction(A)) {
4492 ToBeExploredFrom.insert(X: &F->getEntryBlock().front());
4493 assumeLive(A, BB: F->getEntryBlock());
4494 }
4495 }
4496
4497 bool isAssumedDeadInternalFunction(Attributor &A) {
4498 if (!getAnchorScope()->hasLocalLinkage())
4499 return false;
4500 bool UsedAssumedInformation = false;
4501 return A.checkForAllCallSites(Pred: [](AbstractCallSite) { return false; }, QueryingAA: *this,
4502 RequireAllCallSites: true, UsedAssumedInformation);
4503 }
4504
4505 /// See AbstractAttribute::getAsStr().
4506 const std::string getAsStr(Attributor *A) const override {
4507 return "Live[#BB " + std::to_string(val: AssumedLiveBlocks.size()) + "/" +
4508 std::to_string(val: getAnchorScope()->size()) + "][#TBEP " +
4509 std::to_string(val: ToBeExploredFrom.size()) + "][#KDE " +
4510 std::to_string(val: KnownDeadEnds.size()) + "]";
4511 }
4512
4513 /// See AbstractAttribute::manifest(...).
4514 ChangeStatus manifest(Attributor &A) override {
4515 assert(getState().isValidState() &&
4516 "Attempted to manifest an invalid state!");
4517
4518 ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
4519 Function &F = *getAnchorScope();
4520
4521 if (AssumedLiveBlocks.empty()) {
4522 A.deleteAfterManifest(F);
4523 return ChangeStatus::CHANGED;
4524 }
4525
4526 // Flag to determine if we can change an invoke to a call assuming the
4527 // callee is nounwind. This is not possible if the personality of the
4528 // function allows to catch asynchronous exceptions.
4529 bool Invoke2CallAllowed = !mayCatchAsynchronousExceptions(F);
4530
4531 KnownDeadEnds.set_union(ToBeExploredFrom);
4532 for (const Instruction *DeadEndI : KnownDeadEnds) {
4533 auto *CB = dyn_cast<CallBase>(Val: DeadEndI);
4534 if (!CB)
4535 continue;
4536 bool IsKnownNoReturn;
4537 bool MayReturn = !AA::hasAssumedIRAttr<Attribute::NoReturn>(
4538 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL,
4539 IsKnown&: IsKnownNoReturn);
4540 if (MayReturn && (!Invoke2CallAllowed || !isa<InvokeInst>(Val: CB)))
4541 continue;
4542
4543 if (auto *II = dyn_cast<InvokeInst>(Val: DeadEndI))
4544 A.registerInvokeWithDeadSuccessor(II&: const_cast<InvokeInst &>(*II));
4545 else
4546 A.changeToUnreachableAfterManifest(
4547 I: const_cast<Instruction *>(DeadEndI->getNextNode()));
4548 HasChanged = ChangeStatus::CHANGED;
4549 }
4550
4551 STATS_DECL(AAIsDead, BasicBlock, "Number of dead basic blocks deleted.");
4552 for (BasicBlock &BB : F)
4553 if (!AssumedLiveBlocks.count(V: &BB)) {
4554 A.deleteAfterManifest(BB);
4555 ++BUILD_STAT_NAME(AAIsDead, BasicBlock);
4556 HasChanged = ChangeStatus::CHANGED;
4557 }
4558
4559 return HasChanged;
4560 }
4561
4562 /// See AbstractAttribute::updateImpl(...).
4563 ChangeStatus updateImpl(Attributor &A) override;
4564
4565 bool isEdgeDead(const BasicBlock *From, const BasicBlock *To) const override {
4566 assert(From->getParent() == getAnchorScope() &&
4567 To->getParent() == getAnchorScope() &&
4568 "Used AAIsDead of the wrong function");
4569 return isValidState() && !AssumedLiveEdges.count(V: std::make_pair(x&: From, y&: To));
4570 }
4571
4572 /// See AbstractAttribute::trackStatistics()
4573 void trackStatistics() const override {}
4574
4575 /// Returns true if the function is assumed dead.
4576 bool isAssumedDead() const override { return false; }
4577
4578 /// See AAIsDead::isKnownDead().
4579 bool isKnownDead() const override { return false; }
4580
4581 /// See AAIsDead::isAssumedDead(BasicBlock *).
4582 bool isAssumedDead(const BasicBlock *BB) const override {
4583 assert(BB->getParent() == getAnchorScope() &&
4584 "BB must be in the same anchor scope function.");
4585
4586 if (!getAssumed())
4587 return false;
4588 return !AssumedLiveBlocks.count(V: BB);
4589 }
4590
4591 /// See AAIsDead::isKnownDead(BasicBlock *).
4592 bool isKnownDead(const BasicBlock *BB) const override {
4593 return getKnown() && isAssumedDead(BB);
4594 }
4595
4596 /// See AAIsDead::isAssumed(Instruction *I).
4597 bool isAssumedDead(const Instruction *I) const override {
4598 assert(I->getParent()->getParent() == getAnchorScope() &&
4599 "Instruction must be in the same anchor scope function.");
4600
4601 if (!getAssumed())
4602 return false;
4603
4604 // If it is not in AssumedLiveBlocks then it for sure dead.
4605 // Otherwise, it can still be after noreturn call in a live block.
4606 if (!AssumedLiveBlocks.count(V: I->getParent()))
4607 return true;
4608
4609 // If it is not after a liveness barrier it is live.
4610 const Instruction *PrevI = I->getPrevNode();
4611 while (PrevI) {
4612 if (KnownDeadEnds.count(key: PrevI) || ToBeExploredFrom.count(key: PrevI))
4613 return true;
4614 PrevI = PrevI->getPrevNode();
4615 }
4616 return false;
4617 }
4618
4619 /// See AAIsDead::isKnownDead(Instruction *I).
4620 bool isKnownDead(const Instruction *I) const override {
4621 return getKnown() && isAssumedDead(I);
4622 }
4623
4624 /// Assume \p BB is (partially) live now and indicate to the Attributor \p A
4625 /// that internal function called from \p BB should now be looked at.
4626 bool assumeLive(Attributor &A, const BasicBlock &BB) {
4627 if (!AssumedLiveBlocks.insert(V: &BB).second)
4628 return false;
4629
4630 // We assume that all of BB is (probably) live now and if there are calls to
4631 // internal functions we will assume that those are now live as well. This
4632 // is a performance optimization for blocks with calls to a lot of internal
4633 // functions. It can however cause dead functions to be treated as live.
4634 for (const Instruction &I : BB)
4635 if (const auto *CB = dyn_cast<CallBase>(Val: &I))
4636 if (auto *F = dyn_cast_if_present<Function>(Val: CB->getCalledOperand()))
4637 if (F->hasLocalLinkage())
4638 A.markLiveInternalFunction(F: *F);
4639 return true;
4640 }
4641
4642 /// Collection of instructions that need to be explored again, e.g., we
4643 /// did assume they do not transfer control to (one of their) successors.
4644 SmallSetVector<const Instruction *, 8> ToBeExploredFrom;
4645
4646 /// Collection of instructions that are known to not transfer control.
4647 SmallSetVector<const Instruction *, 8> KnownDeadEnds;
4648
4649 /// Collection of all assumed live edges
4650 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> AssumedLiveEdges;
4651
4652 /// Collection of all assumed live BasicBlocks.
4653 DenseSet<const BasicBlock *> AssumedLiveBlocks;
4654};
4655
4656static bool
4657identifyAliveSuccessors(Attributor &A, const CallBase &CB,
4658 AbstractAttribute &AA,
4659 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4660 const IRPosition &IPos = IRPosition::callsite_function(CB);
4661
4662 bool IsKnownNoReturn;
4663 if (AA::hasAssumedIRAttr<Attribute::NoReturn>(
4664 A, QueryingAA: &AA, IRP: IPos, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoReturn))
4665 return !IsKnownNoReturn;
4666 if (CB.isTerminator())
4667 AliveSuccessors.push_back(Elt: &CB.getSuccessor(Idx: 0)->front());
4668 else
4669 AliveSuccessors.push_back(Elt: CB.getNextNode());
4670 return false;
4671}
4672
4673static bool
4674identifyAliveSuccessors(Attributor &A, const InvokeInst &II,
4675 AbstractAttribute &AA,
4676 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4677 bool UsedAssumedInformation =
4678 identifyAliveSuccessors(A, CB: cast<CallBase>(Val: II), AA, AliveSuccessors);
4679
4680 // First, determine if we can change an invoke to a call assuming the
4681 // callee is nounwind. This is not possible if the personality of the
4682 // function allows to catch asynchronous exceptions.
4683 if (AAIsDeadFunction::mayCatchAsynchronousExceptions(F: *II.getFunction())) {
4684 AliveSuccessors.push_back(Elt: &II.getUnwindDest()->front());
4685 } else {
4686 const IRPosition &IPos = IRPosition::callsite_function(CB: II);
4687
4688 bool IsKnownNoUnwind;
4689 if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
4690 A, QueryingAA: &AA, IRP: IPos, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoUnwind)) {
4691 UsedAssumedInformation |= !IsKnownNoUnwind;
4692 } else {
4693 AliveSuccessors.push_back(Elt: &II.getUnwindDest()->front());
4694 }
4695 }
4696 return UsedAssumedInformation;
4697}
4698
4699static bool
4700identifyAliveSuccessors(Attributor &, const UncondBrInst &BI,
4701 AbstractAttribute &,
4702 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4703 AliveSuccessors.push_back(Elt: &BI.getSuccessor()->front());
4704 return false;
4705}
4706
4707static bool
4708identifyAliveSuccessors(Attributor &A, const CondBrInst &BI,
4709 AbstractAttribute &AA,
4710 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4711 bool UsedAssumedInformation = false;
4712 std::optional<Constant *> C =
4713 A.getAssumedConstant(V: *BI.getCondition(), AA, UsedAssumedInformation);
4714 if (!C || isa_and_nonnull<UndefValue>(Val: *C)) {
4715 // No value yet, assume both edges are dead.
4716 } else if (isa_and_nonnull<ConstantInt>(Val: *C)) {
4717 const BasicBlock *SuccBB =
4718 BI.getSuccessor(i: 1 - cast<ConstantInt>(Val: *C)->getValue().getZExtValue());
4719 AliveSuccessors.push_back(Elt: &SuccBB->front());
4720 } else {
4721 AliveSuccessors.push_back(Elt: &BI.getSuccessor(i: 0)->front());
4722 AliveSuccessors.push_back(Elt: &BI.getSuccessor(i: 1)->front());
4723 UsedAssumedInformation = false;
4724 }
4725 return UsedAssumedInformation;
4726}
4727
4728static bool
4729identifyAliveSuccessors(Attributor &A, const SwitchInst &SI,
4730 AbstractAttribute &AA,
4731 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4732 bool UsedAssumedInformation = false;
4733 SmallVector<AA::ValueAndContext> Values;
4734 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *SI.getCondition()), AA: &AA,
4735 Values, S: AA::AnyScope,
4736 UsedAssumedInformation)) {
4737 // Something went wrong, assume all successors are live.
4738 for (const BasicBlock *SuccBB : successors(BB: SI.getParent()))
4739 AliveSuccessors.push_back(Elt: &SuccBB->front());
4740 return false;
4741 }
4742
4743 if (Values.empty() ||
4744 (Values.size() == 1 &&
4745 isa_and_nonnull<UndefValue>(Val: Values.front().getValue()))) {
4746 // No valid value yet, assume all edges are dead.
4747 return UsedAssumedInformation;
4748 }
4749
4750 Type &Ty = *SI.getCondition()->getType();
4751 SmallPtrSet<ConstantInt *, 8> Constants;
4752 auto CheckForConstantInt = [&](Value *V) {
4753 if (auto *CI = dyn_cast_if_present<ConstantInt>(Val: AA::getWithType(V&: *V, Ty))) {
4754 Constants.insert(Ptr: CI);
4755 return true;
4756 }
4757 return false;
4758 };
4759
4760 if (!all_of(Range&: Values, P: [&](AA::ValueAndContext &VAC) {
4761 return CheckForConstantInt(VAC.getValue());
4762 })) {
4763 for (const BasicBlock *SuccBB : successors(BB: SI.getParent()))
4764 AliveSuccessors.push_back(Elt: &SuccBB->front());
4765 return UsedAssumedInformation;
4766 }
4767
4768 unsigned MatchedCases = 0;
4769 for (const auto &CaseIt : SI.cases()) {
4770 if (Constants.count(Ptr: CaseIt.getCaseValue())) {
4771 ++MatchedCases;
4772 AliveSuccessors.push_back(Elt: &CaseIt.getCaseSuccessor()->front());
4773 }
4774 }
4775
4776 // If all potential values have been matched, we will not visit the default
4777 // case.
4778 if (MatchedCases < Constants.size())
4779 AliveSuccessors.push_back(Elt: &SI.getDefaultDest()->front());
4780 return UsedAssumedInformation;
4781}
4782
4783ChangeStatus AAIsDeadFunction::updateImpl(Attributor &A) {
4784 ChangeStatus Change = ChangeStatus::UNCHANGED;
4785
4786 if (AssumedLiveBlocks.empty()) {
4787 if (isAssumedDeadInternalFunction(A))
4788 return ChangeStatus::UNCHANGED;
4789
4790 Function *F = getAnchorScope();
4791 ToBeExploredFrom.insert(X: &F->getEntryBlock().front());
4792 assumeLive(A, BB: F->getEntryBlock());
4793 Change = ChangeStatus::CHANGED;
4794 }
4795
4796 LLVM_DEBUG(dbgs() << "[AAIsDead] Live [" << AssumedLiveBlocks.size() << "/"
4797 << getAnchorScope()->size() << "] BBs and "
4798 << ToBeExploredFrom.size() << " exploration points and "
4799 << KnownDeadEnds.size() << " known dead ends\n");
4800
4801 // Copy and clear the list of instructions we need to explore from. It is
4802 // refilled with instructions the next update has to look at.
4803 SmallVector<const Instruction *, 8> Worklist(ToBeExploredFrom.begin(),
4804 ToBeExploredFrom.end());
4805 decltype(ToBeExploredFrom) NewToBeExploredFrom;
4806
4807 SmallVector<const Instruction *, 8> AliveSuccessors;
4808 while (!Worklist.empty()) {
4809 const Instruction *I = Worklist.pop_back_val();
4810 LLVM_DEBUG(dbgs() << "[AAIsDead] Exploration inst: " << *I << "\n");
4811
4812 // Fast forward for uninteresting instructions. We could look for UB here
4813 // though.
4814 while (!I->isTerminator() && !isa<CallBase>(Val: I))
4815 I = I->getNextNode();
4816
4817 AliveSuccessors.clear();
4818
4819 bool UsedAssumedInformation = false;
4820 switch (I->getOpcode()) {
4821 // TODO: look for (assumed) UB to backwards propagate "deadness".
4822 default:
4823 assert(I->isTerminator() &&
4824 "Expected non-terminators to be handled already!");
4825 for (const BasicBlock *SuccBB : successors(BB: I->getParent()))
4826 AliveSuccessors.push_back(Elt: &SuccBB->front());
4827 break;
4828 case Instruction::Call:
4829 UsedAssumedInformation = identifyAliveSuccessors(A, CB: cast<CallInst>(Val: *I),
4830 AA&: *this, AliveSuccessors);
4831 break;
4832 case Instruction::Invoke:
4833 UsedAssumedInformation = identifyAliveSuccessors(A, II: cast<InvokeInst>(Val: *I),
4834 AA&: *this, AliveSuccessors);
4835 break;
4836 case Instruction::UncondBr:
4837 UsedAssumedInformation = identifyAliveSuccessors(
4838 A, BI: cast<UncondBrInst>(Val: *I), *this, AliveSuccessors);
4839 break;
4840 case Instruction::CondBr:
4841 UsedAssumedInformation = identifyAliveSuccessors(A, BI: cast<CondBrInst>(Val: *I),
4842 AA&: *this, AliveSuccessors);
4843 break;
4844 case Instruction::Switch:
4845 UsedAssumedInformation = identifyAliveSuccessors(A, SI: cast<SwitchInst>(Val: *I),
4846 AA&: *this, AliveSuccessors);
4847 break;
4848 }
4849
4850 if (UsedAssumedInformation) {
4851 NewToBeExploredFrom.insert(X: I);
4852 } else if (AliveSuccessors.empty() ||
4853 (I->isTerminator() &&
4854 AliveSuccessors.size() < I->getNumSuccessors())) {
4855 if (KnownDeadEnds.insert(X: I))
4856 Change = ChangeStatus::CHANGED;
4857 }
4858
4859 LLVM_DEBUG(dbgs() << "[AAIsDead] #AliveSuccessors: "
4860 << AliveSuccessors.size() << " UsedAssumedInformation: "
4861 << UsedAssumedInformation << "\n");
4862
4863 for (const Instruction *AliveSuccessor : AliveSuccessors) {
4864 if (!I->isTerminator()) {
4865 assert(AliveSuccessors.size() == 1 &&
4866 "Non-terminator expected to have a single successor!");
4867 Worklist.push_back(Elt: AliveSuccessor);
4868 } else {
4869 // record the assumed live edge
4870 auto Edge = std::make_pair(x: I->getParent(), y: AliveSuccessor->getParent());
4871 if (AssumedLiveEdges.insert(V: Edge).second)
4872 Change = ChangeStatus::CHANGED;
4873 if (assumeLive(A, BB: *AliveSuccessor->getParent()))
4874 Worklist.push_back(Elt: AliveSuccessor);
4875 }
4876 }
4877 }
4878
4879 // Check if the content of ToBeExploredFrom changed, ignore the order.
4880 if (NewToBeExploredFrom.size() != ToBeExploredFrom.size() ||
4881 llvm::any_of(Range&: NewToBeExploredFrom, P: [&](const Instruction *I) {
4882 return !ToBeExploredFrom.count(key: I);
4883 })) {
4884 Change = ChangeStatus::CHANGED;
4885 ToBeExploredFrom = std::move(NewToBeExploredFrom);
4886 }
4887
4888 // If we know everything is live there is no need to query for liveness.
4889 // Instead, indicating a pessimistic fixpoint will cause the state to be
4890 // "invalid" and all queries to be answered conservatively without lookups.
4891 // To be in this state we have to (1) finished the exploration and (3) not
4892 // discovered any non-trivial dead end and (2) not ruled unreachable code
4893 // dead.
4894 if (ToBeExploredFrom.empty() &&
4895 getAnchorScope()->size() == AssumedLiveBlocks.size() &&
4896 llvm::all_of(Range&: KnownDeadEnds, P: [](const Instruction *DeadEndI) {
4897 return DeadEndI->isTerminator() && DeadEndI->getNumSuccessors() == 0;
4898 }))
4899 return indicatePessimisticFixpoint();
4900 return Change;
4901}
4902
4903/// Liveness information for a call sites.
4904struct AAIsDeadCallSite final : AAIsDeadFunction {
4905 AAIsDeadCallSite(const IRPosition &IRP, Attributor &A)
4906 : AAIsDeadFunction(IRP, A) {}
4907
4908 /// See AbstractAttribute::initialize(...).
4909 void initialize(Attributor &A) override {
4910 // TODO: Once we have call site specific value information we can provide
4911 // call site specific liveness information and then it makes
4912 // sense to specialize attributes for call sites instead of
4913 // redirecting requests to the callee.
4914 llvm_unreachable("Abstract attributes for liveness are not "
4915 "supported for call sites yet!");
4916 }
4917
4918 /// See AbstractAttribute::updateImpl(...).
4919 ChangeStatus updateImpl(Attributor &A) override {
4920 return indicatePessimisticFixpoint();
4921 }
4922
4923 /// See AbstractAttribute::trackStatistics()
4924 void trackStatistics() const override {}
4925};
4926} // namespace
4927
4928/// -------------------- Dereferenceable Argument Attribute --------------------
4929
4930namespace {
4931struct AADereferenceableImpl : AADereferenceable {
4932 AADereferenceableImpl(const IRPosition &IRP, Attributor &A)
4933 : AADereferenceable(IRP, A) {}
4934 using StateType = DerefState;
4935
4936 /// See AbstractAttribute::initialize(...).
4937 void initialize(Attributor &A) override {
4938 Value &V = *getAssociatedValue().stripPointerCasts();
4939 SmallVector<Attribute, 4> Attrs;
4940 A.getAttrs(IRP: getIRPosition(),
4941 AKs: {Attribute::Dereferenceable, Attribute::DereferenceableOrNull},
4942 Attrs, /* IgnoreSubsumingPositions */ false);
4943 for (const Attribute &Attr : Attrs)
4944 takeKnownDerefBytesMaximum(Bytes: Attr.getValueAsInt());
4945
4946 // Ensure we initialize the non-null AA (if necessary).
4947 bool IsKnownNonNull;
4948 AA::hasAssumedIRAttr<Attribute::NonNull>(
4949 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNonNull);
4950
4951 bool CanBeNull, CanBeFreed;
4952 takeKnownDerefBytesMaximum(Bytes: V.getPointerDereferenceableBytes(
4953 DL: A.getDataLayout(), CanBeNull, CanBeFreed));
4954
4955 if (Instruction *CtxI = getCtxI())
4956 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
4957 }
4958
4959 /// See AbstractAttribute::getState()
4960 /// {
4961 StateType &getState() override { return *this; }
4962 const StateType &getState() const override { return *this; }
4963 /// }
4964
4965 /// Helper function for collecting accessed bytes in must-be-executed-context
4966 void addAccessedBytesForUse(Attributor &A, const Use *U, const Instruction *I,
4967 DerefState &State) {
4968 const Value *UseV = U->get();
4969 if (!UseV->getType()->isPointerTy())
4970 return;
4971
4972 std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(Inst: I);
4973 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() || I->isVolatile())
4974 return;
4975
4976 int64_t Offset;
4977 const Value *Base = GetPointerBaseWithConstantOffset(
4978 Ptr: Loc->Ptr, Offset, DL: A.getDataLayout(), /*AllowNonInbounds*/ true);
4979 if (Base && Base == &getAssociatedValue())
4980 State.addAccessedBytes(Offset, Size: Loc->Size.getValue());
4981 }
4982
4983 /// See followUsesInMBEC
4984 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
4985 AADereferenceable::StateType &State) {
4986 bool IsNonNull = false;
4987 bool TrackUse = false;
4988 int64_t DerefBytes = getKnownNonNullAndDerefBytesForUse(
4989 A, QueryingAA: *this, AssociatedValue&: getAssociatedValue(), U, I, IsNonNull, TrackUse);
4990 LLVM_DEBUG(dbgs() << "[AADereferenceable] Deref bytes: " << DerefBytes
4991 << " for instruction " << *I << "\n");
4992
4993 addAccessedBytesForUse(A, U, I, State);
4994 State.takeKnownDerefBytesMaximum(Bytes: DerefBytes);
4995 return TrackUse;
4996 }
4997
4998 /// See AbstractAttribute::manifest(...).
4999 ChangeStatus manifest(Attributor &A) override {
5000 ChangeStatus Change = AADereferenceable::manifest(A);
5001 bool IsKnownNonNull;
5002 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
5003 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
5004 if (IsAssumedNonNull &&
5005 A.hasAttr(IRP: getIRPosition(), AKs: Attribute::DereferenceableOrNull)) {
5006 A.removeAttrs(IRP: getIRPosition(), AttrKinds: {Attribute::DereferenceableOrNull});
5007 return ChangeStatus::CHANGED;
5008 }
5009 return Change;
5010 }
5011
5012 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5013 SmallVectorImpl<Attribute> &Attrs) const override {
5014 // TODO: Add *_globally support
5015 bool IsKnownNonNull;
5016 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
5017 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
5018 if (IsAssumedNonNull)
5019 Attrs.emplace_back(Args: Attribute::getWithDereferenceableBytes(
5020 Context&: Ctx, Bytes: getAssumedDereferenceableBytes()));
5021 else
5022 Attrs.emplace_back(Args: Attribute::getWithDereferenceableOrNullBytes(
5023 Context&: Ctx, Bytes: getAssumedDereferenceableBytes()));
5024 }
5025
5026 /// See AbstractAttribute::getAsStr().
5027 const std::string getAsStr(Attributor *A) const override {
5028 if (!getAssumedDereferenceableBytes())
5029 return "unknown-dereferenceable";
5030 bool IsKnownNonNull;
5031 bool IsAssumedNonNull = false;
5032 if (A)
5033 IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
5034 A&: *A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
5035 return std::string("dereferenceable") +
5036 (IsAssumedNonNull ? "" : "_or_null") +
5037 (isAssumedGlobal() ? "_globally" : "") + "<" +
5038 std::to_string(val: getKnownDereferenceableBytes()) + "-" +
5039 std::to_string(val: getAssumedDereferenceableBytes()) + ">" +
5040 (!A ? " [non-null is unknown]" : "");
5041 }
5042};
5043
5044/// Dereferenceable attribute for a floating value.
5045struct AADereferenceableFloating : AADereferenceableImpl {
5046 AADereferenceableFloating(const IRPosition &IRP, Attributor &A)
5047 : AADereferenceableImpl(IRP, A) {}
5048
5049 /// See AbstractAttribute::updateImpl(...).
5050 ChangeStatus updateImpl(Attributor &A) override {
5051 bool Stripped;
5052 bool UsedAssumedInformation = false;
5053 SmallVector<AA::ValueAndContext> Values;
5054 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
5055 S: AA::AnyScope, UsedAssumedInformation)) {
5056 Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
5057 Stripped = false;
5058 } else {
5059 Stripped = Values.size() != 1 ||
5060 Values.front().getValue() != &getAssociatedValue();
5061 }
5062
5063 const DataLayout &DL = A.getDataLayout();
5064 DerefState T;
5065
5066 auto VisitValueCB = [&](const Value &V) -> bool {
5067 unsigned IdxWidth =
5068 DL.getIndexSizeInBits(AS: V.getType()->getPointerAddressSpace());
5069 APInt Offset(IdxWidth, 0);
5070 const Value *Base = stripAndAccumulateOffsets(
5071 A, QueryingAA: *this, Val: &V, DL, Offset, /* GetMinOffset */ false,
5072 /* AllowNonInbounds */ true);
5073
5074 const auto *AA = A.getAAFor<AADereferenceable>(
5075 QueryingAA: *this, IRP: IRPosition::value(V: *Base), DepClass: DepClassTy::REQUIRED);
5076 int64_t DerefBytes = 0;
5077 if (!AA || (!Stripped && this == AA)) {
5078 // Use IR information if we did not strip anything.
5079 // TODO: track globally.
5080 bool CanBeNull, CanBeFreed;
5081 DerefBytes =
5082 Base->getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed);
5083 T.GlobalState.indicatePessimisticFixpoint();
5084 } else {
5085 const DerefState &DS = AA->getState();
5086 DerefBytes = DS.DerefBytesState.getAssumed();
5087 T.GlobalState &= DS.GlobalState;
5088 }
5089
5090 // For now we do not try to "increase" dereferenceability due to negative
5091 // indices as we first have to come up with code to deal with loops and
5092 // for overflows of the dereferenceable bytes.
5093 int64_t OffsetSExt = Offset.getSExtValue();
5094 if (OffsetSExt < 0)
5095 OffsetSExt = 0;
5096
5097 T.takeAssumedDerefBytesMinimum(
5098 Bytes: std::max(a: int64_t(0), b: DerefBytes - OffsetSExt));
5099
5100 if (this == AA) {
5101 if (!Stripped) {
5102 // If nothing was stripped IR information is all we got.
5103 T.takeKnownDerefBytesMaximum(
5104 Bytes: std::max(a: int64_t(0), b: DerefBytes - OffsetSExt));
5105 T.indicatePessimisticFixpoint();
5106 } else if (OffsetSExt > 0) {
5107 // If something was stripped but there is circular reasoning we look
5108 // for the offset. If it is positive we basically decrease the
5109 // dereferenceable bytes in a circular loop now, which will simply
5110 // drive them down to the known value in a very slow way which we
5111 // can accelerate.
5112 T.indicatePessimisticFixpoint();
5113 }
5114 }
5115
5116 return T.isValidState();
5117 };
5118
5119 for (const auto &VAC : Values)
5120 if (!VisitValueCB(*VAC.getValue()))
5121 return indicatePessimisticFixpoint();
5122
5123 return clampStateAndIndicateChange(S&: getState(), R: T);
5124 }
5125
5126 /// See AbstractAttribute::trackStatistics()
5127 void trackStatistics() const override {
5128 STATS_DECLTRACK_FLOATING_ATTR(dereferenceable)
5129 }
5130};
5131
5132/// Dereferenceable attribute for a return value.
5133struct AADereferenceableReturned final
5134 : AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl> {
5135 using Base =
5136 AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl>;
5137 AADereferenceableReturned(const IRPosition &IRP, Attributor &A)
5138 : Base(IRP, A) {}
5139
5140 /// See AbstractAttribute::trackStatistics()
5141 void trackStatistics() const override {
5142 STATS_DECLTRACK_FNRET_ATTR(dereferenceable)
5143 }
5144};
5145
5146/// Dereferenceable attribute for an argument
5147struct AADereferenceableArgument final
5148 : AAArgumentFromCallSiteArguments<AADereferenceable,
5149 AADereferenceableImpl> {
5150 using Base =
5151 AAArgumentFromCallSiteArguments<AADereferenceable, AADereferenceableImpl>;
5152 AADereferenceableArgument(const IRPosition &IRP, Attributor &A)
5153 : Base(IRP, A) {}
5154
5155 /// See AbstractAttribute::trackStatistics()
5156 void trackStatistics() const override {
5157 STATS_DECLTRACK_ARG_ATTR(dereferenceable)
5158 }
5159};
5160
5161/// Dereferenceable attribute for a call site argument.
5162struct AADereferenceableCallSiteArgument final : AADereferenceableFloating {
5163 AADereferenceableCallSiteArgument(const IRPosition &IRP, Attributor &A)
5164 : AADereferenceableFloating(IRP, A) {}
5165
5166 /// See AbstractAttribute::trackStatistics()
5167 void trackStatistics() const override {
5168 STATS_DECLTRACK_CSARG_ATTR(dereferenceable)
5169 }
5170};
5171
5172/// Dereferenceable attribute deduction for a call site return value.
5173struct AADereferenceableCallSiteReturned final
5174 : AACalleeToCallSite<AADereferenceable, AADereferenceableImpl> {
5175 using Base = AACalleeToCallSite<AADereferenceable, AADereferenceableImpl>;
5176 AADereferenceableCallSiteReturned(const IRPosition &IRP, Attributor &A)
5177 : Base(IRP, A) {}
5178
5179 /// See AbstractAttribute::trackStatistics()
5180 void trackStatistics() const override {
5181 STATS_DECLTRACK_CS_ATTR(dereferenceable);
5182 }
5183};
5184} // namespace
5185
5186// ------------------------ Align Argument Attribute ------------------------
5187
5188namespace {
5189
5190static unsigned getKnownAlignForUse(Attributor &A, AAAlign &QueryingAA,
5191 Value &AssociatedValue, const Use *U,
5192 const Instruction *I, bool &TrackUse) {
5193 // We need to follow common pointer manipulation uses to the accesses they
5194 // feed into.
5195 if (isa<CastInst>(Val: I)) {
5196 // Follow all but ptr2int casts.
5197 TrackUse = !isa<PtrToIntInst>(Val: I);
5198 return 0;
5199 }
5200 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: I)) {
5201 if (GEP->hasAllConstantIndices())
5202 TrackUse = true;
5203 return 0;
5204 }
5205 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I))
5206 switch (II->getIntrinsicID()) {
5207 case Intrinsic::ptrmask: {
5208 // Is it appropriate to pull attribute in initialization?
5209 const auto *ConstVals = A.getAAFor<AAPotentialConstantValues>(
5210 QueryingAA, IRP: IRPosition::value(V: *II->getOperand(i_nocapture: 1)), DepClass: DepClassTy::NONE);
5211 const auto *AlignAA = A.getAAFor<AAAlign>(
5212 QueryingAA, IRP: IRPosition::value(V: *II), DepClass: DepClassTy::NONE);
5213 if (ConstVals && ConstVals->isValidState() && ConstVals->isAtFixpoint()) {
5214 unsigned ShiftValue = std::min(a: ConstVals->getAssumedMinTrailingZeros(),
5215 b: Value::MaxAlignmentExponent);
5216 Align ConstAlign(UINT64_C(1) << ShiftValue);
5217 if (ConstAlign >= AlignAA->getKnownAlign())
5218 return Align(1).value();
5219 }
5220 if (AlignAA)
5221 return AlignAA->getKnownAlign().value();
5222 break;
5223 }
5224 case Intrinsic::amdgcn_make_buffer_rsrc: {
5225 const auto *AlignAA = A.getAAFor<AAAlign>(
5226 QueryingAA, IRP: IRPosition::value(V: *II), DepClass: DepClassTy::NONE);
5227 if (AlignAA)
5228 return AlignAA->getKnownAlign().value();
5229 break;
5230 }
5231 default:
5232 break;
5233 }
5234
5235 MaybeAlign MA;
5236 if (const auto *CB = dyn_cast<CallBase>(Val: I)) {
5237 if (CB->isBundleOperand(U) || CB->isCallee(U))
5238 return 0;
5239
5240 unsigned ArgNo = CB->getArgOperandNo(U);
5241 IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
5242 // As long as we only use known information there is no need to track
5243 // dependences here.
5244 auto *AlignAA = A.getAAFor<AAAlign>(QueryingAA, IRP, DepClass: DepClassTy::NONE);
5245 if (AlignAA)
5246 MA = MaybeAlign(AlignAA->getKnownAlign());
5247 }
5248
5249 const DataLayout &DL = A.getDataLayout();
5250 const Value *UseV = U->get();
5251 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
5252 if (SI->getPointerOperand() == UseV)
5253 MA = SI->getAlign();
5254 } else if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
5255 if (LI->getPointerOperand() == UseV)
5256 MA = LI->getAlign();
5257 } else if (auto *AI = dyn_cast<AtomicRMWInst>(Val: I)) {
5258 if (AI->getPointerOperand() == UseV)
5259 MA = AI->getAlign();
5260 } else if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
5261 if (AI->getPointerOperand() == UseV)
5262 MA = AI->getAlign();
5263 }
5264
5265 if (!MA || *MA <= QueryingAA.getKnownAlign())
5266 return 0;
5267
5268 unsigned Alignment = MA->value();
5269 int64_t Offset;
5270
5271 if (const Value *Base = GetPointerBaseWithConstantOffset(Ptr: UseV, Offset, DL)) {
5272 if (Base == &AssociatedValue) {
5273 // BasePointerAddr + Offset = Alignment * Q for some integer Q.
5274 // So we can say that the maximum power of two which is a divisor of
5275 // gcd(Offset, Alignment) is an alignment.
5276
5277 uint32_t gcd = std::gcd(m: uint32_t(abs(x: (int32_t)Offset)), n: Alignment);
5278 Alignment = llvm::bit_floor(Value: gcd);
5279 }
5280 }
5281
5282 return Alignment;
5283}
5284
5285struct AAAlignImpl : AAAlign {
5286 AAAlignImpl(const IRPosition &IRP, Attributor &A) : AAAlign(IRP, A) {}
5287
5288 /// See AbstractAttribute::initialize(...).
5289 void initialize(Attributor &A) override {
5290 SmallVector<Attribute, 4> Attrs;
5291 A.getAttrs(IRP: getIRPosition(), AKs: {Attribute::Alignment}, Attrs);
5292 for (const Attribute &Attr : Attrs)
5293 takeKnownMaximum(Value: Attr.getValueAsInt());
5294
5295 Value &V = *getAssociatedValue().stripPointerCasts();
5296 takeKnownMaximum(Value: V.getPointerAlignment(DL: A.getDataLayout()).value());
5297
5298 if (Instruction *CtxI = getCtxI())
5299 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
5300 }
5301
5302 /// See AbstractAttribute::manifest(...).
5303 ChangeStatus manifest(Attributor &A) override {
5304 ChangeStatus InstrChanged = ChangeStatus::UNCHANGED;
5305
5306 // Check for users that allow alignment annotations.
5307 Value &AssociatedValue = getAssociatedValue();
5308 if (isa<ConstantData>(Val: AssociatedValue))
5309 return ChangeStatus::UNCHANGED;
5310
5311 for (const Use &U : AssociatedValue.uses()) {
5312 if (auto *SI = dyn_cast<StoreInst>(Val: U.getUser())) {
5313 if (SI->getPointerOperand() == &AssociatedValue)
5314 if (SI->getAlign() < getAssumedAlign()) {
5315 STATS_DECLTRACK(AAAlign, Store,
5316 "Number of times alignment added to a store");
5317 SI->setAlignment(getAssumedAlign());
5318 InstrChanged = ChangeStatus::CHANGED;
5319 }
5320 } else if (auto *LI = dyn_cast<LoadInst>(Val: U.getUser())) {
5321 if (LI->getPointerOperand() == &AssociatedValue)
5322 if (LI->getAlign() < getAssumedAlign()) {
5323 LI->setAlignment(getAssumedAlign());
5324 STATS_DECLTRACK(AAAlign, Load,
5325 "Number of times alignment added to a load");
5326 InstrChanged = ChangeStatus::CHANGED;
5327 }
5328 } else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: U.getUser())) {
5329 if (RMW->getPointerOperand() == &AssociatedValue) {
5330 if (RMW->getAlign() < getAssumedAlign()) {
5331 STATS_DECLTRACK(AAAlign, AtomicRMW,
5332 "Number of times alignment added to atomicrmw");
5333
5334 RMW->setAlignment(getAssumedAlign());
5335 InstrChanged = ChangeStatus::CHANGED;
5336 }
5337 }
5338 } else if (auto *CAS = dyn_cast<AtomicCmpXchgInst>(Val: U.getUser())) {
5339 if (CAS->getPointerOperand() == &AssociatedValue) {
5340 if (CAS->getAlign() < getAssumedAlign()) {
5341 STATS_DECLTRACK(AAAlign, AtomicCmpXchg,
5342 "Number of times alignment added to cmpxchg");
5343 CAS->setAlignment(getAssumedAlign());
5344 InstrChanged = ChangeStatus::CHANGED;
5345 }
5346 }
5347 }
5348 }
5349
5350 ChangeStatus Changed = AAAlign::manifest(A);
5351
5352 Align InheritAlign =
5353 getAssociatedValue().getPointerAlignment(DL: A.getDataLayout());
5354 if (InheritAlign >= getAssumedAlign())
5355 return InstrChanged;
5356 return Changed | InstrChanged;
5357 }
5358
5359 // TODO: Provide a helper to determine the implied ABI alignment and check in
5360 // the existing manifest method and a new one for AAAlignImpl that value
5361 // to avoid making the alignment explicit if it did not improve.
5362
5363 /// See AbstractAttribute::getDeducedAttributes
5364 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5365 SmallVectorImpl<Attribute> &Attrs) const override {
5366 if (getAssumedAlign() > 1)
5367 Attrs.emplace_back(
5368 Args: Attribute::getWithAlignment(Context&: Ctx, Alignment: Align(getAssumedAlign())));
5369 }
5370
5371 /// See followUsesInMBEC
5372 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
5373 AAAlign::StateType &State) {
5374 bool TrackUse = false;
5375
5376 unsigned int KnownAlign =
5377 getKnownAlignForUse(A, QueryingAA&: *this, AssociatedValue&: getAssociatedValue(), U, I, TrackUse);
5378 State.takeKnownMaximum(Value: KnownAlign);
5379
5380 return TrackUse;
5381 }
5382
5383 /// See AbstractAttribute::getAsStr().
5384 const std::string getAsStr(Attributor *A) const override {
5385 return "align<" + std::to_string(val: getKnownAlign().value()) + "-" +
5386 std::to_string(val: getAssumedAlign().value()) + ">";
5387 }
5388};
5389
5390/// Align attribute for a floating value.
5391struct AAAlignFloating : AAAlignImpl {
5392 AAAlignFloating(const IRPosition &IRP, Attributor &A) : AAAlignImpl(IRP, A) {}
5393
5394 /// See AbstractAttribute::updateImpl(...).
5395 ChangeStatus updateImpl(Attributor &A) override {
5396 const DataLayout &DL = A.getDataLayout();
5397
5398 bool Stripped;
5399 bool UsedAssumedInformation = false;
5400 SmallVector<AA::ValueAndContext> Values;
5401 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
5402 S: AA::AnyScope, UsedAssumedInformation)) {
5403 Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
5404 Stripped = false;
5405 } else {
5406 Stripped = Values.size() != 1 ||
5407 Values.front().getValue() != &getAssociatedValue();
5408 }
5409
5410 StateType T;
5411 auto VisitValueCB = [&](Value &V) -> bool {
5412 if (isa<UndefValue>(Val: V) || isa<ConstantPointerNull>(Val: V))
5413 return true;
5414 const auto *AA = A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V),
5415 DepClass: DepClassTy::REQUIRED);
5416 if (!AA || (!Stripped && this == AA)) {
5417 int64_t Offset;
5418 unsigned Alignment = 1;
5419 if (const Value *Base =
5420 GetPointerBaseWithConstantOffset(Ptr: &V, Offset, DL)) {
5421 // TODO: Use AAAlign for the base too.
5422 Align PA = Base->getPointerAlignment(DL);
5423 // BasePointerAddr + Offset = Alignment * Q for some integer Q.
5424 // So we can say that the maximum power of two which is a divisor of
5425 // gcd(Offset, Alignment) is an alignment.
5426
5427 uint32_t gcd =
5428 std::gcd(m: uint32_t(abs(x: (int32_t)Offset)), n: uint32_t(PA.value()));
5429 Alignment = llvm::bit_floor(Value: gcd);
5430 } else {
5431 Alignment = V.getPointerAlignment(DL).value();
5432 }
5433 // Use only IR information if we did not strip anything.
5434 T.takeKnownMaximum(Value: Alignment);
5435 T.indicatePessimisticFixpoint();
5436 } else {
5437 // Use abstract attribute information.
5438 const AAAlign::StateType &DS = AA->getState();
5439 T ^= DS;
5440 }
5441 return T.isValidState();
5442 };
5443
5444 for (const auto &VAC : Values) {
5445 if (!VisitValueCB(*VAC.getValue()))
5446 return indicatePessimisticFixpoint();
5447 }
5448
5449 // TODO: If we know we visited all incoming values, thus no are assumed
5450 // dead, we can take the known information from the state T.
5451 return clampStateAndIndicateChange(S&: getState(), R: T);
5452 }
5453
5454 /// See AbstractAttribute::trackStatistics()
5455 void trackStatistics() const override { STATS_DECLTRACK_FLOATING_ATTR(align) }
5456};
5457
5458/// Align attribute for function return value.
5459struct AAAlignReturned final
5460 : AAReturnedFromReturnedValues<AAAlign, AAAlignImpl> {
5461 using Base = AAReturnedFromReturnedValues<AAAlign, AAAlignImpl>;
5462 AAAlignReturned(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5463
5464 /// See AbstractAttribute::trackStatistics()
5465 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(aligned) }
5466};
5467
5468/// Align attribute for function argument.
5469struct AAAlignArgument final
5470 : AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl> {
5471 using Base = AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl>;
5472 AAAlignArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5473
5474 /// See AbstractAttribute::manifest(...).
5475 ChangeStatus manifest(Attributor &A) override {
5476 // If the associated argument is involved in a must-tail call we give up
5477 // because we would need to keep the argument alignments of caller and
5478 // callee in-sync. Just does not seem worth the trouble right now.
5479 if (A.getInfoCache().isInvolvedInMustTailCall(Arg: *getAssociatedArgument()))
5480 return ChangeStatus::UNCHANGED;
5481 return Base::manifest(A);
5482 }
5483
5484 /// See AbstractAttribute::trackStatistics()
5485 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(aligned) }
5486};
5487
5488struct AAAlignCallSiteArgument final : AAAlignFloating {
5489 AAAlignCallSiteArgument(const IRPosition &IRP, Attributor &A)
5490 : AAAlignFloating(IRP, A) {}
5491
5492 /// See AbstractAttribute::manifest(...).
5493 ChangeStatus manifest(Attributor &A) override {
5494 // If the associated argument is involved in a must-tail call we give up
5495 // because we would need to keep the argument alignments of caller and
5496 // callee in-sync. Just does not seem worth the trouble right now.
5497 if (Argument *Arg = getAssociatedArgument())
5498 if (A.getInfoCache().isInvolvedInMustTailCall(Arg: *Arg))
5499 return ChangeStatus::UNCHANGED;
5500 ChangeStatus Changed = AAAlignImpl::manifest(A);
5501 Align InheritAlign =
5502 getAssociatedValue().getPointerAlignment(DL: A.getDataLayout());
5503 if (InheritAlign >= getAssumedAlign())
5504 Changed = ChangeStatus::UNCHANGED;
5505 return Changed;
5506 }
5507
5508 /// See AbstractAttribute::updateImpl(Attributor &A).
5509 ChangeStatus updateImpl(Attributor &A) override {
5510 ChangeStatus Changed = AAAlignFloating::updateImpl(A);
5511 if (Argument *Arg = getAssociatedArgument()) {
5512 // We only take known information from the argument
5513 // so we do not need to track a dependence.
5514 const auto *ArgAlignAA = A.getAAFor<AAAlign>(
5515 QueryingAA: *this, IRP: IRPosition::argument(Arg: *Arg), DepClass: DepClassTy::NONE);
5516 if (ArgAlignAA)
5517 takeKnownMaximum(Value: ArgAlignAA->getKnownAlign().value());
5518 }
5519 return Changed;
5520 }
5521
5522 /// See AbstractAttribute::trackStatistics()
5523 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(aligned) }
5524};
5525
5526/// Align attribute deduction for a call site return value.
5527struct AAAlignCallSiteReturned final
5528 : AACalleeToCallSite<AAAlign, AAAlignImpl> {
5529 using Base = AACalleeToCallSite<AAAlign, AAAlignImpl>;
5530 AAAlignCallSiteReturned(const IRPosition &IRP, Attributor &A)
5531 : Base(IRP, A) {}
5532
5533 ChangeStatus updateImpl(Attributor &A) override {
5534 Instruction *I = getIRPosition().getCtxI();
5535 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
5536 switch (II->getIntrinsicID()) {
5537 case Intrinsic::ptrmask: {
5538 Align Alignment;
5539 bool Valid = false;
5540
5541 const auto *ConstVals = A.getAAFor<AAPotentialConstantValues>(
5542 QueryingAA: *this, IRP: IRPosition::value(V: *II->getOperand(i_nocapture: 1)), DepClass: DepClassTy::REQUIRED);
5543 if (ConstVals && ConstVals->isValidState()) {
5544 unsigned ShiftValue =
5545 std::min(a: ConstVals->getAssumedMinTrailingZeros(),
5546 b: Value::MaxAlignmentExponent);
5547 Alignment = Align(UINT64_C(1) << ShiftValue);
5548 Valid = true;
5549 }
5550
5551 const auto *AlignAA =
5552 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *(II->getOperand(i_nocapture: 0))),
5553 DepClass: DepClassTy::REQUIRED);
5554 if (AlignAA) {
5555 Alignment = std::max(a: AlignAA->getAssumedAlign(), b: Alignment);
5556 Valid = true;
5557 }
5558
5559 if (Valid)
5560 return clampStateAndIndicateChange<StateType>(
5561 S&: this->getState(),
5562 R: std::min(a: this->getAssumedAlign(), b: Alignment).value());
5563 break;
5564 }
5565 // FIXME: Should introduce target specific sub-attributes and letting
5566 // getAAfor<AAAlign> lead to create sub-attribute to handle target
5567 // specific intrinsics.
5568 case Intrinsic::amdgcn_make_buffer_rsrc: {
5569 const auto *AlignAA =
5570 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *(II->getOperand(i_nocapture: 0))),
5571 DepClass: DepClassTy::REQUIRED);
5572 if (AlignAA)
5573 return clampStateAndIndicateChange<StateType>(
5574 S&: this->getState(), R: AlignAA->getAssumedAlign().value());
5575 break;
5576 }
5577 default:
5578 break;
5579 }
5580 }
5581 return Base::updateImpl(A);
5582 };
5583 /// See AbstractAttribute::trackStatistics()
5584 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(align); }
5585};
5586} // namespace
5587
5588/// ------------------ Function No-Return Attribute ----------------------------
5589namespace {
5590struct AANoReturnImpl : public AANoReturn {
5591 AANoReturnImpl(const IRPosition &IRP, Attributor &A) : AANoReturn(IRP, A) {}
5592
5593 /// See AbstractAttribute::initialize(...).
5594 void initialize(Attributor &A) override {
5595 bool IsKnown;
5596 assert(!AA::hasAssumedIRAttr<Attribute::NoReturn>(
5597 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5598 (void)IsKnown;
5599 }
5600
5601 /// See AbstractAttribute::getAsStr().
5602 const std::string getAsStr(Attributor *A) const override {
5603 return getAssumed() ? "noreturn" : "may-return";
5604 }
5605
5606 /// See AbstractAttribute::updateImpl(Attributor &A).
5607 ChangeStatus updateImpl(Attributor &A) override {
5608 auto CheckForNoReturn = [](Instruction &) { return false; };
5609 bool UsedAssumedInformation = false;
5610 if (!A.checkForAllInstructions(Pred: CheckForNoReturn, QueryingAA: *this,
5611 Opcodes: {(unsigned)Instruction::Ret},
5612 UsedAssumedInformation))
5613 return indicatePessimisticFixpoint();
5614 return ChangeStatus::UNCHANGED;
5615 }
5616};
5617
5618struct AANoReturnFunction final : AANoReturnImpl {
5619 AANoReturnFunction(const IRPosition &IRP, Attributor &A)
5620 : AANoReturnImpl(IRP, A) {}
5621
5622 /// See AbstractAttribute::trackStatistics()
5623 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(noreturn) }
5624};
5625
5626/// NoReturn attribute deduction for a call sites.
5627struct AANoReturnCallSite final
5628 : AACalleeToCallSite<AANoReturn, AANoReturnImpl> {
5629 AANoReturnCallSite(const IRPosition &IRP, Attributor &A)
5630 : AACalleeToCallSite<AANoReturn, AANoReturnImpl>(IRP, A) {}
5631
5632 /// See AbstractAttribute::trackStatistics()
5633 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(noreturn); }
5634};
5635} // namespace
5636
5637/// ----------------------- Instance Info ---------------------------------
5638
5639namespace {
5640/// A class to hold the state of for no-capture attributes.
5641struct AAInstanceInfoImpl : public AAInstanceInfo {
5642 AAInstanceInfoImpl(const IRPosition &IRP, Attributor &A)
5643 : AAInstanceInfo(IRP, A) {}
5644
5645 /// See AbstractAttribute::initialize(...).
5646 void initialize(Attributor &A) override {
5647 Value &V = getAssociatedValue();
5648 if (auto *C = dyn_cast<Constant>(Val: &V)) {
5649 if (C->isThreadDependent())
5650 indicatePessimisticFixpoint();
5651 else
5652 indicateOptimisticFixpoint();
5653 return;
5654 }
5655 if (auto *CB = dyn_cast<CallBase>(Val: &V))
5656 if (CB->arg_size() == 0 && !CB->mayHaveSideEffects() &&
5657 !CB->mayReadFromMemory()) {
5658 indicateOptimisticFixpoint();
5659 return;
5660 }
5661 if (auto *I = dyn_cast<Instruction>(Val: &V)) {
5662 const auto *CI =
5663 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
5664 F: *I->getFunction());
5665 if (mayBeInCycle(CI, I, /* HeaderOnly */ false)) {
5666 indicatePessimisticFixpoint();
5667 return;
5668 }
5669 }
5670 }
5671
5672 /// See AbstractAttribute::updateImpl(...).
5673 ChangeStatus updateImpl(Attributor &A) override {
5674 ChangeStatus Changed = ChangeStatus::UNCHANGED;
5675
5676 Value &V = getAssociatedValue();
5677 const Function *Scope = nullptr;
5678 if (auto *I = dyn_cast<Instruction>(Val: &V))
5679 Scope = I->getFunction();
5680 if (auto *A = dyn_cast<Argument>(Val: &V)) {
5681 Scope = A->getParent();
5682 if (!Scope->hasLocalLinkage())
5683 return Changed;
5684 }
5685 if (!Scope)
5686 return indicateOptimisticFixpoint();
5687
5688 bool IsKnownNoRecurse;
5689 if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
5690 A, QueryingAA: this, IRP: IRPosition::function(F: *Scope), DepClass: DepClassTy::OPTIONAL,
5691 IsKnown&: IsKnownNoRecurse))
5692 return Changed;
5693
5694 auto UsePred = [&](const Use &U, bool &Follow) {
5695 const Instruction *UserI = dyn_cast<Instruction>(Val: U.getUser());
5696 if (!UserI || isa<GetElementPtrInst>(Val: UserI) || isa<CastInst>(Val: UserI) ||
5697 isa<PHINode>(Val: UserI) || isa<SelectInst>(Val: UserI)) {
5698 Follow = true;
5699 return true;
5700 }
5701 if (isa<LoadInst>(Val: UserI) || isa<CmpInst>(Val: UserI) ||
5702 (isa<StoreInst>(Val: UserI) &&
5703 cast<StoreInst>(Val: UserI)->getValueOperand() != U.get()))
5704 return true;
5705 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
5706 // This check is not guaranteeing uniqueness but for now that we cannot
5707 // end up with two versions of \p U thinking it was one.
5708 auto *Callee = dyn_cast_if_present<Function>(Val: CB->getCalledOperand());
5709 if (!Callee || !Callee->hasLocalLinkage())
5710 return true;
5711 if (!CB->isArgOperand(U: &U))
5712 return false;
5713 const auto *ArgInstanceInfoAA = A.getAAFor<AAInstanceInfo>(
5714 QueryingAA: *this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U)),
5715 DepClass: DepClassTy::OPTIONAL);
5716 if (!ArgInstanceInfoAA ||
5717 !ArgInstanceInfoAA->isAssumedUniqueForAnalysis())
5718 return false;
5719 // If this call base might reach the scope again we might forward the
5720 // argument back here. This is very conservative.
5721 if (AA::isPotentiallyReachable(
5722 A, FromI: *CB, ToFn: *Scope, QueryingAA: *this, /* ExclusionSet */ nullptr,
5723 GoBackwardsCB: [Scope](const Function &Fn) { return &Fn != Scope; }))
5724 return false;
5725 return true;
5726 }
5727 return false;
5728 };
5729
5730 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
5731 if (auto *SI = dyn_cast<StoreInst>(Val: OldU.getUser())) {
5732 auto *Ptr = SI->getPointerOperand()->stripPointerCasts();
5733 if ((isa<AllocaInst>(Val: Ptr) || isNoAliasCall(V: Ptr)) &&
5734 AA::isDynamicallyUnique(A, QueryingAA: *this, V: *Ptr))
5735 return true;
5736 }
5737 return false;
5738 };
5739
5740 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V, /* CheckBBLivenessOnly */ true,
5741 LivenessDepClass: DepClassTy::OPTIONAL,
5742 /* IgnoreDroppableUses */ true, EquivalentUseCB))
5743 return indicatePessimisticFixpoint();
5744
5745 return Changed;
5746 }
5747
5748 /// See AbstractState::getAsStr().
5749 const std::string getAsStr(Attributor *A) const override {
5750 return isAssumedUniqueForAnalysis() ? "<unique [fAa]>" : "<unknown>";
5751 }
5752
5753 /// See AbstractAttribute::trackStatistics()
5754 void trackStatistics() const override {}
5755};
5756
5757/// InstanceInfo attribute for floating values.
5758struct AAInstanceInfoFloating : AAInstanceInfoImpl {
5759 AAInstanceInfoFloating(const IRPosition &IRP, Attributor &A)
5760 : AAInstanceInfoImpl(IRP, A) {}
5761};
5762
5763/// NoCapture attribute for function arguments.
5764struct AAInstanceInfoArgument final : AAInstanceInfoFloating {
5765 AAInstanceInfoArgument(const IRPosition &IRP, Attributor &A)
5766 : AAInstanceInfoFloating(IRP, A) {}
5767};
5768
5769/// InstanceInfo attribute for call site arguments.
5770struct AAInstanceInfoCallSiteArgument final : AAInstanceInfoImpl {
5771 AAInstanceInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
5772 : AAInstanceInfoImpl(IRP, A) {}
5773
5774 /// See AbstractAttribute::updateImpl(...).
5775 ChangeStatus updateImpl(Attributor &A) override {
5776 // TODO: Once we have call site specific value information we can provide
5777 // call site specific liveness information and then it makes
5778 // sense to specialize attributes for call sites arguments instead of
5779 // redirecting requests to the callee argument.
5780 Argument *Arg = getAssociatedArgument();
5781 if (!Arg)
5782 return indicatePessimisticFixpoint();
5783 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
5784 auto *ArgAA =
5785 A.getAAFor<AAInstanceInfo>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
5786 if (!ArgAA)
5787 return indicatePessimisticFixpoint();
5788 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
5789 }
5790};
5791
5792/// InstanceInfo attribute for function return value.
5793struct AAInstanceInfoReturned final : AAInstanceInfoImpl {
5794 AAInstanceInfoReturned(const IRPosition &IRP, Attributor &A)
5795 : AAInstanceInfoImpl(IRP, A) {
5796 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5797 }
5798
5799 /// See AbstractAttribute::initialize(...).
5800 void initialize(Attributor &A) override {
5801 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5802 }
5803
5804 /// See AbstractAttribute::updateImpl(...).
5805 ChangeStatus updateImpl(Attributor &A) override {
5806 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5807 }
5808};
5809
5810/// InstanceInfo attribute deduction for a call site return value.
5811struct AAInstanceInfoCallSiteReturned final : AAInstanceInfoFloating {
5812 AAInstanceInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
5813 : AAInstanceInfoFloating(IRP, A) {}
5814};
5815} // namespace
5816
5817/// ----------------------- Variable Capturing ---------------------------------
5818bool AANoCapture::isImpliedByIR(Attributor &A, const IRPosition &IRP,
5819 Attribute::AttrKind ImpliedAttributeKind,
5820 bool IgnoreSubsumingPositions) {
5821 assert(ImpliedAttributeKind == Attribute::Captures &&
5822 "Unexpected attribute kind");
5823 Value &V = IRP.getAssociatedValue();
5824 if (!isa<Constant>(Val: V) && !IRP.isArgumentPosition())
5825 return V.use_empty();
5826
5827 // You cannot "capture" null in the default address space.
5828 //
5829 // FIXME: This should use NullPointerIsDefined to account for the function
5830 // attribute.
5831 if (isa<UndefValue>(Val: V) || (isa<ConstantPointerNull>(Val: V) &&
5832 V.getType()->getPointerAddressSpace() == 0)) {
5833 return true;
5834 }
5835
5836 SmallVector<Attribute, 1> Attrs;
5837 A.getAttrs(IRP, AKs: {Attribute::Captures}, Attrs,
5838 /* IgnoreSubsumingPositions */ true);
5839 for (const Attribute &Attr : Attrs)
5840 if (capturesNothing(CC: Attr.getCaptureInfo()))
5841 return true;
5842
5843 if (IRP.getPositionKind() == IRP_CALL_SITE_ARGUMENT)
5844 if (Argument *Arg = IRP.getAssociatedArgument()) {
5845 SmallVector<Attribute, 1> Attrs;
5846 A.getAttrs(IRP: IRPosition::argument(Arg: *Arg),
5847 AKs: {Attribute::Captures, Attribute::ByVal}, Attrs,
5848 /* IgnoreSubsumingPositions */ true);
5849 bool ArgNoCapture = any_of(Range&: Attrs, P: [](Attribute Attr) {
5850 return Attr.getKindAsEnum() == Attribute::ByVal ||
5851 capturesNothing(CC: Attr.getCaptureInfo());
5852 });
5853 if (ArgNoCapture) {
5854 A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithCaptureInfo(
5855 Context&: V.getContext(), CI: CaptureInfo::none()));
5856 return true;
5857 }
5858 }
5859
5860 if (const Function *F = IRP.getAssociatedFunction()) {
5861 // Check what state the associated function can actually capture.
5862 AANoCapture::StateType State;
5863 determineFunctionCaptureCapabilities(IRP, F: *F, State);
5864 if (State.isKnown(BitsEncoding: NO_CAPTURE)) {
5865 A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithCaptureInfo(Context&: V.getContext(),
5866 CI: CaptureInfo::none()));
5867 return true;
5868 }
5869 }
5870
5871 return false;
5872}
5873
5874/// Set the NOT_CAPTURED_IN_MEM and NOT_CAPTURED_IN_RET bits in \p Known
5875/// depending on the ability of the function associated with \p IRP to capture
5876/// state in memory and through "returning/throwing", respectively.
5877void AANoCapture::determineFunctionCaptureCapabilities(const IRPosition &IRP,
5878 const Function &F,
5879 BitIntegerState &State) {
5880 // TODO: Once we have memory behavior attributes we should use them here.
5881
5882 // If we know we cannot communicate or write to memory, we do not care about
5883 // ptr2int anymore.
5884 bool ReadOnly = F.onlyReadsMemory();
5885 bool NoThrow = F.doesNotThrow();
5886 bool IsVoidReturn = F.getReturnType()->isVoidTy();
5887 if (ReadOnly && NoThrow && IsVoidReturn) {
5888 State.addKnownBits(Bits: NO_CAPTURE);
5889 return;
5890 }
5891
5892 // A function cannot capture state in memory if it only reads memory, it can
5893 // however return/throw state and the state might be influenced by the
5894 // pointer value, e.g., loading from a returned pointer might reveal a bit.
5895 if (ReadOnly)
5896 State.addKnownBits(Bits: NOT_CAPTURED_IN_MEM);
5897
5898 // A function cannot communicate state back if it does not through
5899 // exceptions and doesn not return values.
5900 if (NoThrow && IsVoidReturn)
5901 State.addKnownBits(Bits: NOT_CAPTURED_IN_RET);
5902
5903 // Check existing "returned" attributes.
5904 int ArgNo = IRP.getCalleeArgNo();
5905 if (!NoThrow || ArgNo < 0 ||
5906 !F.getAttributes().hasAttrSomewhere(Kind: Attribute::Returned))
5907 return;
5908
5909 for (unsigned U = 0, E = F.arg_size(); U < E; ++U)
5910 if (F.hasParamAttribute(ArgNo: U, Kind: Attribute::Returned)) {
5911 if (U == unsigned(ArgNo))
5912 State.removeAssumedBits(BitsEncoding: NOT_CAPTURED_IN_RET);
5913 else if (ReadOnly)
5914 State.addKnownBits(Bits: NO_CAPTURE);
5915 else
5916 State.addKnownBits(Bits: NOT_CAPTURED_IN_RET);
5917 break;
5918 }
5919}
5920
5921namespace {
5922/// A class to hold the state of for no-capture attributes.
5923struct AANoCaptureImpl : public AANoCapture {
5924 AANoCaptureImpl(const IRPosition &IRP, Attributor &A) : AANoCapture(IRP, A) {}
5925
5926 /// See AbstractAttribute::initialize(...).
5927 void initialize(Attributor &A) override {
5928 bool IsKnown;
5929 assert(!AA::hasAssumedIRAttr<Attribute::Captures>(
5930 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5931 (void)IsKnown;
5932 }
5933
5934 /// See AbstractAttribute::updateImpl(...).
5935 ChangeStatus updateImpl(Attributor &A) override;
5936
5937 /// see AbstractAttribute::isAssumedNoCaptureMaybeReturned(...).
5938 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5939 SmallVectorImpl<Attribute> &Attrs) const override {
5940 if (!isAssumedNoCaptureMaybeReturned())
5941 return;
5942
5943 if (isArgumentPosition()) {
5944 if (isAssumedNoCapture())
5945 Attrs.emplace_back(Args: Attribute::get(Context&: Ctx, Kind: Attribute::Captures));
5946 else if (ManifestInternal)
5947 Attrs.emplace_back(Args: Attribute::get(Context&: Ctx, Kind: "no-capture-maybe-returned"));
5948 }
5949 }
5950
5951 /// See AbstractState::getAsStr().
5952 const std::string getAsStr(Attributor *A) const override {
5953 if (isKnownNoCapture())
5954 return "known not-captured";
5955 if (isAssumedNoCapture())
5956 return "assumed not-captured";
5957 if (isKnownNoCaptureMaybeReturned())
5958 return "known not-captured-maybe-returned";
5959 if (isAssumedNoCaptureMaybeReturned())
5960 return "assumed not-captured-maybe-returned";
5961 return "assumed-captured";
5962 }
5963
5964 /// Check the use \p U and update \p State accordingly. Return true if we
5965 /// should continue to update the state.
5966 bool checkUse(Attributor &A, AANoCapture::StateType &State, const Use &U,
5967 bool &Follow) {
5968 Instruction *UInst = cast<Instruction>(Val: U.getUser());
5969 LLVM_DEBUG(dbgs() << "[AANoCapture] Check use: " << *U.get() << " in "
5970 << *UInst << "\n");
5971
5972 // Deal with ptr2int by following uses.
5973 if (isa<PtrToIntInst>(Val: UInst)) {
5974 LLVM_DEBUG(dbgs() << " - ptr2int assume the worst!\n");
5975 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
5976 /* Return */ CapturedInRet: true);
5977 }
5978
5979 // For stores we already checked if we can follow them, if they make it
5980 // here we give up.
5981 if (isa<StoreInst>(Val: UInst))
5982 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
5983 /* Return */ CapturedInRet: true);
5984
5985 // Explicitly catch return instructions.
5986 if (isa<ReturnInst>(Val: UInst)) {
5987 if (UInst->getFunction() == getAnchorScope())
5988 return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
5989 /* Return */ CapturedInRet: true);
5990 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
5991 /* Return */ CapturedInRet: true);
5992 }
5993
5994 // For now we only use special logic for call sites. However, the tracker
5995 // itself knows about a lot of other non-capturing cases already.
5996 auto *CB = dyn_cast<CallBase>(Val: UInst);
5997 if (!CB || !CB->isArgOperand(U: &U))
5998 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
5999 /* Return */ CapturedInRet: true);
6000
6001 unsigned ArgNo = CB->getArgOperandNo(U: &U);
6002 const IRPosition &CSArgPos = IRPosition::callsite_argument(CB: *CB, ArgNo);
6003 // If we have a abstract no-capture attribute for the argument we can use
6004 // it to justify a non-capture attribute here. This allows recursion!
6005 bool IsKnownNoCapture;
6006 const AANoCapture *ArgNoCaptureAA = nullptr;
6007 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
6008 A, QueryingAA: this, IRP: CSArgPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
6009 AAPtr: &ArgNoCaptureAA);
6010 if (IsAssumedNoCapture)
6011 return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
6012 /* Return */ CapturedInRet: false);
6013 if (ArgNoCaptureAA && ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned()) {
6014 Follow = true;
6015 return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
6016 /* Return */ CapturedInRet: false);
6017 }
6018
6019 // Lastly, we could not find a reason no-capture can be assumed so we don't.
6020 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
6021 /* Return */ CapturedInRet: true);
6022 }
6023
6024 /// Update \p State according to \p CapturedInMem, \p CapturedInInt, and
6025 /// \p CapturedInRet, then return true if we should continue updating the
6026 /// state.
6027 static bool isCapturedIn(AANoCapture::StateType &State, bool CapturedInMem,
6028 bool CapturedInInt, bool CapturedInRet) {
6029 LLVM_DEBUG(dbgs() << " - captures [Mem " << CapturedInMem << "|Int "
6030 << CapturedInInt << "|Ret " << CapturedInRet << "]\n");
6031 if (CapturedInMem)
6032 State.removeAssumedBits(BitsEncoding: AANoCapture::NOT_CAPTURED_IN_MEM);
6033 if (CapturedInInt)
6034 State.removeAssumedBits(BitsEncoding: AANoCapture::NOT_CAPTURED_IN_INT);
6035 if (CapturedInRet)
6036 State.removeAssumedBits(BitsEncoding: AANoCapture::NOT_CAPTURED_IN_RET);
6037 return State.isAssumed(BitsEncoding: AANoCapture::NO_CAPTURE_MAYBE_RETURNED);
6038 }
6039};
6040
6041ChangeStatus AANoCaptureImpl::updateImpl(Attributor &A) {
6042 const IRPosition &IRP = getIRPosition();
6043 Value *V = isArgumentPosition() ? IRP.getAssociatedArgument()
6044 : &IRP.getAssociatedValue();
6045 if (!V)
6046 return indicatePessimisticFixpoint();
6047
6048 const Function *F =
6049 isArgumentPosition() ? IRP.getAssociatedFunction() : IRP.getAnchorScope();
6050
6051 // TODO: Is the checkForAllUses below useful for constants?
6052 if (!F)
6053 return indicatePessimisticFixpoint();
6054
6055 AANoCapture::StateType T;
6056 const IRPosition &FnPos = IRPosition::function(F: *F);
6057
6058 // Readonly means we cannot capture through memory.
6059 bool IsKnown;
6060 if (AA::isAssumedReadOnly(A, IRP: FnPos, QueryingAA: *this, IsKnown)) {
6061 T.addKnownBits(Bits: NOT_CAPTURED_IN_MEM);
6062 if (IsKnown)
6063 addKnownBits(Bits: NOT_CAPTURED_IN_MEM);
6064 }
6065
6066 // Make sure all returned values are different than the underlying value.
6067 // TODO: we could do this in a more sophisticated way inside
6068 // AAReturnedValues, e.g., track all values that escape through returns
6069 // directly somehow.
6070 auto CheckReturnedArgs = [&](bool &UsedAssumedInformation) {
6071 SmallVector<AA::ValueAndContext> Values;
6072 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::returned(F: *F), AA: this, Values,
6073 S: AA::ValueScope::Intraprocedural,
6074 UsedAssumedInformation))
6075 return false;
6076 bool SeenConstant = false;
6077 for (const AA::ValueAndContext &VAC : Values) {
6078 if (isa<Constant>(Val: VAC.getValue())) {
6079 if (SeenConstant)
6080 return false;
6081 SeenConstant = true;
6082 } else if (!isa<Argument>(Val: VAC.getValue()) ||
6083 VAC.getValue() == getAssociatedArgument())
6084 return false;
6085 }
6086 return true;
6087 };
6088
6089 bool IsKnownNoUnwind;
6090 if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
6091 A, QueryingAA: this, IRP: FnPos, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoUnwind)) {
6092 bool IsVoidTy = F->getReturnType()->isVoidTy();
6093 bool UsedAssumedInformation = false;
6094 if (IsVoidTy || CheckReturnedArgs(UsedAssumedInformation)) {
6095 T.addKnownBits(Bits: NOT_CAPTURED_IN_RET);
6096 if (T.isKnown(BitsEncoding: NOT_CAPTURED_IN_MEM))
6097 return ChangeStatus::UNCHANGED;
6098 if (IsKnownNoUnwind && (IsVoidTy || !UsedAssumedInformation)) {
6099 addKnownBits(Bits: NOT_CAPTURED_IN_RET);
6100 if (isKnown(BitsEncoding: NOT_CAPTURED_IN_MEM))
6101 return indicateOptimisticFixpoint();
6102 }
6103 }
6104 }
6105
6106 auto UseCheck = [&](const Use &U, bool &Follow) -> bool {
6107 // TODO(captures): Make this more precise.
6108 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
6109 if (capturesNothing(CC: CI))
6110 return true;
6111 if (CI.isPassthrough()) {
6112 Follow = true;
6113 return true;
6114 }
6115 return checkUse(A, State&: T, U, Follow);
6116 };
6117
6118 if (!A.checkForAllUses(Pred: UseCheck, QueryingAA: *this, V: *V))
6119 return indicatePessimisticFixpoint();
6120
6121 AANoCapture::StateType &S = getState();
6122 auto Assumed = S.getAssumed();
6123 S.intersectAssumedBits(BitsEncoding: T.getAssumed());
6124 if (!isAssumedNoCaptureMaybeReturned())
6125 return indicatePessimisticFixpoint();
6126 return Assumed == S.getAssumed() ? ChangeStatus::UNCHANGED
6127 : ChangeStatus::CHANGED;
6128}
6129
6130/// NoCapture attribute for function arguments.
6131struct AANoCaptureArgument final : AANoCaptureImpl {
6132 AANoCaptureArgument(const IRPosition &IRP, Attributor &A)
6133 : AANoCaptureImpl(IRP, A) {}
6134
6135 /// See AbstractAttribute::trackStatistics()
6136 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nocapture) }
6137};
6138
6139/// NoCapture attribute for call site arguments.
6140struct AANoCaptureCallSiteArgument final : AANoCaptureImpl {
6141 AANoCaptureCallSiteArgument(const IRPosition &IRP, Attributor &A)
6142 : AANoCaptureImpl(IRP, A) {}
6143
6144 /// See AbstractAttribute::updateImpl(...).
6145 ChangeStatus updateImpl(Attributor &A) override {
6146 // TODO: Once we have call site specific value information we can provide
6147 // call site specific liveness information and then it makes
6148 // sense to specialize attributes for call sites arguments instead of
6149 // redirecting requests to the callee argument.
6150 Argument *Arg = getAssociatedArgument();
6151 if (!Arg)
6152 return indicatePessimisticFixpoint();
6153 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
6154 bool IsKnownNoCapture;
6155 const AANoCapture *ArgAA = nullptr;
6156 if (AA::hasAssumedIRAttr<Attribute::Captures>(
6157 A, QueryingAA: this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
6158 AAPtr: &ArgAA))
6159 return ChangeStatus::UNCHANGED;
6160 if (!ArgAA || !ArgAA->isAssumedNoCaptureMaybeReturned())
6161 return indicatePessimisticFixpoint();
6162 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
6163 }
6164
6165 /// See AbstractAttribute::trackStatistics()
6166 void trackStatistics() const override {
6167 STATS_DECLTRACK_CSARG_ATTR(nocapture)
6168 };
6169};
6170
6171/// NoCapture attribute for floating values.
6172struct AANoCaptureFloating final : AANoCaptureImpl {
6173 AANoCaptureFloating(const IRPosition &IRP, Attributor &A)
6174 : AANoCaptureImpl(IRP, A) {}
6175
6176 /// See AbstractAttribute::trackStatistics()
6177 void trackStatistics() const override {
6178 STATS_DECLTRACK_FLOATING_ATTR(nocapture)
6179 }
6180};
6181
6182/// NoCapture attribute for function return value.
6183struct AANoCaptureReturned final : AANoCaptureImpl {
6184 AANoCaptureReturned(const IRPosition &IRP, Attributor &A)
6185 : AANoCaptureImpl(IRP, A) {
6186 llvm_unreachable("NoCapture is not applicable to function returns!");
6187 }
6188
6189 /// See AbstractAttribute::initialize(...).
6190 void initialize(Attributor &A) override {
6191 llvm_unreachable("NoCapture is not applicable to function returns!");
6192 }
6193
6194 /// See AbstractAttribute::updateImpl(...).
6195 ChangeStatus updateImpl(Attributor &A) override {
6196 llvm_unreachable("NoCapture is not applicable to function returns!");
6197 }
6198
6199 /// See AbstractAttribute::trackStatistics()
6200 void trackStatistics() const override {}
6201};
6202
6203/// NoCapture attribute deduction for a call site return value.
6204struct AANoCaptureCallSiteReturned final : AANoCaptureImpl {
6205 AANoCaptureCallSiteReturned(const IRPosition &IRP, Attributor &A)
6206 : AANoCaptureImpl(IRP, A) {}
6207
6208 /// See AbstractAttribute::initialize(...).
6209 void initialize(Attributor &A) override {
6210 const Function *F = getAnchorScope();
6211 // Check what state the associated function can actually capture.
6212 determineFunctionCaptureCapabilities(IRP: getIRPosition(), F: *F, State&: *this);
6213 }
6214
6215 /// See AbstractAttribute::trackStatistics()
6216 void trackStatistics() const override {
6217 STATS_DECLTRACK_CSRET_ATTR(nocapture)
6218 }
6219};
6220} // namespace
6221
6222/// ------------------ Value Simplify Attribute ----------------------------
6223
6224bool ValueSimplifyStateType::unionAssumed(std::optional<Value *> Other) {
6225 // FIXME: Add a typecast support.
6226 SimplifiedAssociatedValue = AA::combineOptionalValuesInAAValueLatice(
6227 A: SimplifiedAssociatedValue, B: Other, Ty);
6228 if (SimplifiedAssociatedValue == std::optional<Value *>(nullptr))
6229 return false;
6230
6231 LLVM_DEBUG({
6232 if (SimplifiedAssociatedValue)
6233 dbgs() << "[ValueSimplify] is assumed to be "
6234 << **SimplifiedAssociatedValue << "\n";
6235 else
6236 dbgs() << "[ValueSimplify] is assumed to be <none>\n";
6237 });
6238 return true;
6239}
6240
6241namespace {
6242struct AAValueSimplifyImpl : AAValueSimplify {
6243 AAValueSimplifyImpl(const IRPosition &IRP, Attributor &A)
6244 : AAValueSimplify(IRP, A) {}
6245
6246 /// See AbstractAttribute::initialize(...).
6247 void initialize(Attributor &A) override {
6248 if (getAssociatedValue().getType()->isVoidTy())
6249 indicatePessimisticFixpoint();
6250 if (A.hasSimplificationCallback(IRP: getIRPosition()))
6251 indicatePessimisticFixpoint();
6252 }
6253
6254 /// See AbstractAttribute::getAsStr().
6255 const std::string getAsStr(Attributor *A) const override {
6256 LLVM_DEBUG({
6257 dbgs() << "SAV: " << (bool)SimplifiedAssociatedValue << " ";
6258 if (SimplifiedAssociatedValue && *SimplifiedAssociatedValue)
6259 dbgs() << "SAV: " << **SimplifiedAssociatedValue << " ";
6260 });
6261 return isValidState() ? (isAtFixpoint() ? "simplified" : "maybe-simple")
6262 : "not-simple";
6263 }
6264
6265 /// See AbstractAttribute::trackStatistics()
6266 void trackStatistics() const override {}
6267
6268 /// See AAValueSimplify::getAssumedSimplifiedValue()
6269 std::optional<Value *>
6270 getAssumedSimplifiedValue(Attributor &A) const override {
6271 return SimplifiedAssociatedValue;
6272 }
6273
6274 /// Ensure the return value is \p V with type \p Ty, if not possible return
6275 /// nullptr. If \p Check is true we will only verify such an operation would
6276 /// suceed and return a non-nullptr value if that is the case. No IR is
6277 /// generated or modified.
6278 static Value *ensureType(Attributor &A, Value &V, Type &Ty, Instruction *CtxI,
6279 bool Check) {
6280 if (auto *TypedV = AA::getWithType(V, Ty))
6281 return TypedV;
6282 if (CtxI && V.getType()->canLosslesslyBitCastTo(Ty: &Ty))
6283 return Check ? &V
6284 : BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6285 S: &V, Ty: &Ty, Name: "", InsertBefore: CtxI->getIterator());
6286 return nullptr;
6287 }
6288
6289 /// Reproduce \p I with type \p Ty or return nullptr if that is not posisble.
6290 /// If \p Check is true we will only verify such an operation would suceed and
6291 /// return a non-nullptr value if that is the case. No IR is generated or
6292 /// modified.
6293 static Value *reproduceInst(Attributor &A,
6294 const AbstractAttribute &QueryingAA,
6295 Instruction &I, Type &Ty, Instruction *CtxI,
6296 bool Check, ValueToValueMapTy &VMap) {
6297 assert(CtxI && "Cannot reproduce an instruction without context!");
6298 if (Check && (I.mayReadFromMemory() ||
6299 !isSafeToSpeculativelyExecute(I: &I, CtxI, /* DT */ AC: nullptr,
6300 /* TLI */ DT: nullptr)))
6301 return nullptr;
6302 for (Value *Op : I.operands()) {
6303 Value *NewOp = reproduceValue(A, QueryingAA, V&: *Op, Ty, CtxI, Check, VMap);
6304 if (!NewOp) {
6305 assert(Check && "Manifest of new value unexpectedly failed!");
6306 return nullptr;
6307 }
6308 if (!Check)
6309 VMap[Op] = NewOp;
6310 }
6311 if (Check)
6312 return &I;
6313
6314 Instruction *CloneI = I.clone();
6315 // TODO: Try to salvage debug information here.
6316 CloneI->setDebugLoc(DebugLoc());
6317 VMap[&I] = CloneI;
6318 CloneI->insertBefore(InsertPos: CtxI->getIterator());
6319 RemapInstruction(I: CloneI, VM&: VMap);
6320 return CloneI;
6321 }
6322
6323 /// Reproduce \p V with type \p Ty or return nullptr if that is not posisble.
6324 /// If \p Check is true we will only verify such an operation would suceed and
6325 /// return a non-nullptr value if that is the case. No IR is generated or
6326 /// modified.
6327 static Value *reproduceValue(Attributor &A,
6328 const AbstractAttribute &QueryingAA, Value &V,
6329 Type &Ty, Instruction *CtxI, bool Check,
6330 ValueToValueMapTy &VMap) {
6331 if (const auto &NewV = VMap.lookup(Val: &V))
6332 return NewV;
6333 bool UsedAssumedInformation = false;
6334 std::optional<Value *> SimpleV = A.getAssumedSimplified(
6335 V, AA: QueryingAA, UsedAssumedInformation, S: AA::Interprocedural);
6336 if (!SimpleV.has_value())
6337 return PoisonValue::get(T: &Ty);
6338 Value *EffectiveV = &V;
6339 if (*SimpleV)
6340 EffectiveV = *SimpleV;
6341 if (auto *C = dyn_cast<Constant>(Val: EffectiveV))
6342 return C;
6343 if (CtxI && AA::isValidAtPosition(VAC: AA::ValueAndContext(*EffectiveV, *CtxI),
6344 InfoCache&: A.getInfoCache()))
6345 return ensureType(A, V&: *EffectiveV, Ty, CtxI, Check);
6346 if (auto *I = dyn_cast<Instruction>(Val: EffectiveV))
6347 if (Value *NewV = reproduceInst(A, QueryingAA, I&: *I, Ty, CtxI, Check, VMap))
6348 return ensureType(A, V&: *NewV, Ty, CtxI, Check);
6349 return nullptr;
6350 }
6351
6352 /// Return a value we can use as replacement for the associated one, or
6353 /// nullptr if we don't have one that makes sense.
6354 Value *manifestReplacementValue(Attributor &A, Instruction *CtxI) const {
6355 Value *NewV = SimplifiedAssociatedValue
6356 ? *SimplifiedAssociatedValue
6357 : UndefValue::get(T: getAssociatedType());
6358 if (NewV && NewV != &getAssociatedValue()) {
6359 ValueToValueMapTy VMap;
6360 // First verify we can reprduce the value with the required type at the
6361 // context location before we actually start modifying the IR.
6362 if (reproduceValue(A, QueryingAA: *this, V&: *NewV, Ty&: *getAssociatedType(), CtxI,
6363 /* CheckOnly */ Check: true, VMap))
6364 return reproduceValue(A, QueryingAA: *this, V&: *NewV, Ty&: *getAssociatedType(), CtxI,
6365 /* CheckOnly */ Check: false, VMap);
6366 }
6367 return nullptr;
6368 }
6369
6370 /// Helper function for querying AAValueSimplify and updating candidate.
6371 /// \param IRP The value position we are trying to unify with SimplifiedValue
6372 bool checkAndUpdate(Attributor &A, const AbstractAttribute &QueryingAA,
6373 const IRPosition &IRP, bool Simplify = true) {
6374 bool UsedAssumedInformation = false;
6375 std::optional<Value *> QueryingValueSimplified = &IRP.getAssociatedValue();
6376 if (Simplify)
6377 QueryingValueSimplified = A.getAssumedSimplified(
6378 IRP, AA: QueryingAA, UsedAssumedInformation, S: AA::Interprocedural);
6379 return unionAssumed(Other: QueryingValueSimplified);
6380 }
6381
6382 /// Returns a candidate is found or not
6383 template <typename AAType> bool askSimplifiedValueFor(Attributor &A) {
6384 if (!getAssociatedValue().getType()->isIntegerTy())
6385 return false;
6386
6387 // This will also pass the call base context.
6388 const auto *AA =
6389 A.getAAFor<AAType>(*this, getIRPosition(), DepClassTy::NONE);
6390 if (!AA)
6391 return false;
6392
6393 std::optional<Constant *> COpt = AA->getAssumedConstant(A);
6394
6395 if (!COpt) {
6396 SimplifiedAssociatedValue = std::nullopt;
6397 A.recordDependence(FromAA: *AA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
6398 return true;
6399 }
6400 if (auto *C = *COpt) {
6401 SimplifiedAssociatedValue = C;
6402 A.recordDependence(FromAA: *AA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
6403 return true;
6404 }
6405 return false;
6406 }
6407
6408 bool askSimplifiedValueForOtherAAs(Attributor &A) {
6409 if (askSimplifiedValueFor<AAValueConstantRange>(A))
6410 return true;
6411 if (askSimplifiedValueFor<AAPotentialConstantValues>(A))
6412 return true;
6413 return false;
6414 }
6415
6416 /// See AbstractAttribute::manifest(...).
6417 ChangeStatus manifest(Attributor &A) override {
6418 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6419 for (auto &U : getAssociatedValue().uses()) {
6420 // Check if we need to adjust the insertion point to make sure the IR is
6421 // valid.
6422 Instruction *IP = dyn_cast<Instruction>(Val: U.getUser());
6423 if (auto *PHI = dyn_cast_or_null<PHINode>(Val: IP))
6424 IP = PHI->getIncomingBlock(U)->getTerminator();
6425 if (auto *NewV = manifestReplacementValue(A, CtxI: IP)) {
6426 LLVM_DEBUG(dbgs() << "[ValueSimplify] " << getAssociatedValue()
6427 << " -> " << *NewV << " :: " << *this << "\n");
6428 if (A.changeUseAfterManifest(U, NV&: *NewV))
6429 Changed = ChangeStatus::CHANGED;
6430 }
6431 }
6432
6433 return Changed | AAValueSimplify::manifest(A);
6434 }
6435
6436 /// See AbstractState::indicatePessimisticFixpoint(...).
6437 ChangeStatus indicatePessimisticFixpoint() override {
6438 SimplifiedAssociatedValue = &getAssociatedValue();
6439 return AAValueSimplify::indicatePessimisticFixpoint();
6440 }
6441};
6442
6443struct AAValueSimplifyArgument final : AAValueSimplifyImpl {
6444 AAValueSimplifyArgument(const IRPosition &IRP, Attributor &A)
6445 : AAValueSimplifyImpl(IRP, A) {}
6446
6447 void initialize(Attributor &A) override {
6448 AAValueSimplifyImpl::initialize(A);
6449 if (A.hasAttr(IRP: getIRPosition(),
6450 AKs: {Attribute::InAlloca, Attribute::Preallocated,
6451 Attribute::StructRet, Attribute::Nest, Attribute::ByVal},
6452 /* IgnoreSubsumingPositions */ true))
6453 indicatePessimisticFixpoint();
6454 }
6455
6456 /// See AbstractAttribute::updateImpl(...).
6457 ChangeStatus updateImpl(Attributor &A) override {
6458 // Byval is only replacable if it is readonly otherwise we would write into
6459 // the replaced value and not the copy that byval creates implicitly.
6460 Argument *Arg = getAssociatedArgument();
6461 if (Arg->hasByValAttr()) {
6462 // TODO: We probably need to verify synchronization is not an issue, e.g.,
6463 // there is no race by not copying a constant byval.
6464 bool IsKnown;
6465 if (!AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
6466 return indicatePessimisticFixpoint();
6467 }
6468
6469 auto Before = SimplifiedAssociatedValue;
6470
6471 auto PredForCallSite = [&](AbstractCallSite ACS) {
6472 const IRPosition &ACSArgPos =
6473 IRPosition::callsite_argument(ACS, ArgNo: getCallSiteArgNo());
6474 // Check if a coresponding argument was found or if it is on not
6475 // associated (which can happen for callback calls).
6476 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
6477 return false;
6478
6479 // Simplify the argument operand explicitly and check if the result is
6480 // valid in the current scope. This avoids refering to simplified values
6481 // in other functions, e.g., we don't want to say a an argument in a
6482 // static function is actually an argument in a different function.
6483 bool UsedAssumedInformation = false;
6484 std::optional<Constant *> SimpleArgOp =
6485 A.getAssumedConstant(IRP: ACSArgPos, AA: *this, UsedAssumedInformation);
6486 if (!SimpleArgOp)
6487 return true;
6488 if (!*SimpleArgOp)
6489 return false;
6490 if (!AA::isDynamicallyUnique(A, QueryingAA: *this, V: **SimpleArgOp))
6491 return false;
6492 return unionAssumed(Other: *SimpleArgOp);
6493 };
6494
6495 // Generate a answer specific to a call site context.
6496 bool Success;
6497 bool UsedAssumedInformation = false;
6498 if (hasCallBaseContext() &&
6499 getCallBaseContext()->getCalledOperand() == Arg->getParent())
6500 Success = PredForCallSite(
6501 AbstractCallSite(&getCallBaseContext()->getCalledOperandUse()));
6502 else
6503 Success = A.checkForAllCallSites(Pred: PredForCallSite, QueryingAA: *this, RequireAllCallSites: true,
6504 UsedAssumedInformation);
6505
6506 if (!Success)
6507 if (!askSimplifiedValueForOtherAAs(A))
6508 return indicatePessimisticFixpoint();
6509
6510 // If a candidate was found in this update, return CHANGED.
6511 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6512 : ChangeStatus ::CHANGED;
6513 }
6514
6515 /// See AbstractAttribute::trackStatistics()
6516 void trackStatistics() const override {
6517 STATS_DECLTRACK_ARG_ATTR(value_simplify)
6518 }
6519};
6520
6521struct AAValueSimplifyReturned : AAValueSimplifyImpl {
6522 AAValueSimplifyReturned(const IRPosition &IRP, Attributor &A)
6523 : AAValueSimplifyImpl(IRP, A) {}
6524
6525 /// See AAValueSimplify::getAssumedSimplifiedValue()
6526 std::optional<Value *>
6527 getAssumedSimplifiedValue(Attributor &A) const override {
6528 if (!isValidState())
6529 return nullptr;
6530 return SimplifiedAssociatedValue;
6531 }
6532
6533 /// See AbstractAttribute::updateImpl(...).
6534 ChangeStatus updateImpl(Attributor &A) override {
6535 auto Before = SimplifiedAssociatedValue;
6536
6537 auto ReturnInstCB = [&](Instruction &I) {
6538 auto &RI = cast<ReturnInst>(Val&: I);
6539 return checkAndUpdate(
6540 A, QueryingAA: *this,
6541 IRP: IRPosition::value(V: *RI.getReturnValue(), CBContext: getCallBaseContext()));
6542 };
6543
6544 bool UsedAssumedInformation = false;
6545 if (!A.checkForAllInstructions(Pred: ReturnInstCB, QueryingAA: *this, Opcodes: {Instruction::Ret},
6546 UsedAssumedInformation))
6547 if (!askSimplifiedValueForOtherAAs(A))
6548 return indicatePessimisticFixpoint();
6549
6550 // If a candidate was found in this update, return CHANGED.
6551 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6552 : ChangeStatus ::CHANGED;
6553 }
6554
6555 ChangeStatus manifest(Attributor &A) override {
6556 // We queried AAValueSimplify for the returned values so they will be
6557 // replaced if a simplified form was found. Nothing to do here.
6558 return ChangeStatus::UNCHANGED;
6559 }
6560
6561 /// See AbstractAttribute::trackStatistics()
6562 void trackStatistics() const override {
6563 STATS_DECLTRACK_FNRET_ATTR(value_simplify)
6564 }
6565};
6566
6567struct AAValueSimplifyFloating : AAValueSimplifyImpl {
6568 AAValueSimplifyFloating(const IRPosition &IRP, Attributor &A)
6569 : AAValueSimplifyImpl(IRP, A) {}
6570
6571 /// See AbstractAttribute::initialize(...).
6572 void initialize(Attributor &A) override {
6573 AAValueSimplifyImpl::initialize(A);
6574 Value &V = getAnchorValue();
6575
6576 // TODO: add other stuffs
6577 if (isa<Constant>(Val: V))
6578 indicatePessimisticFixpoint();
6579 }
6580
6581 /// See AbstractAttribute::updateImpl(...).
6582 ChangeStatus updateImpl(Attributor &A) override {
6583 auto Before = SimplifiedAssociatedValue;
6584 if (!askSimplifiedValueForOtherAAs(A))
6585 return indicatePessimisticFixpoint();
6586
6587 // If a candidate was found in this update, return CHANGED.
6588 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6589 : ChangeStatus ::CHANGED;
6590 }
6591
6592 /// See AbstractAttribute::trackStatistics()
6593 void trackStatistics() const override {
6594 STATS_DECLTRACK_FLOATING_ATTR(value_simplify)
6595 }
6596};
6597
6598struct AAValueSimplifyFunction : AAValueSimplifyImpl {
6599 AAValueSimplifyFunction(const IRPosition &IRP, Attributor &A)
6600 : AAValueSimplifyImpl(IRP, A) {}
6601
6602 /// See AbstractAttribute::initialize(...).
6603 void initialize(Attributor &A) override {
6604 SimplifiedAssociatedValue = nullptr;
6605 indicateOptimisticFixpoint();
6606 }
6607 /// See AbstractAttribute::initialize(...).
6608 ChangeStatus updateImpl(Attributor &A) override {
6609 llvm_unreachable(
6610 "AAValueSimplify(Function|CallSite)::updateImpl will not be called");
6611 }
6612 /// See AbstractAttribute::trackStatistics()
6613 void trackStatistics() const override {
6614 STATS_DECLTRACK_FN_ATTR(value_simplify)
6615 }
6616};
6617
6618struct AAValueSimplifyCallSite : AAValueSimplifyFunction {
6619 AAValueSimplifyCallSite(const IRPosition &IRP, Attributor &A)
6620 : AAValueSimplifyFunction(IRP, A) {}
6621 /// See AbstractAttribute::trackStatistics()
6622 void trackStatistics() const override {
6623 STATS_DECLTRACK_CS_ATTR(value_simplify)
6624 }
6625};
6626
6627struct AAValueSimplifyCallSiteReturned : AAValueSimplifyImpl {
6628 AAValueSimplifyCallSiteReturned(const IRPosition &IRP, Attributor &A)
6629 : AAValueSimplifyImpl(IRP, A) {}
6630
6631 void initialize(Attributor &A) override {
6632 AAValueSimplifyImpl::initialize(A);
6633 Function *Fn = getAssociatedFunction();
6634 assert(Fn && "Did expect an associted function");
6635 for (Argument &Arg : Fn->args()) {
6636 if (Arg.hasReturnedAttr()) {
6637 auto IRP = IRPosition::callsite_argument(CB: *cast<CallBase>(Val: getCtxI()),
6638 ArgNo: Arg.getArgNo());
6639 if (IRP.getPositionKind() == IRPosition::IRP_CALL_SITE_ARGUMENT &&
6640 checkAndUpdate(A, QueryingAA: *this, IRP))
6641 indicateOptimisticFixpoint();
6642 else
6643 indicatePessimisticFixpoint();
6644 return;
6645 }
6646 }
6647 }
6648
6649 /// See AbstractAttribute::updateImpl(...).
6650 ChangeStatus updateImpl(Attributor &A) override {
6651 return indicatePessimisticFixpoint();
6652 }
6653
6654 void trackStatistics() const override {
6655 STATS_DECLTRACK_CSRET_ATTR(value_simplify)
6656 }
6657};
6658
6659struct AAValueSimplifyCallSiteArgument : AAValueSimplifyFloating {
6660 AAValueSimplifyCallSiteArgument(const IRPosition &IRP, Attributor &A)
6661 : AAValueSimplifyFloating(IRP, A) {}
6662
6663 /// See AbstractAttribute::manifest(...).
6664 ChangeStatus manifest(Attributor &A) override {
6665 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6666 // TODO: We should avoid simplification duplication to begin with.
6667 auto *FloatAA = A.lookupAAFor<AAValueSimplify>(
6668 IRP: IRPosition::value(V: getAssociatedValue()), QueryingAA: this, DepClass: DepClassTy::NONE);
6669 if (FloatAA && FloatAA->getState().isValidState())
6670 return Changed;
6671
6672 if (auto *NewV = manifestReplacementValue(A, CtxI: getCtxI())) {
6673 Use &U = cast<CallBase>(Val: &getAnchorValue())
6674 ->getArgOperandUse(i: getCallSiteArgNo());
6675 if (A.changeUseAfterManifest(U, NV&: *NewV))
6676 Changed = ChangeStatus::CHANGED;
6677 }
6678
6679 return Changed | AAValueSimplify::manifest(A);
6680 }
6681
6682 void trackStatistics() const override {
6683 STATS_DECLTRACK_CSARG_ATTR(value_simplify)
6684 }
6685};
6686} // namespace
6687
6688/// ----------------------- Heap-To-Stack Conversion ---------------------------
6689namespace {
6690struct AAHeapToStackFunction final : public AAHeapToStack {
6691
6692 struct AllocationInfo {
6693 /// The call that allocates the memory.
6694 CallBase *const CB;
6695
6696 /// The library function id for the allocation.
6697 LibFunc LibraryFunctionId = NotLibFunc;
6698
6699 /// The status wrt. a rewrite.
6700 enum {
6701 STACK_DUE_TO_USE,
6702 STACK_DUE_TO_FREE,
6703 INVALID,
6704 } Status = STACK_DUE_TO_USE;
6705
6706 /// Flag to indicate if we encountered a use that might free this allocation
6707 /// but which is not in the deallocation infos.
6708 bool HasPotentiallyFreeingUnknownUses = false;
6709
6710 /// Flag to indicate that we should place the new alloca in the function
6711 /// entry block rather than where the call site (CB) is.
6712 bool MoveAllocaIntoEntry = true;
6713
6714 /// The set of free calls that use this allocation.
6715 SmallSetVector<CallBase *, 1> PotentialFreeCalls{};
6716 };
6717
6718 struct DeallocationInfo {
6719 /// The call that deallocates the memory.
6720 CallBase *const CB;
6721 /// The value freed by the call.
6722 Value *FreedOp;
6723
6724 /// Flag to indicate if we don't know all objects this deallocation might
6725 /// free.
6726 bool MightFreeUnknownObjects = false;
6727
6728 /// The set of allocation calls that are potentially freed.
6729 SmallSetVector<CallBase *, 1> PotentialAllocationCalls{};
6730 };
6731
6732 AAHeapToStackFunction(const IRPosition &IRP, Attributor &A)
6733 : AAHeapToStack(IRP, A) {}
6734
6735 ~AAHeapToStackFunction() override {
6736 // Ensure we call the destructor so we release any memory allocated in the
6737 // sets.
6738 for (auto &It : AllocationInfos)
6739 It.second->~AllocationInfo();
6740 for (auto &It : DeallocationInfos)
6741 It.second->~DeallocationInfo();
6742 }
6743
6744 void initialize(Attributor &A) override {
6745 AAHeapToStack::initialize(A);
6746
6747 const Function *F = getAnchorScope();
6748 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6749
6750 auto AllocationIdentifierCB = [&](Instruction &I) {
6751 CallBase *CB = dyn_cast<CallBase>(Val: &I);
6752 if (!CB)
6753 return true;
6754 if (Value *FreedOp = getFreedOperand(CB, TLI)) {
6755 DeallocationInfos[CB] = new (A.Allocator) DeallocationInfo{.CB: CB, .FreedOp: FreedOp};
6756 return true;
6757 }
6758 // To do heap to stack, we need to know that the allocation itself is
6759 // removable once uses are rewritten, and that we can initialize the
6760 // alloca to the same pattern as the original allocation result.
6761 if (isRemovableAlloc(V: CB, TLI)) {
6762 auto *I8Ty = Type::getInt8Ty(C&: CB->getParent()->getContext());
6763 if (nullptr != getInitialValueOfAllocation(V: CB, TLI, Ty: I8Ty)) {
6764 AllocationInfo *AI = new (A.Allocator) AllocationInfo{.CB: CB};
6765 AllocationInfos[CB] = AI;
6766 if (TLI)
6767 TLI->getLibFunc(CB: *CB, F&: AI->LibraryFunctionId);
6768 }
6769 }
6770 return true;
6771 };
6772
6773 bool UsedAssumedInformation = false;
6774 bool Success = A.checkForAllCallLikeInstructions(
6775 Pred: AllocationIdentifierCB, QueryingAA: *this, UsedAssumedInformation,
6776 /* CheckBBLivenessOnly */ false,
6777 /* CheckPotentiallyDead */ true);
6778 (void)Success;
6779 assert(Success && "Did not expect the call base visit callback to fail!");
6780
6781 Attributor::SimplifictionCallbackTy SCB =
6782 [](const IRPosition &, const AbstractAttribute *,
6783 bool &) -> std::optional<Value *> { return nullptr; };
6784 for (const auto &It : AllocationInfos)
6785 A.registerSimplificationCallback(IRP: IRPosition::callsite_returned(CB: *It.first),
6786 CB: SCB);
6787 for (const auto &It : DeallocationInfos)
6788 A.registerSimplificationCallback(IRP: IRPosition::callsite_returned(CB: *It.first),
6789 CB: SCB);
6790 }
6791
6792 const std::string getAsStr(Attributor *A) const override {
6793 unsigned NumH2SMallocs = 0, NumInvalidMallocs = 0;
6794 for (const auto &It : AllocationInfos) {
6795 if (It.second->Status == AllocationInfo::INVALID)
6796 ++NumInvalidMallocs;
6797 else
6798 ++NumH2SMallocs;
6799 }
6800 return "[H2S] Mallocs Good/Bad: " + std::to_string(val: NumH2SMallocs) + "/" +
6801 std::to_string(val: NumInvalidMallocs);
6802 }
6803
6804 /// See AbstractAttribute::trackStatistics().
6805 void trackStatistics() const override {
6806 STATS_DECL(
6807 MallocCalls, Function,
6808 "Number of malloc/calloc/aligned_alloc calls converted to allocas");
6809 for (const auto &It : AllocationInfos)
6810 if (It.second->Status != AllocationInfo::INVALID)
6811 ++BUILD_STAT_NAME(MallocCalls, Function);
6812 }
6813
6814 bool isAssumedHeapToStack(const CallBase &CB) const override {
6815 if (isValidState())
6816 if (AllocationInfo *AI =
6817 AllocationInfos.lookup(Key: const_cast<CallBase *>(&CB)))
6818 return AI->Status != AllocationInfo::INVALID;
6819 return false;
6820 }
6821
6822 bool isAssumedHeapToStackRemovedFree(CallBase &CB) const override {
6823 if (!isValidState())
6824 return false;
6825
6826 for (const auto &It : AllocationInfos) {
6827 AllocationInfo &AI = *It.second;
6828 if (AI.Status == AllocationInfo::INVALID)
6829 continue;
6830
6831 if (AI.PotentialFreeCalls.count(key: &CB))
6832 return true;
6833 }
6834
6835 return false;
6836 }
6837
6838 ChangeStatus manifest(Attributor &A) override {
6839 assert(getState().isValidState() &&
6840 "Attempted to manifest an invalid state!");
6841
6842 ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
6843 Function *F = getAnchorScope();
6844 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6845
6846 for (auto &It : AllocationInfos) {
6847 AllocationInfo &AI = *It.second;
6848 if (AI.Status == AllocationInfo::INVALID)
6849 continue;
6850
6851 for (CallBase *FreeCall : AI.PotentialFreeCalls) {
6852 LLVM_DEBUG(dbgs() << "H2S: Removing free call: " << *FreeCall << "\n");
6853 A.deleteAfterManifest(I&: *FreeCall);
6854 HasChanged = ChangeStatus::CHANGED;
6855 }
6856
6857 LLVM_DEBUG(dbgs() << "H2S: Removing malloc-like call: " << *AI.CB
6858 << "\n");
6859
6860 auto Remark = [&](OptimizationRemark OR) {
6861 LibFunc IsAllocShared;
6862 if (TLI->getLibFunc(CB: *AI.CB, F&: IsAllocShared))
6863 if (IsAllocShared == LibFunc___kmpc_alloc_shared)
6864 return OR << "Moving globalized variable to the stack.";
6865 return OR << "Moving memory allocation from the heap to the stack.";
6866 };
6867 if (AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared)
6868 A.emitRemark<OptimizationRemark>(I: AI.CB, RemarkName: "OMP110", RemarkCB&: Remark);
6869 else
6870 A.emitRemark<OptimizationRemark>(I: AI.CB, RemarkName: "HeapToStack", RemarkCB&: Remark);
6871
6872 const DataLayout &DL = A.getInfoCache().getDL();
6873 Value *Size;
6874 std::optional<APInt> SizeAPI = getSize(A, AA: *this, AI);
6875 if (SizeAPI) {
6876 Size = ConstantInt::get(Context&: AI.CB->getContext(), V: *SizeAPI);
6877 } else {
6878 LLVMContext &Ctx = AI.CB->getContext();
6879 ObjectSizeOpts Opts;
6880 ObjectSizeOffsetEvaluator Eval(DL, TLI, Ctx, Opts);
6881 SizeOffsetValue SizeOffsetPair = Eval.compute(V: AI.CB);
6882 assert(SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown() &&
6883 cast<ConstantInt>(SizeOffsetPair.Offset)->isZero());
6884 Size = SizeOffsetPair.Size;
6885 }
6886
6887 BasicBlock::iterator IP = AI.MoveAllocaIntoEntry
6888 ? F->getEntryBlock().begin()
6889 : AI.CB->getIterator();
6890
6891 Align Alignment(1);
6892 if (MaybeAlign RetAlign = AI.CB->getRetAlign())
6893 Alignment = std::max(a: Alignment, b: *RetAlign);
6894 if (Value *Align = getAllocAlignment(V: AI.CB, TLI)) {
6895 std::optional<APInt> AlignmentAPI = getAPInt(A, AA: *this, V&: *Align);
6896 assert(AlignmentAPI && AlignmentAPI->getZExtValue() > 0 &&
6897 "Expected an alignment during manifest!");
6898 Alignment =
6899 std::max(a: Alignment, b: assumeAligned(Value: AlignmentAPI->getZExtValue()));
6900 }
6901
6902 // TODO: Hoist the alloca towards the function entry.
6903 unsigned AS = DL.getAllocaAddrSpace();
6904 Instruction *Alloca =
6905 new AllocaInst(Type::getInt8Ty(C&: F->getContext()), AS, Size, Alignment,
6906 AI.CB->getName() + ".h2s", IP);
6907
6908 if (Alloca->getType() != AI.CB->getType())
6909 Alloca = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6910 S: Alloca, Ty: AI.CB->getType(), Name: "malloc_cast", InsertBefore: AI.CB->getIterator());
6911
6912 auto *I8Ty = Type::getInt8Ty(C&: F->getContext());
6913 auto *InitVal = getInitialValueOfAllocation(V: AI.CB, TLI, Ty: I8Ty);
6914 assert(InitVal &&
6915 "Must be able to materialize initial memory state of allocation");
6916
6917 A.changeAfterManifest(IRP: IRPosition::inst(I: *AI.CB), NV&: *Alloca);
6918
6919 if (auto *II = dyn_cast<InvokeInst>(Val: AI.CB)) {
6920 auto *NBB = II->getNormalDest();
6921 UncondBrInst::Create(IfTrue: NBB, InsertBefore: AI.CB->getParent());
6922 A.deleteAfterManifest(I&: *AI.CB);
6923 } else {
6924 A.deleteAfterManifest(I&: *AI.CB);
6925 }
6926
6927 // Initialize the alloca with the same value as used by the allocation
6928 // function. We can skip undef as the initial value of an alloc is
6929 // undef, and the memset would simply end up being DSEd.
6930 if (!isa<UndefValue>(Val: InitVal)) {
6931 IRBuilder<> Builder(Alloca->getNextNode());
6932 // TODO: Use alignment above if align!=1
6933 Builder.CreateMemSet(Ptr: Alloca, Val: InitVal, Size, Align: std::nullopt);
6934 }
6935 HasChanged = ChangeStatus::CHANGED;
6936 }
6937
6938 return HasChanged;
6939 }
6940
6941 std::optional<APInt> getAPInt(Attributor &A, const AbstractAttribute &AA,
6942 Value &V) {
6943 bool UsedAssumedInformation = false;
6944 std::optional<Constant *> SimpleV =
6945 A.getAssumedConstant(V, AA, UsedAssumedInformation);
6946 if (!SimpleV)
6947 return APInt(64, 0);
6948 if (auto *CI = dyn_cast_or_null<ConstantInt>(Val: *SimpleV))
6949 return CI->getValue();
6950 return std::nullopt;
6951 }
6952
6953 std::optional<APInt> getSize(Attributor &A, const AbstractAttribute &AA,
6954 AllocationInfo &AI) {
6955 auto Mapper = [&](const Value *V) -> const Value * {
6956 bool UsedAssumedInformation = false;
6957 if (std::optional<Constant *> SimpleV =
6958 A.getAssumedConstant(V: *V, AA, UsedAssumedInformation))
6959 if (*SimpleV)
6960 return *SimpleV;
6961 return V;
6962 };
6963
6964 const Function *F = getAnchorScope();
6965 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6966 return getAllocSize(CB: AI.CB, TLI, Mapper);
6967 }
6968
6969 /// Collection of all malloc-like calls in a function with associated
6970 /// information.
6971 MapVector<CallBase *, AllocationInfo *> AllocationInfos;
6972
6973 /// Collection of all free-like calls in a function with associated
6974 /// information.
6975 MapVector<CallBase *, DeallocationInfo *> DeallocationInfos;
6976
6977 ChangeStatus updateImpl(Attributor &A) override;
6978};
6979
6980ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) {
6981 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6982 const Function *F = getAnchorScope();
6983 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6984
6985 const auto *LivenessAA =
6986 A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: IRPosition::function(F: *F), DepClass: DepClassTy::NONE);
6987
6988 MustBeExecutedContextExplorer *Explorer =
6989 A.getInfoCache().getMustBeExecutedContextExplorer();
6990
6991 bool StackIsAccessibleByOtherThreads =
6992 A.getInfoCache().stackIsAccessibleByOtherThreads();
6993
6994 LoopInfo *LI =
6995 A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F: *F);
6996 std::optional<bool> MayContainIrreducibleControl;
6997 auto IsInLoop = [&](BasicBlock &BB) {
6998 if (&F->getEntryBlock() == &BB)
6999 return false;
7000 if (!MayContainIrreducibleControl.has_value())
7001 MayContainIrreducibleControl = mayContainIrreducibleControl(F: *F, LI);
7002 if (*MayContainIrreducibleControl)
7003 return true;
7004 if (!LI)
7005 return true;
7006 return LI->getLoopFor(BB: &BB) != nullptr;
7007 };
7008
7009 // Flag to ensure we update our deallocation information at most once per
7010 // updateImpl call and only if we use the free check reasoning.
7011 bool HasUpdatedFrees = false;
7012
7013 auto UpdateFrees = [&]() {
7014 HasUpdatedFrees = true;
7015
7016 for (auto &It : DeallocationInfos) {
7017 DeallocationInfo &DI = *It.second;
7018 // For now we cannot use deallocations that have unknown inputs, skip
7019 // them.
7020 if (DI.MightFreeUnknownObjects)
7021 continue;
7022
7023 // No need to analyze dead calls, ignore them instead.
7024 bool UsedAssumedInformation = false;
7025 if (A.isAssumedDead(I: *DI.CB, QueryingAA: this, LivenessAA, UsedAssumedInformation,
7026 /* CheckBBLivenessOnly */ true))
7027 continue;
7028
7029 // Use the non-optimistic version to get the freed object.
7030 Value *Obj = getUnderlyingObject(V: DI.FreedOp);
7031 if (!Obj) {
7032 LLVM_DEBUG(dbgs() << "[H2S] Unknown underlying object for free!\n");
7033 DI.MightFreeUnknownObjects = true;
7034 continue;
7035 }
7036
7037 // Free of null and undef can be ignored as no-ops (or UB in the latter
7038 // case).
7039 if (isa<ConstantPointerNull>(Val: Obj) || isa<UndefValue>(Val: Obj))
7040 continue;
7041
7042 CallBase *ObjCB = dyn_cast<CallBase>(Val: Obj);
7043 if (!ObjCB) {
7044 LLVM_DEBUG(dbgs() << "[H2S] Free of a non-call object: " << *Obj
7045 << "\n");
7046 DI.MightFreeUnknownObjects = true;
7047 continue;
7048 }
7049
7050 AllocationInfo *AI = AllocationInfos.lookup(Key: ObjCB);
7051 if (!AI) {
7052 LLVM_DEBUG(dbgs() << "[H2S] Free of a non-allocation object: " << *Obj
7053 << "\n");
7054 DI.MightFreeUnknownObjects = true;
7055 continue;
7056 }
7057
7058 DI.PotentialAllocationCalls.insert(X: ObjCB);
7059 }
7060 };
7061
7062 auto FreeCheck = [&](AllocationInfo &AI) {
7063 // If the stack is not accessible by other threads, the "must-free" logic
7064 // doesn't apply as the pointer could be shared and needs to be places in
7065 // "shareable" memory.
7066 if (!StackIsAccessibleByOtherThreads) {
7067 bool IsKnownNoSycn;
7068 if (!AA::hasAssumedIRAttr<Attribute::NoSync>(
7069 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoSycn)) {
7070 LLVM_DEBUG(
7071 dbgs() << "[H2S] found an escaping use, stack is not accessible by "
7072 "other threads and function is not nosync:\n");
7073 return false;
7074 }
7075 }
7076 if (!HasUpdatedFrees)
7077 UpdateFrees();
7078
7079 // TODO: Allow multi exit functions that have different free calls.
7080 if (AI.PotentialFreeCalls.size() != 1) {
7081 LLVM_DEBUG(dbgs() << "[H2S] did not find one free call but "
7082 << AI.PotentialFreeCalls.size() << "\n");
7083 return false;
7084 }
7085 CallBase *UniqueFree = *AI.PotentialFreeCalls.begin();
7086 DeallocationInfo *DI = DeallocationInfos.lookup(Key: UniqueFree);
7087 if (!DI) {
7088 LLVM_DEBUG(
7089 dbgs() << "[H2S] unique free call was not known as deallocation call "
7090 << *UniqueFree << "\n");
7091 return false;
7092 }
7093 if (DI->MightFreeUnknownObjects) {
7094 LLVM_DEBUG(
7095 dbgs() << "[H2S] unique free call might free unknown allocations\n");
7096 return false;
7097 }
7098 if (DI->PotentialAllocationCalls.empty())
7099 return true;
7100 if (DI->PotentialAllocationCalls.size() > 1) {
7101 LLVM_DEBUG(dbgs() << "[H2S] unique free call might free "
7102 << DI->PotentialAllocationCalls.size()
7103 << " different allocations\n");
7104 return false;
7105 }
7106 if (*DI->PotentialAllocationCalls.begin() != AI.CB) {
7107 LLVM_DEBUG(
7108 dbgs()
7109 << "[H2S] unique free call not known to free this allocation but "
7110 << **DI->PotentialAllocationCalls.begin() << "\n");
7111 return false;
7112 }
7113
7114 // __kmpc_alloc_shared and __kmpc_alloc_free are by construction matched.
7115 if (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared) {
7116 Instruction *CtxI = isa<InvokeInst>(Val: AI.CB) ? AI.CB : AI.CB->getNextNode();
7117 if (!Explorer || !Explorer->findInContextOf(I: UniqueFree, PP: CtxI)) {
7118 LLVM_DEBUG(dbgs() << "[H2S] unique free call might not be executed "
7119 "with the allocation "
7120 << *UniqueFree << "\n");
7121 return false;
7122 }
7123 }
7124 return true;
7125 };
7126
7127 auto UsesCheck = [&](AllocationInfo &AI) {
7128 bool ValidUsesOnly = true;
7129
7130 auto Pred = [&](const Use &U, bool &Follow) -> bool {
7131 Instruction *UserI = cast<Instruction>(Val: U.getUser());
7132 if (isa<LoadInst>(Val: UserI))
7133 return true;
7134 if (auto *SI = dyn_cast<StoreInst>(Val: UserI)) {
7135 if (SI->getValueOperand() == U.get()) {
7136 LLVM_DEBUG(dbgs()
7137 << "[H2S] escaping store to memory: " << *UserI << "\n");
7138 ValidUsesOnly = false;
7139 } else {
7140 // A store into the malloc'ed memory is fine.
7141 }
7142 return true;
7143 }
7144 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
7145 if (!CB->isArgOperand(U: &U) || CB->isLifetimeStartOrEnd())
7146 return true;
7147 if (DeallocationInfos.count(Key: CB)) {
7148 AI.PotentialFreeCalls.insert(X: CB);
7149 return true;
7150 }
7151
7152 unsigned ArgNo = CB->getArgOperandNo(U: &U);
7153 auto CBIRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
7154
7155 bool IsKnownNoCapture;
7156 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
7157 A, QueryingAA: this, IRP: CBIRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture);
7158
7159 // If a call site argument use is nofree, we are fine.
7160 bool IsKnownNoFree;
7161 bool IsAssumedNoFree = AA::hasAssumedIRAttr<Attribute::NoFree>(
7162 A, QueryingAA: this, IRP: CBIRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoFree);
7163
7164 if (!IsAssumedNoCapture ||
7165 (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared &&
7166 !IsAssumedNoFree)) {
7167 AI.HasPotentiallyFreeingUnknownUses |= !IsAssumedNoFree;
7168
7169 // Emit a missed remark if this is missed OpenMP globalization.
7170 auto Remark = [&](OptimizationRemarkMissed ORM) {
7171 return ORM
7172 << "Could not move globalized variable to the stack. "
7173 "Variable is potentially captured in call. Mark "
7174 "parameter as `__attribute__((noescape))` to override.";
7175 };
7176
7177 if (ValidUsesOnly &&
7178 AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared)
7179 A.emitRemark<OptimizationRemarkMissed>(I: CB, RemarkName: "OMP113", RemarkCB&: Remark);
7180
7181 LLVM_DEBUG(dbgs() << "[H2S] Bad user: " << *UserI << "\n");
7182 ValidUsesOnly = false;
7183 }
7184 return true;
7185 }
7186
7187 if (isa<GetElementPtrInst>(Val: UserI) || isa<BitCastInst>(Val: UserI) ||
7188 isa<PHINode>(Val: UserI) || isa<SelectInst>(Val: UserI)) {
7189 Follow = true;
7190 return true;
7191 }
7192 // Unknown user for which we can not track uses further (in a way that
7193 // makes sense).
7194 LLVM_DEBUG(dbgs() << "[H2S] Unknown user: " << *UserI << "\n");
7195 ValidUsesOnly = false;
7196 return true;
7197 };
7198 if (!A.checkForAllUses(Pred, QueryingAA: *this, V: *AI.CB, /* CheckBBLivenessOnly */ false,
7199 LivenessDepClass: DepClassTy::OPTIONAL, /* IgnoreDroppableUses */ true,
7200 EquivalentUseCB: [&](const Use &OldU, const Use &NewU) {
7201 auto *SI = dyn_cast<StoreInst>(Val: OldU.getUser());
7202 return !SI || StackIsAccessibleByOtherThreads ||
7203 AA::isAssumedThreadLocalObject(
7204 A, Obj&: *SI->getPointerOperand(), QueryingAA: *this);
7205 }))
7206 return false;
7207 return ValidUsesOnly;
7208 };
7209
7210 // The actual update starts here. We look at all allocations and depending on
7211 // their status perform the appropriate check(s).
7212 for (auto &It : AllocationInfos) {
7213 AllocationInfo &AI = *It.second;
7214 if (AI.Status == AllocationInfo::INVALID)
7215 continue;
7216
7217 if (Value *Align = getAllocAlignment(V: AI.CB, TLI)) {
7218 std::optional<APInt> APAlign = getAPInt(A, AA: *this, V&: *Align);
7219 if (!APAlign) {
7220 // Can't generate an alloca which respects the required alignment
7221 // on the allocation.
7222 LLVM_DEBUG(dbgs() << "[H2S] Unknown allocation alignment: " << *AI.CB
7223 << "\n");
7224 AI.Status = AllocationInfo::INVALID;
7225 Changed = ChangeStatus::CHANGED;
7226 continue;
7227 }
7228 if (APAlign->ugt(RHS: llvm::Value::MaximumAlignment) ||
7229 !APAlign->isPowerOf2()) {
7230 LLVM_DEBUG(dbgs() << "[H2S] Invalid allocation alignment: " << APAlign
7231 << "\n");
7232 AI.Status = AllocationInfo::INVALID;
7233 Changed = ChangeStatus::CHANGED;
7234 continue;
7235 }
7236 }
7237
7238 std::optional<APInt> Size = getSize(A, AA: *this, AI);
7239 if (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared &&
7240 MaxHeapToStackSize != -1) {
7241 if (!Size || Size->ugt(RHS: MaxHeapToStackSize)) {
7242 LLVM_DEBUG({
7243 if (!Size)
7244 dbgs() << "[H2S] Unknown allocation size: " << *AI.CB << "\n";
7245 else
7246 dbgs() << "[H2S] Allocation size too large: " << *AI.CB << " vs. "
7247 << MaxHeapToStackSize << "\n";
7248 });
7249
7250 AI.Status = AllocationInfo::INVALID;
7251 Changed = ChangeStatus::CHANGED;
7252 continue;
7253 }
7254 }
7255
7256 switch (AI.Status) {
7257 case AllocationInfo::STACK_DUE_TO_USE:
7258 if (UsesCheck(AI))
7259 break;
7260 AI.Status = AllocationInfo::STACK_DUE_TO_FREE;
7261 [[fallthrough]];
7262 case AllocationInfo::STACK_DUE_TO_FREE:
7263 if (FreeCheck(AI))
7264 break;
7265 AI.Status = AllocationInfo::INVALID;
7266 Changed = ChangeStatus::CHANGED;
7267 break;
7268 case AllocationInfo::INVALID:
7269 llvm_unreachable("Invalid allocations should never reach this point!");
7270 };
7271
7272 // Check if we still think we can move it into the entry block. If the
7273 // alloca comes from a converted __kmpc_alloc_shared then we can usually
7274 // ignore the potential compilations associated with loops.
7275 bool IsGlobalizedLocal =
7276 AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared;
7277 if (AI.MoveAllocaIntoEntry &&
7278 (!Size.has_value() ||
7279 (!IsGlobalizedLocal && IsInLoop(*AI.CB->getParent()))))
7280 AI.MoveAllocaIntoEntry = false;
7281 }
7282
7283 return Changed;
7284}
7285} // namespace
7286
7287/// ----------------------- Privatizable Pointers ------------------------------
7288namespace {
7289struct AAPrivatizablePtrImpl : public AAPrivatizablePtr {
7290 AAPrivatizablePtrImpl(const IRPosition &IRP, Attributor &A)
7291 : AAPrivatizablePtr(IRP, A), PrivatizableType(std::nullopt) {}
7292
7293 ChangeStatus indicatePessimisticFixpoint() override {
7294 AAPrivatizablePtr::indicatePessimisticFixpoint();
7295 PrivatizableType = nullptr;
7296 return ChangeStatus::CHANGED;
7297 }
7298
7299 /// Identify the type we can chose for a private copy of the underlying
7300 /// argument. std::nullopt means it is not clear yet, nullptr means there is
7301 /// none.
7302 virtual std::optional<Type *> identifyPrivatizableType(Attributor &A) = 0;
7303
7304 /// Return a privatizable type that encloses both T0 and T1.
7305 /// TODO: This is merely a stub for now as we should manage a mapping as well.
7306 std::optional<Type *> combineTypes(std::optional<Type *> T0,
7307 std::optional<Type *> T1) {
7308 if (!T0)
7309 return T1;
7310 if (!T1)
7311 return T0;
7312 if (T0 == T1)
7313 return T0;
7314 return nullptr;
7315 }
7316
7317 std::optional<Type *> getPrivatizableType() const override {
7318 return PrivatizableType;
7319 }
7320
7321 const std::string getAsStr(Attributor *A) const override {
7322 return isAssumedPrivatizablePtr() ? "[priv]" : "[no-priv]";
7323 }
7324
7325protected:
7326 std::optional<Type *> PrivatizableType;
7327};
7328
7329// TODO: Do this for call site arguments (probably also other values) as well.
7330
7331struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl {
7332 AAPrivatizablePtrArgument(const IRPosition &IRP, Attributor &A)
7333 : AAPrivatizablePtrImpl(IRP, A) {}
7334
7335 /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
7336 std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7337 // If this is a byval argument and we know all the call sites (so we can
7338 // rewrite them), there is no need to check them explicitly.
7339 bool UsedAssumedInformation = false;
7340 SmallVector<Attribute, 1> Attrs;
7341 A.getAttrs(IRP: getIRPosition(), AKs: {Attribute::ByVal}, Attrs,
7342 /* IgnoreSubsumingPositions */ true);
7343 if (!Attrs.empty() &&
7344 A.checkForAllCallSites(Pred: [](AbstractCallSite ACS) { return true; }, QueryingAA: *this,
7345 RequireAllCallSites: true, UsedAssumedInformation))
7346 return Attrs[0].getValueAsType();
7347
7348 std::optional<Type *> Ty;
7349 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
7350
7351 // Make sure the associated call site argument has the same type at all call
7352 // sites and it is an allocation we know is safe to privatize, for now that
7353 // means we only allow alloca instructions.
7354 // TODO: We can additionally analyze the accesses in the callee to create
7355 // the type from that information instead. That is a little more
7356 // involved and will be done in a follow up patch.
7357 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7358 IRPosition ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
7359 // Check if a coresponding argument was found or if it is one not
7360 // associated (which can happen for callback calls).
7361 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
7362 return false;
7363
7364 // Check that all call sites agree on a type.
7365 auto *PrivCSArgAA =
7366 A.getAAFor<AAPrivatizablePtr>(QueryingAA: *this, IRP: ACSArgPos, DepClass: DepClassTy::REQUIRED);
7367 if (!PrivCSArgAA)
7368 return false;
7369 std::optional<Type *> CSTy = PrivCSArgAA->getPrivatizableType();
7370
7371 LLVM_DEBUG({
7372 dbgs() << "[AAPrivatizablePtr] ACSPos: " << ACSArgPos << ", CSTy: ";
7373 if (CSTy && *CSTy)
7374 (*CSTy)->print(dbgs());
7375 else if (CSTy)
7376 dbgs() << "<nullptr>";
7377 else
7378 dbgs() << "<none>";
7379 });
7380
7381 Ty = combineTypes(T0: Ty, T1: CSTy);
7382
7383 LLVM_DEBUG({
7384 dbgs() << " : New Type: ";
7385 if (Ty && *Ty)
7386 (*Ty)->print(dbgs());
7387 else if (Ty)
7388 dbgs() << "<nullptr>";
7389 else
7390 dbgs() << "<none>";
7391 dbgs() << "\n";
7392 });
7393
7394 return !Ty || *Ty;
7395 };
7396
7397 if (!A.checkForAllCallSites(Pred: CallSiteCheck, QueryingAA: *this, RequireAllCallSites: true,
7398 UsedAssumedInformation))
7399 return nullptr;
7400 return Ty;
7401 }
7402
7403 /// See AbstractAttribute::updateImpl(...).
7404 ChangeStatus updateImpl(Attributor &A) override {
7405 PrivatizableType = identifyPrivatizableType(A);
7406 if (!PrivatizableType)
7407 return ChangeStatus::UNCHANGED;
7408 if (!*PrivatizableType)
7409 return indicatePessimisticFixpoint();
7410
7411 // The dependence is optional so we don't give up once we give up on the
7412 // alignment.
7413 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: getAssociatedValue()),
7414 DepClass: DepClassTy::OPTIONAL);
7415
7416 // Avoid arguments with padding for now.
7417 if (!A.hasAttr(IRP: getIRPosition(), AKs: Attribute::ByVal) &&
7418 !isDenselyPacked(Ty: *PrivatizableType, DL: A.getInfoCache().getDL())) {
7419 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Padding detected\n");
7420 return indicatePessimisticFixpoint();
7421 }
7422
7423 // Collect the types that will replace the privatizable type in the function
7424 // signature.
7425 SmallVector<Type *, 16> ReplacementTypes;
7426 identifyReplacementTypes(PrivType: *PrivatizableType, ReplacementTypes);
7427
7428 // Verify callee and caller agree on how the promoted argument would be
7429 // passed.
7430 Function &Fn = *getIRPosition().getAnchorScope();
7431 const auto *TTI =
7432 A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(F: Fn);
7433 if (!TTI) {
7434 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Missing TTI for function "
7435 << Fn.getName() << "\n");
7436 return indicatePessimisticFixpoint();
7437 }
7438
7439 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7440 CallBase *CB = ACS.getInstruction();
7441 return TTI->areTypesABICompatible(
7442 Caller: CB->getCaller(),
7443 Callee: dyn_cast_if_present<Function>(Val: CB->getCalledOperand()),
7444 Types: ReplacementTypes);
7445 };
7446 bool UsedAssumedInformation = false;
7447 if (!A.checkForAllCallSites(Pred: CallSiteCheck, QueryingAA: *this, RequireAllCallSites: true,
7448 UsedAssumedInformation)) {
7449 LLVM_DEBUG(
7450 dbgs() << "[AAPrivatizablePtr] ABI incompatibility detected for "
7451 << Fn.getName() << "\n");
7452 return indicatePessimisticFixpoint();
7453 }
7454
7455 // Register a rewrite of the argument.
7456 Argument *Arg = getAssociatedArgument();
7457 if (!A.isValidFunctionSignatureRewrite(Arg&: *Arg, ReplacementTypes)) {
7458 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Rewrite not valid\n");
7459 return indicatePessimisticFixpoint();
7460 }
7461
7462 unsigned ArgNo = Arg->getArgNo();
7463
7464 // Helper to check if for the given call site the associated argument is
7465 // passed to a callback where the privatization would be different.
7466 auto IsCompatiblePrivArgOfCallback = [&](CallBase &CB) {
7467 SmallVector<const Use *, 4> CallbackUses;
7468 AbstractCallSite::getCallbackUses(CB, CallbackUses);
7469 for (const Use *U : CallbackUses) {
7470 AbstractCallSite CBACS(U);
7471 assert(CBACS && CBACS.isCallbackCall());
7472 for (Argument &CBArg : CBACS.getCalledFunction()->args()) {
7473 int CBArgNo = CBACS.getCallArgOperandNo(Arg&: CBArg);
7474
7475 LLVM_DEBUG({
7476 dbgs()
7477 << "[AAPrivatizablePtr] Argument " << *Arg
7478 << "check if can be privatized in the context of its parent ("
7479 << Arg->getParent()->getName()
7480 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7481 "callback ("
7482 << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7483 << ")\n[AAPrivatizablePtr] " << CBArg << " : "
7484 << CBACS.getCallArgOperand(CBArg) << " vs "
7485 << CB.getArgOperand(ArgNo) << "\n"
7486 << "[AAPrivatizablePtr] " << CBArg << " : "
7487 << CBACS.getCallArgOperandNo(CBArg) << " vs " << ArgNo << "\n";
7488 });
7489
7490 if (CBArgNo != int(ArgNo))
7491 continue;
7492 const auto *CBArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7493 QueryingAA: *this, IRP: IRPosition::argument(Arg: CBArg), DepClass: DepClassTy::REQUIRED);
7494 if (CBArgPrivAA && CBArgPrivAA->isValidState()) {
7495 auto CBArgPrivTy = CBArgPrivAA->getPrivatizableType();
7496 if (!CBArgPrivTy)
7497 continue;
7498 if (*CBArgPrivTy == PrivatizableType)
7499 continue;
7500 }
7501
7502 LLVM_DEBUG({
7503 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7504 << " cannot be privatized in the context of its parent ("
7505 << Arg->getParent()->getName()
7506 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7507 "callback ("
7508 << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7509 << ").\n[AAPrivatizablePtr] for which the argument "
7510 "privatization is not compatible.\n";
7511 });
7512 return false;
7513 }
7514 }
7515 return true;
7516 };
7517
7518 // Helper to check if for the given call site the associated argument is
7519 // passed to a direct call where the privatization would be different.
7520 auto IsCompatiblePrivArgOfDirectCS = [&](AbstractCallSite ACS) {
7521 CallBase *DC = cast<CallBase>(Val: ACS.getInstruction());
7522 int DCArgNo = ACS.getCallArgOperandNo(ArgNo);
7523 assert(DCArgNo >= 0 && unsigned(DCArgNo) < DC->arg_size() &&
7524 "Expected a direct call operand for callback call operand");
7525
7526 Function *DCCallee =
7527 dyn_cast_if_present<Function>(Val: DC->getCalledOperand());
7528 LLVM_DEBUG({
7529 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7530 << " check if be privatized in the context of its parent ("
7531 << Arg->getParent()->getName()
7532 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7533 "direct call of ("
7534 << DCArgNo << "@" << DCCallee->getName() << ").\n";
7535 });
7536
7537 if (unsigned(DCArgNo) < DCCallee->arg_size()) {
7538 const auto *DCArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7539 QueryingAA: *this, IRP: IRPosition::argument(Arg: *DCCallee->getArg(i: DCArgNo)),
7540 DepClass: DepClassTy::REQUIRED);
7541 if (DCArgPrivAA && DCArgPrivAA->isValidState()) {
7542 auto DCArgPrivTy = DCArgPrivAA->getPrivatizableType();
7543 if (!DCArgPrivTy)
7544 return true;
7545 if (*DCArgPrivTy == PrivatizableType)
7546 return true;
7547 }
7548 }
7549
7550 LLVM_DEBUG({
7551 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7552 << " cannot be privatized in the context of its parent ("
7553 << Arg->getParent()->getName()
7554 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7555 "direct call of ("
7556 << ACS.getInstruction()->getCalledOperand()->getName()
7557 << ").\n[AAPrivatizablePtr] for which the argument "
7558 "privatization is not compatible.\n";
7559 });
7560 return false;
7561 };
7562
7563 // Helper to check if the associated argument is used at the given abstract
7564 // call site in a way that is incompatible with the privatization assumed
7565 // here.
7566 auto IsCompatiblePrivArgOfOtherCallSite = [&](AbstractCallSite ACS) {
7567 if (ACS.isDirectCall())
7568 return IsCompatiblePrivArgOfCallback(*ACS.getInstruction());
7569 if (ACS.isCallbackCall())
7570 return IsCompatiblePrivArgOfDirectCS(ACS);
7571 return false;
7572 };
7573
7574 if (!A.checkForAllCallSites(Pred: IsCompatiblePrivArgOfOtherCallSite, QueryingAA: *this, RequireAllCallSites: true,
7575 UsedAssumedInformation))
7576 return indicatePessimisticFixpoint();
7577
7578 return ChangeStatus::UNCHANGED;
7579 }
7580
7581 /// Given a type to private \p PrivType, collect the constituates (which are
7582 /// used) in \p ReplacementTypes.
7583 static void
7584 identifyReplacementTypes(Type *PrivType,
7585 SmallVectorImpl<Type *> &ReplacementTypes) {
7586 // TODO: For now we expand the privatization type to the fullest which can
7587 // lead to dead arguments that need to be removed later.
7588 assert(PrivType && "Expected privatizable type!");
7589
7590 // Traverse the type, extract constituate types on the outermost level.
7591 if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7592 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++)
7593 ReplacementTypes.push_back(Elt: PrivStructType->getElementType(N: u));
7594 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7595 ReplacementTypes.append(NumInputs: PrivArrayType->getNumElements(),
7596 Elt: PrivArrayType->getElementType());
7597 } else {
7598 ReplacementTypes.push_back(Elt: PrivType);
7599 }
7600 }
7601
7602 /// Initialize \p Base according to the type \p PrivType at position \p IP.
7603 /// The values needed are taken from the arguments of \p F starting at
7604 /// position \p ArgNo.
7605 static void createInitialization(Type *PrivType, Value &Base, Function &F,
7606 unsigned ArgNo, BasicBlock::iterator IP) {
7607 assert(PrivType && "Expected privatizable type!");
7608
7609 IRBuilder<NoFolder> IRB(IP->getParent(), IP);
7610 const DataLayout &DL = F.getDataLayout();
7611
7612 // Traverse the type, build GEPs and stores.
7613 if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7614 const StructLayout *PrivStructLayout = DL.getStructLayout(Ty: PrivStructType);
7615 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7616 Value *Ptr =
7617 constructPointer(Ptr: &Base, Offset: PrivStructLayout->getElementOffset(Idx: u), IRB);
7618 new StoreInst(F.getArg(i: ArgNo + u), Ptr, IP);
7619 }
7620 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7621 Type *PointeeTy = PrivArrayType->getElementType();
7622 uint64_t PointeeTySize = DL.getTypeStoreSize(Ty: PointeeTy);
7623 for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7624 Value *Ptr = constructPointer(Ptr: &Base, Offset: u * PointeeTySize, IRB);
7625 new StoreInst(F.getArg(i: ArgNo + u), Ptr, IP);
7626 }
7627 } else {
7628 new StoreInst(F.getArg(i: ArgNo), &Base, IP);
7629 }
7630 }
7631
7632 /// Extract values from \p Base according to the type \p PrivType at the
7633 /// call position \p ACS. The values are appended to \p ReplacementValues.
7634 void createReplacementValues(Align Alignment, Type *PrivType,
7635 AbstractCallSite ACS, Value *Base,
7636 SmallVectorImpl<Value *> &ReplacementValues) {
7637 assert(Base && "Expected base value!");
7638 assert(PrivType && "Expected privatizable type!");
7639 Instruction *IP = ACS.getInstruction();
7640
7641 IRBuilder<NoFolder> IRB(IP);
7642 const DataLayout &DL = IP->getDataLayout();
7643
7644 // Traverse the type, build GEPs and loads.
7645 if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7646 const StructLayout *PrivStructLayout = DL.getStructLayout(Ty: PrivStructType);
7647 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7648 Type *PointeeTy = PrivStructType->getElementType(N: u);
7649 Value *Ptr =
7650 constructPointer(Ptr: Base, Offset: PrivStructLayout->getElementOffset(Idx: u), IRB);
7651 LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7652 L->setAlignment(Alignment);
7653 ReplacementValues.push_back(Elt: L);
7654 }
7655 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7656 Type *PointeeTy = PrivArrayType->getElementType();
7657 uint64_t PointeeTySize = DL.getTypeStoreSize(Ty: PointeeTy);
7658 for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7659 Value *Ptr = constructPointer(Ptr: Base, Offset: u * PointeeTySize, IRB);
7660 LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7661 L->setAlignment(Alignment);
7662 ReplacementValues.push_back(Elt: L);
7663 }
7664 } else {
7665 LoadInst *L = new LoadInst(PrivType, Base, "", IP->getIterator());
7666 L->setAlignment(Alignment);
7667 ReplacementValues.push_back(Elt: L);
7668 }
7669 }
7670
7671 /// See AbstractAttribute::manifest(...)
7672 ChangeStatus manifest(Attributor &A) override {
7673 if (!PrivatizableType)
7674 return ChangeStatus::UNCHANGED;
7675 assert(*PrivatizableType && "Expected privatizable type!");
7676
7677 // Collect all tail calls in the function as we cannot allow new allocas to
7678 // escape into tail recursion.
7679 // TODO: Be smarter about new allocas escaping into tail calls.
7680 SmallVector<CallInst *, 16> TailCalls;
7681 bool UsedAssumedInformation = false;
7682 if (!A.checkForAllInstructions(
7683 Pred: [&](Instruction &I) {
7684 CallInst &CI = cast<CallInst>(Val&: I);
7685 if (CI.isTailCall())
7686 TailCalls.push_back(Elt: &CI);
7687 return true;
7688 },
7689 QueryingAA: *this, Opcodes: {Instruction::Call}, UsedAssumedInformation))
7690 return ChangeStatus::UNCHANGED;
7691
7692 Argument *Arg = getAssociatedArgument();
7693 // Query AAAlign attribute for alignment of associated argument to
7694 // determine the best alignment of loads.
7695 const auto *AlignAA =
7696 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *Arg), DepClass: DepClassTy::NONE);
7697
7698 // Callback to repair the associated function. A new alloca is placed at the
7699 // beginning and initialized with the values passed through arguments. The
7700 // new alloca replaces the use of the old pointer argument.
7701 Attributor::ArgumentReplacementInfo::CalleeRepairCBTy FnRepairCB =
7702 [=](const Attributor::ArgumentReplacementInfo &ARI,
7703 Function &ReplacementFn, Function::arg_iterator ArgIt) {
7704 BasicBlock &EntryBB = ReplacementFn.getEntryBlock();
7705 BasicBlock::iterator IP = EntryBB.getFirstInsertionPt();
7706 const DataLayout &DL = IP->getDataLayout();
7707 unsigned AS = DL.getAllocaAddrSpace();
7708 Instruction *AI = new AllocaInst(*PrivatizableType, AS,
7709 Arg->getName() + ".priv", IP);
7710 createInitialization(PrivType: *PrivatizableType, Base&: *AI, F&: ReplacementFn,
7711 ArgNo: ArgIt->getArgNo(), IP);
7712
7713 if (AI->getType() != Arg->getType())
7714 AI = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
7715 S: AI, Ty: Arg->getType(), Name: "", InsertBefore: IP);
7716 Arg->replaceAllUsesWith(V: AI);
7717
7718 for (CallInst *CI : TailCalls)
7719 CI->setTailCall(false);
7720 };
7721
7722 // Callback to repair a call site of the associated function. The elements
7723 // of the privatizable type are loaded prior to the call and passed to the
7724 // new function version.
7725 Attributor::ArgumentReplacementInfo::ACSRepairCBTy ACSRepairCB =
7726 [=](const Attributor::ArgumentReplacementInfo &ARI,
7727 AbstractCallSite ACS, SmallVectorImpl<Value *> &NewArgOperands) {
7728 // When no alignment is specified for the load instruction,
7729 // natural alignment is assumed.
7730 createReplacementValues(
7731 Alignment: AlignAA ? AlignAA->getAssumedAlign() : Align(0),
7732 PrivType: *PrivatizableType, ACS,
7733 Base: ACS.getCallArgOperand(ArgNo: ARI.getReplacedArg().getArgNo()),
7734 ReplacementValues&: NewArgOperands);
7735 };
7736
7737 // Collect the types that will replace the privatizable type in the function
7738 // signature.
7739 SmallVector<Type *, 16> ReplacementTypes;
7740 identifyReplacementTypes(PrivType: *PrivatizableType, ReplacementTypes);
7741
7742 // Register a rewrite of the argument.
7743 if (A.registerFunctionSignatureRewrite(Arg&: *Arg, ReplacementTypes,
7744 CalleeRepairCB: std::move(FnRepairCB),
7745 ACSRepairCB: std::move(ACSRepairCB)))
7746 return ChangeStatus::CHANGED;
7747 return ChangeStatus::UNCHANGED;
7748 }
7749
7750 /// See AbstractAttribute::trackStatistics()
7751 void trackStatistics() const override {
7752 STATS_DECLTRACK_ARG_ATTR(privatizable_ptr);
7753 }
7754};
7755
7756struct AAPrivatizablePtrFloating : public AAPrivatizablePtrImpl {
7757 AAPrivatizablePtrFloating(const IRPosition &IRP, Attributor &A)
7758 : AAPrivatizablePtrImpl(IRP, A) {}
7759
7760 /// See AbstractAttribute::initialize(...).
7761 void initialize(Attributor &A) override {
7762 // TODO: We can privatize more than arguments.
7763 indicatePessimisticFixpoint();
7764 }
7765
7766 ChangeStatus updateImpl(Attributor &A) override {
7767 llvm_unreachable("AAPrivatizablePtr(Floating|Returned|CallSiteReturned)::"
7768 "updateImpl will not be called");
7769 }
7770
7771 /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
7772 std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7773 Value *Obj = getUnderlyingObject(V: &getAssociatedValue());
7774 if (!Obj) {
7775 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] No underlying object found!\n");
7776 return nullptr;
7777 }
7778
7779 if (auto *AI = dyn_cast<AllocaInst>(Val: Obj))
7780 if (auto *CI = dyn_cast<ConstantInt>(Val: AI->getArraySize()))
7781 if (CI->isOne())
7782 return AI->getAllocatedType();
7783 if (auto *Arg = dyn_cast<Argument>(Val: Obj)) {
7784 auto *PrivArgAA = A.getAAFor<AAPrivatizablePtr>(
7785 QueryingAA: *this, IRP: IRPosition::argument(Arg: *Arg), DepClass: DepClassTy::REQUIRED);
7786 if (PrivArgAA && PrivArgAA->isAssumedPrivatizablePtr())
7787 return PrivArgAA->getPrivatizableType();
7788 }
7789
7790 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Underlying object neither valid "
7791 "alloca nor privatizable argument: "
7792 << *Obj << "!\n");
7793 return nullptr;
7794 }
7795
7796 /// See AbstractAttribute::trackStatistics()
7797 void trackStatistics() const override {
7798 STATS_DECLTRACK_FLOATING_ATTR(privatizable_ptr);
7799 }
7800};
7801
7802struct AAPrivatizablePtrCallSiteArgument final
7803 : public AAPrivatizablePtrFloating {
7804 AAPrivatizablePtrCallSiteArgument(const IRPosition &IRP, Attributor &A)
7805 : AAPrivatizablePtrFloating(IRP, A) {}
7806
7807 /// See AbstractAttribute::initialize(...).
7808 void initialize(Attributor &A) override {
7809 if (A.hasAttr(IRP: getIRPosition(), AKs: Attribute::ByVal))
7810 indicateOptimisticFixpoint();
7811 }
7812
7813 /// See AbstractAttribute::updateImpl(...).
7814 ChangeStatus updateImpl(Attributor &A) override {
7815 PrivatizableType = identifyPrivatizableType(A);
7816 if (!PrivatizableType)
7817 return ChangeStatus::UNCHANGED;
7818 if (!*PrivatizableType)
7819 return indicatePessimisticFixpoint();
7820
7821 const IRPosition &IRP = getIRPosition();
7822 bool IsKnownNoCapture;
7823 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
7824 A, QueryingAA: this, IRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture);
7825 if (!IsAssumedNoCapture) {
7826 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might be captured!\n");
7827 return indicatePessimisticFixpoint();
7828 }
7829
7830 bool IsKnownNoAlias;
7831 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
7832 A, QueryingAA: this, IRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoAlias)) {
7833 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might alias!\n");
7834 return indicatePessimisticFixpoint();
7835 }
7836
7837 bool IsKnown;
7838 if (!AA::isAssumedReadOnly(A, IRP, QueryingAA: *this, IsKnown)) {
7839 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer is written!\n");
7840 return indicatePessimisticFixpoint();
7841 }
7842
7843 return ChangeStatus::UNCHANGED;
7844 }
7845
7846 /// See AbstractAttribute::trackStatistics()
7847 void trackStatistics() const override {
7848 STATS_DECLTRACK_CSARG_ATTR(privatizable_ptr);
7849 }
7850};
7851
7852struct AAPrivatizablePtrCallSiteReturned final
7853 : public AAPrivatizablePtrFloating {
7854 AAPrivatizablePtrCallSiteReturned(const IRPosition &IRP, Attributor &A)
7855 : AAPrivatizablePtrFloating(IRP, A) {}
7856
7857 /// See AbstractAttribute::initialize(...).
7858 void initialize(Attributor &A) override {
7859 // TODO: We can privatize more than arguments.
7860 indicatePessimisticFixpoint();
7861 }
7862
7863 /// See AbstractAttribute::trackStatistics()
7864 void trackStatistics() const override {
7865 STATS_DECLTRACK_CSRET_ATTR(privatizable_ptr);
7866 }
7867};
7868
7869struct AAPrivatizablePtrReturned final : public AAPrivatizablePtrFloating {
7870 AAPrivatizablePtrReturned(const IRPosition &IRP, Attributor &A)
7871 : AAPrivatizablePtrFloating(IRP, A) {}
7872
7873 /// See AbstractAttribute::initialize(...).
7874 void initialize(Attributor &A) override {
7875 // TODO: We can privatize more than arguments.
7876 indicatePessimisticFixpoint();
7877 }
7878
7879 /// See AbstractAttribute::trackStatistics()
7880 void trackStatistics() const override {
7881 STATS_DECLTRACK_FNRET_ATTR(privatizable_ptr);
7882 }
7883};
7884} // namespace
7885
7886/// -------------------- Memory Behavior Attributes ----------------------------
7887/// Includes read-none, read-only, and write-only.
7888/// ----------------------------------------------------------------------------
7889namespace {
7890struct AAMemoryBehaviorImpl : public AAMemoryBehavior {
7891 AAMemoryBehaviorImpl(const IRPosition &IRP, Attributor &A)
7892 : AAMemoryBehavior(IRP, A) {}
7893
7894 /// See AbstractAttribute::initialize(...).
7895 void initialize(Attributor &A) override {
7896 intersectAssumedBits(BitsEncoding: BEST_STATE);
7897 getKnownStateFromValue(A, IRP: getIRPosition(), State&: getState());
7898 AAMemoryBehavior::initialize(A);
7899 }
7900
7901 /// Return the memory behavior information encoded in the IR for \p IRP.
7902 static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
7903 BitIntegerState &State,
7904 bool IgnoreSubsumingPositions = false) {
7905 SmallVector<Attribute, 2> Attrs;
7906 A.getAttrs(IRP, AKs: AttrKinds, Attrs, IgnoreSubsumingPositions);
7907 for (const Attribute &Attr : Attrs) {
7908 switch (Attr.getKindAsEnum()) {
7909 case Attribute::ReadNone:
7910 State.addKnownBits(Bits: NO_ACCESSES);
7911 break;
7912 case Attribute::ReadOnly:
7913 State.addKnownBits(Bits: NO_WRITES);
7914 break;
7915 case Attribute::WriteOnly:
7916 State.addKnownBits(Bits: NO_READS);
7917 break;
7918 default:
7919 llvm_unreachable("Unexpected attribute!");
7920 }
7921 }
7922
7923 if (auto *I = dyn_cast<Instruction>(Val: &IRP.getAnchorValue())) {
7924 if (!I->mayReadFromMemory())
7925 State.addKnownBits(Bits: NO_READS);
7926 if (!I->mayWriteToMemory())
7927 State.addKnownBits(Bits: NO_WRITES);
7928 }
7929 }
7930
7931 /// See AbstractAttribute::getDeducedAttributes(...).
7932 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
7933 SmallVectorImpl<Attribute> &Attrs) const override {
7934 assert(Attrs.size() == 0);
7935 if (isAssumedReadNone())
7936 Attrs.push_back(Elt: Attribute::get(Context&: Ctx, Kind: Attribute::ReadNone));
7937 else if (isAssumedReadOnly())
7938 Attrs.push_back(Elt: Attribute::get(Context&: Ctx, Kind: Attribute::ReadOnly));
7939 else if (isAssumedWriteOnly())
7940 Attrs.push_back(Elt: Attribute::get(Context&: Ctx, Kind: Attribute::WriteOnly));
7941 assert(Attrs.size() <= 1);
7942 }
7943
7944 /// See AbstractAttribute::manifest(...).
7945 ChangeStatus manifest(Attributor &A) override {
7946 const IRPosition &IRP = getIRPosition();
7947
7948 if (A.hasAttr(IRP, AKs: Attribute::ReadNone,
7949 /* IgnoreSubsumingPositions */ true))
7950 return ChangeStatus::UNCHANGED;
7951
7952 // Check if we would improve the existing attributes first.
7953 SmallVector<Attribute, 4> DeducedAttrs;
7954 getDeducedAttributes(A, Ctx&: IRP.getAnchorValue().getContext(), Attrs&: DeducedAttrs);
7955 if (llvm::all_of(Range&: DeducedAttrs, P: [&](const Attribute &Attr) {
7956 return A.hasAttr(IRP, AKs: Attr.getKindAsEnum(),
7957 /* IgnoreSubsumingPositions */ true);
7958 }))
7959 return ChangeStatus::UNCHANGED;
7960
7961 // Clear existing attributes.
7962 A.removeAttrs(IRP, AttrKinds);
7963 // Clear conflicting writable attribute.
7964 if (isAssumedReadOnly())
7965 A.removeAttrs(IRP, AttrKinds: Attribute::Writable);
7966
7967 // Use the generic manifest method.
7968 return IRAttribute::manifest(A);
7969 }
7970
7971 /// See AbstractState::getAsStr().
7972 const std::string getAsStr(Attributor *A) const override {
7973 if (isAssumedReadNone())
7974 return "readnone";
7975 if (isAssumedReadOnly())
7976 return "readonly";
7977 if (isAssumedWriteOnly())
7978 return "writeonly";
7979 return "may-read/write";
7980 }
7981
7982 /// The set of IR attributes AAMemoryBehavior deals with.
7983 static const Attribute::AttrKind AttrKinds[3];
7984};
7985
7986const Attribute::AttrKind AAMemoryBehaviorImpl::AttrKinds[] = {
7987 Attribute::ReadNone, Attribute::ReadOnly, Attribute::WriteOnly};
7988
7989/// Memory behavior attribute for a floating value.
7990struct AAMemoryBehaviorFloating : AAMemoryBehaviorImpl {
7991 AAMemoryBehaviorFloating(const IRPosition &IRP, Attributor &A)
7992 : AAMemoryBehaviorImpl(IRP, A) {}
7993
7994 /// See AbstractAttribute::updateImpl(...).
7995 ChangeStatus updateImpl(Attributor &A) override;
7996
7997 /// See AbstractAttribute::trackStatistics()
7998 void trackStatistics() const override {
7999 if (isAssumedReadNone())
8000 STATS_DECLTRACK_FLOATING_ATTR(readnone)
8001 else if (isAssumedReadOnly())
8002 STATS_DECLTRACK_FLOATING_ATTR(readonly)
8003 else if (isAssumedWriteOnly())
8004 STATS_DECLTRACK_FLOATING_ATTR(writeonly)
8005 }
8006
8007private:
8008 /// Return true if users of \p UserI might access the underlying
8009 /// variable/location described by \p U and should therefore be analyzed.
8010 bool followUsersOfUseIn(Attributor &A, const Use &U,
8011 const Instruction *UserI);
8012
8013 /// Update the state according to the effect of use \p U in \p UserI.
8014 void analyzeUseIn(Attributor &A, const Use &U, const Instruction *UserI);
8015};
8016
8017/// Memory behavior attribute for function argument.
8018struct AAMemoryBehaviorArgument : AAMemoryBehaviorFloating {
8019 AAMemoryBehaviorArgument(const IRPosition &IRP, Attributor &A)
8020 : AAMemoryBehaviorFloating(IRP, A) {}
8021
8022 /// See AbstractAttribute::initialize(...).
8023 void initialize(Attributor &A) override {
8024 intersectAssumedBits(BitsEncoding: BEST_STATE);
8025 const IRPosition &IRP = getIRPosition();
8026 // TODO: Make IgnoreSubsumingPositions a property of an IRAttribute so we
8027 // can query it when we use has/getAttr. That would allow us to reuse the
8028 // initialize of the base class here.
8029 bool HasByVal = A.hasAttr(IRP, AKs: {Attribute::ByVal},
8030 /* IgnoreSubsumingPositions */ true);
8031 getKnownStateFromValue(A, IRP, State&: getState(),
8032 /* IgnoreSubsumingPositions */ HasByVal);
8033 }
8034
8035 ChangeStatus manifest(Attributor &A) override {
8036 // TODO: Pointer arguments are not supported on vectors of pointers yet.
8037 if (!getAssociatedValue().getType()->isPointerTy())
8038 return ChangeStatus::UNCHANGED;
8039
8040 // TODO: From readattrs.ll: "inalloca parameters are always
8041 // considered written"
8042 if (A.hasAttr(IRP: getIRPosition(),
8043 AKs: {Attribute::InAlloca, Attribute::Preallocated})) {
8044 removeKnownBits(BitsEncoding: NO_WRITES);
8045 removeAssumedBits(BitsEncoding: NO_WRITES);
8046 }
8047 A.removeAttrs(IRP: getIRPosition(), AttrKinds);
8048 return AAMemoryBehaviorFloating::manifest(A);
8049 }
8050
8051 /// See AbstractAttribute::trackStatistics()
8052 void trackStatistics() const override {
8053 if (isAssumedReadNone())
8054 STATS_DECLTRACK_ARG_ATTR(readnone)
8055 else if (isAssumedReadOnly())
8056 STATS_DECLTRACK_ARG_ATTR(readonly)
8057 else if (isAssumedWriteOnly())
8058 STATS_DECLTRACK_ARG_ATTR(writeonly)
8059 }
8060};
8061
8062struct AAMemoryBehaviorCallSiteArgument final : AAMemoryBehaviorArgument {
8063 AAMemoryBehaviorCallSiteArgument(const IRPosition &IRP, Attributor &A)
8064 : AAMemoryBehaviorArgument(IRP, A) {}
8065
8066 /// See AbstractAttribute::initialize(...).
8067 void initialize(Attributor &A) override {
8068 // If we don't have an associated attribute this is either a variadic call
8069 // or an indirect call, either way, nothing to do here.
8070 Argument *Arg = getAssociatedArgument();
8071 if (!Arg) {
8072 indicatePessimisticFixpoint();
8073 return;
8074 }
8075 if (Arg->hasByValAttr()) {
8076 addKnownBits(Bits: NO_WRITES);
8077 removeKnownBits(BitsEncoding: NO_READS);
8078 removeAssumedBits(BitsEncoding: NO_READS);
8079 }
8080 AAMemoryBehaviorArgument::initialize(A);
8081 if (getAssociatedFunction()->isDeclaration())
8082 indicatePessimisticFixpoint();
8083 }
8084
8085 /// See AbstractAttribute::updateImpl(...).
8086 ChangeStatus updateImpl(Attributor &A) override {
8087 // TODO: Once we have call site specific value information we can provide
8088 // call site specific liveness liveness information and then it makes
8089 // sense to specialize attributes for call sites arguments instead of
8090 // redirecting requests to the callee argument.
8091 Argument *Arg = getAssociatedArgument();
8092 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
8093 auto *ArgAA =
8094 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
8095 if (!ArgAA)
8096 return indicatePessimisticFixpoint();
8097 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
8098 }
8099
8100 /// See AbstractAttribute::trackStatistics()
8101 void trackStatistics() const override {
8102 if (isAssumedReadNone())
8103 STATS_DECLTRACK_CSARG_ATTR(readnone)
8104 else if (isAssumedReadOnly())
8105 STATS_DECLTRACK_CSARG_ATTR(readonly)
8106 else if (isAssumedWriteOnly())
8107 STATS_DECLTRACK_CSARG_ATTR(writeonly)
8108 }
8109};
8110
8111/// Memory behavior attribute for a call site return position.
8112struct AAMemoryBehaviorCallSiteReturned final : AAMemoryBehaviorFloating {
8113 AAMemoryBehaviorCallSiteReturned(const IRPosition &IRP, Attributor &A)
8114 : AAMemoryBehaviorFloating(IRP, A) {}
8115
8116 /// See AbstractAttribute::initialize(...).
8117 void initialize(Attributor &A) override {
8118 AAMemoryBehaviorImpl::initialize(A);
8119 }
8120 /// See AbstractAttribute::manifest(...).
8121 ChangeStatus manifest(Attributor &A) override {
8122 // We do not annotate returned values.
8123 return ChangeStatus::UNCHANGED;
8124 }
8125
8126 /// See AbstractAttribute::trackStatistics()
8127 void trackStatistics() const override {}
8128};
8129
8130/// An AA to represent the memory behavior function attributes.
8131struct AAMemoryBehaviorFunction final : public AAMemoryBehaviorImpl {
8132 AAMemoryBehaviorFunction(const IRPosition &IRP, Attributor &A)
8133 : AAMemoryBehaviorImpl(IRP, A) {}
8134
8135 /// See AbstractAttribute::updateImpl(Attributor &A).
8136 ChangeStatus updateImpl(Attributor &A) override;
8137
8138 /// See AbstractAttribute::manifest(...).
8139 ChangeStatus manifest(Attributor &A) override {
8140 // TODO: It would be better to merge this with AAMemoryLocation, so that
8141 // we could determine read/write per location. This would also have the
8142 // benefit of only one place trying to manifest the memory attribute.
8143 Function &F = cast<Function>(Val&: getAnchorValue());
8144 MemoryEffects ME = MemoryEffects::unknown();
8145 if (isAssumedReadNone())
8146 ME = MemoryEffects::none();
8147 else if (isAssumedReadOnly())
8148 ME = MemoryEffects::readOnly();
8149 else if (isAssumedWriteOnly())
8150 ME = MemoryEffects::writeOnly();
8151
8152 A.removeAttrs(IRP: getIRPosition(), AttrKinds);
8153 // Clear conflicting writable attribute.
8154 if (ME.onlyReadsMemory())
8155 for (Argument &Arg : F.args())
8156 A.removeAttrs(IRP: IRPosition::argument(Arg), AttrKinds: Attribute::Writable);
8157 return A.manifestAttrs(IRP: getIRPosition(),
8158 DeducedAttrs: Attribute::getWithMemoryEffects(Context&: F.getContext(), ME));
8159 }
8160
8161 /// See AbstractAttribute::trackStatistics()
8162 void trackStatistics() const override {
8163 if (isAssumedReadNone())
8164 STATS_DECLTRACK_FN_ATTR(readnone)
8165 else if (isAssumedReadOnly())
8166 STATS_DECLTRACK_FN_ATTR(readonly)
8167 else if (isAssumedWriteOnly())
8168 STATS_DECLTRACK_FN_ATTR(writeonly)
8169 }
8170};
8171
8172/// AAMemoryBehavior attribute for call sites.
8173struct AAMemoryBehaviorCallSite final
8174 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl> {
8175 AAMemoryBehaviorCallSite(const IRPosition &IRP, Attributor &A)
8176 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl>(IRP, A) {}
8177
8178 /// See AbstractAttribute::manifest(...).
8179 ChangeStatus manifest(Attributor &A) override {
8180 // TODO: Deduplicate this with AAMemoryBehaviorFunction.
8181 CallBase &CB = cast<CallBase>(Val&: getAnchorValue());
8182 MemoryEffects ME = MemoryEffects::unknown();
8183 if (isAssumedReadNone())
8184 ME = MemoryEffects::none();
8185 else if (isAssumedReadOnly())
8186 ME = MemoryEffects::readOnly();
8187 else if (isAssumedWriteOnly())
8188 ME = MemoryEffects::writeOnly();
8189
8190 A.removeAttrs(IRP: getIRPosition(), AttrKinds);
8191 // Clear conflicting writable attribute.
8192 if (ME.onlyReadsMemory())
8193 for (Use &U : CB.args())
8194 A.removeAttrs(IRP: IRPosition::callsite_argument(CB, ArgNo: U.getOperandNo()),
8195 AttrKinds: Attribute::Writable);
8196 return A.manifestAttrs(
8197 IRP: getIRPosition(), DeducedAttrs: Attribute::getWithMemoryEffects(Context&: CB.getContext(), ME));
8198 }
8199
8200 /// See AbstractAttribute::trackStatistics()
8201 void trackStatistics() const override {
8202 if (isAssumedReadNone())
8203 STATS_DECLTRACK_CS_ATTR(readnone)
8204 else if (isAssumedReadOnly())
8205 STATS_DECLTRACK_CS_ATTR(readonly)
8206 else if (isAssumedWriteOnly())
8207 STATS_DECLTRACK_CS_ATTR(writeonly)
8208 }
8209};
8210
8211ChangeStatus AAMemoryBehaviorFunction::updateImpl(Attributor &A) {
8212
8213 // The current assumed state used to determine a change.
8214 auto AssumedState = getAssumed();
8215
8216 auto CheckRWInst = [&](Instruction &I) {
8217 // If the instruction has an own memory behavior state, use it to restrict
8218 // the local state. No further analysis is required as the other memory
8219 // state is as optimistic as it gets.
8220 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
8221 const auto *MemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
8222 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::REQUIRED);
8223 if (MemBehaviorAA) {
8224 intersectAssumedBits(BitsEncoding: MemBehaviorAA->getAssumed());
8225 return !isAtFixpoint();
8226 }
8227 }
8228
8229 // Remove access kind modifiers if necessary.
8230 if (I.mayReadFromMemory())
8231 removeAssumedBits(BitsEncoding: NO_READS);
8232 if (I.mayWriteToMemory())
8233 removeAssumedBits(BitsEncoding: NO_WRITES);
8234 return !isAtFixpoint();
8235 };
8236
8237 bool UsedAssumedInformation = false;
8238 if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInst, QueryingAA&: *this,
8239 UsedAssumedInformation))
8240 return indicatePessimisticFixpoint();
8241
8242 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8243 : ChangeStatus::UNCHANGED;
8244}
8245
8246ChangeStatus AAMemoryBehaviorFloating::updateImpl(Attributor &A) {
8247
8248 const IRPosition &IRP = getIRPosition();
8249 const IRPosition &FnPos = IRPosition::function_scope(IRP);
8250 AAMemoryBehavior::StateType &S = getState();
8251
8252 // First, check the function scope. We take the known information and we avoid
8253 // work if the assumed information implies the current assumed information for
8254 // this attribute. This is a valid for all but byval arguments.
8255 Argument *Arg = IRP.getAssociatedArgument();
8256 AAMemoryBehavior::base_t FnMemAssumedState =
8257 AAMemoryBehavior::StateType::getWorstState();
8258 if (!Arg || !Arg->hasByValAttr()) {
8259 const auto *FnMemAA =
8260 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::OPTIONAL);
8261 if (FnMemAA) {
8262 FnMemAssumedState = FnMemAA->getAssumed();
8263 S.addKnownBits(Bits: FnMemAA->getKnown());
8264 if ((S.getAssumed() & FnMemAA->getAssumed()) == S.getAssumed())
8265 return ChangeStatus::UNCHANGED;
8266 }
8267 }
8268
8269 // The current assumed state used to determine a change.
8270 auto AssumedState = S.getAssumed();
8271
8272 // Make sure the value is not captured (except through "return"), if
8273 // it is, any information derived would be irrelevant anyway as we cannot
8274 // check the potential aliases introduced by the capture. However, no need
8275 // to fall back to anythign less optimistic than the function state.
8276 bool IsKnownNoCapture;
8277 const AANoCapture *ArgNoCaptureAA = nullptr;
8278 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
8279 A, QueryingAA: this, IRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
8280 AAPtr: &ArgNoCaptureAA);
8281
8282 if (!IsAssumedNoCapture &&
8283 (!ArgNoCaptureAA || !ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
8284 S.intersectAssumedBits(BitsEncoding: FnMemAssumedState);
8285 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8286 : ChangeStatus::UNCHANGED;
8287 }
8288
8289 // Visit and expand uses until all are analyzed or a fixpoint is reached.
8290 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
8291 Instruction *UserI = cast<Instruction>(Val: U.getUser());
8292 LLVM_DEBUG(dbgs() << "[AAMemoryBehavior] Use: " << *U << " in " << *UserI
8293 << " \n");
8294
8295 // Droppable users, e.g., llvm::assume does not actually perform any action.
8296 if (UserI->isDroppable())
8297 return true;
8298
8299 // Check if the users of UserI should also be visited.
8300 Follow = followUsersOfUseIn(A, U, UserI);
8301
8302 // If UserI might touch memory we analyze the use in detail.
8303 if (UserI->mayReadOrWriteMemory())
8304 analyzeUseIn(A, U, UserI);
8305
8306 return !isAtFixpoint();
8307 };
8308
8309 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: getAssociatedValue()))
8310 return indicatePessimisticFixpoint();
8311
8312 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8313 : ChangeStatus::UNCHANGED;
8314}
8315
8316bool AAMemoryBehaviorFloating::followUsersOfUseIn(Attributor &A, const Use &U,
8317 const Instruction *UserI) {
8318 // The loaded value is unrelated to the pointer argument, no need to
8319 // follow the users of the load.
8320 if (isa<LoadInst>(Val: UserI) || isa<ReturnInst>(Val: UserI))
8321 return false;
8322
8323 // By default we follow all uses assuming UserI might leak information on U,
8324 // we have special handling for call sites operands though.
8325 const auto *CB = dyn_cast<CallBase>(Val: UserI);
8326 if (!CB || !CB->isArgOperand(U: &U))
8327 return true;
8328
8329 // If the use is a call argument known not to be captured, the users of
8330 // the call do not need to be visited because they have to be unrelated to
8331 // the input. Note that this check is not trivial even though we disallow
8332 // general capturing of the underlying argument. The reason is that the
8333 // call might the argument "through return", which we allow and for which we
8334 // need to check call users.
8335 if (U.get()->getType()->isPointerTy()) {
8336 unsigned ArgNo = CB->getArgOperandNo(U: &U);
8337 bool IsKnownNoCapture;
8338 return !AA::hasAssumedIRAttr<Attribute::Captures>(
8339 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
8340 DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture);
8341 }
8342
8343 return true;
8344}
8345
8346void AAMemoryBehaviorFloating::analyzeUseIn(Attributor &A, const Use &U,
8347 const Instruction *UserI) {
8348 assert(UserI->mayReadOrWriteMemory());
8349
8350 switch (UserI->getOpcode()) {
8351 default:
8352 // TODO: Handle all atomics and other side-effect operations we know of.
8353 break;
8354 case Instruction::Load:
8355 // Loads cause the NO_READS property to disappear.
8356 removeAssumedBits(BitsEncoding: NO_READS);
8357 return;
8358
8359 case Instruction::Store:
8360 // Stores cause the NO_WRITES property to disappear if the use is the
8361 // pointer operand. Note that while capturing was taken care of somewhere
8362 // else we need to deal with stores of the value that is not looked through.
8363 if (cast<StoreInst>(Val: UserI)->getPointerOperand() == U.get())
8364 removeAssumedBits(BitsEncoding: NO_WRITES);
8365 else
8366 indicatePessimisticFixpoint();
8367 return;
8368
8369 case Instruction::Call:
8370 case Instruction::CallBr:
8371 case Instruction::Invoke: {
8372 // For call sites we look at the argument memory behavior attribute (this
8373 // could be recursive!) in order to restrict our own state.
8374 const auto *CB = cast<CallBase>(Val: UserI);
8375
8376 // Give up on operand bundles.
8377 if (CB->isBundleOperand(U: &U)) {
8378 indicatePessimisticFixpoint();
8379 return;
8380 }
8381
8382 // Calling a function does read the function pointer, maybe write it if the
8383 // function is self-modifying.
8384 if (CB->isCallee(U: &U)) {
8385 removeAssumedBits(BitsEncoding: NO_READS);
8386 break;
8387 }
8388
8389 // Adjust the possible access behavior based on the information on the
8390 // argument.
8391 IRPosition Pos;
8392 if (U.get()->getType()->isPointerTy())
8393 Pos = IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U));
8394 else
8395 Pos = IRPosition::callsite_function(CB: *CB);
8396 const auto *MemBehaviorAA =
8397 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: Pos, DepClass: DepClassTy::OPTIONAL);
8398 if (!MemBehaviorAA)
8399 break;
8400 // "assumed" has at most the same bits as the MemBehaviorAA assumed
8401 // and at least "known".
8402 intersectAssumedBits(BitsEncoding: MemBehaviorAA->getAssumed());
8403 return;
8404 }
8405 };
8406
8407 // Generally, look at the "may-properties" and adjust the assumed state if we
8408 // did not trigger special handling before.
8409 if (UserI->mayReadFromMemory())
8410 removeAssumedBits(BitsEncoding: NO_READS);
8411 if (UserI->mayWriteToMemory())
8412 removeAssumedBits(BitsEncoding: NO_WRITES);
8413}
8414} // namespace
8415
8416/// -------------------- Memory Locations Attributes ---------------------------
8417/// Includes read-none, argmemonly, inaccessiblememonly,
8418/// inaccessiblememorargmemonly
8419/// ----------------------------------------------------------------------------
8420
8421std::string AAMemoryLocation::getMemoryLocationsAsStr(
8422 AAMemoryLocation::MemoryLocationsKind MLK) {
8423 if (0 == (MLK & AAMemoryLocation::NO_LOCATIONS))
8424 return "all memory";
8425 if (MLK == AAMemoryLocation::NO_LOCATIONS)
8426 return "no memory";
8427 std::string S = "memory:";
8428 if (0 == (MLK & AAMemoryLocation::NO_LOCAL_MEM))
8429 S += "stack,";
8430 if (0 == (MLK & AAMemoryLocation::NO_CONST_MEM))
8431 S += "constant,";
8432 if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_INTERNAL_MEM))
8433 S += "internal global,";
8434 if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_EXTERNAL_MEM))
8435 S += "external global,";
8436 if (0 == (MLK & AAMemoryLocation::NO_ARGUMENT_MEM))
8437 S += "argument,";
8438 if (0 == (MLK & AAMemoryLocation::NO_INACCESSIBLE_MEM))
8439 S += "inaccessible,";
8440 if (0 == (MLK & AAMemoryLocation::NO_MALLOCED_MEM))
8441 S += "malloced,";
8442 if (0 == (MLK & AAMemoryLocation::NO_UNKOWN_MEM))
8443 S += "unknown,";
8444 S.pop_back();
8445 return S;
8446}
8447
8448namespace {
8449struct AAMemoryLocationImpl : public AAMemoryLocation {
8450
8451 AAMemoryLocationImpl(const IRPosition &IRP, Attributor &A)
8452 : AAMemoryLocation(IRP, A), Allocator(A.Allocator) {
8453 AccessKind2Accesses.fill(u: nullptr);
8454 }
8455
8456 ~AAMemoryLocationImpl() override {
8457 // The AccessSets are allocated via a BumpPtrAllocator, we call
8458 // the destructor manually.
8459 for (AccessSet *AS : AccessKind2Accesses)
8460 if (AS)
8461 AS->~AccessSet();
8462 }
8463
8464 /// See AbstractAttribute::initialize(...).
8465 void initialize(Attributor &A) override {
8466 intersectAssumedBits(BitsEncoding: BEST_STATE);
8467 getKnownStateFromValue(A, IRP: getIRPosition(), State&: getState());
8468 AAMemoryLocation::initialize(A);
8469 }
8470
8471 /// Return the memory behavior information encoded in the IR for \p IRP.
8472 static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
8473 BitIntegerState &State,
8474 bool IgnoreSubsumingPositions = false) {
8475 // For internal functions we ignore `argmemonly` and
8476 // `inaccessiblememorargmemonly` as we might break it via interprocedural
8477 // constant propagation. It is unclear if this is the best way but it is
8478 // unlikely this will cause real performance problems. If we are deriving
8479 // attributes for the anchor function we even remove the attribute in
8480 // addition to ignoring it.
8481 // TODO: A better way to handle this would be to add ~NO_GLOBAL_MEM /
8482 // MemoryEffects::Other as a possible location.
8483 bool UseArgMemOnly = true;
8484 Function *AnchorFn = IRP.getAnchorScope();
8485 if (AnchorFn && A.isRunOn(Fn&: *AnchorFn))
8486 UseArgMemOnly = !AnchorFn->hasLocalLinkage();
8487
8488 SmallVector<Attribute, 2> Attrs;
8489 A.getAttrs(IRP, AKs: {Attribute::Memory}, Attrs, IgnoreSubsumingPositions);
8490 for (const Attribute &Attr : Attrs) {
8491 // TODO: We can map MemoryEffects to Attributor locations more precisely.
8492 MemoryEffects ME = Attr.getMemoryEffects();
8493 if (ME.doesNotAccessMemory()) {
8494 State.addKnownBits(Bits: NO_LOCAL_MEM | NO_CONST_MEM);
8495 continue;
8496 }
8497 if (ME.onlyAccessesInaccessibleMem()) {
8498 State.addKnownBits(Bits: inverseLocation(Loc: NO_INACCESSIBLE_MEM, AndLocalMem: true, AndConstMem: true));
8499 continue;
8500 }
8501 if (ME.onlyAccessesArgPointees()) {
8502 if (UseArgMemOnly)
8503 State.addKnownBits(Bits: inverseLocation(Loc: NO_ARGUMENT_MEM, AndLocalMem: true, AndConstMem: true));
8504 else {
8505 // Remove location information, only keep read/write info.
8506 ME = MemoryEffects(ME.getModRef());
8507 A.manifestAttrs(IRP,
8508 DeducedAttrs: Attribute::getWithMemoryEffects(
8509 Context&: IRP.getAnchorValue().getContext(), ME),
8510 /*ForceReplace*/ true);
8511 }
8512 continue;
8513 }
8514 if (ME.onlyAccessesInaccessibleOrArgMem()) {
8515 if (UseArgMemOnly)
8516 State.addKnownBits(Bits: inverseLocation(
8517 Loc: NO_INACCESSIBLE_MEM | NO_ARGUMENT_MEM, AndLocalMem: true, AndConstMem: true));
8518 else {
8519 // Remove location information, only keep read/write info.
8520 ME = MemoryEffects(ME.getModRef());
8521 A.manifestAttrs(IRP,
8522 DeducedAttrs: Attribute::getWithMemoryEffects(
8523 Context&: IRP.getAnchorValue().getContext(), ME),
8524 /*ForceReplace*/ true);
8525 }
8526 continue;
8527 }
8528 }
8529 }
8530
8531 /// See AbstractAttribute::getDeducedAttributes(...).
8532 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
8533 SmallVectorImpl<Attribute> &Attrs) const override {
8534 // TODO: We can map Attributor locations to MemoryEffects more precisely.
8535 assert(Attrs.size() == 0);
8536 if (getIRPosition().getPositionKind() == IRPosition::IRP_FUNCTION) {
8537 if (isAssumedReadNone())
8538 Attrs.push_back(
8539 Elt: Attribute::getWithMemoryEffects(Context&: Ctx, ME: MemoryEffects::none()));
8540 else if (isAssumedInaccessibleMemOnly())
8541 Attrs.push_back(Elt: Attribute::getWithMemoryEffects(
8542 Context&: Ctx, ME: MemoryEffects::inaccessibleMemOnly()));
8543 else if (isAssumedArgMemOnly())
8544 Attrs.push_back(
8545 Elt: Attribute::getWithMemoryEffects(Context&: Ctx, ME: MemoryEffects::argMemOnly()));
8546 else if (isAssumedInaccessibleOrArgMemOnly())
8547 Attrs.push_back(Elt: Attribute::getWithMemoryEffects(
8548 Context&: Ctx, ME: MemoryEffects::inaccessibleOrArgMemOnly()));
8549 }
8550 assert(Attrs.size() <= 1);
8551 }
8552
8553 /// See AbstractAttribute::manifest(...).
8554 ChangeStatus manifest(Attributor &A) override {
8555 // TODO: If AAMemoryLocation and AAMemoryBehavior are merged, we could
8556 // provide per-location modref information here.
8557 const IRPosition &IRP = getIRPosition();
8558
8559 SmallVector<Attribute, 1> DeducedAttrs;
8560 getDeducedAttributes(A, Ctx&: IRP.getAnchorValue().getContext(), Attrs&: DeducedAttrs);
8561 if (DeducedAttrs.size() != 1)
8562 return ChangeStatus::UNCHANGED;
8563 MemoryEffects ME = DeducedAttrs[0].getMemoryEffects();
8564
8565 return A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithMemoryEffects(
8566 Context&: IRP.getAnchorValue().getContext(), ME));
8567 }
8568
8569 /// See AAMemoryLocation::checkForAllAccessesToMemoryKind(...).
8570 bool checkForAllAccessesToMemoryKind(
8571 function_ref<bool(const Instruction *, const Value *, AccessKind,
8572 MemoryLocationsKind)>
8573 Pred,
8574 MemoryLocationsKind RequestedMLK) const override {
8575 if (!isValidState())
8576 return false;
8577
8578 MemoryLocationsKind AssumedMLK = getAssumedNotAccessedLocation();
8579 if (AssumedMLK == NO_LOCATIONS)
8580 return true;
8581
8582 unsigned Idx = 0;
8583 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS;
8584 CurMLK *= 2, ++Idx) {
8585 if (CurMLK & RequestedMLK)
8586 continue;
8587
8588 if (const AccessSet *Accesses = AccessKind2Accesses[Idx])
8589 for (const AccessInfo &AI : *Accesses)
8590 if (!Pred(AI.I, AI.Ptr, AI.Kind, CurMLK))
8591 return false;
8592 }
8593
8594 return true;
8595 }
8596
8597 ChangeStatus indicatePessimisticFixpoint() override {
8598 // If we give up and indicate a pessimistic fixpoint this instruction will
8599 // become an access for all potential access kinds:
8600 // TODO: Add pointers for argmemonly and globals to improve the results of
8601 // checkForAllAccessesToMemoryKind.
8602 bool Changed = false;
8603 MemoryLocationsKind KnownMLK = getKnown();
8604 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
8605 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2)
8606 if (!(CurMLK & KnownMLK))
8607 updateStateAndAccessesMap(State&: getState(), MLK: CurMLK, I, Ptr: nullptr, Changed,
8608 AK: getAccessKindFromInst(I));
8609 return AAMemoryLocation::indicatePessimisticFixpoint();
8610 }
8611
8612protected:
8613 /// Helper struct to tie together an instruction that has a read or write
8614 /// effect with the pointer it accesses (if any).
8615 struct AccessInfo {
8616
8617 /// The instruction that caused the access.
8618 const Instruction *I;
8619
8620 /// The base pointer that is accessed, or null if unknown.
8621 const Value *Ptr;
8622
8623 /// The kind of access (read/write/read+write).
8624 AccessKind Kind;
8625
8626 bool operator==(const AccessInfo &RHS) const {
8627 return I == RHS.I && Ptr == RHS.Ptr && Kind == RHS.Kind;
8628 }
8629 bool operator()(const AccessInfo &LHS, const AccessInfo &RHS) const {
8630 if (LHS.I != RHS.I)
8631 return LHS.I < RHS.I;
8632 if (LHS.Ptr != RHS.Ptr)
8633 return LHS.Ptr < RHS.Ptr;
8634 if (LHS.Kind != RHS.Kind)
8635 return LHS.Kind < RHS.Kind;
8636 return false;
8637 }
8638 };
8639
8640 /// Mapping from *single* memory location kinds, e.g., LOCAL_MEM with the
8641 /// value of NO_LOCAL_MEM, to the accesses encountered for this memory kind.
8642 using AccessSet = SmallSet<AccessInfo, 2, AccessInfo>;
8643 std::array<AccessSet *, llvm::ConstantLog2<VALID_STATE>()>
8644 AccessKind2Accesses;
8645
8646 /// Categorize the pointer arguments of CB that might access memory in
8647 /// AccessedLoc and update the state and access map accordingly.
8648 void
8649 categorizeArgumentPointerLocations(Attributor &A, CallBase &CB,
8650 AAMemoryLocation::StateType &AccessedLocs,
8651 bool &Changed);
8652
8653 /// Return the kind(s) of location that may be accessed by \p V.
8654 AAMemoryLocation::MemoryLocationsKind
8655 categorizeAccessedLocations(Attributor &A, Instruction &I, bool &Changed);
8656
8657 /// Return the access kind as determined by \p I.
8658 AccessKind getAccessKindFromInst(const Instruction *I) {
8659 AccessKind AK = READ_WRITE;
8660 if (I) {
8661 AK = I->mayReadFromMemory() ? READ : NONE;
8662 AK = AccessKind(AK | (I->mayWriteToMemory() ? WRITE : NONE));
8663 }
8664 return AK;
8665 }
8666
8667 /// Update the state \p State and the AccessKind2Accesses given that \p I is
8668 /// an access of kind \p AK to a \p MLK memory location with the access
8669 /// pointer \p Ptr.
8670 void updateStateAndAccessesMap(AAMemoryLocation::StateType &State,
8671 MemoryLocationsKind MLK, const Instruction *I,
8672 const Value *Ptr, bool &Changed,
8673 AccessKind AK = READ_WRITE) {
8674
8675 assert(isPowerOf2_32(MLK) && "Expected a single location set!");
8676 auto *&Accesses = AccessKind2Accesses[llvm::Log2_32(Value: MLK)];
8677 if (!Accesses)
8678 Accesses = new (Allocator) AccessSet();
8679 Changed |= Accesses->insert(V: AccessInfo{.I: I, .Ptr: Ptr, .Kind: AK}).second;
8680 if (MLK == NO_UNKOWN_MEM)
8681 MLK = NO_LOCATIONS;
8682 State.removeAssumedBits(BitsEncoding: MLK);
8683 }
8684
8685 /// Determine the underlying locations kinds for \p Ptr, e.g., globals or
8686 /// arguments, and update the state and access map accordingly.
8687 void categorizePtrValue(Attributor &A, const Instruction &I, const Value &Ptr,
8688 AAMemoryLocation::StateType &State, bool &Changed,
8689 unsigned AccessAS = 0);
8690
8691 /// Used to allocate access sets.
8692 BumpPtrAllocator &Allocator;
8693};
8694
8695void AAMemoryLocationImpl::categorizePtrValue(
8696 Attributor &A, const Instruction &I, const Value &Ptr,
8697 AAMemoryLocation::StateType &State, bool &Changed, unsigned AccessAS) {
8698 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize pointer locations for "
8699 << Ptr << " ["
8700 << getMemoryLocationsAsStr(State.getAssumed()) << "]\n");
8701
8702 auto Pred = [&](Value &Obj) {
8703 unsigned ObjectAS = Obj.getType()->getPointerAddressSpace();
8704 // TODO: recognize the TBAA used for constant accesses.
8705 MemoryLocationsKind MLK = NO_LOCATIONS;
8706
8707 // Filter accesses to constant (GPU) memory if we have an AS at the access
8708 // site or the object is known to actually have the associated AS.
8709 if ((AccessAS == (unsigned)AA::GPUAddressSpace::Constant ||
8710 (ObjectAS == (unsigned)AA::GPUAddressSpace::Constant &&
8711 isIdentifiedObject(V: &Obj))) &&
8712 AA::isGPU(M: *I.getModule()))
8713 return true;
8714
8715 if (isa<UndefValue>(Val: &Obj))
8716 return true;
8717 if (isa<Argument>(Val: &Obj)) {
8718 // TODO: For now we do not treat byval arguments as local copies performed
8719 // on the call edge, though, we should. To make that happen we need to
8720 // teach various passes, e.g., DSE, about the copy effect of a byval. That
8721 // would also allow us to mark functions only accessing byval arguments as
8722 // readnone again, arguably their accesses have no effect outside of the
8723 // function, like accesses to allocas.
8724 MLK = NO_ARGUMENT_MEM;
8725 } else if (auto *GV = dyn_cast<GlobalValue>(Val: &Obj)) {
8726 // Reading constant memory is not treated as a read "effect" by the
8727 // function attr pass so we won't neither. Constants defined by TBAA are
8728 // similar. (We know we do not write it because it is constant.)
8729 if (auto *GVar = dyn_cast<GlobalVariable>(Val: GV))
8730 if (GVar->isConstant())
8731 return true;
8732
8733 if (GV->hasLocalLinkage())
8734 MLK = NO_GLOBAL_INTERNAL_MEM;
8735 else
8736 MLK = NO_GLOBAL_EXTERNAL_MEM;
8737 } else if (isa<ConstantPointerNull>(Val: &Obj) &&
8738 (!NullPointerIsDefined(F: getAssociatedFunction(), AS: AccessAS) ||
8739 !NullPointerIsDefined(F: getAssociatedFunction(), AS: ObjectAS))) {
8740 return true;
8741 } else if (isa<AllocaInst>(Val: &Obj)) {
8742 MLK = NO_LOCAL_MEM;
8743 } else if (const auto *CB = dyn_cast<CallBase>(Val: &Obj)) {
8744 bool IsKnownNoAlias;
8745 if (AA::hasAssumedIRAttr<Attribute::NoAlias>(
8746 A, QueryingAA: this, IRP: IRPosition::callsite_returned(CB: *CB), DepClass: DepClassTy::OPTIONAL,
8747 IsKnown&: IsKnownNoAlias))
8748 MLK = NO_MALLOCED_MEM;
8749 else
8750 MLK = NO_UNKOWN_MEM;
8751 } else {
8752 MLK = NO_UNKOWN_MEM;
8753 }
8754
8755 assert(MLK != NO_LOCATIONS && "No location specified!");
8756 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Ptr value can be categorized: "
8757 << Obj << " -> " << getMemoryLocationsAsStr(MLK) << "\n");
8758 updateStateAndAccessesMap(State, MLK, I: &I, Ptr: &Obj, Changed,
8759 AK: getAccessKindFromInst(I: &I));
8760
8761 return true;
8762 };
8763
8764 const auto *AA = A.getAAFor<AAUnderlyingObjects>(
8765 QueryingAA: *this, IRP: IRPosition::value(V: Ptr), DepClass: DepClassTy::OPTIONAL);
8766 if (!AA || !AA->forallUnderlyingObjects(Pred, Scope: AA::Intraprocedural)) {
8767 LLVM_DEBUG(
8768 dbgs() << "[AAMemoryLocation] Pointer locations not categorized\n");
8769 updateStateAndAccessesMap(State, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr, Changed,
8770 AK: getAccessKindFromInst(I: &I));
8771 return;
8772 }
8773
8774 LLVM_DEBUG(
8775 dbgs() << "[AAMemoryLocation] Accessed locations with pointer locations: "
8776 << getMemoryLocationsAsStr(State.getAssumed()) << "\n");
8777}
8778
8779void AAMemoryLocationImpl::categorizeArgumentPointerLocations(
8780 Attributor &A, CallBase &CB, AAMemoryLocation::StateType &AccessedLocs,
8781 bool &Changed) {
8782 for (unsigned ArgNo = 0, E = CB.arg_size(); ArgNo < E; ++ArgNo) {
8783
8784 // Skip non-pointer arguments.
8785 const Value *ArgOp = CB.getArgOperand(i: ArgNo);
8786 if (!ArgOp->getType()->isPtrOrPtrVectorTy())
8787 continue;
8788
8789 // Skip readnone arguments.
8790 const IRPosition &ArgOpIRP = IRPosition::callsite_argument(CB, ArgNo);
8791 const auto *ArgOpMemLocationAA =
8792 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: ArgOpIRP, DepClass: DepClassTy::OPTIONAL);
8793
8794 if (ArgOpMemLocationAA && ArgOpMemLocationAA->isAssumedReadNone())
8795 continue;
8796
8797 // Categorize potentially accessed pointer arguments as if there was an
8798 // access instruction with them as pointer.
8799 categorizePtrValue(A, I: CB, Ptr: *ArgOp, State&: AccessedLocs, Changed);
8800 }
8801}
8802
8803AAMemoryLocation::MemoryLocationsKind
8804AAMemoryLocationImpl::categorizeAccessedLocations(Attributor &A, Instruction &I,
8805 bool &Changed) {
8806 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize accessed locations for "
8807 << I << "\n");
8808
8809 AAMemoryLocation::StateType AccessedLocs;
8810 AccessedLocs.intersectAssumedBits(BitsEncoding: NO_LOCATIONS);
8811
8812 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
8813
8814 // First check if we assume any memory is access is visible.
8815 const auto *CBMemLocationAA = A.getAAFor<AAMemoryLocation>(
8816 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL);
8817 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize call site: " << I
8818 << " [" << CBMemLocationAA << "]\n");
8819 if (!CBMemLocationAA) {
8820 updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr,
8821 Changed, AK: getAccessKindFromInst(I: &I));
8822 return NO_UNKOWN_MEM;
8823 }
8824
8825 if (CBMemLocationAA->isAssumedReadNone())
8826 return NO_LOCATIONS;
8827
8828 if (CBMemLocationAA->isAssumedInaccessibleMemOnly()) {
8829 updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_INACCESSIBLE_MEM, I: &I, Ptr: nullptr,
8830 Changed, AK: getAccessKindFromInst(I: &I));
8831 return AccessedLocs.getAssumed();
8832 }
8833
8834 uint32_t CBAssumedNotAccessedLocs =
8835 CBMemLocationAA->getAssumedNotAccessedLocation();
8836
8837 // Set the argmemonly and global bit as we handle them separately below.
8838 uint32_t CBAssumedNotAccessedLocsNoArgMem =
8839 CBAssumedNotAccessedLocs | NO_ARGUMENT_MEM | NO_GLOBAL_MEM;
8840
8841 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2) {
8842 if (CBAssumedNotAccessedLocsNoArgMem & CurMLK)
8843 continue;
8844 updateStateAndAccessesMap(State&: AccessedLocs, MLK: CurMLK, I: &I, Ptr: nullptr, Changed,
8845 AK: getAccessKindFromInst(I: &I));
8846 }
8847
8848 // Now handle global memory if it might be accessed. This is slightly tricky
8849 // as NO_GLOBAL_MEM has multiple bits set.
8850 bool HasGlobalAccesses = ((~CBAssumedNotAccessedLocs) & NO_GLOBAL_MEM);
8851 if (HasGlobalAccesses) {
8852 auto AccessPred = [&](const Instruction *, const Value *Ptr,
8853 AccessKind Kind, MemoryLocationsKind MLK) {
8854 updateStateAndAccessesMap(State&: AccessedLocs, MLK, I: &I, Ptr, Changed,
8855 AK: getAccessKindFromInst(I: &I));
8856 return true;
8857 };
8858 if (!CBMemLocationAA->checkForAllAccessesToMemoryKind(
8859 Pred: AccessPred, MLK: inverseLocation(Loc: NO_GLOBAL_MEM, AndLocalMem: false, AndConstMem: false)))
8860 return AccessedLocs.getWorstState();
8861 }
8862
8863 LLVM_DEBUG(
8864 dbgs() << "[AAMemoryLocation] Accessed state before argument handling: "
8865 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8866
8867 // Now handle argument memory if it might be accessed.
8868 bool HasArgAccesses = ((~CBAssumedNotAccessedLocs) & NO_ARGUMENT_MEM);
8869 if (HasArgAccesses)
8870 categorizeArgumentPointerLocations(A, CB&: *CB, AccessedLocs, Changed);
8871
8872 LLVM_DEBUG(
8873 dbgs() << "[AAMemoryLocation] Accessed state after argument handling: "
8874 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8875
8876 return AccessedLocs.getAssumed();
8877 }
8878
8879 if (const Value *Ptr = getPointerOperand(I: &I, /* AllowVolatile */ true)) {
8880 LLVM_DEBUG(
8881 dbgs() << "[AAMemoryLocation] Categorize memory access with pointer: "
8882 << I << " [" << *Ptr << "]\n");
8883 categorizePtrValue(A, I, Ptr: *Ptr, State&: AccessedLocs, Changed,
8884 AccessAS: Ptr->getType()->getPointerAddressSpace());
8885 return AccessedLocs.getAssumed();
8886 }
8887
8888 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Failed to categorize instruction: "
8889 << I << "\n");
8890 updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr, Changed,
8891 AK: getAccessKindFromInst(I: &I));
8892 return AccessedLocs.getAssumed();
8893}
8894
8895/// An AA to represent the memory behavior function attributes.
8896struct AAMemoryLocationFunction final : public AAMemoryLocationImpl {
8897 AAMemoryLocationFunction(const IRPosition &IRP, Attributor &A)
8898 : AAMemoryLocationImpl(IRP, A) {}
8899
8900 /// See AbstractAttribute::updateImpl(Attributor &A).
8901 ChangeStatus updateImpl(Attributor &A) override {
8902
8903 const auto *MemBehaviorAA =
8904 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::NONE);
8905 if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
8906 if (MemBehaviorAA->isKnownReadNone())
8907 return indicateOptimisticFixpoint();
8908 assert(isAssumedReadNone() &&
8909 "AAMemoryLocation was not read-none but AAMemoryBehavior was!");
8910 A.recordDependence(FromAA: *MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
8911 return ChangeStatus::UNCHANGED;
8912 }
8913
8914 // The current assumed state used to determine a change.
8915 auto AssumedState = getAssumed();
8916 bool Changed = false;
8917
8918 auto CheckRWInst = [&](Instruction &I) {
8919 MemoryLocationsKind MLK = categorizeAccessedLocations(A, I, Changed);
8920 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Accessed locations for " << I
8921 << ": " << getMemoryLocationsAsStr(MLK) << "\n");
8922 removeAssumedBits(BitsEncoding: inverseLocation(Loc: MLK, AndLocalMem: false, AndConstMem: false));
8923 // Stop once only the valid bit set in the *not assumed location*, thus
8924 // once we don't actually exclude any memory locations in the state.
8925 return getAssumedNotAccessedLocation() != VALID_STATE;
8926 };
8927
8928 bool UsedAssumedInformation = false;
8929 if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInst, QueryingAA&: *this,
8930 UsedAssumedInformation))
8931 return indicatePessimisticFixpoint();
8932
8933 Changed |= AssumedState != getAssumed();
8934 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
8935 }
8936
8937 /// See AbstractAttribute::trackStatistics()
8938 void trackStatistics() const override {
8939 if (isAssumedReadNone())
8940 STATS_DECLTRACK_FN_ATTR(readnone)
8941 else if (isAssumedArgMemOnly())
8942 STATS_DECLTRACK_FN_ATTR(argmemonly)
8943 else if (isAssumedInaccessibleMemOnly())
8944 STATS_DECLTRACK_FN_ATTR(inaccessiblememonly)
8945 else if (isAssumedInaccessibleOrArgMemOnly())
8946 STATS_DECLTRACK_FN_ATTR(inaccessiblememorargmemonly)
8947 }
8948};
8949
8950/// AAMemoryLocation attribute for call sites.
8951struct AAMemoryLocationCallSite final : AAMemoryLocationImpl {
8952 AAMemoryLocationCallSite(const IRPosition &IRP, Attributor &A)
8953 : AAMemoryLocationImpl(IRP, A) {}
8954
8955 /// See AbstractAttribute::updateImpl(...).
8956 ChangeStatus updateImpl(Attributor &A) override {
8957 // TODO: Once we have call site specific value information we can provide
8958 // call site specific liveness liveness information and then it makes
8959 // sense to specialize attributes for call sites arguments instead of
8960 // redirecting requests to the callee argument.
8961 Function *F = getAssociatedFunction();
8962 const IRPosition &FnPos = IRPosition::function(F: *F);
8963 auto *FnAA =
8964 A.getAAFor<AAMemoryLocation>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
8965 if (!FnAA)
8966 return indicatePessimisticFixpoint();
8967 bool Changed = false;
8968 auto AccessPred = [&](const Instruction *I, const Value *Ptr,
8969 AccessKind Kind, MemoryLocationsKind MLK) {
8970 updateStateAndAccessesMap(State&: getState(), MLK, I, Ptr, Changed,
8971 AK: getAccessKindFromInst(I));
8972 return true;
8973 };
8974 if (!FnAA->checkForAllAccessesToMemoryKind(Pred: AccessPred, MLK: ALL_LOCATIONS))
8975 return indicatePessimisticFixpoint();
8976 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
8977 }
8978
8979 /// See AbstractAttribute::trackStatistics()
8980 void trackStatistics() const override {
8981 if (isAssumedReadNone())
8982 STATS_DECLTRACK_CS_ATTR(readnone)
8983 }
8984};
8985} // namespace
8986
8987/// ------------------ denormal-fp-math Attribute -------------------------
8988
8989namespace {
8990struct AADenormalFPMathImpl : public AADenormalFPMath {
8991 AADenormalFPMathImpl(const IRPosition &IRP, Attributor &A)
8992 : AADenormalFPMath(IRP, A) {}
8993
8994 const std::string getAsStr(Attributor *A) const override {
8995 std::string Str("AADenormalFPMath[");
8996 raw_string_ostream OS(Str);
8997
8998 DenormalState Known = getKnown();
8999 if (Known.Mode.isValid())
9000 OS << "denormal-fp-math=" << Known.Mode;
9001 else
9002 OS << "invalid";
9003
9004 if (Known.ModeF32.isValid())
9005 OS << " denormal-fp-math-f32=" << Known.ModeF32;
9006 OS << ']';
9007 return Str;
9008 }
9009};
9010
9011struct AADenormalFPMathFunction final : AADenormalFPMathImpl {
9012 AADenormalFPMathFunction(const IRPosition &IRP, Attributor &A)
9013 : AADenormalFPMathImpl(IRP, A) {}
9014
9015 void initialize(Attributor &A) override {
9016 const Function *F = getAnchorScope();
9017 DenormalFPEnv DenormEnv = F->getDenormalFPEnv();
9018
9019 Known = DenormalState{.Mode: DenormEnv.DefaultMode, .ModeF32: DenormEnv.F32Mode};
9020 if (isModeFixed())
9021 indicateFixpoint();
9022 }
9023
9024 ChangeStatus updateImpl(Attributor &A) override {
9025 ChangeStatus Change = ChangeStatus::UNCHANGED;
9026
9027 auto CheckCallSite = [=, &Change, &A](AbstractCallSite CS) {
9028 Function *Caller = CS.getInstruction()->getFunction();
9029 LLVM_DEBUG(dbgs() << "[AADenormalFPMath] Call " << Caller->getName()
9030 << "->" << getAssociatedFunction()->getName() << '\n');
9031
9032 const auto *CallerInfo = A.getAAFor<AADenormalFPMath>(
9033 QueryingAA: *this, IRP: IRPosition::function(F: *Caller), DepClass: DepClassTy::REQUIRED);
9034 if (!CallerInfo)
9035 return false;
9036
9037 Change = Change | clampStateAndIndicateChange(S&: this->getState(),
9038 R: CallerInfo->getState());
9039 return true;
9040 };
9041
9042 bool AllCallSitesKnown = true;
9043 if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this, RequireAllCallSites: true, UsedAssumedInformation&: AllCallSitesKnown))
9044 return indicatePessimisticFixpoint();
9045
9046 if (Change == ChangeStatus::CHANGED && isModeFixed())
9047 indicateFixpoint();
9048 return Change;
9049 }
9050
9051 ChangeStatus manifest(Attributor &A) override {
9052 LLVMContext &Ctx = getAssociatedFunction()->getContext();
9053
9054 SmallVector<Attribute, 2> AttrToAdd;
9055 SmallVector<Attribute::AttrKind, 2> AttrToRemove;
9056
9057 // TODO: Change to use DenormalFPEnv everywhere.
9058 DenormalFPEnv KnownEnv(Known.Mode, Known.ModeF32);
9059
9060 if (KnownEnv == DenormalFPEnv::getDefault()) {
9061 AttrToRemove.push_back(Elt: Attribute::DenormalFPEnv);
9062 } else {
9063 AttrToAdd.push_back(Elt: Attribute::get(
9064 Context&: Ctx, Kind: Attribute::DenormalFPEnv,
9065 Val: DenormalFPEnv(Known.Mode, Known.ModeF32).toIntValue()));
9066 }
9067
9068 auto &IRP = getIRPosition();
9069
9070 // TODO: There should be a combined add and remove API.
9071 return A.removeAttrs(IRP, AttrKinds: AttrToRemove) |
9072 A.manifestAttrs(IRP, DeducedAttrs: AttrToAdd, /*ForceReplace=*/true);
9073 }
9074
9075 void trackStatistics() const override {
9076 STATS_DECLTRACK_FN_ATTR(denormal_fpenv)
9077 }
9078};
9079} // namespace
9080
9081/// ------------------ Value Constant Range Attribute -------------------------
9082
9083namespace {
9084struct AAValueConstantRangeImpl : AAValueConstantRange {
9085 using StateType = IntegerRangeState;
9086 AAValueConstantRangeImpl(const IRPosition &IRP, Attributor &A)
9087 : AAValueConstantRange(IRP, A) {}
9088
9089 /// See AbstractAttribute::initialize(..).
9090 void initialize(Attributor &A) override {
9091 if (A.hasSimplificationCallback(IRP: getIRPosition())) {
9092 indicatePessimisticFixpoint();
9093 return;
9094 }
9095
9096 // Intersect a range given by SCEV.
9097 intersectKnown(R: getConstantRangeFromSCEV(A, I: getCtxI()));
9098
9099 // Intersect a range given by LVI.
9100 intersectKnown(R: getConstantRangeFromLVI(A, CtxI: getCtxI()));
9101 }
9102
9103 /// See AbstractAttribute::getAsStr().
9104 const std::string getAsStr(Attributor *A) const override {
9105 std::string Str;
9106 llvm::raw_string_ostream OS(Str);
9107 OS << "range(" << getBitWidth() << ")<";
9108 getKnown().print(OS);
9109 OS << " / ";
9110 getAssumed().print(OS);
9111 OS << ">";
9112 return Str;
9113 }
9114
9115 /// Helper function to get a SCEV expr for the associated value at program
9116 /// point \p I.
9117 const SCEV *getSCEV(Attributor &A, const Instruction *I = nullptr) const {
9118 if (!getAnchorScope())
9119 return nullptr;
9120
9121 ScalarEvolution *SE =
9122 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
9123 F: *getAnchorScope());
9124
9125 LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(
9126 F: *getAnchorScope());
9127
9128 if (!SE || !LI)
9129 return nullptr;
9130
9131 const SCEV *S = SE->getSCEV(V: &getAssociatedValue());
9132 if (!I)
9133 return S;
9134
9135 return SE->getSCEVAtScope(S, L: LI->getLoopFor(BB: I->getParent()));
9136 }
9137
9138 /// Helper function to get a range from SCEV for the associated value at
9139 /// program point \p I.
9140 ConstantRange getConstantRangeFromSCEV(Attributor &A,
9141 const Instruction *I = nullptr) const {
9142 if (!getAnchorScope())
9143 return getWorstState(BitWidth: getBitWidth());
9144
9145 ScalarEvolution *SE =
9146 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
9147 F: *getAnchorScope());
9148
9149 const SCEV *S = getSCEV(A, I);
9150 if (!SE || !S)
9151 return getWorstState(BitWidth: getBitWidth());
9152
9153 return SE->getUnsignedRange(S);
9154 }
9155
9156 /// Helper function to get a range from LVI for the associated value at
9157 /// program point \p I.
9158 ConstantRange
9159 getConstantRangeFromLVI(Attributor &A,
9160 const Instruction *CtxI = nullptr) const {
9161 if (!getAnchorScope())
9162 return getWorstState(BitWidth: getBitWidth());
9163
9164 LazyValueInfo *LVI =
9165 A.getInfoCache().getAnalysisResultForFunction<LazyValueAnalysis>(
9166 F: *getAnchorScope());
9167
9168 if (!LVI || !CtxI)
9169 return getWorstState(BitWidth: getBitWidth());
9170 return LVI->getConstantRange(V: &getAssociatedValue(),
9171 CxtI: const_cast<Instruction *>(CtxI),
9172 /*UndefAllowed*/ false);
9173 }
9174
9175 /// Return true if \p CtxI is valid for querying outside analyses.
9176 /// This basically makes sure we do not ask intra-procedural analysis
9177 /// about a context in the wrong function or a context that violates
9178 /// dominance assumptions they might have. The \p AllowAACtxI flag indicates
9179 /// if the original context of this AA is OK or should be considered invalid.
9180 bool isValidCtxInstructionForOutsideAnalysis(Attributor &A,
9181 const Instruction *CtxI,
9182 bool AllowAACtxI) const {
9183 if (!CtxI || (!AllowAACtxI && CtxI == getCtxI()))
9184 return false;
9185
9186 // Our context might be in a different function, neither intra-procedural
9187 // analysis (ScalarEvolution nor LazyValueInfo) can handle that.
9188 if (!AA::isValidInScope(V: getAssociatedValue(), Scope: CtxI->getFunction()))
9189 return false;
9190
9191 // If the context is not dominated by the value there are paths to the
9192 // context that do not define the value. This cannot be handled by
9193 // LazyValueInfo so we need to bail.
9194 if (auto *I = dyn_cast<Instruction>(Val: &getAssociatedValue())) {
9195 InformationCache &InfoCache = A.getInfoCache();
9196 const DominatorTree *DT =
9197 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(
9198 F: *I->getFunction());
9199 return DT && DT->dominates(Def: I, User: CtxI);
9200 }
9201
9202 return true;
9203 }
9204
9205 /// See AAValueConstantRange::getKnownConstantRange(..).
9206 ConstantRange
9207 getKnownConstantRange(Attributor &A,
9208 const Instruction *CtxI = nullptr) const override {
9209 if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
9210 /* AllowAACtxI */ false))
9211 return getKnown();
9212
9213 ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
9214 ConstantRange SCEVR = getConstantRangeFromSCEV(A, I: CtxI);
9215 return getKnown().intersectWith(CR: SCEVR).intersectWith(CR: LVIR);
9216 }
9217
9218 /// See AAValueConstantRange::getAssumedConstantRange(..).
9219 ConstantRange
9220 getAssumedConstantRange(Attributor &A,
9221 const Instruction *CtxI = nullptr) const override {
9222 // TODO: Make SCEV use Attributor assumption.
9223 // We may be able to bound a variable range via assumptions in
9224 // Attributor. ex.) If x is assumed to be in [1, 3] and y is known to
9225 // evolve to x^2 + x, then we can say that y is in [2, 12].
9226 if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
9227 /* AllowAACtxI */ false))
9228 return getAssumed();
9229
9230 ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
9231 ConstantRange SCEVR = getConstantRangeFromSCEV(A, I: CtxI);
9232 return getAssumed().intersectWith(CR: SCEVR).intersectWith(CR: LVIR);
9233 }
9234
9235 /// Helper function to create MDNode for range metadata.
9236 static MDNode *
9237 getMDNodeForConstantRange(Type *Ty, LLVMContext &Ctx,
9238 const ConstantRange &AssumedConstantRange) {
9239 Metadata *LowAndHigh[] = {ConstantAsMetadata::get(C: ConstantInt::get(
9240 Ty, V: AssumedConstantRange.getLower())),
9241 ConstantAsMetadata::get(C: ConstantInt::get(
9242 Ty, V: AssumedConstantRange.getUpper()))};
9243 return MDNode::get(Context&: Ctx, MDs: LowAndHigh);
9244 }
9245
9246 /// Return true if \p Assumed is included in ranges from instruction \p I.
9247 static bool isBetterRange(const ConstantRange &Assumed,
9248 const Instruction &I) {
9249 if (Assumed.isFullSet())
9250 return false;
9251
9252 std::optional<ConstantRange> Known;
9253
9254 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
9255 Known = CB->getRange();
9256 } else if (MDNode *KnownRanges = I.getMetadata(KindID: LLVMContext::MD_range)) {
9257 // If multiple ranges are annotated in IR, we give up to annotate assumed
9258 // range for now.
9259
9260 // TODO: If there exists a known range which containts assumed range, we
9261 // can say assumed range is better.
9262 if (KnownRanges->getNumOperands() > 2)
9263 return false;
9264
9265 ConstantInt *Lower =
9266 mdconst::extract<ConstantInt>(MD: KnownRanges->getOperand(I: 0));
9267 ConstantInt *Upper =
9268 mdconst::extract<ConstantInt>(MD: KnownRanges->getOperand(I: 1));
9269
9270 Known.emplace(args: Lower->getValue(), args: Upper->getValue());
9271 }
9272 return !Known || (*Known != Assumed && Known->contains(CR: Assumed));
9273 }
9274
9275 /// Helper function to set range metadata.
9276 static bool
9277 setRangeMetadataIfisBetterRange(Instruction *I,
9278 const ConstantRange &AssumedConstantRange) {
9279 if (isBetterRange(Assumed: AssumedConstantRange, I: *I)) {
9280 I->setMetadata(KindID: LLVMContext::MD_range,
9281 Node: getMDNodeForConstantRange(Ty: I->getType(), Ctx&: I->getContext(),
9282 AssumedConstantRange));
9283 return true;
9284 }
9285 return false;
9286 }
9287 /// Helper function to set range return attribute.
9288 static bool
9289 setRangeRetAttrIfisBetterRange(Attributor &A, const IRPosition &IRP,
9290 Instruction *I,
9291 const ConstantRange &AssumedConstantRange) {
9292 if (isBetterRange(Assumed: AssumedConstantRange, I: *I)) {
9293 A.manifestAttrs(IRP,
9294 DeducedAttrs: Attribute::get(Context&: I->getContext(), Kind: Attribute::Range,
9295 CR: AssumedConstantRange),
9296 /*ForceReplace*/ true);
9297 return true;
9298 }
9299 return false;
9300 }
9301
9302 /// See AbstractAttribute::manifest()
9303 ChangeStatus manifest(Attributor &A) override {
9304 ChangeStatus Changed = ChangeStatus::UNCHANGED;
9305 ConstantRange AssumedConstantRange = getAssumedConstantRange(A);
9306 assert(!AssumedConstantRange.isFullSet() && "Invalid state");
9307
9308 auto &V = getAssociatedValue();
9309 if (!AssumedConstantRange.isEmptySet() &&
9310 !AssumedConstantRange.isSingleElement()) {
9311 if (Instruction *I = dyn_cast<Instruction>(Val: &V)) {
9312 assert(I == getCtxI() && "Should not annotate an instruction which is "
9313 "not the context instruction");
9314 if (isa<LoadInst>(Val: I))
9315 if (setRangeMetadataIfisBetterRange(I, AssumedConstantRange))
9316 Changed = ChangeStatus::CHANGED;
9317 if (isa<CallInst>(Val: I))
9318 if (setRangeRetAttrIfisBetterRange(A, IRP: getIRPosition(), I,
9319 AssumedConstantRange))
9320 Changed = ChangeStatus::CHANGED;
9321 }
9322 }
9323
9324 return Changed;
9325 }
9326};
9327
9328struct AAValueConstantRangeArgument final
9329 : AAArgumentFromCallSiteArguments<
9330 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9331 true /* BridgeCallBaseContext */> {
9332 using Base = AAArgumentFromCallSiteArguments<
9333 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9334 true /* BridgeCallBaseContext */>;
9335 AAValueConstantRangeArgument(const IRPosition &IRP, Attributor &A)
9336 : Base(IRP, A) {}
9337
9338 /// See AbstractAttribute::trackStatistics()
9339 void trackStatistics() const override {
9340 STATS_DECLTRACK_ARG_ATTR(value_range)
9341 }
9342};
9343
9344struct AAValueConstantRangeReturned
9345 : AAReturnedFromReturnedValues<AAValueConstantRange,
9346 AAValueConstantRangeImpl,
9347 AAValueConstantRangeImpl::StateType,
9348 /* PropagateCallBaseContext */ true> {
9349 using Base =
9350 AAReturnedFromReturnedValues<AAValueConstantRange,
9351 AAValueConstantRangeImpl,
9352 AAValueConstantRangeImpl::StateType,
9353 /* PropagateCallBaseContext */ true>;
9354 AAValueConstantRangeReturned(const IRPosition &IRP, Attributor &A)
9355 : Base(IRP, A) {}
9356
9357 /// See AbstractAttribute::initialize(...).
9358 void initialize(Attributor &A) override {
9359 if (!A.isFunctionIPOAmendable(F: *getAssociatedFunction()))
9360 indicatePessimisticFixpoint();
9361 }
9362
9363 /// See AbstractAttribute::trackStatistics()
9364 void trackStatistics() const override {
9365 STATS_DECLTRACK_FNRET_ATTR(value_range)
9366 }
9367};
9368
9369struct AAValueConstantRangeFloating : AAValueConstantRangeImpl {
9370 AAValueConstantRangeFloating(const IRPosition &IRP, Attributor &A)
9371 : AAValueConstantRangeImpl(IRP, A) {}
9372
9373 /// See AbstractAttribute::initialize(...).
9374 void initialize(Attributor &A) override {
9375 AAValueConstantRangeImpl::initialize(A);
9376 if (isAtFixpoint())
9377 return;
9378
9379 Value &V = getAssociatedValue();
9380
9381 if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
9382 unionAssumed(R: ConstantRange(C->getValue()));
9383 indicateOptimisticFixpoint();
9384 return;
9385 }
9386
9387 if (isa<UndefValue>(Val: &V)) {
9388 // Collapse the undef state to 0.
9389 unionAssumed(R: ConstantRange(APInt(getBitWidth(), 0)));
9390 indicateOptimisticFixpoint();
9391 return;
9392 }
9393
9394 if (isa<CallBase>(Val: &V))
9395 return;
9396
9397 if (isa<BinaryOperator>(Val: &V) || isa<CmpInst>(Val: &V) || isa<CastInst>(Val: &V))
9398 return;
9399
9400 // If it is a load instruction with range metadata, use it.
9401 if (LoadInst *LI = dyn_cast<LoadInst>(Val: &V))
9402 if (auto *RangeMD = LI->getMetadata(KindID: LLVMContext::MD_range)) {
9403 intersectKnown(R: getConstantRangeFromMetadata(RangeMD: *RangeMD));
9404 return;
9405 }
9406
9407 // We can work with PHI and select instruction as we traverse their operands
9408 // during update.
9409 if (isa<SelectInst>(Val: V) || isa<PHINode>(Val: V))
9410 return;
9411
9412 // Otherwise we give up.
9413 indicatePessimisticFixpoint();
9414
9415 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] We give up: "
9416 << getAssociatedValue() << "\n");
9417 }
9418
9419 bool calculateBinaryOperator(
9420 Attributor &A, BinaryOperator *BinOp, IntegerRangeState &T,
9421 const Instruction *CtxI,
9422 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9423 Value *LHS = BinOp->getOperand(i_nocapture: 0);
9424 Value *RHS = BinOp->getOperand(i_nocapture: 1);
9425
9426 // Simplify the operands first.
9427 bool UsedAssumedInformation = false;
9428 const auto &SimplifiedLHS = A.getAssumedSimplified(
9429 IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()), AA: *this,
9430 UsedAssumedInformation, S: AA::Interprocedural);
9431 if (!SimplifiedLHS.has_value())
9432 return true;
9433 if (!*SimplifiedLHS)
9434 return false;
9435 LHS = *SimplifiedLHS;
9436
9437 const auto &SimplifiedRHS = A.getAssumedSimplified(
9438 IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()), AA: *this,
9439 UsedAssumedInformation, S: AA::Interprocedural);
9440 if (!SimplifiedRHS.has_value())
9441 return true;
9442 if (!*SimplifiedRHS)
9443 return false;
9444 RHS = *SimplifiedRHS;
9445
9446 // TODO: Allow non integers as well.
9447 if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9448 return false;
9449
9450 auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9451 QueryingAA: *this, IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()),
9452 DepClass: DepClassTy::REQUIRED);
9453 if (!LHSAA)
9454 return false;
9455 QuerriedAAs.push_back(Elt: LHSAA);
9456 auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9457
9458 auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9459 QueryingAA: *this, IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()),
9460 DepClass: DepClassTy::REQUIRED);
9461 if (!RHSAA)
9462 return false;
9463 QuerriedAAs.push_back(Elt: RHSAA);
9464 auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9465
9466 auto AssumedRange = LHSAARange.binaryOp(BinOp: BinOp->getOpcode(), Other: RHSAARange);
9467
9468 T.unionAssumed(R: AssumedRange);
9469
9470 // TODO: Track a known state too.
9471
9472 return T.isValidState();
9473 }
9474
9475 bool calculateCastInst(
9476 Attributor &A, CastInst *CastI, IntegerRangeState &T,
9477 const Instruction *CtxI,
9478 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9479 assert(CastI->getNumOperands() == 1 && "Expected cast to be unary!");
9480 // TODO: Allow non integers as well.
9481 Value *OpV = CastI->getOperand(i_nocapture: 0);
9482
9483 // Simplify the operand first.
9484 bool UsedAssumedInformation = false;
9485 const auto &SimplifiedOpV = A.getAssumedSimplified(
9486 IRP: IRPosition::value(V: *OpV, CBContext: getCallBaseContext()), AA: *this,
9487 UsedAssumedInformation, S: AA::Interprocedural);
9488 if (!SimplifiedOpV.has_value())
9489 return true;
9490 if (!*SimplifiedOpV)
9491 return false;
9492 OpV = *SimplifiedOpV;
9493
9494 if (!OpV->getType()->isIntegerTy())
9495 return false;
9496
9497 auto *OpAA = A.getAAFor<AAValueConstantRange>(
9498 QueryingAA: *this, IRP: IRPosition::value(V: *OpV, CBContext: getCallBaseContext()),
9499 DepClass: DepClassTy::REQUIRED);
9500 if (!OpAA)
9501 return false;
9502 QuerriedAAs.push_back(Elt: OpAA);
9503 T.unionAssumed(R: OpAA->getAssumed().castOp(CastOp: CastI->getOpcode(),
9504 BitWidth: getState().getBitWidth()));
9505 return T.isValidState();
9506 }
9507
9508 bool
9509 calculateCmpInst(Attributor &A, CmpInst *CmpI, IntegerRangeState &T,
9510 const Instruction *CtxI,
9511 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9512 Value *LHS = CmpI->getOperand(i_nocapture: 0);
9513 Value *RHS = CmpI->getOperand(i_nocapture: 1);
9514
9515 // Simplify the operands first.
9516 bool UsedAssumedInformation = false;
9517 const auto &SimplifiedLHS = A.getAssumedSimplified(
9518 IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()), AA: *this,
9519 UsedAssumedInformation, S: AA::Interprocedural);
9520 if (!SimplifiedLHS.has_value())
9521 return true;
9522 if (!*SimplifiedLHS)
9523 return false;
9524 LHS = *SimplifiedLHS;
9525
9526 const auto &SimplifiedRHS = A.getAssumedSimplified(
9527 IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()), AA: *this,
9528 UsedAssumedInformation, S: AA::Interprocedural);
9529 if (!SimplifiedRHS.has_value())
9530 return true;
9531 if (!*SimplifiedRHS)
9532 return false;
9533 RHS = *SimplifiedRHS;
9534
9535 // TODO: Allow non integers as well.
9536 if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9537 return false;
9538
9539 auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9540 QueryingAA: *this, IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()),
9541 DepClass: DepClassTy::REQUIRED);
9542 if (!LHSAA)
9543 return false;
9544 QuerriedAAs.push_back(Elt: LHSAA);
9545 auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9546 QueryingAA: *this, IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()),
9547 DepClass: DepClassTy::REQUIRED);
9548 if (!RHSAA)
9549 return false;
9550 QuerriedAAs.push_back(Elt: RHSAA);
9551 auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9552 auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9553
9554 // If one of them is empty set, we can't decide.
9555 if (LHSAARange.isEmptySet() || RHSAARange.isEmptySet())
9556 return true;
9557
9558 bool MustTrue = false, MustFalse = false;
9559
9560 auto AllowedRegion =
9561 ConstantRange::makeAllowedICmpRegion(Pred: CmpI->getPredicate(), Other: RHSAARange);
9562
9563 if (AllowedRegion.intersectWith(CR: LHSAARange).isEmptySet())
9564 MustFalse = true;
9565
9566 if (LHSAARange.icmp(Pred: CmpI->getPredicate(), Other: RHSAARange))
9567 MustTrue = true;
9568
9569 assert((!MustTrue || !MustFalse) &&
9570 "Either MustTrue or MustFalse should be false!");
9571
9572 if (MustTrue)
9573 T.unionAssumed(R: ConstantRange(APInt(/* numBits */ 1, /* val */ 1)));
9574 else if (MustFalse)
9575 T.unionAssumed(R: ConstantRange(APInt(/* numBits */ 1, /* val */ 0)));
9576 else
9577 T.unionAssumed(R: ConstantRange(/* BitWidth */ 1, /* isFullSet */ true));
9578
9579 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] " << *CmpI << " after "
9580 << (MustTrue ? "true" : (MustFalse ? "false" : "unknown"))
9581 << ": " << T << "\n\t" << *LHSAA << "\t<op>\n\t"
9582 << *RHSAA);
9583
9584 // TODO: Track a known state too.
9585 return T.isValidState();
9586 }
9587
9588 /// See AbstractAttribute::updateImpl(...).
9589 ChangeStatus updateImpl(Attributor &A) override {
9590
9591 IntegerRangeState T(getBitWidth());
9592 auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
9593 Instruction *I = dyn_cast<Instruction>(Val: &V);
9594 if (!I || isa<CallBase>(Val: I)) {
9595
9596 // Simplify the operand first.
9597 bool UsedAssumedInformation = false;
9598 const auto &SimplifiedOpV = A.getAssumedSimplified(
9599 IRP: IRPosition::value(V, CBContext: getCallBaseContext()), AA: *this,
9600 UsedAssumedInformation, S: AA::Interprocedural);
9601 if (!SimplifiedOpV.has_value())
9602 return true;
9603 if (!*SimplifiedOpV)
9604 return false;
9605 Value *VPtr = *SimplifiedOpV;
9606
9607 // If the value is not instruction, we query AA to Attributor.
9608 const auto *AA = A.getAAFor<AAValueConstantRange>(
9609 QueryingAA: *this, IRP: IRPosition::value(V: *VPtr, CBContext: getCallBaseContext()),
9610 DepClass: DepClassTy::REQUIRED);
9611
9612 // Clamp operator is not used to utilize a program point CtxI.
9613 if (AA)
9614 T.unionAssumed(R: AA->getAssumedConstantRange(A, CtxI));
9615 else
9616 return false;
9617
9618 return T.isValidState();
9619 }
9620
9621 SmallVector<const AAValueConstantRange *, 4> QuerriedAAs;
9622 if (auto *BinOp = dyn_cast<BinaryOperator>(Val: I)) {
9623 if (!calculateBinaryOperator(A, BinOp, T, CtxI, QuerriedAAs))
9624 return false;
9625 } else if (auto *CmpI = dyn_cast<CmpInst>(Val: I)) {
9626 if (!calculateCmpInst(A, CmpI, T, CtxI, QuerriedAAs))
9627 return false;
9628 } else if (auto *CastI = dyn_cast<CastInst>(Val: I)) {
9629 if (!calculateCastInst(A, CastI, T, CtxI, QuerriedAAs))
9630 return false;
9631 } else {
9632 // Give up with other instructions.
9633 // TODO: Add other instructions
9634
9635 T.indicatePessimisticFixpoint();
9636 return false;
9637 }
9638
9639 // Catch circular reasoning in a pessimistic way for now.
9640 // TODO: Check how the range evolves and if we stripped anything, see also
9641 // AADereferenceable or AAAlign for similar situations.
9642 for (const AAValueConstantRange *QueriedAA : QuerriedAAs) {
9643 if (QueriedAA != this)
9644 continue;
9645 // If we are in a stady state we do not need to worry.
9646 if (T.getAssumed() == getState().getAssumed())
9647 continue;
9648 T.indicatePessimisticFixpoint();
9649 }
9650
9651 return T.isValidState();
9652 };
9653
9654 if (!VisitValueCB(getAssociatedValue(), getCtxI()))
9655 return indicatePessimisticFixpoint();
9656
9657 // Ensure that long def-use chains can't cause circular reasoning either by
9658 // introducing a cutoff below.
9659 if (clampStateAndIndicateChange(S&: getState(), R: T) == ChangeStatus::UNCHANGED)
9660 return ChangeStatus::UNCHANGED;
9661 if (++NumChanges > MaxNumChanges) {
9662 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] performed " << NumChanges
9663 << " but only " << MaxNumChanges
9664 << " are allowed to avoid cyclic reasoning.");
9665 return indicatePessimisticFixpoint();
9666 }
9667 return ChangeStatus::CHANGED;
9668 }
9669
9670 /// See AbstractAttribute::trackStatistics()
9671 void trackStatistics() const override {
9672 STATS_DECLTRACK_FLOATING_ATTR(value_range)
9673 }
9674
9675 /// Tracker to bail after too many widening steps of the constant range.
9676 int NumChanges = 0;
9677
9678 /// Upper bound for the number of allowed changes (=widening steps) for the
9679 /// constant range before we give up.
9680 static constexpr int MaxNumChanges = 5;
9681};
9682
9683struct AAValueConstantRangeFunction : AAValueConstantRangeImpl {
9684 AAValueConstantRangeFunction(const IRPosition &IRP, Attributor &A)
9685 : AAValueConstantRangeImpl(IRP, A) {}
9686
9687 /// See AbstractAttribute::initialize(...).
9688 ChangeStatus updateImpl(Attributor &A) override {
9689 llvm_unreachable("AAValueConstantRange(Function|CallSite)::updateImpl will "
9690 "not be called");
9691 }
9692
9693 /// See AbstractAttribute::trackStatistics()
9694 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(value_range) }
9695};
9696
9697struct AAValueConstantRangeCallSite : AAValueConstantRangeFunction {
9698 AAValueConstantRangeCallSite(const IRPosition &IRP, Attributor &A)
9699 : AAValueConstantRangeFunction(IRP, A) {}
9700
9701 /// See AbstractAttribute::trackStatistics()
9702 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(value_range) }
9703};
9704
9705struct AAValueConstantRangeCallSiteReturned
9706 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9707 AAValueConstantRangeImpl::StateType,
9708 /* IntroduceCallBaseContext */ true> {
9709 AAValueConstantRangeCallSiteReturned(const IRPosition &IRP, Attributor &A)
9710 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9711 AAValueConstantRangeImpl::StateType,
9712 /* IntroduceCallBaseContext */ true>(IRP, A) {}
9713
9714 /// See AbstractAttribute::initialize(...).
9715 void initialize(Attributor &A) override {
9716 // If it is a call instruction with range attribute, use the range.
9717 if (CallInst *CI = dyn_cast<CallInst>(Val: &getAssociatedValue())) {
9718 if (std::optional<ConstantRange> Range = CI->getRange())
9719 intersectKnown(R: *Range);
9720 }
9721
9722 AAValueConstantRangeImpl::initialize(A);
9723 }
9724
9725 /// See AbstractAttribute::trackStatistics()
9726 void trackStatistics() const override {
9727 STATS_DECLTRACK_CSRET_ATTR(value_range)
9728 }
9729};
9730struct AAValueConstantRangeCallSiteArgument : AAValueConstantRangeFloating {
9731 AAValueConstantRangeCallSiteArgument(const IRPosition &IRP, Attributor &A)
9732 : AAValueConstantRangeFloating(IRP, A) {}
9733
9734 /// See AbstractAttribute::manifest()
9735 ChangeStatus manifest(Attributor &A) override {
9736 return ChangeStatus::UNCHANGED;
9737 }
9738
9739 /// See AbstractAttribute::trackStatistics()
9740 void trackStatistics() const override {
9741 STATS_DECLTRACK_CSARG_ATTR(value_range)
9742 }
9743};
9744} // namespace
9745
9746/// ------------------ Potential Values Attribute -------------------------
9747
9748namespace {
9749struct AAPotentialConstantValuesImpl : AAPotentialConstantValues {
9750 using StateType = PotentialConstantIntValuesState;
9751
9752 AAPotentialConstantValuesImpl(const IRPosition &IRP, Attributor &A)
9753 : AAPotentialConstantValues(IRP, A) {}
9754
9755 /// See AbstractAttribute::initialize(..).
9756 void initialize(Attributor &A) override {
9757 if (A.hasSimplificationCallback(IRP: getIRPosition()))
9758 indicatePessimisticFixpoint();
9759 else
9760 AAPotentialConstantValues::initialize(A);
9761 }
9762
9763 bool fillSetWithConstantValues(Attributor &A, const IRPosition &IRP, SetTy &S,
9764 bool &ContainsUndef, bool ForSelf) {
9765 SmallVector<AA::ValueAndContext> Values;
9766 bool UsedAssumedInformation = false;
9767 if (!A.getAssumedSimplifiedValues(IRP, AA: *this, Values, S: AA::Interprocedural,
9768 UsedAssumedInformation)) {
9769 // Avoid recursion when the caller is computing constant values for this
9770 // IRP itself.
9771 if (ForSelf)
9772 return false;
9773 if (!IRP.getAssociatedType()->isIntegerTy())
9774 return false;
9775 auto *PotentialValuesAA = A.getAAFor<AAPotentialConstantValues>(
9776 QueryingAA: *this, IRP, DepClass: DepClassTy::REQUIRED);
9777 if (!PotentialValuesAA || !PotentialValuesAA->getState().isValidState())
9778 return false;
9779 ContainsUndef = PotentialValuesAA->getState().undefIsContained();
9780 S = PotentialValuesAA->getState().getAssumedSet();
9781 return true;
9782 }
9783
9784 // Copy all the constant values, except UndefValue. ContainsUndef is true
9785 // iff Values contains only UndefValue instances. If there are other known
9786 // constants, then UndefValue is dropped.
9787 ContainsUndef = false;
9788 for (auto &It : Values) {
9789 if (isa<UndefValue>(Val: It.getValue())) {
9790 ContainsUndef = true;
9791 continue;
9792 }
9793 auto *CI = dyn_cast<ConstantInt>(Val: It.getValue());
9794 if (!CI)
9795 return false;
9796 S.insert(X: CI->getValue());
9797 }
9798 ContainsUndef &= S.empty();
9799
9800 return true;
9801 }
9802
9803 /// See AbstractAttribute::getAsStr().
9804 const std::string getAsStr(Attributor *A) const override {
9805 std::string Str;
9806 llvm::raw_string_ostream OS(Str);
9807 OS << getState();
9808 return Str;
9809 }
9810
9811 /// See AbstractAttribute::updateImpl(...).
9812 ChangeStatus updateImpl(Attributor &A) override {
9813 return indicatePessimisticFixpoint();
9814 }
9815};
9816
9817struct AAPotentialConstantValuesArgument final
9818 : AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9819 AAPotentialConstantValuesImpl,
9820 PotentialConstantIntValuesState> {
9821 using Base = AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9822 AAPotentialConstantValuesImpl,
9823 PotentialConstantIntValuesState>;
9824 AAPotentialConstantValuesArgument(const IRPosition &IRP, Attributor &A)
9825 : Base(IRP, A) {}
9826
9827 /// See AbstractAttribute::trackStatistics()
9828 void trackStatistics() const override {
9829 STATS_DECLTRACK_ARG_ATTR(potential_values)
9830 }
9831};
9832
9833struct AAPotentialConstantValuesReturned
9834 : AAReturnedFromReturnedValues<AAPotentialConstantValues,
9835 AAPotentialConstantValuesImpl> {
9836 using Base = AAReturnedFromReturnedValues<AAPotentialConstantValues,
9837 AAPotentialConstantValuesImpl>;
9838 AAPotentialConstantValuesReturned(const IRPosition &IRP, Attributor &A)
9839 : Base(IRP, A) {}
9840
9841 void initialize(Attributor &A) override {
9842 if (!A.isFunctionIPOAmendable(F: *getAssociatedFunction()))
9843 indicatePessimisticFixpoint();
9844 Base::initialize(A);
9845 }
9846
9847 /// See AbstractAttribute::trackStatistics()
9848 void trackStatistics() const override {
9849 STATS_DECLTRACK_FNRET_ATTR(potential_values)
9850 }
9851};
9852
9853struct AAPotentialConstantValuesFloating : AAPotentialConstantValuesImpl {
9854 AAPotentialConstantValuesFloating(const IRPosition &IRP, Attributor &A)
9855 : AAPotentialConstantValuesImpl(IRP, A) {}
9856
9857 /// See AbstractAttribute::initialize(..).
9858 void initialize(Attributor &A) override {
9859 AAPotentialConstantValuesImpl::initialize(A);
9860 if (isAtFixpoint())
9861 return;
9862
9863 Value &V = getAssociatedValue();
9864
9865 if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
9866 unionAssumed(C: C->getValue());
9867 indicateOptimisticFixpoint();
9868 return;
9869 }
9870
9871 if (isa<UndefValue>(Val: &V)) {
9872 unionAssumedWithUndef();
9873 indicateOptimisticFixpoint();
9874 return;
9875 }
9876
9877 if (isa<BinaryOperator>(Val: &V) || isa<ICmpInst>(Val: &V) || isa<CastInst>(Val: &V))
9878 return;
9879
9880 if (isa<SelectInst>(Val: V) || isa<PHINode>(Val: V) || isa<LoadInst>(Val: V))
9881 return;
9882
9883 indicatePessimisticFixpoint();
9884
9885 LLVM_DEBUG(dbgs() << "[AAPotentialConstantValues] We give up: "
9886 << getAssociatedValue() << "\n");
9887 }
9888
9889 static bool calculateICmpInst(const ICmpInst *ICI, const APInt &LHS,
9890 const APInt &RHS) {
9891 return ICmpInst::compare(LHS, RHS, Pred: ICI->getPredicate());
9892 }
9893
9894 static APInt calculateCastInst(const CastInst *CI, const APInt &Src,
9895 uint32_t ResultBitWidth) {
9896 Instruction::CastOps CastOp = CI->getOpcode();
9897 switch (CastOp) {
9898 default:
9899 llvm_unreachable("unsupported or not integer cast");
9900 case Instruction::Trunc:
9901 return Src.trunc(width: ResultBitWidth);
9902 case Instruction::SExt:
9903 return Src.sext(width: ResultBitWidth);
9904 case Instruction::ZExt:
9905 return Src.zext(width: ResultBitWidth);
9906 case Instruction::BitCast:
9907 return Src;
9908 }
9909 }
9910
9911 static APInt calculateBinaryOperator(const BinaryOperator *BinOp,
9912 const APInt &LHS, const APInt &RHS,
9913 bool &SkipOperation, bool &Unsupported) {
9914 Instruction::BinaryOps BinOpcode = BinOp->getOpcode();
9915 // Unsupported is set to true when the binary operator is not supported.
9916 // SkipOperation is set to true when UB occur with the given operand pair
9917 // (LHS, RHS).
9918 // TODO: we should look at nsw and nuw keywords to handle operations
9919 // that create poison or undef value.
9920 switch (BinOpcode) {
9921 default:
9922 Unsupported = true;
9923 return LHS;
9924 case Instruction::Add:
9925 return LHS + RHS;
9926 case Instruction::Sub:
9927 return LHS - RHS;
9928 case Instruction::Mul:
9929 return LHS * RHS;
9930 case Instruction::UDiv:
9931 if (RHS.isZero()) {
9932 SkipOperation = true;
9933 return LHS;
9934 }
9935 return LHS.udiv(RHS);
9936 case Instruction::SDiv:
9937 if (RHS.isZero()) {
9938 SkipOperation = true;
9939 return LHS;
9940 }
9941 return LHS.sdiv(RHS);
9942 case Instruction::URem:
9943 if (RHS.isZero()) {
9944 SkipOperation = true;
9945 return LHS;
9946 }
9947 return LHS.urem(RHS);
9948 case Instruction::SRem:
9949 if (RHS.isZero()) {
9950 SkipOperation = true;
9951 return LHS;
9952 }
9953 return LHS.srem(RHS);
9954 case Instruction::Shl:
9955 return LHS.shl(ShiftAmt: RHS);
9956 case Instruction::LShr:
9957 return LHS.lshr(ShiftAmt: RHS);
9958 case Instruction::AShr:
9959 return LHS.ashr(ShiftAmt: RHS);
9960 case Instruction::And:
9961 return LHS & RHS;
9962 case Instruction::Or:
9963 return LHS | RHS;
9964 case Instruction::Xor:
9965 return LHS ^ RHS;
9966 }
9967 }
9968
9969 bool calculateBinaryOperatorAndTakeUnion(const BinaryOperator *BinOp,
9970 const APInt &LHS, const APInt &RHS) {
9971 bool SkipOperation = false;
9972 bool Unsupported = false;
9973 APInt Result =
9974 calculateBinaryOperator(BinOp, LHS, RHS, SkipOperation, Unsupported);
9975 if (Unsupported)
9976 return false;
9977 // If SkipOperation is true, we can ignore this operand pair (L, R).
9978 if (!SkipOperation)
9979 unionAssumed(C: Result);
9980 return isValidState();
9981 }
9982
9983 ChangeStatus updateWithICmpInst(Attributor &A, ICmpInst *ICI) {
9984 auto AssumedBefore = getAssumed();
9985 Value *LHS = ICI->getOperand(i_nocapture: 0);
9986 Value *RHS = ICI->getOperand(i_nocapture: 1);
9987
9988 bool LHSContainsUndef = false, RHSContainsUndef = false;
9989 SetTy LHSAAPVS, RHSAAPVS;
9990 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
9991 ContainsUndef&: LHSContainsUndef, /* ForSelf */ false) ||
9992 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
9993 ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
9994 return indicatePessimisticFixpoint();
9995
9996 // TODO: make use of undef flag to limit potential values aggressively.
9997 bool MaybeTrue = false, MaybeFalse = false;
9998 const APInt Zero(RHS->getType()->getIntegerBitWidth(), 0);
9999 if (LHSContainsUndef && RHSContainsUndef) {
10000 // The result of any comparison between undefs can be soundly replaced
10001 // with undef.
10002 unionAssumedWithUndef();
10003 } else if (LHSContainsUndef) {
10004 for (const APInt &R : RHSAAPVS) {
10005 bool CmpResult = calculateICmpInst(ICI, LHS: Zero, RHS: R);
10006 MaybeTrue |= CmpResult;
10007 MaybeFalse |= !CmpResult;
10008 if (MaybeTrue & MaybeFalse)
10009 return indicatePessimisticFixpoint();
10010 }
10011 } else if (RHSContainsUndef) {
10012 for (const APInt &L : LHSAAPVS) {
10013 bool CmpResult = calculateICmpInst(ICI, LHS: L, RHS: Zero);
10014 MaybeTrue |= CmpResult;
10015 MaybeFalse |= !CmpResult;
10016 if (MaybeTrue & MaybeFalse)
10017 return indicatePessimisticFixpoint();
10018 }
10019 } else {
10020 for (const APInt &L : LHSAAPVS) {
10021 for (const APInt &R : RHSAAPVS) {
10022 bool CmpResult = calculateICmpInst(ICI, LHS: L, RHS: R);
10023 MaybeTrue |= CmpResult;
10024 MaybeFalse |= !CmpResult;
10025 if (MaybeTrue & MaybeFalse)
10026 return indicatePessimisticFixpoint();
10027 }
10028 }
10029 }
10030 if (MaybeTrue)
10031 unionAssumed(C: APInt(/* numBits */ 1, /* val */ 1));
10032 if (MaybeFalse)
10033 unionAssumed(C: APInt(/* numBits */ 1, /* val */ 0));
10034 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10035 : ChangeStatus::CHANGED;
10036 }
10037
10038 ChangeStatus updateWithSelectInst(Attributor &A, SelectInst *SI) {
10039 auto AssumedBefore = getAssumed();
10040 Value *LHS = SI->getTrueValue();
10041 Value *RHS = SI->getFalseValue();
10042
10043 bool UsedAssumedInformation = false;
10044 std::optional<Constant *> C = A.getAssumedConstant(
10045 V: *SI->getCondition(), AA: *this, UsedAssumedInformation);
10046
10047 // Check if we only need one operand.
10048 bool OnlyLeft = false, OnlyRight = false;
10049 if (C && *C && (*C)->isOneValue())
10050 OnlyLeft = true;
10051 else if (C && *C && (*C)->isNullValue())
10052 OnlyRight = true;
10053
10054 bool LHSContainsUndef = false, RHSContainsUndef = false;
10055 SetTy LHSAAPVS, RHSAAPVS;
10056 if (!OnlyRight &&
10057 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
10058 ContainsUndef&: LHSContainsUndef, /* ForSelf */ false))
10059 return indicatePessimisticFixpoint();
10060
10061 if (!OnlyLeft &&
10062 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
10063 ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
10064 return indicatePessimisticFixpoint();
10065
10066 if (OnlyLeft || OnlyRight) {
10067 // select (true/false), lhs, rhs
10068 auto *OpAA = OnlyLeft ? &LHSAAPVS : &RHSAAPVS;
10069 auto Undef = OnlyLeft ? LHSContainsUndef : RHSContainsUndef;
10070
10071 if (Undef)
10072 unionAssumedWithUndef();
10073 else {
10074 for (const auto &It : *OpAA)
10075 unionAssumed(C: It);
10076 }
10077
10078 } else if (LHSContainsUndef && RHSContainsUndef) {
10079 // select i1 *, undef , undef => undef
10080 unionAssumedWithUndef();
10081 } else {
10082 for (const auto &It : LHSAAPVS)
10083 unionAssumed(C: It);
10084 for (const auto &It : RHSAAPVS)
10085 unionAssumed(C: It);
10086 }
10087 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10088 : ChangeStatus::CHANGED;
10089 }
10090
10091 ChangeStatus updateWithCastInst(Attributor &A, CastInst *CI) {
10092 auto AssumedBefore = getAssumed();
10093 if (!CI->isIntegerCast())
10094 return indicatePessimisticFixpoint();
10095 assert(CI->getNumOperands() == 1 && "Expected cast to be unary!");
10096 uint32_t ResultBitWidth = CI->getDestTy()->getIntegerBitWidth();
10097 Value *Src = CI->getOperand(i_nocapture: 0);
10098
10099 bool SrcContainsUndef = false;
10100 SetTy SrcPVS;
10101 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *Src), S&: SrcPVS,
10102 ContainsUndef&: SrcContainsUndef, /* ForSelf */ false))
10103 return indicatePessimisticFixpoint();
10104
10105 if (SrcContainsUndef)
10106 unionAssumedWithUndef();
10107 else {
10108 for (const APInt &S : SrcPVS) {
10109 APInt T = calculateCastInst(CI, Src: S, ResultBitWidth);
10110 unionAssumed(C: T);
10111 }
10112 }
10113 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10114 : ChangeStatus::CHANGED;
10115 }
10116
10117 ChangeStatus updateWithBinaryOperator(Attributor &A, BinaryOperator *BinOp) {
10118 auto AssumedBefore = getAssumed();
10119 Value *LHS = BinOp->getOperand(i_nocapture: 0);
10120 Value *RHS = BinOp->getOperand(i_nocapture: 1);
10121
10122 bool LHSContainsUndef = false, RHSContainsUndef = false;
10123 SetTy LHSAAPVS, RHSAAPVS;
10124 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
10125 ContainsUndef&: LHSContainsUndef, /* ForSelf */ false) ||
10126 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
10127 ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
10128 return indicatePessimisticFixpoint();
10129
10130 const APInt Zero = APInt(LHS->getType()->getIntegerBitWidth(), 0);
10131
10132 // TODO: make use of undef flag to limit potential values aggressively.
10133 if (LHSContainsUndef && RHSContainsUndef) {
10134 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: Zero, RHS: Zero))
10135 return indicatePessimisticFixpoint();
10136 } else if (LHSContainsUndef) {
10137 for (const APInt &R : RHSAAPVS) {
10138 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: Zero, RHS: R))
10139 return indicatePessimisticFixpoint();
10140 }
10141 } else if (RHSContainsUndef) {
10142 for (const APInt &L : LHSAAPVS) {
10143 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: L, RHS: Zero))
10144 return indicatePessimisticFixpoint();
10145 }
10146 } else {
10147 for (const APInt &L : LHSAAPVS) {
10148 for (const APInt &R : RHSAAPVS) {
10149 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: L, RHS: R))
10150 return indicatePessimisticFixpoint();
10151 }
10152 }
10153 }
10154 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10155 : ChangeStatus::CHANGED;
10156 }
10157
10158 ChangeStatus updateWithInstruction(Attributor &A, Instruction *Inst) {
10159 auto AssumedBefore = getAssumed();
10160 SetTy Incoming;
10161 bool ContainsUndef;
10162 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *Inst), S&: Incoming,
10163 ContainsUndef, /* ForSelf */ true))
10164 return indicatePessimisticFixpoint();
10165 if (ContainsUndef) {
10166 unionAssumedWithUndef();
10167 } else {
10168 for (const auto &It : Incoming)
10169 unionAssumed(C: It);
10170 }
10171 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10172 : ChangeStatus::CHANGED;
10173 }
10174
10175 /// See AbstractAttribute::updateImpl(...).
10176 ChangeStatus updateImpl(Attributor &A) override {
10177 Value &V = getAssociatedValue();
10178 Instruction *I = dyn_cast<Instruction>(Val: &V);
10179
10180 if (auto *ICI = dyn_cast<ICmpInst>(Val: I))
10181 return updateWithICmpInst(A, ICI);
10182
10183 if (auto *SI = dyn_cast<SelectInst>(Val: I))
10184 return updateWithSelectInst(A, SI);
10185
10186 if (auto *CI = dyn_cast<CastInst>(Val: I))
10187 return updateWithCastInst(A, CI);
10188
10189 if (auto *BinOp = dyn_cast<BinaryOperator>(Val: I))
10190 return updateWithBinaryOperator(A, BinOp);
10191
10192 if (isa<PHINode>(Val: I) || isa<LoadInst>(Val: I))
10193 return updateWithInstruction(A, Inst: I);
10194
10195 return indicatePessimisticFixpoint();
10196 }
10197
10198 /// See AbstractAttribute::trackStatistics()
10199 void trackStatistics() const override {
10200 STATS_DECLTRACK_FLOATING_ATTR(potential_values)
10201 }
10202};
10203
10204struct AAPotentialConstantValuesFunction : AAPotentialConstantValuesImpl {
10205 AAPotentialConstantValuesFunction(const IRPosition &IRP, Attributor &A)
10206 : AAPotentialConstantValuesImpl(IRP, A) {}
10207
10208 /// See AbstractAttribute::initialize(...).
10209 ChangeStatus updateImpl(Attributor &A) override {
10210 llvm_unreachable(
10211 "AAPotentialConstantValues(Function|CallSite)::updateImpl will "
10212 "not be called");
10213 }
10214
10215 /// See AbstractAttribute::trackStatistics()
10216 void trackStatistics() const override {
10217 STATS_DECLTRACK_FN_ATTR(potential_values)
10218 }
10219};
10220
10221struct AAPotentialConstantValuesCallSite : AAPotentialConstantValuesFunction {
10222 AAPotentialConstantValuesCallSite(const IRPosition &IRP, Attributor &A)
10223 : AAPotentialConstantValuesFunction(IRP, A) {}
10224
10225 /// See AbstractAttribute::trackStatistics()
10226 void trackStatistics() const override {
10227 STATS_DECLTRACK_CS_ATTR(potential_values)
10228 }
10229};
10230
10231struct AAPotentialConstantValuesCallSiteReturned
10232 : AACalleeToCallSite<AAPotentialConstantValues,
10233 AAPotentialConstantValuesImpl> {
10234 AAPotentialConstantValuesCallSiteReturned(const IRPosition &IRP,
10235 Attributor &A)
10236 : AACalleeToCallSite<AAPotentialConstantValues,
10237 AAPotentialConstantValuesImpl>(IRP, A) {}
10238
10239 /// See AbstractAttribute::trackStatistics()
10240 void trackStatistics() const override {
10241 STATS_DECLTRACK_CSRET_ATTR(potential_values)
10242 }
10243};
10244
10245struct AAPotentialConstantValuesCallSiteArgument
10246 : AAPotentialConstantValuesFloating {
10247 AAPotentialConstantValuesCallSiteArgument(const IRPosition &IRP,
10248 Attributor &A)
10249 : AAPotentialConstantValuesFloating(IRP, A) {}
10250
10251 /// See AbstractAttribute::initialize(..).
10252 void initialize(Attributor &A) override {
10253 AAPotentialConstantValuesImpl::initialize(A);
10254 if (isAtFixpoint())
10255 return;
10256
10257 Value &V = getAssociatedValue();
10258
10259 if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
10260 unionAssumed(C: C->getValue());
10261 indicateOptimisticFixpoint();
10262 return;
10263 }
10264
10265 if (isa<UndefValue>(Val: &V)) {
10266 unionAssumedWithUndef();
10267 indicateOptimisticFixpoint();
10268 return;
10269 }
10270 }
10271
10272 /// See AbstractAttribute::updateImpl(...).
10273 ChangeStatus updateImpl(Attributor &A) override {
10274 Value &V = getAssociatedValue();
10275 auto AssumedBefore = getAssumed();
10276 auto *AA = A.getAAFor<AAPotentialConstantValues>(
10277 QueryingAA: *this, IRP: IRPosition::value(V), DepClass: DepClassTy::REQUIRED);
10278 if (!AA)
10279 return indicatePessimisticFixpoint();
10280 const auto &S = AA->getAssumed();
10281 unionAssumed(PVS: S);
10282 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10283 : ChangeStatus::CHANGED;
10284 }
10285
10286 /// See AbstractAttribute::trackStatistics()
10287 void trackStatistics() const override {
10288 STATS_DECLTRACK_CSARG_ATTR(potential_values)
10289 }
10290};
10291} // namespace
10292
10293/// ------------------------ NoUndef Attribute ---------------------------------
10294bool AANoUndef::isImpliedByIR(Attributor &A, const IRPosition &IRP,
10295 Attribute::AttrKind ImpliedAttributeKind,
10296 bool IgnoreSubsumingPositions) {
10297 assert(ImpliedAttributeKind == Attribute::NoUndef &&
10298 "Unexpected attribute kind");
10299 if (A.hasAttr(IRP, AKs: {Attribute::NoUndef}, IgnoreSubsumingPositions,
10300 ImpliedAttributeKind: Attribute::NoUndef))
10301 return true;
10302
10303 Value &Val = IRP.getAssociatedValue();
10304 if (IRP.getPositionKind() != IRPosition::IRP_RETURNED &&
10305 isGuaranteedNotToBeUndefOrPoison(V: &Val)) {
10306 LLVMContext &Ctx = Val.getContext();
10307 A.manifestAttrs(IRP, DeducedAttrs: Attribute::get(Context&: Ctx, Kind: Attribute::NoUndef));
10308 return true;
10309 }
10310
10311 return false;
10312}
10313
10314namespace {
10315struct AANoUndefImpl : AANoUndef {
10316 AANoUndefImpl(const IRPosition &IRP, Attributor &A) : AANoUndef(IRP, A) {}
10317
10318 /// See AbstractAttribute::initialize(...).
10319 void initialize(Attributor &A) override {
10320 Value &V = getAssociatedValue();
10321 if (isa<UndefValue>(Val: V))
10322 indicatePessimisticFixpoint();
10323 assert(!isImpliedByIR(A, getIRPosition(), Attribute::NoUndef));
10324 }
10325
10326 /// See followUsesInMBEC
10327 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10328 AANoUndef::StateType &State) {
10329 const Value *UseV = U->get();
10330 const DominatorTree *DT = nullptr;
10331 AssumptionCache *AC = nullptr;
10332 InformationCache &InfoCache = A.getInfoCache();
10333 if (Function *F = getAnchorScope()) {
10334 DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
10335 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
10336 }
10337 State.setKnown(isGuaranteedNotToBeUndefOrPoison(V: UseV, AC, CtxI: I, DT));
10338 bool TrackUse = false;
10339 // Track use for instructions which must produce undef or poison bits when
10340 // at least one operand contains such bits.
10341 if (isa<CastInst>(Val: *I) || isa<GetElementPtrInst>(Val: *I))
10342 TrackUse = true;
10343 return TrackUse;
10344 }
10345
10346 /// See AbstractAttribute::getAsStr().
10347 const std::string getAsStr(Attributor *A) const override {
10348 return getAssumed() ? "noundef" : "may-undef-or-poison";
10349 }
10350
10351 ChangeStatus manifest(Attributor &A) override {
10352 // We don't manifest noundef attribute for dead positions because the
10353 // associated values with dead positions would be replaced with undef
10354 // values.
10355 bool UsedAssumedInformation = false;
10356 if (A.isAssumedDead(IRP: getIRPosition(), QueryingAA: nullptr, FnLivenessAA: nullptr,
10357 UsedAssumedInformation))
10358 return ChangeStatus::UNCHANGED;
10359 // A position whose simplified value does not have any value is
10360 // considered to be dead. We don't manifest noundef in such positions for
10361 // the same reason above.
10362 if (!A.getAssumedSimplified(IRP: getIRPosition(), AA: *this, UsedAssumedInformation,
10363 S: AA::Interprocedural)
10364 .has_value())
10365 return ChangeStatus::UNCHANGED;
10366 return AANoUndef::manifest(A);
10367 }
10368};
10369
10370struct AANoUndefFloating : public AANoUndefImpl {
10371 AANoUndefFloating(const IRPosition &IRP, Attributor &A)
10372 : AANoUndefImpl(IRP, A) {}
10373
10374 /// See AbstractAttribute::initialize(...).
10375 void initialize(Attributor &A) override {
10376 AANoUndefImpl::initialize(A);
10377 if (!getState().isAtFixpoint() && getAnchorScope() &&
10378 !getAnchorScope()->isDeclaration())
10379 if (Instruction *CtxI = getCtxI())
10380 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
10381 }
10382
10383 /// See AbstractAttribute::updateImpl(...).
10384 ChangeStatus updateImpl(Attributor &A) override {
10385 auto VisitValueCB = [&](const IRPosition &IRP) -> bool {
10386 bool IsKnownNoUndef;
10387 return AA::hasAssumedIRAttr<Attribute::NoUndef>(
10388 A, QueryingAA: this, IRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoUndef);
10389 };
10390
10391 bool Stripped;
10392 bool UsedAssumedInformation = false;
10393 Value *AssociatedValue = &getAssociatedValue();
10394 SmallVector<AA::ValueAndContext> Values;
10395 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
10396 S: AA::AnyScope, UsedAssumedInformation))
10397 Stripped = false;
10398 else
10399 Stripped =
10400 Values.size() != 1 || Values.front().getValue() != AssociatedValue;
10401
10402 if (!Stripped) {
10403 // If we haven't stripped anything we might still be able to use a
10404 // different AA, but only if the IRP changes. Effectively when we
10405 // interpret this not as a call site value but as a floating/argument
10406 // value.
10407 const IRPosition AVIRP = IRPosition::value(V: *AssociatedValue);
10408 if (AVIRP == getIRPosition() || !VisitValueCB(AVIRP))
10409 return indicatePessimisticFixpoint();
10410 return ChangeStatus::UNCHANGED;
10411 }
10412
10413 for (const auto &VAC : Values)
10414 if (!VisitValueCB(IRPosition::value(V: *VAC.getValue())))
10415 return indicatePessimisticFixpoint();
10416
10417 return ChangeStatus::UNCHANGED;
10418 }
10419
10420 /// See AbstractAttribute::trackStatistics()
10421 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10422};
10423
10424struct AANoUndefReturned final
10425 : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl> {
10426 AANoUndefReturned(const IRPosition &IRP, Attributor &A)
10427 : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl>(IRP, A) {}
10428
10429 /// See AbstractAttribute::trackStatistics()
10430 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10431};
10432
10433struct AANoUndefArgument final
10434 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl> {
10435 AANoUndefArgument(const IRPosition &IRP, Attributor &A)
10436 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl>(IRP, A) {}
10437
10438 /// See AbstractAttribute::trackStatistics()
10439 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noundef) }
10440};
10441
10442struct AANoUndefCallSiteArgument final : AANoUndefFloating {
10443 AANoUndefCallSiteArgument(const IRPosition &IRP, Attributor &A)
10444 : AANoUndefFloating(IRP, A) {}
10445
10446 /// See AbstractAttribute::trackStatistics()
10447 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noundef) }
10448};
10449
10450struct AANoUndefCallSiteReturned final
10451 : AACalleeToCallSite<AANoUndef, AANoUndefImpl> {
10452 AANoUndefCallSiteReturned(const IRPosition &IRP, Attributor &A)
10453 : AACalleeToCallSite<AANoUndef, AANoUndefImpl>(IRP, A) {}
10454
10455 /// See AbstractAttribute::trackStatistics()
10456 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noundef) }
10457};
10458
10459/// ------------------------ NoFPClass Attribute -------------------------------
10460
10461struct AANoFPClassImpl : AANoFPClass {
10462 AANoFPClassImpl(const IRPosition &IRP, Attributor &A) : AANoFPClass(IRP, A) {}
10463
10464 void initialize(Attributor &A) override {
10465 const IRPosition &IRP = getIRPosition();
10466
10467 Value &V = IRP.getAssociatedValue();
10468 if (isa<UndefValue>(Val: V)) {
10469 indicateOptimisticFixpoint();
10470 return;
10471 }
10472
10473 SmallVector<Attribute> Attrs;
10474 A.getAttrs(IRP: getIRPosition(), AKs: {Attribute::NoFPClass}, Attrs, IgnoreSubsumingPositions: false);
10475 for (const auto &Attr : Attrs) {
10476 addKnownBits(Bits: Attr.getNoFPClass());
10477 }
10478
10479 Instruction *CtxI = getCtxI();
10480
10481 if (getPositionKind() != IRPosition::IRP_RETURNED) {
10482 const DataLayout &DL = A.getDataLayout();
10483 InformationCache &InfoCache = A.getInfoCache();
10484
10485 const DominatorTree *DT = nullptr;
10486 AssumptionCache *AC = nullptr;
10487 const TargetLibraryInfo *TLI = nullptr;
10488 Function *F = getAnchorScope();
10489 if (F) {
10490 TLI = InfoCache.getTargetLibraryInfoForFunction(F: *F);
10491 if (!F->isDeclaration()) {
10492 DT =
10493 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
10494 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
10495 }
10496 }
10497
10498 SimplifyQuery Q(DL, TLI, DT, AC, CtxI);
10499
10500 KnownFPClass KnownFPClass = computeKnownFPClass(V: &V, InterestedClasses: fcAllFlags, SQ: Q);
10501 addKnownBits(Bits: ~KnownFPClass.KnownFPClasses);
10502 }
10503
10504 if (CtxI)
10505 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
10506 }
10507
10508 /// See followUsesInMBEC
10509 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10510 AANoFPClass::StateType &State) {
10511 // TODO: Determine what instructions can be looked through.
10512 auto *CB = dyn_cast<CallBase>(Val: I);
10513 if (!CB)
10514 return false;
10515
10516 if (!CB->isArgOperand(U))
10517 return false;
10518
10519 unsigned ArgNo = CB->getArgOperandNo(U);
10520 IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
10521 if (auto *NoFPAA = A.getAAFor<AANoFPClass>(QueryingAA: *this, IRP, DepClass: DepClassTy::NONE))
10522 State.addKnownBits(Bits: NoFPAA->getState().getKnown());
10523 return false;
10524 }
10525
10526 const std::string getAsStr(Attributor *A) const override {
10527 std::string Result = "nofpclass";
10528 raw_string_ostream OS(Result);
10529 OS << getKnownNoFPClass() << '/' << getAssumedNoFPClass();
10530 return Result;
10531 }
10532
10533 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
10534 SmallVectorImpl<Attribute> &Attrs) const override {
10535 Attrs.emplace_back(Args: Attribute::getWithNoFPClass(Context&: Ctx, Mask: getAssumedNoFPClass()));
10536 }
10537};
10538
10539struct AANoFPClassFloating : public AANoFPClassImpl {
10540 AANoFPClassFloating(const IRPosition &IRP, Attributor &A)
10541 : AANoFPClassImpl(IRP, A) {}
10542
10543 /// See AbstractAttribute::updateImpl(...).
10544 ChangeStatus updateImpl(Attributor &A) override {
10545 SmallVector<AA::ValueAndContext> Values;
10546 bool UsedAssumedInformation = false;
10547 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
10548 S: AA::AnyScope, UsedAssumedInformation)) {
10549 Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
10550 }
10551
10552 StateType T;
10553 auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
10554 const auto *AA = A.getAAFor<AANoFPClass>(QueryingAA: *this, IRP: IRPosition::value(V),
10555 DepClass: DepClassTy::REQUIRED);
10556 if (!AA || this == AA) {
10557 T.indicatePessimisticFixpoint();
10558 } else {
10559 const AANoFPClass::StateType &S =
10560 static_cast<const AANoFPClass::StateType &>(AA->getState());
10561 T ^= S;
10562 }
10563 return T.isValidState();
10564 };
10565
10566 for (const auto &VAC : Values)
10567 if (!VisitValueCB(*VAC.getValue(), VAC.getCtxI()))
10568 return indicatePessimisticFixpoint();
10569
10570 return clampStateAndIndicateChange(S&: getState(), R: T);
10571 }
10572
10573 /// See AbstractAttribute::trackStatistics()
10574 void trackStatistics() const override {
10575 STATS_DECLTRACK_FNRET_ATTR(nofpclass)
10576 }
10577};
10578
10579struct AANoFPClassReturned final
10580 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10581 AANoFPClassImpl::StateType, false,
10582 Attribute::None, false> {
10583 AANoFPClassReturned(const IRPosition &IRP, Attributor &A)
10584 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10585 AANoFPClassImpl::StateType, false,
10586 Attribute::None, false>(IRP, A) {}
10587
10588 /// See AbstractAttribute::trackStatistics()
10589 void trackStatistics() const override {
10590 STATS_DECLTRACK_FNRET_ATTR(nofpclass)
10591 }
10592};
10593
10594struct AANoFPClassArgument final
10595 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl> {
10596 AANoFPClassArgument(const IRPosition &IRP, Attributor &A)
10597 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10598
10599 /// See AbstractAttribute::trackStatistics()
10600 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofpclass) }
10601};
10602
10603struct AANoFPClassCallSiteArgument final : AANoFPClassFloating {
10604 AANoFPClassCallSiteArgument(const IRPosition &IRP, Attributor &A)
10605 : AANoFPClassFloating(IRP, A) {}
10606
10607 /// See AbstractAttribute::trackStatistics()
10608 void trackStatistics() const override {
10609 STATS_DECLTRACK_CSARG_ATTR(nofpclass)
10610 }
10611};
10612
10613struct AANoFPClassCallSiteReturned final
10614 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl> {
10615 AANoFPClassCallSiteReturned(const IRPosition &IRP, Attributor &A)
10616 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10617
10618 /// See AbstractAttribute::trackStatistics()
10619 void trackStatistics() const override {
10620 STATS_DECLTRACK_CSRET_ATTR(nofpclass)
10621 }
10622};
10623
10624struct AACallEdgesImpl : public AACallEdges {
10625 AACallEdgesImpl(const IRPosition &IRP, Attributor &A) : AACallEdges(IRP, A) {}
10626
10627 const SetVector<Function *> &getOptimisticEdges() const override {
10628 return CalledFunctions;
10629 }
10630
10631 bool hasUnknownCallee() const override { return HasUnknownCallee; }
10632
10633 bool hasNonAsmUnknownCallee() const override {
10634 return HasUnknownCalleeNonAsm;
10635 }
10636
10637 const std::string getAsStr(Attributor *A) const override {
10638 return "CallEdges[" + std::to_string(val: HasUnknownCallee) + "," +
10639 std::to_string(val: CalledFunctions.size()) + "]";
10640 }
10641
10642 void trackStatistics() const override {}
10643
10644protected:
10645 void addCalledFunction(Function *Fn, ChangeStatus &Change) {
10646 if (CalledFunctions.insert(X: Fn)) {
10647 Change = ChangeStatus::CHANGED;
10648 LLVM_DEBUG(dbgs() << "[AACallEdges] New call edge: " << Fn->getName()
10649 << "\n");
10650 }
10651 }
10652
10653 void setHasUnknownCallee(bool NonAsm, ChangeStatus &Change) {
10654 if (!HasUnknownCallee)
10655 Change = ChangeStatus::CHANGED;
10656 if (NonAsm && !HasUnknownCalleeNonAsm)
10657 Change = ChangeStatus::CHANGED;
10658 HasUnknownCalleeNonAsm |= NonAsm;
10659 HasUnknownCallee = true;
10660 }
10661
10662private:
10663 /// Optimistic set of functions that might be called by this position.
10664 SetVector<Function *> CalledFunctions;
10665
10666 /// Is there any call with a unknown callee.
10667 bool HasUnknownCallee = false;
10668
10669 /// Is there any call with a unknown callee, excluding any inline asm.
10670 bool HasUnknownCalleeNonAsm = false;
10671};
10672
10673struct AACallEdgesCallSite : public AACallEdgesImpl {
10674 AACallEdgesCallSite(const IRPosition &IRP, Attributor &A)
10675 : AACallEdgesImpl(IRP, A) {}
10676 /// See AbstractAttribute::updateImpl(...).
10677 ChangeStatus updateImpl(Attributor &A) override {
10678 ChangeStatus Change = ChangeStatus::UNCHANGED;
10679
10680 auto VisitValue = [&](Value &V, const Instruction *CtxI) -> bool {
10681 if (Function *Fn = dyn_cast<Function>(Val: &V)) {
10682 addCalledFunction(Fn, Change);
10683 } else {
10684 LLVM_DEBUG(dbgs() << "[AACallEdges] Unrecognized value: " << V << "\n");
10685 setHasUnknownCallee(NonAsm: true, Change);
10686 }
10687
10688 // Explore all values.
10689 return true;
10690 };
10691
10692 SmallVector<AA::ValueAndContext> Values;
10693 // Process any value that we might call.
10694 auto ProcessCalledOperand = [&](Value *V, Instruction *CtxI) {
10695 if (isa<Constant>(Val: V)) {
10696 VisitValue(*V, CtxI);
10697 return;
10698 }
10699
10700 bool UsedAssumedInformation = false;
10701 Values.clear();
10702 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *V), AA: *this, Values,
10703 S: AA::AnyScope, UsedAssumedInformation)) {
10704 Values.push_back(Elt: {*V, CtxI});
10705 }
10706 for (auto &VAC : Values)
10707 VisitValue(*VAC.getValue(), VAC.getCtxI());
10708 };
10709
10710 CallBase *CB = cast<CallBase>(Val: getCtxI());
10711
10712 if (auto *IA = dyn_cast<InlineAsm>(Val: CB->getCalledOperand())) {
10713 if (IA->hasSideEffects() &&
10714 !hasAssumption(F: *CB->getCaller(), AssumptionStr: "ompx_no_call_asm") &&
10715 !hasAssumption(CB: *CB, AssumptionStr: "ompx_no_call_asm")) {
10716 setHasUnknownCallee(NonAsm: false, Change);
10717 }
10718 return Change;
10719 }
10720
10721 if (CB->isIndirectCall())
10722 if (auto *IndirectCallAA = A.getAAFor<AAIndirectCallInfo>(
10723 QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL))
10724 if (IndirectCallAA->foreachCallee(
10725 CB: [&](Function *Fn) { return VisitValue(*Fn, CB); }))
10726 return Change;
10727
10728 // The most simple case.
10729 ProcessCalledOperand(CB->getCalledOperand(), CB);
10730
10731 // Process callback functions.
10732 SmallVector<const Use *, 4u> CallbackUses;
10733 AbstractCallSite::getCallbackUses(CB: *CB, CallbackUses);
10734 for (const Use *U : CallbackUses)
10735 ProcessCalledOperand(U->get(), CB);
10736
10737 return Change;
10738 }
10739};
10740
10741struct AACallEdgesFunction : public AACallEdgesImpl {
10742 AACallEdgesFunction(const IRPosition &IRP, Attributor &A)
10743 : AACallEdgesImpl(IRP, A) {}
10744
10745 /// See AbstractAttribute::updateImpl(...).
10746 ChangeStatus updateImpl(Attributor &A) override {
10747 ChangeStatus Change = ChangeStatus::UNCHANGED;
10748
10749 auto ProcessCallInst = [&](Instruction &Inst) {
10750 CallBase &CB = cast<CallBase>(Val&: Inst);
10751
10752 auto *CBEdges = A.getAAFor<AACallEdges>(
10753 QueryingAA: *this, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::REQUIRED);
10754 if (!CBEdges)
10755 return false;
10756 if (CBEdges->hasNonAsmUnknownCallee())
10757 setHasUnknownCallee(NonAsm: true, Change);
10758 if (CBEdges->hasUnknownCallee())
10759 setHasUnknownCallee(NonAsm: false, Change);
10760
10761 for (Function *F : CBEdges->getOptimisticEdges())
10762 addCalledFunction(Fn: F, Change);
10763
10764 return true;
10765 };
10766
10767 // Visit all callable instructions.
10768 bool UsedAssumedInformation = false;
10769 if (!A.checkForAllCallLikeInstructions(Pred: ProcessCallInst, QueryingAA: *this,
10770 UsedAssumedInformation,
10771 /* CheckBBLivenessOnly */ true)) {
10772 // If we haven't looked at all call like instructions, assume that there
10773 // are unknown callees.
10774 setHasUnknownCallee(NonAsm: true, Change);
10775 }
10776
10777 return Change;
10778 }
10779};
10780
10781/// -------------------AAInterFnReachability Attribute--------------------------
10782
10783struct AAInterFnReachabilityFunction
10784 : public CachedReachabilityAA<AAInterFnReachability, Function> {
10785 using Base = CachedReachabilityAA<AAInterFnReachability, Function>;
10786 AAInterFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
10787 : Base(IRP, A) {}
10788
10789 bool instructionCanReach(
10790 Attributor &A, const Instruction &From, const Function &To,
10791 const AA::InstExclusionSetTy *ExclusionSet) const override {
10792 assert(From.getFunction() == getAnchorScope() && "Queried the wrong AA!");
10793 auto *NonConstThis = const_cast<AAInterFnReachabilityFunction *>(this);
10794
10795 RQITy StackRQI(A, From, To, ExclusionSet, false);
10796 RQITy::Reachable Result;
10797 if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
10798 return NonConstThis->isReachableImpl(A, RQI&: StackRQI,
10799 /*IsTemporaryRQI=*/true);
10800 return Result == RQITy::Reachable::Yes;
10801 }
10802
10803 bool isReachableImpl(Attributor &A, RQITy &RQI,
10804 bool IsTemporaryRQI) override {
10805 const Instruction *EntryI =
10806 &RQI.From->getFunction()->getEntryBlock().front();
10807 if (EntryI != RQI.From &&
10808 !instructionCanReach(A, From: *EntryI, To: *RQI.To, ExclusionSet: nullptr))
10809 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet: false,
10810 IsTemporaryRQI);
10811
10812 auto CheckReachableCallBase = [&](CallBase *CB) {
10813 auto *CBEdges = A.getAAFor<AACallEdges>(
10814 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL);
10815 if (!CBEdges || !CBEdges->getState().isValidState())
10816 return false;
10817 // TODO Check To backwards in this case.
10818 if (CBEdges->hasUnknownCallee())
10819 return false;
10820
10821 for (Function *Fn : CBEdges->getOptimisticEdges()) {
10822 if (Fn == RQI.To)
10823 return false;
10824
10825 if (Fn->isDeclaration()) {
10826 if (Fn->hasFnAttribute(Kind: Attribute::NoCallback))
10827 continue;
10828 // TODO Check To backwards in this case.
10829 return false;
10830 }
10831
10832 if (Fn == getAnchorScope()) {
10833 if (EntryI == RQI.From)
10834 continue;
10835 return false;
10836 }
10837
10838 const AAInterFnReachability *InterFnReachability =
10839 A.getAAFor<AAInterFnReachability>(QueryingAA: *this, IRP: IRPosition::function(F: *Fn),
10840 DepClass: DepClassTy::OPTIONAL);
10841
10842 const Instruction &FnFirstInst = Fn->getEntryBlock().front();
10843 if (!InterFnReachability ||
10844 InterFnReachability->instructionCanReach(A, Inst: FnFirstInst, Fn: *RQI.To,
10845 ExclusionSet: RQI.ExclusionSet))
10846 return false;
10847 }
10848 return true;
10849 };
10850
10851 const auto *IntraFnReachability = A.getAAFor<AAIntraFnReachability>(
10852 QueryingAA: *this, IRP: IRPosition::function(F: *RQI.From->getFunction()),
10853 DepClass: DepClassTy::OPTIONAL);
10854
10855 // Determine call like instructions that we can reach from the inst.
10856 auto CheckCallBase = [&](Instruction &CBInst) {
10857 // There are usually less nodes in the call graph, check inter function
10858 // reachability first.
10859 if (CheckReachableCallBase(cast<CallBase>(Val: &CBInst)))
10860 return true;
10861 return IntraFnReachability && !IntraFnReachability->isAssumedReachable(
10862 A, From: *RQI.From, To: CBInst, ExclusionSet: RQI.ExclusionSet);
10863 };
10864
10865 bool UsedExclusionSet = /* conservative */ true;
10866 bool UsedAssumedInformation = false;
10867 if (!A.checkForAllCallLikeInstructions(Pred: CheckCallBase, QueryingAA: *this,
10868 UsedAssumedInformation,
10869 /* CheckBBLivenessOnly */ true))
10870 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
10871 IsTemporaryRQI);
10872
10873 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
10874 IsTemporaryRQI);
10875 }
10876
10877 void trackStatistics() const override {}
10878};
10879} // namespace
10880
10881template <typename AAType>
10882static std::optional<Constant *>
10883askForAssumedConstant(Attributor &A, const AbstractAttribute &QueryingAA,
10884 const IRPosition &IRP, Type &Ty) {
10885 if (!Ty.isIntegerTy())
10886 return nullptr;
10887
10888 // This will also pass the call base context.
10889 const auto *AA = A.getAAFor<AAType>(QueryingAA, IRP, DepClassTy::NONE);
10890 if (!AA)
10891 return nullptr;
10892
10893 std::optional<Constant *> COpt = AA->getAssumedConstant(A);
10894
10895 if (!COpt.has_value()) {
10896 A.recordDependence(FromAA: *AA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
10897 return std::nullopt;
10898 }
10899 if (auto *C = *COpt) {
10900 A.recordDependence(FromAA: *AA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
10901 return C;
10902 }
10903 return nullptr;
10904}
10905
10906Value *AAPotentialValues::getSingleValue(
10907 Attributor &A, const AbstractAttribute &AA, const IRPosition &IRP,
10908 SmallVectorImpl<AA::ValueAndContext> &Values) {
10909 Type &Ty = *IRP.getAssociatedType();
10910 std::optional<Value *> V;
10911 for (auto &It : Values) {
10912 V = AA::combineOptionalValuesInAAValueLatice(A: V, B: It.getValue(), Ty: &Ty);
10913 if (V.has_value() && !*V)
10914 break;
10915 }
10916 if (!V.has_value())
10917 return UndefValue::get(T: &Ty);
10918 return *V;
10919}
10920
10921namespace {
10922struct AAPotentialValuesImpl : AAPotentialValues {
10923 using StateType = PotentialLLVMValuesState;
10924
10925 AAPotentialValuesImpl(const IRPosition &IRP, Attributor &A)
10926 : AAPotentialValues(IRP, A) {}
10927
10928 /// See AbstractAttribute::initialize(..).
10929 void initialize(Attributor &A) override {
10930 if (A.hasSimplificationCallback(IRP: getIRPosition())) {
10931 indicatePessimisticFixpoint();
10932 return;
10933 }
10934 Value *Stripped = getAssociatedValue().stripPointerCasts();
10935 if (isa<Constant>(Val: Stripped) && !isa<ConstantExpr>(Val: Stripped)) {
10936 addValue(A, State&: getState(), V&: *Stripped, CtxI: getCtxI(), S: AA::AnyScope,
10937 AnchorScope: getAnchorScope());
10938 indicateOptimisticFixpoint();
10939 return;
10940 }
10941 AAPotentialValues::initialize(A);
10942 }
10943
10944 /// See AbstractAttribute::getAsStr().
10945 const std::string getAsStr(Attributor *A) const override {
10946 std::string Str;
10947 llvm::raw_string_ostream OS(Str);
10948 OS << getState();
10949 return Str;
10950 }
10951
10952 template <typename AAType>
10953 static std::optional<Value *> askOtherAA(Attributor &A,
10954 const AbstractAttribute &AA,
10955 const IRPosition &IRP, Type &Ty) {
10956 if (isa<Constant>(Val: IRP.getAssociatedValue()))
10957 return &IRP.getAssociatedValue();
10958 std::optional<Constant *> C = askForAssumedConstant<AAType>(A, AA, IRP, Ty);
10959 if (!C)
10960 return std::nullopt;
10961 if (*C)
10962 if (auto *CC = AA::getWithType(V&: **C, Ty))
10963 return CC;
10964 return nullptr;
10965 }
10966
10967 virtual void addValue(Attributor &A, StateType &State, Value &V,
10968 const Instruction *CtxI, AA::ValueScope S,
10969 Function *AnchorScope) const {
10970
10971 IRPosition ValIRP = IRPosition::value(V);
10972 if (auto *CB = dyn_cast_or_null<CallBase>(Val: CtxI)) {
10973 for (const auto &U : CB->args()) {
10974 if (U.get() != &V)
10975 continue;
10976 ValIRP = IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U));
10977 break;
10978 }
10979 }
10980
10981 Value *VPtr = &V;
10982 if (ValIRP.getAssociatedType()->isIntegerTy()) {
10983 Type &Ty = *getAssociatedType();
10984 std::optional<Value *> SimpleV =
10985 askOtherAA<AAValueConstantRange>(A, AA: *this, IRP: ValIRP, Ty);
10986 if (SimpleV.has_value() && !*SimpleV) {
10987 auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
10988 QueryingAA: *this, IRP: ValIRP, DepClass: DepClassTy::OPTIONAL);
10989 if (PotentialConstantsAA && PotentialConstantsAA->isValidState()) {
10990 for (const auto &It : PotentialConstantsAA->getAssumedSet())
10991 State.unionAssumed(C: {{*ConstantInt::get(Ty: &Ty, V: It), nullptr}, S});
10992 if (PotentialConstantsAA->undefIsContained())
10993 State.unionAssumed(C: {{*UndefValue::get(T: &Ty), nullptr}, S});
10994 return;
10995 }
10996 }
10997 if (!SimpleV.has_value())
10998 return;
10999
11000 if (*SimpleV)
11001 VPtr = *SimpleV;
11002 }
11003
11004 if (isa<ConstantInt>(Val: VPtr))
11005 CtxI = nullptr;
11006 if (!AA::isValidInScope(V: *VPtr, Scope: AnchorScope))
11007 S = AA::ValueScope(S | AA::Interprocedural);
11008
11009 State.unionAssumed(C: {{*VPtr, CtxI}, S});
11010 }
11011
11012 /// Helper struct to tie a value+context pair together with the scope for
11013 /// which this is the simplified version.
11014 struct ItemInfo {
11015 AA::ValueAndContext I;
11016 AA::ValueScope S;
11017
11018 bool operator==(const ItemInfo &II) const {
11019 return II.I == I && II.S == S;
11020 };
11021 bool operator<(const ItemInfo &II) const {
11022 return std::tie(args: I, args: S) < std::tie(args: II.I, args: II.S);
11023 };
11024 };
11025
11026 bool recurseForValue(Attributor &A, const IRPosition &IRP, AA::ValueScope S) {
11027 SmallMapVector<AA::ValueAndContext, int, 8> ValueScopeMap;
11028 for (auto CS : {AA::Intraprocedural, AA::Interprocedural}) {
11029 if (!(CS & S))
11030 continue;
11031
11032 bool UsedAssumedInformation = false;
11033 SmallVector<AA::ValueAndContext> Values;
11034 if (!A.getAssumedSimplifiedValues(IRP, AA: this, Values, S: CS,
11035 UsedAssumedInformation))
11036 return false;
11037
11038 for (auto &It : Values)
11039 ValueScopeMap[It] += CS;
11040 }
11041 for (auto &It : ValueScopeMap)
11042 addValue(A, State&: getState(), V&: *It.first.getValue(), CtxI: It.first.getCtxI(),
11043 S: AA::ValueScope(It.second), AnchorScope: getAnchorScope());
11044
11045 return true;
11046 }
11047
11048 void giveUpOnIntraprocedural(Attributor &A) {
11049 auto NewS = StateType::getBestState(PVS: getState());
11050 for (const auto &It : getAssumedSet()) {
11051 if (It.second == AA::Intraprocedural)
11052 continue;
11053 addValue(A, State&: NewS, V&: *It.first.getValue(), CtxI: It.first.getCtxI(),
11054 S: AA::Interprocedural, AnchorScope: getAnchorScope());
11055 }
11056 assert(!undefIsContained() && "Undef should be an explicit value!");
11057 addValue(A, State&: NewS, V&: getAssociatedValue(), CtxI: getCtxI(), S: AA::Intraprocedural,
11058 AnchorScope: getAnchorScope());
11059 getState() = NewS;
11060 }
11061
11062 /// See AbstractState::indicatePessimisticFixpoint(...).
11063 ChangeStatus indicatePessimisticFixpoint() override {
11064 getState() = StateType::getBestState(PVS: getState());
11065 getState().unionAssumed(C: {{getAssociatedValue(), getCtxI()}, AA::AnyScope});
11066 AAPotentialValues::indicateOptimisticFixpoint();
11067 return ChangeStatus::CHANGED;
11068 }
11069
11070 /// See AbstractAttribute::updateImpl(...).
11071 ChangeStatus updateImpl(Attributor &A) override {
11072 return indicatePessimisticFixpoint();
11073 }
11074
11075 /// See AbstractAttribute::manifest(...).
11076 ChangeStatus manifest(Attributor &A) override {
11077 SmallVector<AA::ValueAndContext> Values;
11078 for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {
11079 Values.clear();
11080 if (!getAssumedSimplifiedValues(A, Values, S))
11081 continue;
11082 Value &OldV = getAssociatedValue();
11083 if (isa<UndefValue>(Val: OldV))
11084 continue;
11085 Value *NewV = getSingleValue(A, AA: *this, IRP: getIRPosition(), Values);
11086 if (!NewV || NewV == &OldV)
11087 continue;
11088 if (getCtxI() &&
11089 !AA::isValidAtPosition(VAC: {*NewV, *getCtxI()}, InfoCache&: A.getInfoCache()))
11090 continue;
11091 if (A.changeAfterManifest(IRP: getIRPosition(), NV&: *NewV))
11092 return ChangeStatus::CHANGED;
11093 }
11094 return ChangeStatus::UNCHANGED;
11095 }
11096
11097 bool getAssumedSimplifiedValues(
11098 Attributor &A, SmallVectorImpl<AA::ValueAndContext> &Values,
11099 AA::ValueScope S, bool RecurseForSelectAndPHI = false) const override {
11100 if (!isValidState())
11101 return false;
11102 bool UsedAssumedInformation = false;
11103 for (const auto &It : getAssumedSet())
11104 if (It.second & S) {
11105 if (RecurseForSelectAndPHI && (isa<PHINode>(Val: It.first.getValue()) ||
11106 isa<SelectInst>(Val: It.first.getValue()))) {
11107 if (A.getAssumedSimplifiedValues(
11108 IRP: IRPosition::inst(I: *cast<Instruction>(Val: It.first.getValue())),
11109 AA: this, Values, S, UsedAssumedInformation))
11110 continue;
11111 }
11112 Values.push_back(Elt: It.first);
11113 }
11114 assert(!undefIsContained() && "Undef should be an explicit value!");
11115 return true;
11116 }
11117};
11118
11119struct AAPotentialValuesFloating : AAPotentialValuesImpl {
11120 AAPotentialValuesFloating(const IRPosition &IRP, Attributor &A)
11121 : AAPotentialValuesImpl(IRP, A) {}
11122
11123 /// See AbstractAttribute::updateImpl(...).
11124 ChangeStatus updateImpl(Attributor &A) override {
11125 auto AssumedBefore = getAssumed();
11126
11127 genericValueTraversal(A, InitialV: &getAssociatedValue());
11128
11129 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11130 : ChangeStatus::CHANGED;
11131 }
11132
11133 /// Helper struct to remember which AAIsDead instances we actually used.
11134 struct LivenessInfo {
11135 const AAIsDead *LivenessAA = nullptr;
11136 bool AnyDead = false;
11137 };
11138
11139 /// Check if \p Cmp is a comparison we can simplify.
11140 ///
11141 /// We handle multiple cases, one in which at least one operand is an
11142 /// (assumed) nullptr. If so, try to simplify it using AANonNull on the other
11143 /// operand. Return true if successful, in that case Worklist will be updated.
11144 bool handleCmp(Attributor &A, Value &Cmp, Value *LHS, Value *RHS,
11145 CmpInst::Predicate Pred, ItemInfo II,
11146 SmallVectorImpl<ItemInfo> &Worklist) {
11147
11148 // Simplify the operands first.
11149 bool UsedAssumedInformation = false;
11150 SmallVector<AA::ValueAndContext> LHSValues, RHSValues;
11151 auto GetSimplifiedValues = [&](Value &V,
11152 SmallVector<AA::ValueAndContext> &Values) {
11153 if (!A.getAssumedSimplifiedValues(
11154 IRP: IRPosition::value(V, CBContext: getCallBaseContext()), AA: this, Values,
11155 S: AA::Intraprocedural, UsedAssumedInformation)) {
11156 Values.clear();
11157 Values.push_back(Elt: AA::ValueAndContext{V, II.I.getCtxI()});
11158 }
11159 return Values.empty();
11160 };
11161 if (GetSimplifiedValues(*LHS, LHSValues))
11162 return true;
11163 if (GetSimplifiedValues(*RHS, RHSValues))
11164 return true;
11165
11166 LLVMContext &Ctx = LHS->getContext();
11167
11168 InformationCache &InfoCache = A.getInfoCache();
11169 Instruction *CmpI = dyn_cast<Instruction>(Val: &Cmp);
11170 Function *F = CmpI ? CmpI->getFunction() : nullptr;
11171 const auto *DT =
11172 F ? InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F)
11173 : nullptr;
11174 const auto *TLI =
11175 F ? A.getInfoCache().getTargetLibraryInfoForFunction(F: *F) : nullptr;
11176 auto *AC =
11177 F ? InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F)
11178 : nullptr;
11179
11180 const DataLayout &DL = A.getDataLayout();
11181 SimplifyQuery Q(DL, TLI, DT, AC, CmpI);
11182
11183 auto CheckPair = [&](Value &LHSV, Value &RHSV) {
11184 if (isa<UndefValue>(Val: LHSV) || isa<UndefValue>(Val: RHSV)) {
11185 addValue(A, State&: getState(), V&: *UndefValue::get(T: Cmp.getType()),
11186 /* CtxI */ nullptr, S: II.S, AnchorScope: getAnchorScope());
11187 return true;
11188 }
11189
11190 // Handle the trivial case first in which we don't even need to think
11191 // about null or non-null.
11192 if (&LHSV == &RHSV &&
11193 (CmpInst::isTrueWhenEqual(predicate: Pred) || CmpInst::isFalseWhenEqual(predicate: Pred))) {
11194 Constant *NewV = ConstantInt::get(Ty: Type::getInt1Ty(C&: Ctx),
11195 V: CmpInst::isTrueWhenEqual(predicate: Pred));
11196 addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11197 AnchorScope: getAnchorScope());
11198 return true;
11199 }
11200
11201 auto *TypedLHS = AA::getWithType(V&: LHSV, Ty&: *LHS->getType());
11202 auto *TypedRHS = AA::getWithType(V&: RHSV, Ty&: *RHS->getType());
11203 if (TypedLHS && TypedRHS) {
11204 Value *NewV = simplifyCmpInst(Predicate: Pred, LHS: TypedLHS, RHS: TypedRHS, Q);
11205 if (NewV && NewV != &Cmp) {
11206 addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11207 AnchorScope: getAnchorScope());
11208 return true;
11209 }
11210 }
11211
11212 // From now on we only handle equalities (==, !=).
11213 if (!CmpInst::isEquality(pred: Pred))
11214 return false;
11215
11216 bool LHSIsNull = isa<ConstantPointerNull>(Val: LHSV);
11217 bool RHSIsNull = isa<ConstantPointerNull>(Val: RHSV);
11218 if (!LHSIsNull && !RHSIsNull)
11219 return false;
11220
11221 // Left is the nullptr ==/!= non-nullptr case. We'll use AANonNull on the
11222 // non-nullptr operand and if we assume it's non-null we can conclude the
11223 // result of the comparison.
11224 assert((LHSIsNull || RHSIsNull) &&
11225 "Expected nullptr versus non-nullptr comparison at this point");
11226
11227 // The index is the operand that we assume is not null.
11228 unsigned PtrIdx = LHSIsNull;
11229 bool IsKnownNonNull;
11230 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
11231 A, QueryingAA: this, IRP: IRPosition::value(V: *(PtrIdx ? &RHSV : &LHSV)),
11232 DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNonNull);
11233 if (!IsAssumedNonNull)
11234 return false;
11235
11236 // The new value depends on the predicate, true for != and false for ==.
11237 Constant *NewV =
11238 ConstantInt::get(Ty: Type::getInt1Ty(C&: Ctx), V: Pred == CmpInst::ICMP_NE);
11239 addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11240 AnchorScope: getAnchorScope());
11241 return true;
11242 };
11243
11244 for (auto &LHSValue : LHSValues)
11245 for (auto &RHSValue : RHSValues)
11246 if (!CheckPair(*LHSValue.getValue(), *RHSValue.getValue()))
11247 return false;
11248 return true;
11249 }
11250
11251 bool handleSelectInst(Attributor &A, SelectInst &SI, ItemInfo II,
11252 SmallVectorImpl<ItemInfo> &Worklist) {
11253 const Instruction *CtxI = II.I.getCtxI();
11254 bool UsedAssumedInformation = false;
11255
11256 std::optional<Constant *> C =
11257 A.getAssumedConstant(V: *SI.getCondition(), AA: *this, UsedAssumedInformation);
11258 bool NoValueYet = !C.has_value();
11259 if (NoValueYet || isa_and_nonnull<UndefValue>(Val: *C))
11260 return true;
11261 if (auto *CI = dyn_cast_or_null<ConstantInt>(Val: *C)) {
11262 if (CI->isZero())
11263 Worklist.push_back(Elt: {.I: {*SI.getFalseValue(), CtxI}, .S: II.S});
11264 else
11265 Worklist.push_back(Elt: {.I: {*SI.getTrueValue(), CtxI}, .S: II.S});
11266 } else if (&SI == &getAssociatedValue()) {
11267 // We could not simplify the condition, assume both values.
11268 Worklist.push_back(Elt: {.I: {*SI.getTrueValue(), CtxI}, .S: II.S});
11269 Worklist.push_back(Elt: {.I: {*SI.getFalseValue(), CtxI}, .S: II.S});
11270 } else {
11271 std::optional<Value *> SimpleV = A.getAssumedSimplified(
11272 IRP: IRPosition::inst(I: SI), AA: *this, UsedAssumedInformation, S: II.S);
11273 if (!SimpleV.has_value())
11274 return true;
11275 if (*SimpleV) {
11276 addValue(A, State&: getState(), V&: **SimpleV, CtxI, S: II.S, AnchorScope: getAnchorScope());
11277 return true;
11278 }
11279 return false;
11280 }
11281 return true;
11282 }
11283
11284 bool handleLoadInst(Attributor &A, LoadInst &LI, ItemInfo II,
11285 SmallVectorImpl<ItemInfo> &Worklist) {
11286 SmallSetVector<Value *, 4> PotentialCopies;
11287 SmallSetVector<Instruction *, 4> PotentialValueOrigins;
11288 bool UsedAssumedInformation = false;
11289 if (!AA::getPotentiallyLoadedValues(A, LI, PotentialValues&: PotentialCopies,
11290 PotentialValueOrigins, QueryingAA: *this,
11291 UsedAssumedInformation,
11292 /* OnlyExact */ true)) {
11293 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Failed to get potentially "
11294 "loaded values for load instruction "
11295 << LI << "\n");
11296 return false;
11297 }
11298
11299 // Do not simplify loads that are only used in llvm.assume if we cannot also
11300 // remove all stores that may feed into the load. The reason is that the
11301 // assume is probably worth something as long as the stores are around.
11302 InformationCache &InfoCache = A.getInfoCache();
11303 if (InfoCache.isOnlyUsedByAssume(I: LI)) {
11304 if (!llvm::all_of(Range&: PotentialValueOrigins, P: [&](Instruction *I) {
11305 if (!I || isa<AssumeInst>(Val: I))
11306 return true;
11307 if (auto *SI = dyn_cast<StoreInst>(Val: I))
11308 return A.isAssumedDead(U: SI->getOperandUse(i: 0), QueryingAA: this,
11309 /* LivenessAA */ FnLivenessAA: nullptr,
11310 UsedAssumedInformation,
11311 /* CheckBBLivenessOnly */ false);
11312 return A.isAssumedDead(I: *I, QueryingAA: this, /* LivenessAA */ nullptr,
11313 UsedAssumedInformation,
11314 /* CheckBBLivenessOnly */ false);
11315 })) {
11316 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Load is onl used by assumes "
11317 "and we cannot delete all the stores: "
11318 << LI << "\n");
11319 return false;
11320 }
11321 }
11322
11323 // Values have to be dynamically unique or we loose the fact that a
11324 // single llvm::Value might represent two runtime values (e.g.,
11325 // stack locations in different recursive calls).
11326 const Instruction *CtxI = II.I.getCtxI();
11327 bool ScopeIsLocal = (II.S & AA::Intraprocedural);
11328 bool AllLocal = ScopeIsLocal;
11329 bool DynamicallyUnique = llvm::all_of(Range&: PotentialCopies, P: [&](Value *PC) {
11330 AllLocal &= AA::isValidInScope(V: *PC, Scope: getAnchorScope());
11331 return AA::isDynamicallyUnique(A, QueryingAA: *this, V: *PC);
11332 });
11333 if (!DynamicallyUnique) {
11334 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Not all potentially loaded "
11335 "values are dynamically unique: "
11336 << LI << "\n");
11337 return false;
11338 }
11339
11340 for (auto *PotentialCopy : PotentialCopies) {
11341 if (AllLocal) {
11342 Worklist.push_back(Elt: {.I: {*PotentialCopy, CtxI}, .S: II.S});
11343 } else {
11344 Worklist.push_back(Elt: {.I: {*PotentialCopy, CtxI}, .S: AA::Interprocedural});
11345 }
11346 }
11347 if (!AllLocal && ScopeIsLocal)
11348 addValue(A, State&: getState(), V&: LI, CtxI, S: AA::Intraprocedural, AnchorScope: getAnchorScope());
11349 return true;
11350 }
11351
11352 bool handlePHINode(
11353 Attributor &A, PHINode &PHI, ItemInfo II,
11354 SmallVectorImpl<ItemInfo> &Worklist,
11355 SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
11356 auto GetLivenessInfo = [&](const Function &F) -> LivenessInfo & {
11357 LivenessInfo &LI = LivenessAAs[&F];
11358 if (!LI.LivenessAA)
11359 LI.LivenessAA = A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: IRPosition::function(F),
11360 DepClass: DepClassTy::NONE);
11361 return LI;
11362 };
11363
11364 if (&PHI == &getAssociatedValue()) {
11365 LivenessInfo &LI = GetLivenessInfo(*PHI.getFunction());
11366 const auto *CI =
11367 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
11368 F: *PHI.getFunction());
11369
11370 Cycle *C = nullptr;
11371 bool CyclePHI = mayBeInCycle(CI, I: &PHI, /* HeaderOnly */ true, CPtr: &C);
11372 for (unsigned u = 0, e = PHI.getNumIncomingValues(); u < e; u++) {
11373 BasicBlock *IncomingBB = PHI.getIncomingBlock(i: u);
11374 if (LI.LivenessAA &&
11375 LI.LivenessAA->isEdgeDead(From: IncomingBB, To: PHI.getParent())) {
11376 LI.AnyDead = true;
11377 continue;
11378 }
11379 Value *V = PHI.getIncomingValue(i: u);
11380 if (V == &PHI)
11381 continue;
11382
11383 // If the incoming value is not the PHI but an instruction in the same
11384 // cycle we might have multiple versions of it flying around.
11385 if (CyclePHI && isa<Instruction>(Val: V) &&
11386 (!C || C->contains(Block: cast<Instruction>(Val: V)->getParent())))
11387 return false;
11388
11389 Worklist.push_back(Elt: {.I: {*V, IncomingBB->getTerminator()}, .S: II.S});
11390 }
11391 return true;
11392 }
11393
11394 bool UsedAssumedInformation = false;
11395 std::optional<Value *> SimpleV = A.getAssumedSimplified(
11396 IRP: IRPosition::inst(I: PHI), AA: *this, UsedAssumedInformation, S: II.S);
11397 if (!SimpleV.has_value())
11398 return true;
11399 if (!(*SimpleV))
11400 return false;
11401 addValue(A, State&: getState(), V&: **SimpleV, CtxI: &PHI, S: II.S, AnchorScope: getAnchorScope());
11402 return true;
11403 }
11404
11405 /// Use the generic, non-optimistic InstSimplfy functionality if we managed to
11406 /// simplify any operand of the instruction \p I. Return true if successful,
11407 /// in that case Worklist will be updated.
11408 bool handleGenericInst(Attributor &A, Instruction &I, ItemInfo II,
11409 SmallVectorImpl<ItemInfo> &Worklist) {
11410 bool SomeSimplified = false;
11411 bool UsedAssumedInformation = false;
11412
11413 SmallVector<Value *, 8> NewOps(I.getNumOperands());
11414 int Idx = 0;
11415 for (Value *Op : I.operands()) {
11416 const auto &SimplifiedOp = A.getAssumedSimplified(
11417 IRP: IRPosition::value(V: *Op, CBContext: getCallBaseContext()), AA: *this,
11418 UsedAssumedInformation, S: AA::Intraprocedural);
11419 // If we are not sure about any operand we are not sure about the entire
11420 // instruction, we'll wait.
11421 if (!SimplifiedOp.has_value())
11422 return true;
11423
11424 if (*SimplifiedOp)
11425 NewOps[Idx] = *SimplifiedOp;
11426 else
11427 NewOps[Idx] = Op;
11428
11429 SomeSimplified |= (NewOps[Idx] != Op);
11430 ++Idx;
11431 }
11432
11433 // We won't bother with the InstSimplify interface if we didn't simplify any
11434 // operand ourselves.
11435 if (!SomeSimplified)
11436 return false;
11437
11438 InformationCache &InfoCache = A.getInfoCache();
11439 Function *F = I.getFunction();
11440 const auto *DT =
11441 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
11442 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
11443 auto *AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
11444
11445 const DataLayout &DL = I.getDataLayout();
11446 SimplifyQuery Q(DL, TLI, DT, AC, &I);
11447 Value *NewV = simplifyInstructionWithOperands(I: &I, NewOps, Q);
11448 if (!NewV || NewV == &I)
11449 return false;
11450
11451 LLVM_DEBUG(dbgs() << "Generic inst " << I << " assumed simplified to "
11452 << *NewV << "\n");
11453 Worklist.push_back(Elt: {.I: {*NewV, II.I.getCtxI()}, .S: II.S});
11454 return true;
11455 }
11456
11457 bool simplifyInstruction(
11458 Attributor &A, Instruction &I, ItemInfo II,
11459 SmallVectorImpl<ItemInfo> &Worklist,
11460 SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
11461 if (auto *CI = dyn_cast<CmpInst>(Val: &I))
11462 return handleCmp(A, Cmp&: *CI, LHS: CI->getOperand(i_nocapture: 0), RHS: CI->getOperand(i_nocapture: 1),
11463 Pred: CI->getPredicate(), II, Worklist);
11464
11465 switch (I.getOpcode()) {
11466 case Instruction::Select:
11467 return handleSelectInst(A, SI&: cast<SelectInst>(Val&: I), II, Worklist);
11468 case Instruction::PHI:
11469 return handlePHINode(A, PHI&: cast<PHINode>(Val&: I), II, Worklist, LivenessAAs);
11470 case Instruction::Load:
11471 return handleLoadInst(A, LI&: cast<LoadInst>(Val&: I), II, Worklist);
11472 default:
11473 return handleGenericInst(A, I, II, Worklist);
11474 };
11475 return false;
11476 }
11477
11478 void genericValueTraversal(Attributor &A, Value *InitialV) {
11479 SmallMapVector<const Function *, LivenessInfo, 4> LivenessAAs;
11480
11481 SmallSet<ItemInfo, 16> Visited;
11482 SmallVector<ItemInfo, 16> Worklist;
11483 Worklist.push_back(Elt: {.I: {*InitialV, getCtxI()}, .S: AA::AnyScope});
11484
11485 int Iteration = 0;
11486 do {
11487 ItemInfo II = Worklist.pop_back_val();
11488 Value *V = II.I.getValue();
11489 assert(V);
11490 const Instruction *CtxI = II.I.getCtxI();
11491 AA::ValueScope S = II.S;
11492
11493 // Check if we should process the current value. To prevent endless
11494 // recursion keep a record of the values we followed!
11495 if (!Visited.insert(V: II).second)
11496 continue;
11497
11498 // Make sure we limit the compile time for complex expressions.
11499 if (Iteration++ >= MaxPotentialValuesIterations) {
11500 LLVM_DEBUG(dbgs() << "Generic value traversal reached iteration limit: "
11501 << Iteration << "!\n");
11502 addValue(A, State&: getState(), V&: *V, CtxI, S, AnchorScope: getAnchorScope());
11503 continue;
11504 }
11505
11506 // Explicitly look through calls with a "returned" attribute if we do
11507 // not have a pointer as stripPointerCasts only works on them.
11508 Value *NewV = nullptr;
11509 if (V->getType()->isPointerTy()) {
11510 NewV = AA::getWithType(V&: *V->stripPointerCasts(), Ty&: *V->getType());
11511 } else {
11512 if (auto *CB = dyn_cast<CallBase>(Val: V))
11513 if (auto *Callee =
11514 dyn_cast_if_present<Function>(Val: CB->getCalledOperand())) {
11515 for (Argument &Arg : Callee->args())
11516 if (Arg.hasReturnedAttr()) {
11517 NewV = CB->getArgOperand(i: Arg.getArgNo());
11518 break;
11519 }
11520 }
11521 }
11522 if (NewV && NewV != V) {
11523 Worklist.push_back(Elt: {.I: {*NewV, CtxI}, .S: S});
11524 continue;
11525 }
11526
11527 if (auto *I = dyn_cast<Instruction>(Val: V)) {
11528 if (simplifyInstruction(A, I&: *I, II, Worklist, LivenessAAs))
11529 continue;
11530 }
11531
11532 if (V != InitialV || isa<Argument>(Val: V))
11533 if (recurseForValue(A, IRP: IRPosition::value(V: *V), S: II.S))
11534 continue;
11535
11536 // If we haven't stripped anything we give up.
11537 if (V == InitialV && CtxI == getCtxI()) {
11538 indicatePessimisticFixpoint();
11539 return;
11540 }
11541
11542 addValue(A, State&: getState(), V&: *V, CtxI, S, AnchorScope: getAnchorScope());
11543 } while (!Worklist.empty());
11544
11545 // If we actually used liveness information so we have to record a
11546 // dependence.
11547 for (auto &It : LivenessAAs)
11548 if (It.second.AnyDead)
11549 A.recordDependence(FromAA: *It.second.LivenessAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
11550 }
11551
11552 /// See AbstractAttribute::trackStatistics()
11553 void trackStatistics() const override {
11554 STATS_DECLTRACK_FLOATING_ATTR(potential_values)
11555 }
11556};
11557
11558struct AAPotentialValuesArgument final : AAPotentialValuesImpl {
11559 using Base = AAPotentialValuesImpl;
11560 AAPotentialValuesArgument(const IRPosition &IRP, Attributor &A)
11561 : Base(IRP, A) {}
11562
11563 /// See AbstractAttribute::initialize(..).
11564 void initialize(Attributor &A) override {
11565 auto &Arg = cast<Argument>(Val&: getAssociatedValue());
11566 if (Arg.hasPointeeInMemoryValueAttr())
11567 indicatePessimisticFixpoint();
11568 }
11569
11570 /// See AbstractAttribute::updateImpl(...).
11571 ChangeStatus updateImpl(Attributor &A) override {
11572 auto AssumedBefore = getAssumed();
11573
11574 unsigned ArgNo = getCalleeArgNo();
11575
11576 bool UsedAssumedInformation = false;
11577 SmallVector<AA::ValueAndContext> Values;
11578 auto CallSitePred = [&](AbstractCallSite ACS) {
11579 const auto CSArgIRP = IRPosition::callsite_argument(ACS, ArgNo);
11580 if (CSArgIRP.getPositionKind() == IRP_INVALID)
11581 return false;
11582
11583 if (!A.getAssumedSimplifiedValues(IRP: CSArgIRP, AA: this, Values,
11584 S: AA::Interprocedural,
11585 UsedAssumedInformation))
11586 return false;
11587
11588 return isValidState();
11589 };
11590
11591 if (!A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this,
11592 /* RequireAllCallSites */ true,
11593 UsedAssumedInformation))
11594 return indicatePessimisticFixpoint();
11595
11596 Function *Fn = getAssociatedFunction();
11597 bool AnyNonLocal = false;
11598 for (auto &It : Values) {
11599 if (isa<Constant>(Val: It.getValue())) {
11600 addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::AnyScope,
11601 AnchorScope: getAnchorScope());
11602 continue;
11603 }
11604 if (!AA::isDynamicallyUnique(A, QueryingAA: *this, V: *It.getValue()))
11605 return indicatePessimisticFixpoint();
11606
11607 if (auto *Arg = dyn_cast<Argument>(Val: It.getValue()))
11608 if (Arg->getParent() == Fn) {
11609 addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::AnyScope,
11610 AnchorScope: getAnchorScope());
11611 continue;
11612 }
11613 addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::Interprocedural,
11614 AnchorScope: getAnchorScope());
11615 AnyNonLocal = true;
11616 }
11617 assert(!undefIsContained() && "Undef should be an explicit value!");
11618 if (AnyNonLocal)
11619 giveUpOnIntraprocedural(A);
11620
11621 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11622 : ChangeStatus::CHANGED;
11623 }
11624
11625 /// See AbstractAttribute::trackStatistics()
11626 void trackStatistics() const override {
11627 STATS_DECLTRACK_ARG_ATTR(potential_values)
11628 }
11629};
11630
11631struct AAPotentialValuesReturned : public AAPotentialValuesFloating {
11632 using Base = AAPotentialValuesFloating;
11633 AAPotentialValuesReturned(const IRPosition &IRP, Attributor &A)
11634 : Base(IRP, A) {}
11635
11636 /// See AbstractAttribute::initialize(..).
11637 void initialize(Attributor &A) override {
11638 Function *F = getAssociatedFunction();
11639 if (!F || F->isDeclaration() || F->getReturnType()->isVoidTy()) {
11640 indicatePessimisticFixpoint();
11641 return;
11642 }
11643
11644 for (Argument &Arg : F->args())
11645 if (Arg.hasReturnedAttr()) {
11646 addValue(A, State&: getState(), V&: Arg, CtxI: nullptr, S: AA::AnyScope, AnchorScope: F);
11647 ReturnedArg = &Arg;
11648 break;
11649 }
11650 if (!A.isFunctionIPOAmendable(F: *F) ||
11651 A.hasSimplificationCallback(IRP: getIRPosition())) {
11652 if (!ReturnedArg)
11653 indicatePessimisticFixpoint();
11654 else
11655 indicateOptimisticFixpoint();
11656 }
11657 }
11658
11659 /// See AbstractAttribute::updateImpl(...).
11660 ChangeStatus updateImpl(Attributor &A) override {
11661 auto AssumedBefore = getAssumed();
11662 bool UsedAssumedInformation = false;
11663
11664 SmallVector<AA::ValueAndContext> Values;
11665 Function *AnchorScope = getAnchorScope();
11666 auto HandleReturnedValue = [&](Value &V, Instruction *CtxI,
11667 bool AddValues) {
11668 for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {
11669 Values.clear();
11670 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V), AA: this, Values, S,
11671 UsedAssumedInformation,
11672 /* RecurseForSelectAndPHI */ true))
11673 return false;
11674 if (!AddValues)
11675 continue;
11676
11677 bool AllInterAreIntra = false;
11678 if (S == AA::Interprocedural)
11679 AllInterAreIntra =
11680 llvm::all_of(Range&: Values, P: [&](const AA::ValueAndContext &VAC) {
11681 return AA::isValidInScope(V: *VAC.getValue(), Scope: AnchorScope);
11682 });
11683
11684 for (const AA::ValueAndContext &VAC : Values) {
11685 addValue(A, State&: getState(), V&: *VAC.getValue(),
11686 CtxI: VAC.getCtxI() ? VAC.getCtxI() : CtxI,
11687 S: AllInterAreIntra ? AA::AnyScope : S, AnchorScope);
11688 }
11689 if (AllInterAreIntra)
11690 break;
11691 }
11692 return true;
11693 };
11694
11695 if (ReturnedArg) {
11696 HandleReturnedValue(*ReturnedArg, nullptr, true);
11697 } else {
11698 auto RetInstPred = [&](Instruction &RetI) {
11699 bool AddValues = true;
11700 if (isa<PHINode>(Val: RetI.getOperand(i: 0)) ||
11701 isa<SelectInst>(Val: RetI.getOperand(i: 0))) {
11702 addValue(A, State&: getState(), V&: *RetI.getOperand(i: 0), CtxI: &RetI, S: AA::AnyScope,
11703 AnchorScope);
11704 AddValues = false;
11705 }
11706 return HandleReturnedValue(*RetI.getOperand(i: 0), &RetI, AddValues);
11707 };
11708
11709 if (!A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
11710 UsedAssumedInformation,
11711 /* CheckBBLivenessOnly */ true))
11712 return indicatePessimisticFixpoint();
11713 }
11714
11715 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11716 : ChangeStatus::CHANGED;
11717 }
11718
11719 ChangeStatus manifest(Attributor &A) override {
11720 if (ReturnedArg)
11721 return ChangeStatus::UNCHANGED;
11722 SmallVector<AA::ValueAndContext> Values;
11723 if (!getAssumedSimplifiedValues(A, Values, S: AA::ValueScope::Intraprocedural,
11724 /* RecurseForSelectAndPHI */ true))
11725 return ChangeStatus::UNCHANGED;
11726 Value *NewVal = getSingleValue(A, AA: *this, IRP: getIRPosition(), Values);
11727 if (!NewVal)
11728 return ChangeStatus::UNCHANGED;
11729
11730 ChangeStatus Changed = ChangeStatus::UNCHANGED;
11731 if (auto *Arg = dyn_cast<Argument>(Val: NewVal)) {
11732 STATS_DECLTRACK(UniqueReturnValue, FunctionReturn,
11733 "Number of function with unique return");
11734 Changed |= A.manifestAttrs(
11735 IRP: IRPosition::argument(Arg: *Arg),
11736 DeducedAttrs: {Attribute::get(Context&: Arg->getContext(), Kind: Attribute::Returned)});
11737 STATS_DECLTRACK_ARG_ATTR(returned);
11738 }
11739
11740 auto RetInstPred = [&](Instruction &RetI) {
11741 Value *RetOp = RetI.getOperand(i: 0);
11742 if (isa<UndefValue>(Val: RetOp) || RetOp == NewVal)
11743 return true;
11744 if (AA::isValidAtPosition(VAC: {*NewVal, RetI}, InfoCache&: A.getInfoCache()))
11745 if (A.changeUseAfterManifest(U&: RetI.getOperandUse(i: 0), NV&: *NewVal))
11746 Changed = ChangeStatus::CHANGED;
11747 return true;
11748 };
11749 bool UsedAssumedInformation = false;
11750 (void)A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
11751 UsedAssumedInformation,
11752 /* CheckBBLivenessOnly */ true);
11753 return Changed;
11754 }
11755
11756 ChangeStatus indicatePessimisticFixpoint() override {
11757 return AAPotentialValues::indicatePessimisticFixpoint();
11758 }
11759
11760 /// See AbstractAttribute::trackStatistics()
11761 void trackStatistics() const override{
11762 STATS_DECLTRACK_FNRET_ATTR(potential_values)}
11763
11764 /// The argumented with an existing `returned` attribute.
11765 Argument *ReturnedArg = nullptr;
11766};
11767
11768struct AAPotentialValuesFunction : AAPotentialValuesImpl {
11769 AAPotentialValuesFunction(const IRPosition &IRP, Attributor &A)
11770 : AAPotentialValuesImpl(IRP, A) {}
11771
11772 /// See AbstractAttribute::updateImpl(...).
11773 ChangeStatus updateImpl(Attributor &A) override {
11774 llvm_unreachable("AAPotentialValues(Function|CallSite)::updateImpl will "
11775 "not be called");
11776 }
11777
11778 /// See AbstractAttribute::trackStatistics()
11779 void trackStatistics() const override {
11780 STATS_DECLTRACK_FN_ATTR(potential_values)
11781 }
11782};
11783
11784struct AAPotentialValuesCallSite : AAPotentialValuesFunction {
11785 AAPotentialValuesCallSite(const IRPosition &IRP, Attributor &A)
11786 : AAPotentialValuesFunction(IRP, A) {}
11787
11788 /// See AbstractAttribute::trackStatistics()
11789 void trackStatistics() const override {
11790 STATS_DECLTRACK_CS_ATTR(potential_values)
11791 }
11792};
11793
11794struct AAPotentialValuesCallSiteReturned : AAPotentialValuesImpl {
11795 AAPotentialValuesCallSiteReturned(const IRPosition &IRP, Attributor &A)
11796 : AAPotentialValuesImpl(IRP, A) {}
11797
11798 /// See AbstractAttribute::updateImpl(...).
11799 ChangeStatus updateImpl(Attributor &A) override {
11800 auto AssumedBefore = getAssumed();
11801
11802 Function *Callee = getAssociatedFunction();
11803 if (!Callee)
11804 return indicatePessimisticFixpoint();
11805
11806 bool UsedAssumedInformation = false;
11807 auto *CB = cast<CallBase>(Val: getCtxI());
11808 if (CB->isMustTailCall() &&
11809 !A.isAssumedDead(IRP: IRPosition::inst(I: *CB), QueryingAA: this, FnLivenessAA: nullptr,
11810 UsedAssumedInformation))
11811 return indicatePessimisticFixpoint();
11812
11813 Function *Caller = CB->getCaller();
11814
11815 auto AddScope = [&](AA::ValueScope S) {
11816 SmallVector<AA::ValueAndContext> Values;
11817 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::returned(F: *Callee), AA: this,
11818 Values, S, UsedAssumedInformation))
11819 return false;
11820
11821 for (auto &It : Values) {
11822 Value *V = It.getValue();
11823 std::optional<Value *> CallerV = A.translateArgumentToCallSiteContent(
11824 V, CB&: *CB, AA: *this, UsedAssumedInformation);
11825 if (!CallerV.has_value()) {
11826 // Nothing to do as long as no value was determined.
11827 continue;
11828 }
11829 V = *CallerV ? *CallerV : V;
11830 if (*CallerV && AA::isDynamicallyUnique(A, QueryingAA: *this, V: *V)) {
11831 if (recurseForValue(A, IRP: IRPosition::value(V: *V), S))
11832 continue;
11833 }
11834 if (S == AA::Intraprocedural && !AA::isValidInScope(V: *V, Scope: Caller)) {
11835 giveUpOnIntraprocedural(A);
11836 return true;
11837 }
11838 addValue(A, State&: getState(), V&: *V, CtxI: CB, S, AnchorScope: getAnchorScope());
11839 }
11840 return true;
11841 };
11842 if (!AddScope(AA::Intraprocedural))
11843 return indicatePessimisticFixpoint();
11844 if (!AddScope(AA::Interprocedural))
11845 return indicatePessimisticFixpoint();
11846 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11847 : ChangeStatus::CHANGED;
11848 }
11849
11850 ChangeStatus indicatePessimisticFixpoint() override {
11851 return AAPotentialValues::indicatePessimisticFixpoint();
11852 }
11853
11854 /// See AbstractAttribute::trackStatistics()
11855 void trackStatistics() const override {
11856 STATS_DECLTRACK_CSRET_ATTR(potential_values)
11857 }
11858};
11859
11860struct AAPotentialValuesCallSiteArgument : AAPotentialValuesFloating {
11861 AAPotentialValuesCallSiteArgument(const IRPosition &IRP, Attributor &A)
11862 : AAPotentialValuesFloating(IRP, A) {}
11863
11864 /// See AbstractAttribute::trackStatistics()
11865 void trackStatistics() const override {
11866 STATS_DECLTRACK_CSARG_ATTR(potential_values)
11867 }
11868};
11869} // namespace
11870
11871/// ---------------------- Assumption Propagation ------------------------------
11872namespace {
11873struct AAAssumptionInfoImpl : public AAAssumptionInfo {
11874 AAAssumptionInfoImpl(const IRPosition &IRP, Attributor &A,
11875 const DenseSet<StringRef> &Known)
11876 : AAAssumptionInfo(IRP, A, Known) {}
11877
11878 /// See AbstractAttribute::manifest(...).
11879 ChangeStatus manifest(Attributor &A) override {
11880 // Don't manifest a universal set if it somehow made it here.
11881 if (getKnown().isUniversal())
11882 return ChangeStatus::UNCHANGED;
11883
11884 const IRPosition &IRP = getIRPosition();
11885 SmallVector<StringRef, 0> Set(getAssumed().getSet().begin(),
11886 getAssumed().getSet().end());
11887 llvm::sort(C&: Set);
11888 return A.manifestAttrs(IRP,
11889 DeducedAttrs: Attribute::get(Context&: IRP.getAnchorValue().getContext(),
11890 Kind: AssumptionAttrKey,
11891 Val: llvm::join(R&: Set, Separator: ",")),
11892 /*ForceReplace=*/true);
11893 }
11894
11895 bool hasAssumption(const StringRef Assumption) const override {
11896 return isValidState() && setContains(Assumption);
11897 }
11898
11899 /// See AbstractAttribute::getAsStr()
11900 const std::string getAsStr(Attributor *A) const override {
11901 const SetContents &Known = getKnown();
11902 const SetContents &Assumed = getAssumed();
11903
11904 SmallVector<StringRef, 0> Set(Known.getSet().begin(), Known.getSet().end());
11905 llvm::sort(C&: Set);
11906 const std::string KnownStr = llvm::join(R&: Set, Separator: ",");
11907
11908 std::string AssumedStr = "Universal";
11909 if (!Assumed.isUniversal()) {
11910 Set.assign(in_start: Assumed.getSet().begin(), in_end: Assumed.getSet().end());
11911 AssumedStr = llvm::join(R&: Set, Separator: ",");
11912 }
11913 return "Known [" + KnownStr + "]," + " Assumed [" + AssumedStr + "]";
11914 }
11915};
11916
11917/// Propagates assumption information from parent functions to all of their
11918/// successors. An assumption can be propagated if the containing function
11919/// dominates the called function.
11920///
11921/// We start with a "known" set of assumptions already valid for the associated
11922/// function and an "assumed" set that initially contains all possible
11923/// assumptions. The assumed set is inter-procedurally updated by narrowing its
11924/// contents as concrete values are known. The concrete values are seeded by the
11925/// first nodes that are either entries into the call graph, or contains no
11926/// assumptions. Each node is updated as the intersection of the assumed state
11927/// with all of its predecessors.
11928struct AAAssumptionInfoFunction final : AAAssumptionInfoImpl {
11929 AAAssumptionInfoFunction(const IRPosition &IRP, Attributor &A)
11930 : AAAssumptionInfoImpl(IRP, A,
11931 getAssumptions(F: *IRP.getAssociatedFunction())) {}
11932
11933 /// See AbstractAttribute::updateImpl(...).
11934 ChangeStatus updateImpl(Attributor &A) override {
11935 bool Changed = false;
11936
11937 auto CallSitePred = [&](AbstractCallSite ACS) {
11938 const auto *AssumptionAA = A.getAAFor<AAAssumptionInfo>(
11939 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *ACS.getInstruction()),
11940 DepClass: DepClassTy::REQUIRED);
11941 if (!AssumptionAA)
11942 return false;
11943 // Get the set of assumptions shared by all of this function's callers.
11944 Changed |= getIntersection(RHS: AssumptionAA->getAssumed());
11945 return !getAssumed().empty() || !getKnown().empty();
11946 };
11947
11948 bool UsedAssumedInformation = false;
11949 // Get the intersection of all assumptions held by this node's predecessors.
11950 // If we don't know all the call sites then this is either an entry into the
11951 // call graph or an empty node. This node is known to only contain its own
11952 // assumptions and can be propagated to its successors.
11953 if (!A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this, RequireAllCallSites: true,
11954 UsedAssumedInformation))
11955 return indicatePessimisticFixpoint();
11956
11957 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
11958 }
11959
11960 void trackStatistics() const override {}
11961};
11962
11963/// Assumption Info defined for call sites.
11964struct AAAssumptionInfoCallSite final : AAAssumptionInfoImpl {
11965
11966 AAAssumptionInfoCallSite(const IRPosition &IRP, Attributor &A)
11967 : AAAssumptionInfoImpl(IRP, A, getInitialAssumptions(IRP)) {}
11968
11969 /// See AbstractAttribute::initialize(...).
11970 void initialize(Attributor &A) override {
11971 const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
11972 A.getAAFor<AAAssumptionInfo>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
11973 }
11974
11975 /// See AbstractAttribute::updateImpl(...).
11976 ChangeStatus updateImpl(Attributor &A) override {
11977 const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
11978 auto *AssumptionAA =
11979 A.getAAFor<AAAssumptionInfo>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
11980 if (!AssumptionAA)
11981 return indicatePessimisticFixpoint();
11982 bool Changed = getIntersection(RHS: AssumptionAA->getAssumed());
11983 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
11984 }
11985
11986 /// See AbstractAttribute::trackStatistics()
11987 void trackStatistics() const override {}
11988
11989private:
11990 /// Helper to initialized the known set as all the assumptions this call and
11991 /// the callee contain.
11992 DenseSet<StringRef> getInitialAssumptions(const IRPosition &IRP) {
11993 const CallBase &CB = cast<CallBase>(Val&: IRP.getAssociatedValue());
11994 auto Assumptions = getAssumptions(CB);
11995 if (const Function *F = CB.getCaller())
11996 set_union(S1&: Assumptions, S2: getAssumptions(F: *F));
11997 if (Function *F = IRP.getAssociatedFunction())
11998 set_union(S1&: Assumptions, S2: getAssumptions(F: *F));
11999 return Assumptions;
12000 }
12001};
12002} // namespace
12003
12004AACallGraphNode *AACallEdgeIterator::operator*() const {
12005 return static_cast<AACallGraphNode *>(const_cast<AACallEdges *>(
12006 A.getOrCreateAAFor<AACallEdges>(IRP: IRPosition::function(F: **I))));
12007}
12008
12009void AttributorCallGraph::print() { llvm::WriteGraph(O&: outs(), G: this); }
12010
12011/// ------------------------ UnderlyingObjects ---------------------------------
12012
12013namespace {
12014struct AAUnderlyingObjectsImpl
12015 : StateWrapper<BooleanState, AAUnderlyingObjects> {
12016 using BaseTy = StateWrapper<BooleanState, AAUnderlyingObjects>;
12017 AAUnderlyingObjectsImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
12018
12019 /// See AbstractAttribute::getAsStr().
12020 const std::string getAsStr(Attributor *A) const override {
12021 if (!isValidState())
12022 return "<invalid>";
12023 std::string Str;
12024 llvm::raw_string_ostream OS(Str);
12025 OS << "underlying objects: inter " << InterAssumedUnderlyingObjects.size()
12026 << " objects, intra " << IntraAssumedUnderlyingObjects.size()
12027 << " objects.\n";
12028 if (!InterAssumedUnderlyingObjects.empty()) {
12029 OS << "inter objects:\n";
12030 for (auto *Obj : InterAssumedUnderlyingObjects)
12031 OS << *Obj << '\n';
12032 }
12033 if (!IntraAssumedUnderlyingObjects.empty()) {
12034 OS << "intra objects:\n";
12035 for (auto *Obj : IntraAssumedUnderlyingObjects)
12036 OS << *Obj << '\n';
12037 }
12038 return Str;
12039 }
12040
12041 /// See AbstractAttribute::trackStatistics()
12042 void trackStatistics() const override {}
12043
12044 /// See AbstractAttribute::updateImpl(...).
12045 ChangeStatus updateImpl(Attributor &A) override {
12046 auto &Ptr = getAssociatedValue();
12047
12048 bool UsedAssumedInformation = false;
12049 auto DoUpdate = [&](SmallSetVector<Value *, 8> &UnderlyingObjects,
12050 AA::ValueScope Scope) {
12051 SmallPtrSet<Value *, 8> SeenObjects;
12052 SmallVector<AA::ValueAndContext> Values;
12053
12054 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: Ptr), AA: *this, Values,
12055 S: Scope, UsedAssumedInformation))
12056 return UnderlyingObjects.insert(X: &Ptr);
12057
12058 bool Changed = false;
12059
12060 for (unsigned I = 0; I < Values.size(); ++I) {
12061 auto &VAC = Values[I];
12062 auto *Obj = VAC.getValue();
12063 Value *UO = getUnderlyingObject(V: Obj);
12064 if (!SeenObjects.insert(Ptr: UO ? UO : Obj).second)
12065 continue;
12066 if (UO && UO != Obj) {
12067 if (isa<AllocaInst>(Val: UO) || isa<GlobalValue>(Val: UO)) {
12068 Changed |= UnderlyingObjects.insert(X: UO);
12069 continue;
12070 }
12071
12072 const auto *OtherAA = A.getAAFor<AAUnderlyingObjects>(
12073 QueryingAA: *this, IRP: IRPosition::value(V: *UO), DepClass: DepClassTy::OPTIONAL);
12074 auto Pred = [&](Value &V) {
12075 if (&V == UO)
12076 Changed |= UnderlyingObjects.insert(X: UO);
12077 else
12078 Values.emplace_back(Args&: V, Args: nullptr);
12079 return true;
12080 };
12081
12082 if (!OtherAA || !OtherAA->forallUnderlyingObjects(Pred, Scope))
12083 llvm_unreachable(
12084 "The forall call should not return false at this position");
12085 UsedAssumedInformation |= !OtherAA->getState().isAtFixpoint();
12086 continue;
12087 }
12088
12089 if (isa<SelectInst>(Val: Obj)) {
12090 Changed |= handleIndirect(A, V&: *Obj, UnderlyingObjects, Scope,
12091 UsedAssumedInformation);
12092 continue;
12093 }
12094 if (auto *PHI = dyn_cast<PHINode>(Val: Obj)) {
12095 // Explicitly look through PHIs as we do not care about dynamically
12096 // uniqueness.
12097 for (unsigned u = 0, e = PHI->getNumIncomingValues(); u < e; u++) {
12098 Changed |=
12099 handleIndirect(A, V&: *PHI->getIncomingValue(i: u), UnderlyingObjects,
12100 Scope, UsedAssumedInformation);
12101 }
12102 continue;
12103 }
12104
12105 Changed |= UnderlyingObjects.insert(X: Obj);
12106 }
12107
12108 return Changed;
12109 };
12110
12111 bool Changed = false;
12112 Changed |= DoUpdate(IntraAssumedUnderlyingObjects, AA::Intraprocedural);
12113 Changed |= DoUpdate(InterAssumedUnderlyingObjects, AA::Interprocedural);
12114 if (!UsedAssumedInformation)
12115 indicateOptimisticFixpoint();
12116 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
12117 }
12118
12119 bool forallUnderlyingObjects(
12120 function_ref<bool(Value &)> Pred,
12121 AA::ValueScope Scope = AA::Interprocedural) const override {
12122 if (!isValidState())
12123 return Pred(getAssociatedValue());
12124
12125 auto &AssumedUnderlyingObjects = Scope == AA::Intraprocedural
12126 ? IntraAssumedUnderlyingObjects
12127 : InterAssumedUnderlyingObjects;
12128 for (Value *Obj : AssumedUnderlyingObjects)
12129 if (!Pred(*Obj))
12130 return false;
12131
12132 return true;
12133 }
12134
12135private:
12136 /// Handle the case where the value is not the actual underlying value, such
12137 /// as a phi node or a select instruction.
12138 bool handleIndirect(Attributor &A, Value &V,
12139 SmallSetVector<Value *, 8> &UnderlyingObjects,
12140 AA::ValueScope Scope, bool &UsedAssumedInformation) {
12141 bool Changed = false;
12142 const auto *AA = A.getAAFor<AAUnderlyingObjects>(
12143 QueryingAA: *this, IRP: IRPosition::value(V), DepClass: DepClassTy::OPTIONAL);
12144 auto Pred = [&](Value &V) {
12145 Changed |= UnderlyingObjects.insert(X: &V);
12146 return true;
12147 };
12148 if (!AA || !AA->forallUnderlyingObjects(Pred, Scope))
12149 llvm_unreachable(
12150 "The forall call should not return false at this position");
12151 UsedAssumedInformation |= !AA->getState().isAtFixpoint();
12152 return Changed;
12153 }
12154
12155 /// All the underlying objects collected so far via intra procedural scope.
12156 SmallSetVector<Value *, 8> IntraAssumedUnderlyingObjects;
12157 /// All the underlying objects collected so far via inter procedural scope.
12158 SmallSetVector<Value *, 8> InterAssumedUnderlyingObjects;
12159};
12160
12161struct AAUnderlyingObjectsFloating final : AAUnderlyingObjectsImpl {
12162 AAUnderlyingObjectsFloating(const IRPosition &IRP, Attributor &A)
12163 : AAUnderlyingObjectsImpl(IRP, A) {}
12164};
12165
12166struct AAUnderlyingObjectsArgument final : AAUnderlyingObjectsImpl {
12167 AAUnderlyingObjectsArgument(const IRPosition &IRP, Attributor &A)
12168 : AAUnderlyingObjectsImpl(IRP, A) {}
12169};
12170
12171struct AAUnderlyingObjectsCallSite final : AAUnderlyingObjectsImpl {
12172 AAUnderlyingObjectsCallSite(const IRPosition &IRP, Attributor &A)
12173 : AAUnderlyingObjectsImpl(IRP, A) {}
12174};
12175
12176struct AAUnderlyingObjectsCallSiteArgument final : AAUnderlyingObjectsImpl {
12177 AAUnderlyingObjectsCallSiteArgument(const IRPosition &IRP, Attributor &A)
12178 : AAUnderlyingObjectsImpl(IRP, A) {}
12179};
12180
12181struct AAUnderlyingObjectsReturned final : AAUnderlyingObjectsImpl {
12182 AAUnderlyingObjectsReturned(const IRPosition &IRP, Attributor &A)
12183 : AAUnderlyingObjectsImpl(IRP, A) {}
12184};
12185
12186struct AAUnderlyingObjectsCallSiteReturned final : AAUnderlyingObjectsImpl {
12187 AAUnderlyingObjectsCallSiteReturned(const IRPosition &IRP, Attributor &A)
12188 : AAUnderlyingObjectsImpl(IRP, A) {}
12189};
12190
12191struct AAUnderlyingObjectsFunction final : AAUnderlyingObjectsImpl {
12192 AAUnderlyingObjectsFunction(const IRPosition &IRP, Attributor &A)
12193 : AAUnderlyingObjectsImpl(IRP, A) {}
12194};
12195} // namespace
12196
12197/// ------------------------ Global Value Info -------------------------------
12198namespace {
12199struct AAGlobalValueInfoFloating : public AAGlobalValueInfo {
12200 AAGlobalValueInfoFloating(const IRPosition &IRP, Attributor &A)
12201 : AAGlobalValueInfo(IRP, A) {}
12202
12203 /// See AbstractAttribute::initialize(...).
12204 void initialize(Attributor &A) override {}
12205
12206 bool checkUse(Attributor &A, const Use &U, bool &Follow,
12207 SmallVectorImpl<const Value *> &Worklist) {
12208 Instruction *UInst = dyn_cast<Instruction>(Val: U.getUser());
12209 if (!UInst) {
12210 Follow = true;
12211 return true;
12212 }
12213
12214 LLVM_DEBUG(dbgs() << "[AAGlobalValueInfo] Check use: " << *U.get() << " in "
12215 << *UInst << "\n");
12216
12217 if (auto *Cmp = dyn_cast<ICmpInst>(Val: U.getUser())) {
12218 int Idx = &Cmp->getOperandUse(i: 0) == &U;
12219 if (isa<Constant>(Val: Cmp->getOperand(i_nocapture: Idx)))
12220 return true;
12221 return U == &getAnchorValue();
12222 }
12223
12224 // Explicitly catch return instructions.
12225 if (isa<ReturnInst>(Val: UInst)) {
12226 auto CallSitePred = [&](AbstractCallSite ACS) {
12227 Worklist.push_back(Elt: ACS.getInstruction());
12228 return true;
12229 };
12230 bool UsedAssumedInformation = false;
12231 // TODO: We should traverse the uses or add a "non-call-site" CB.
12232 if (!A.checkForAllCallSites(Pred: CallSitePred, Fn: *UInst->getFunction(),
12233 /*RequireAllCallSites=*/true, QueryingAA: this,
12234 UsedAssumedInformation))
12235 return false;
12236 return true;
12237 }
12238
12239 // For now we only use special logic for call sites. However, the tracker
12240 // itself knows about a lot of other non-capturing cases already.
12241 auto *CB = dyn_cast<CallBase>(Val: UInst);
12242 if (!CB)
12243 return false;
12244 // Direct calls are OK uses.
12245 if (CB->isCallee(U: &U))
12246 return true;
12247 // Non-argument uses are scary.
12248 if (!CB->isArgOperand(U: &U))
12249 return false;
12250 // TODO: Iterate callees.
12251 auto *Fn = dyn_cast<Function>(Val: CB->getCalledOperand());
12252 if (!Fn || !A.isFunctionIPOAmendable(F: *Fn))
12253 return false;
12254
12255 unsigned ArgNo = CB->getArgOperandNo(U: &U);
12256 Worklist.push_back(Elt: Fn->getArg(i: ArgNo));
12257 return true;
12258 }
12259
12260 ChangeStatus updateImpl(Attributor &A) override {
12261 unsigned NumUsesBefore = Uses.size();
12262
12263 SmallPtrSet<const Value *, 8> Visited;
12264 SmallVector<const Value *> Worklist;
12265 Worklist.push_back(Elt: &getAnchorValue());
12266
12267 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
12268 Uses.insert(Ptr: &U);
12269 // TODO(captures): Make this more precise.
12270 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
12271 if (CI.isPassthrough()) {
12272 Follow = true;
12273 return true;
12274 }
12275 return checkUse(A, U, Follow, Worklist);
12276 };
12277 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
12278 Uses.insert(Ptr: &OldU);
12279 return true;
12280 };
12281
12282 while (!Worklist.empty()) {
12283 const Value *V = Worklist.pop_back_val();
12284 if (!Visited.insert(Ptr: V).second)
12285 continue;
12286 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: *V,
12287 /* CheckBBLivenessOnly */ true,
12288 LivenessDepClass: DepClassTy::OPTIONAL,
12289 /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
12290 return indicatePessimisticFixpoint();
12291 }
12292 }
12293
12294 return Uses.size() == NumUsesBefore ? ChangeStatus::UNCHANGED
12295 : ChangeStatus::CHANGED;
12296 }
12297
12298 bool isPotentialUse(const Use &U) const override {
12299 return !isValidState() || Uses.contains(Ptr: &U);
12300 }
12301
12302 /// See AbstractAttribute::manifest(...).
12303 ChangeStatus manifest(Attributor &A) override {
12304 return ChangeStatus::UNCHANGED;
12305 }
12306
12307 /// See AbstractAttribute::getAsStr().
12308 const std::string getAsStr(Attributor *A) const override {
12309 return "[" + std::to_string(val: Uses.size()) + " uses]";
12310 }
12311
12312 void trackStatistics() const override {
12313 STATS_DECLTRACK_FLOATING_ATTR(GlobalValuesTracked);
12314 }
12315
12316private:
12317 /// Set of (transitive) uses of this GlobalValue.
12318 SmallPtrSet<const Use *, 8> Uses;
12319};
12320} // namespace
12321
12322/// ------------------------ Indirect Call Info -------------------------------
12323namespace {
12324struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo {
12325 AAIndirectCallInfoCallSite(const IRPosition &IRP, Attributor &A)
12326 : AAIndirectCallInfo(IRP, A) {}
12327
12328 /// See AbstractAttribute::initialize(...).
12329 void initialize(Attributor &A) override {
12330 auto *MD = getCtxI()->getMetadata(KindID: LLVMContext::MD_callees);
12331 if (!MD && !A.isClosedWorldModule())
12332 return;
12333
12334 if (MD) {
12335 for (const auto &Op : MD->operands())
12336 if (Function *Callee = mdconst::dyn_extract_or_null<Function>(MD: Op))
12337 PotentialCallees.insert(X: Callee);
12338 } else if (A.isClosedWorldModule()) {
12339 ArrayRef<Function *> IndirectlyCallableFunctions =
12340 A.getInfoCache().getIndirectlyCallableFunctions(A);
12341 PotentialCallees.insert_range(R&: IndirectlyCallableFunctions);
12342 }
12343
12344 if (PotentialCallees.empty())
12345 indicateOptimisticFixpoint();
12346 }
12347
12348 ChangeStatus updateImpl(Attributor &A) override {
12349 CallBase *CB = cast<CallBase>(Val: getCtxI());
12350 const Use &CalleeUse = CB->getCalledOperandUse();
12351 Value *FP = CB->getCalledOperand();
12352
12353 SmallSetVector<Function *, 4> AssumedCalleesNow;
12354 bool AllCalleesKnownNow = AllCalleesKnown;
12355
12356 auto CheckPotentialCalleeUse = [&](Function &PotentialCallee,
12357 bool &UsedAssumedInformation) {
12358 const auto *GIAA = A.getAAFor<AAGlobalValueInfo>(
12359 QueryingAA: *this, IRP: IRPosition::value(V: PotentialCallee), DepClass: DepClassTy::OPTIONAL);
12360 if (!GIAA || GIAA->isPotentialUse(U: CalleeUse))
12361 return true;
12362 UsedAssumedInformation = !GIAA->isAtFixpoint();
12363 return false;
12364 };
12365
12366 auto AddPotentialCallees = [&]() {
12367 for (auto *PotentialCallee : PotentialCallees) {
12368 bool UsedAssumedInformation = false;
12369 if (CheckPotentialCalleeUse(*PotentialCallee, UsedAssumedInformation))
12370 AssumedCalleesNow.insert(X: PotentialCallee);
12371 }
12372 };
12373
12374 // Use simplification to find potential callees, if !callees was present,
12375 // fallback to that set if necessary.
12376 bool UsedAssumedInformation = false;
12377 SmallVector<AA::ValueAndContext> Values;
12378 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *FP), AA: this, Values,
12379 S: AA::ValueScope::AnyScope,
12380 UsedAssumedInformation)) {
12381 if (PotentialCallees.empty())
12382 return indicatePessimisticFixpoint();
12383 AddPotentialCallees();
12384 }
12385
12386 // Try to find a reason for \p Fn not to be a potential callee. If none was
12387 // found, add it to the assumed callees set.
12388 auto CheckPotentialCallee = [&](Function &Fn) {
12389 if (!PotentialCallees.empty() && !PotentialCallees.count(key: &Fn))
12390 return false;
12391
12392 auto &CachedResult = FilterResults[&Fn];
12393 if (CachedResult.has_value())
12394 return CachedResult.value();
12395
12396 bool UsedAssumedInformation = false;
12397 if (!CheckPotentialCalleeUse(Fn, UsedAssumedInformation)) {
12398 if (!UsedAssumedInformation)
12399 CachedResult = false;
12400 return false;
12401 }
12402
12403 int NumFnArgs = Fn.arg_size();
12404 int NumCBArgs = CB->arg_size();
12405
12406 // Check if any excess argument (which we fill up with poison) is known to
12407 // be UB on undef.
12408 for (int I = NumCBArgs; I < NumFnArgs; ++I) {
12409 bool IsKnown = false;
12410 if (AA::hasAssumedIRAttr<Attribute::NoUndef>(
12411 A, QueryingAA: this, IRP: IRPosition::argument(Arg: *Fn.getArg(i: I)),
12412 DepClass: DepClassTy::OPTIONAL, IsKnown)) {
12413 if (IsKnown)
12414 CachedResult = false;
12415 return false;
12416 }
12417 }
12418
12419 CachedResult = true;
12420 return true;
12421 };
12422
12423 // Check simplification result, prune known UB callees, also restrict it to
12424 // the !callees set, if present.
12425 for (auto &VAC : Values) {
12426 if (isa<UndefValue>(Val: VAC.getValue()))
12427 continue;
12428 if (isa<ConstantPointerNull>(Val: VAC.getValue()) &&
12429 VAC.getValue()->getType()->getPointerAddressSpace() == 0)
12430 continue;
12431 // TODO: Check for known UB, e.g., poison + noundef.
12432 if (auto *VACFn = dyn_cast<Function>(Val: VAC.getValue())) {
12433 if (CheckPotentialCallee(*VACFn))
12434 AssumedCalleesNow.insert(X: VACFn);
12435 continue;
12436 }
12437 if (!PotentialCallees.empty()) {
12438 AddPotentialCallees();
12439 break;
12440 }
12441 AllCalleesKnownNow = false;
12442 }
12443
12444 if (AssumedCalleesNow == AssumedCallees &&
12445 AllCalleesKnown == AllCalleesKnownNow)
12446 return ChangeStatus::UNCHANGED;
12447
12448 std::swap(LHS&: AssumedCallees, RHS&: AssumedCalleesNow);
12449 AllCalleesKnown = AllCalleesKnownNow;
12450 return ChangeStatus::CHANGED;
12451 }
12452
12453 /// See AbstractAttribute::manifest(...).
12454 ChangeStatus manifest(Attributor &A) override {
12455 // If we can't specialize at all, give up now.
12456 if (!AllCalleesKnown && AssumedCallees.empty())
12457 return ChangeStatus::UNCHANGED;
12458
12459 CallBase *CB = cast<CallBase>(Val: getCtxI());
12460 bool UsedAssumedInformation = false;
12461 if (A.isAssumedDead(I: *CB, QueryingAA: this, /*LivenessAA=*/nullptr,
12462 UsedAssumedInformation))
12463 return ChangeStatus::UNCHANGED;
12464
12465 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12466 Value *FP = CB->getCalledOperand();
12467 if (FP->getType()->getPointerAddressSpace())
12468 FP = new AddrSpaceCastInst(FP, PointerType::get(C&: FP->getContext(), AddressSpace: 0),
12469 FP->getName() + ".as0", CB->getIterator());
12470
12471 bool CBIsVoid = CB->getType()->isVoidTy();
12472 BasicBlock::iterator IP = CB->getIterator();
12473 FunctionType *CSFT = CB->getFunctionType();
12474 SmallVector<Value *> CSArgs(CB->args());
12475
12476 // If we know all callees and there are none, the call site is (effectively)
12477 // dead (or UB).
12478 if (AssumedCallees.empty()) {
12479 assert(AllCalleesKnown &&
12480 "Expected all callees to be known if there are none.");
12481 A.changeToUnreachableAfterManifest(I: CB);
12482 return ChangeStatus::CHANGED;
12483 }
12484
12485 // Special handling for the single callee case.
12486 if (AllCalleesKnown && AssumedCallees.size() == 1) {
12487 auto *NewCallee = AssumedCallees.front();
12488 if (isLegalToPromote(CB: *CB, Callee: NewCallee)) {
12489 promoteCall(CB&: *CB, Callee: NewCallee, RetBitCast: nullptr);
12490 NumIndirectCallsPromoted++;
12491 return ChangeStatus::CHANGED;
12492 }
12493 Instruction *NewCall =
12494 CallInst::Create(Func: FunctionCallee(CSFT, NewCallee), Args: CSArgs,
12495 NameStr: CB->getName(), InsertBefore: CB->getIterator());
12496 if (!CBIsVoid)
12497 A.changeAfterManifest(IRP: IRPosition::callsite_returned(CB: *CB), NV&: *NewCall);
12498 A.deleteAfterManifest(I&: *CB);
12499 return ChangeStatus::CHANGED;
12500 }
12501
12502 // For each potential value we create a conditional
12503 //
12504 // ```
12505 // if (ptr == value) value(args);
12506 // else ...
12507 // ```
12508 //
12509 bool SpecializedForAnyCallees = false;
12510 bool SpecializedForAllCallees = AllCalleesKnown;
12511 ICmpInst *LastCmp = nullptr;
12512 SmallVector<Function *, 8> SkippedAssumedCallees;
12513 SmallVector<std::pair<CallInst *, Instruction *>> NewCalls;
12514 for (Function *NewCallee : AssumedCallees) {
12515 if (!A.shouldSpecializeCallSiteForCallee(AA: *this, CB&: *CB, Callee&: *NewCallee,
12516 NumAssumedCallees: AssumedCallees.size())) {
12517 SkippedAssumedCallees.push_back(Elt: NewCallee);
12518 SpecializedForAllCallees = false;
12519 continue;
12520 }
12521 SpecializedForAnyCallees = true;
12522
12523 LastCmp = new ICmpInst(IP, llvm::CmpInst::ICMP_EQ, FP, NewCallee);
12524 Instruction *ThenTI =
12525 SplitBlockAndInsertIfThen(Cond: LastCmp, SplitBefore: IP, /* Unreachable */ false);
12526 BasicBlock *CBBB = CB->getParent();
12527 A.registerManifestAddedBasicBlock(BB&: *ThenTI->getParent());
12528 A.registerManifestAddedBasicBlock(BB&: *IP->getParent());
12529 auto *SplitTI = cast<CondBrInst>(Val: LastCmp->getNextNode());
12530 BasicBlock *ElseBB;
12531 if (&*IP == CB) {
12532 ElseBB = BasicBlock::Create(Context&: ThenTI->getContext(), Name: "",
12533 Parent: ThenTI->getFunction(), InsertBefore: CBBB);
12534 A.registerManifestAddedBasicBlock(BB&: *ElseBB);
12535 IP = UncondBrInst::Create(IfTrue: CBBB, InsertBefore: ElseBB)->getIterator();
12536 SplitTI->replaceUsesOfWith(From: CBBB, To: ElseBB);
12537 } else {
12538 ElseBB = IP->getParent();
12539 ThenTI->replaceUsesOfWith(From: ElseBB, To: CBBB);
12540 }
12541 CastInst *RetBC = nullptr;
12542 CallInst *NewCall = nullptr;
12543 if (isLegalToPromote(CB: *CB, Callee: NewCallee)) {
12544 auto *CBClone = cast<CallBase>(Val: CB->clone());
12545 CBClone->insertBefore(InsertPos: ThenTI->getIterator());
12546 NewCall = &cast<CallInst>(Val&: promoteCall(CB&: *CBClone, Callee: NewCallee, RetBitCast: &RetBC));
12547 NumIndirectCallsPromoted++;
12548 } else {
12549 NewCall = CallInst::Create(Func: FunctionCallee(CSFT, NewCallee), Args: CSArgs,
12550 NameStr: CB->getName(), InsertBefore: ThenTI->getIterator());
12551 }
12552 NewCalls.push_back(Elt: {NewCall, RetBC});
12553 }
12554
12555 auto AttachCalleeMetadata = [&](CallBase &IndirectCB) {
12556 if (!AllCalleesKnown)
12557 return ChangeStatus::UNCHANGED;
12558 MDBuilder MDB(IndirectCB.getContext());
12559 MDNode *Callees = MDB.createCallees(Callees: SkippedAssumedCallees);
12560 IndirectCB.setMetadata(KindID: LLVMContext::MD_callees, Node: Callees);
12561 return ChangeStatus::CHANGED;
12562 };
12563
12564 if (!SpecializedForAnyCallees)
12565 return AttachCalleeMetadata(*CB);
12566
12567 // Check if we need the fallback indirect call still.
12568 if (SpecializedForAllCallees) {
12569 LastCmp->replaceAllUsesWith(V: ConstantInt::getTrue(Context&: LastCmp->getContext()));
12570 LastCmp->eraseFromParent();
12571 new UnreachableInst(IP->getContext(), IP);
12572 IP->eraseFromParent();
12573 } else {
12574 auto *CBClone = cast<CallInst>(Val: CB->clone());
12575 CBClone->setName(CB->getName());
12576 CBClone->insertBefore(BB&: *IP->getParent(), InsertPos: IP);
12577 NewCalls.push_back(Elt: {CBClone, nullptr});
12578 AttachCalleeMetadata(*CBClone);
12579 }
12580
12581 // Check if we need a PHI to merge the results.
12582 if (!CBIsVoid) {
12583 auto *PHI = PHINode::Create(Ty: CB->getType(), NumReservedValues: NewCalls.size(),
12584 NameStr: CB->getName() + ".phi",
12585 InsertBefore: CB->getParent()->getFirstInsertionPt());
12586 for (auto &It : NewCalls) {
12587 CallBase *NewCall = It.first;
12588 Instruction *CallRet = It.second ? It.second : It.first;
12589 if (CallRet->getType() == CB->getType())
12590 PHI->addIncoming(V: CallRet, BB: CallRet->getParent());
12591 else if (NewCall->getType()->isVoidTy())
12592 PHI->addIncoming(V: PoisonValue::get(T: CB->getType()),
12593 BB: NewCall->getParent());
12594 else
12595 llvm_unreachable("Call return should match or be void!");
12596 }
12597 A.changeAfterManifest(IRP: IRPosition::callsite_returned(CB: *CB), NV&: *PHI);
12598 }
12599
12600 A.deleteAfterManifest(I&: *CB);
12601 Changed = ChangeStatus::CHANGED;
12602
12603 return Changed;
12604 }
12605
12606 /// See AbstractAttribute::getAsStr().
12607 const std::string getAsStr(Attributor *A) const override {
12608 return std::string(AllCalleesKnown ? "eliminate" : "specialize") +
12609 " indirect call site with " + std::to_string(val: AssumedCallees.size()) +
12610 " functions";
12611 }
12612
12613 void trackStatistics() const override {
12614 if (AllCalleesKnown) {
12615 STATS_DECLTRACK(
12616 Eliminated, CallSites,
12617 "Number of indirect call sites eliminated via specialization")
12618 } else {
12619 STATS_DECLTRACK(Specialized, CallSites,
12620 "Number of indirect call sites specialized")
12621 }
12622 }
12623
12624 bool foreachCallee(function_ref<bool(Function *)> CB) const override {
12625 return isValidState() && AllCalleesKnown && all_of(Range: AssumedCallees, P: CB);
12626 }
12627
12628private:
12629 /// Map to remember filter results.
12630 DenseMap<Function *, std::optional<bool>> FilterResults;
12631
12632 /// If the !callee metadata was present, this set will contain all potential
12633 /// callees (superset).
12634 SmallSetVector<Function *, 4> PotentialCallees;
12635
12636 /// This set contains all currently assumed calllees, which might grow over
12637 /// time.
12638 SmallSetVector<Function *, 4> AssumedCallees;
12639
12640 /// Flag to indicate if all possible callees are in the AssumedCallees set or
12641 /// if there could be others.
12642 bool AllCalleesKnown = true;
12643};
12644} // namespace
12645
12646/// --------------------- Invariant Load Pointer -------------------------------
12647namespace {
12648
12649struct AAInvariantLoadPointerImpl
12650 : public StateWrapper<BitIntegerState<uint8_t, 15>,
12651 AAInvariantLoadPointer> {
12652
12653 enum {
12654 // pointer does not alias within the bounds of the function
12655 IS_NOALIAS = 1 << 0,
12656 // pointer is not involved in any effectful instructions within the bounds
12657 // of the function
12658 IS_NOEFFECT = 1 << 1,
12659 // loads are invariant within the bounds of the function
12660 IS_LOCALLY_INVARIANT = 1 << 2,
12661 // memory lifetime is constrained within the bounds of the function
12662 IS_LOCALLY_CONSTRAINED = 1 << 3,
12663
12664 IS_BEST_STATE = IS_NOALIAS | IS_NOEFFECT | IS_LOCALLY_INVARIANT |
12665 IS_LOCALLY_CONSTRAINED,
12666 };
12667 static_assert(getBestState() == IS_BEST_STATE, "Unexpected best state");
12668
12669 using Base =
12670 StateWrapper<BitIntegerState<uint8_t, 15>, AAInvariantLoadPointer>;
12671
12672 // the BitIntegerState is optimistic about IS_NOALIAS and IS_NOEFFECT, but
12673 // pessimistic about IS_KNOWN_INVARIANT
12674 AAInvariantLoadPointerImpl(const IRPosition &IRP, Attributor &A)
12675 : Base(IRP) {}
12676
12677 bool isKnownInvariant() const final {
12678 return isKnownLocallyInvariant() && isKnown(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12679 }
12680
12681 bool isKnownLocallyInvariant() const final {
12682 if (isKnown(BitsEncoding: IS_LOCALLY_INVARIANT))
12683 return true;
12684 return isKnown(BitsEncoding: IS_NOALIAS | IS_NOEFFECT);
12685 }
12686
12687 bool isAssumedInvariant() const final {
12688 return isAssumedLocallyInvariant() && isAssumed(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12689 }
12690
12691 bool isAssumedLocallyInvariant() const final {
12692 if (isAssumed(BitsEncoding: IS_LOCALLY_INVARIANT))
12693 return true;
12694 return isAssumed(BitsEncoding: IS_NOALIAS | IS_NOEFFECT);
12695 }
12696
12697 ChangeStatus updateImpl(Attributor &A) override {
12698 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12699
12700 Changed |= updateNoAlias(A);
12701 if (requiresNoAlias() && !isAssumed(BitsEncoding: IS_NOALIAS))
12702 return indicatePessimisticFixpoint();
12703
12704 Changed |= updateNoEffect(A);
12705
12706 Changed |= updateLocalInvariance(A);
12707
12708 return Changed;
12709 }
12710
12711 ChangeStatus manifest(Attributor &A) override {
12712 if (!isKnownInvariant())
12713 return ChangeStatus::UNCHANGED;
12714
12715 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12716 const Value *Ptr = &getAssociatedValue();
12717 const auto TagInvariantLoads = [&](const Use &U, bool &) {
12718 if (U.get() != Ptr)
12719 return true;
12720 auto *I = dyn_cast<Instruction>(Val: U.getUser());
12721 if (!I)
12722 return true;
12723
12724 // Ensure that we are only changing uses from the corresponding callgraph
12725 // SSC in the case that the AA isn't run on the entire module
12726 if (!A.isRunOn(Fn: I->getFunction()))
12727 return true;
12728
12729 if (I->hasMetadata(KindID: LLVMContext::MD_invariant_load))
12730 return true;
12731
12732 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
12733 LI->setMetadata(KindID: LLVMContext::MD_invariant_load,
12734 Node: MDNode::get(Context&: LI->getContext(), MDs: {}));
12735 Changed = ChangeStatus::CHANGED;
12736 }
12737 return true;
12738 };
12739
12740 (void)A.checkForAllUses(Pred: TagInvariantLoads, QueryingAA: *this, V: *Ptr);
12741 return Changed;
12742 }
12743
12744 /// See AbstractAttribute::getAsStr().
12745 const std::string getAsStr(Attributor *) const override {
12746 if (isKnownInvariant())
12747 return "load-invariant pointer";
12748 return "non-invariant pointer";
12749 }
12750
12751 /// See AbstractAttribute::trackStatistics().
12752 void trackStatistics() const override {}
12753
12754private:
12755 /// Indicate that noalias is required for the pointer to be invariant.
12756 bool requiresNoAlias() const {
12757 switch (getPositionKind()) {
12758 default:
12759 // Conservatively default to require noalias.
12760 return true;
12761 case IRP_FLOAT:
12762 case IRP_RETURNED:
12763 case IRP_CALL_SITE:
12764 return false;
12765 case IRP_CALL_SITE_RETURNED: {
12766 const auto &CB = cast<CallBase>(Val&: getAnchorValue());
12767 return !isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
12768 Call: &CB, /*MustPreserveNullness=*/false);
12769 }
12770 case IRP_ARGUMENT: {
12771 const Function *F = getAssociatedFunction();
12772 assert(F && "no associated function for argument");
12773 return !isCallableCC(CC: F->getCallingConv());
12774 }
12775 }
12776 }
12777
12778 bool isExternal() const {
12779 const Function *F = getAssociatedFunction();
12780 if (!F)
12781 return true;
12782 return isCallableCC(CC: F->getCallingConv()) &&
12783 getPositionKind() != IRP_CALL_SITE_RETURNED;
12784 }
12785
12786 ChangeStatus updateNoAlias(Attributor &A) {
12787 if (isKnown(BitsEncoding: IS_NOALIAS) || !isAssumed(BitsEncoding: IS_NOALIAS))
12788 return ChangeStatus::UNCHANGED;
12789
12790 // Try to use AANoAlias.
12791 if (const auto *ANoAlias = A.getOrCreateAAFor<AANoAlias>(
12792 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED)) {
12793 if (ANoAlias->isKnownNoAlias()) {
12794 addKnownBits(Bits: IS_NOALIAS);
12795 return ChangeStatus::CHANGED;
12796 }
12797
12798 if (!ANoAlias->isAssumedNoAlias()) {
12799 removeAssumedBits(BitsEncoding: IS_NOALIAS);
12800 return ChangeStatus::CHANGED;
12801 }
12802
12803 return ChangeStatus::UNCHANGED;
12804 }
12805
12806 // Try to infer noalias from argument attribute, since it is applicable for
12807 // the duration of the function.
12808 if (const Argument *Arg = getAssociatedArgument()) {
12809 if (Arg->hasNoAliasAttr()) {
12810 addKnownBits(Bits: IS_NOALIAS);
12811 return ChangeStatus::UNCHANGED;
12812 }
12813
12814 // Noalias information is not provided, and cannot be inferred,
12815 // so we conservatively assume the pointer aliases.
12816 removeAssumedBits(BitsEncoding: IS_NOALIAS);
12817 return ChangeStatus::CHANGED;
12818 }
12819
12820 return ChangeStatus::UNCHANGED;
12821 }
12822
12823 ChangeStatus updateNoEffect(Attributor &A) {
12824 if (isKnown(BitsEncoding: IS_NOEFFECT) || !isAssumed(BitsEncoding: IS_NOEFFECT))
12825 return ChangeStatus::UNCHANGED;
12826
12827 if (!getAssociatedFunction())
12828 return indicatePessimisticFixpoint();
12829
12830 if (isa<AllocaInst>(Val: &getAssociatedValue()))
12831 return indicatePessimisticFixpoint();
12832
12833 const auto HasNoEffectLoads = [&](const Use &U, bool &) {
12834 const auto *LI = dyn_cast<LoadInst>(Val: U.getUser());
12835 return !LI || !LI->mayHaveSideEffects();
12836 };
12837 if (!A.checkForAllUses(Pred: HasNoEffectLoads, QueryingAA: *this, V: getAssociatedValue()))
12838 return indicatePessimisticFixpoint();
12839
12840 if (const auto *AMemoryBehavior = A.getOrCreateAAFor<AAMemoryBehavior>(
12841 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED)) {
12842 // For non-instructions, try to use AAMemoryBehavior to infer the readonly
12843 // attribute
12844 if (!AMemoryBehavior->isAssumedReadOnly())
12845 return indicatePessimisticFixpoint();
12846
12847 if (AMemoryBehavior->isKnownReadOnly()) {
12848 addKnownBits(Bits: IS_NOEFFECT);
12849 return ChangeStatus::UNCHANGED;
12850 }
12851
12852 return ChangeStatus::UNCHANGED;
12853 }
12854
12855 if (const Argument *Arg = getAssociatedArgument()) {
12856 if (Arg->onlyReadsMemory()) {
12857 addKnownBits(Bits: IS_NOEFFECT);
12858 return ChangeStatus::UNCHANGED;
12859 }
12860
12861 // Readonly information is not provided, and cannot be inferred from
12862 // AAMemoryBehavior.
12863 return indicatePessimisticFixpoint();
12864 }
12865
12866 return ChangeStatus::UNCHANGED;
12867 }
12868
12869 ChangeStatus updateLocalInvariance(Attributor &A) {
12870 if (isKnown(BitsEncoding: IS_LOCALLY_INVARIANT) || !isAssumed(BitsEncoding: IS_LOCALLY_INVARIANT))
12871 return ChangeStatus::UNCHANGED;
12872
12873 // try to infer invariance from underlying objects
12874 const auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(
12875 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED);
12876 if (!AUO)
12877 return ChangeStatus::UNCHANGED;
12878
12879 bool UsedAssumedInformation = false;
12880 const auto IsLocallyInvariantLoadIfPointer = [&](const Value &V) {
12881 if (!V.getType()->isPointerTy())
12882 return true;
12883 const auto *IsInvariantLoadPointer =
12884 A.getOrCreateAAFor<AAInvariantLoadPointer>(IRP: IRPosition::value(V), QueryingAA: this,
12885 DepClass: DepClassTy::REQUIRED);
12886 // Conservatively fail if invariance cannot be inferred.
12887 if (!IsInvariantLoadPointer)
12888 return false;
12889
12890 if (IsInvariantLoadPointer->isKnownLocallyInvariant())
12891 return true;
12892 if (!IsInvariantLoadPointer->isAssumedLocallyInvariant())
12893 return false;
12894
12895 UsedAssumedInformation = true;
12896 return true;
12897 };
12898 if (!AUO->forallUnderlyingObjects(Pred: IsLocallyInvariantLoadIfPointer))
12899 return indicatePessimisticFixpoint();
12900
12901 if (const auto *CB = dyn_cast<CallBase>(Val: &getAnchorValue())) {
12902 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
12903 Call: CB, /*MustPreserveNullness=*/false)) {
12904 for (const Value *Arg : CB->args()) {
12905 if (!IsLocallyInvariantLoadIfPointer(*Arg))
12906 return indicatePessimisticFixpoint();
12907 }
12908 }
12909 }
12910
12911 if (!UsedAssumedInformation) {
12912 // Pointer is known and not just assumed to be locally invariant.
12913 addKnownBits(Bits: IS_LOCALLY_INVARIANT);
12914 return ChangeStatus::CHANGED;
12915 }
12916
12917 return ChangeStatus::UNCHANGED;
12918 }
12919};
12920
12921struct AAInvariantLoadPointerFloating final : AAInvariantLoadPointerImpl {
12922 AAInvariantLoadPointerFloating(const IRPosition &IRP, Attributor &A)
12923 : AAInvariantLoadPointerImpl(IRP, A) {}
12924};
12925
12926struct AAInvariantLoadPointerReturned final : AAInvariantLoadPointerImpl {
12927 AAInvariantLoadPointerReturned(const IRPosition &IRP, Attributor &A)
12928 : AAInvariantLoadPointerImpl(IRP, A) {}
12929
12930 void initialize(Attributor &) override {
12931 removeAssumedBits(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12932 }
12933};
12934
12935struct AAInvariantLoadPointerCallSiteReturned final
12936 : AAInvariantLoadPointerImpl {
12937 AAInvariantLoadPointerCallSiteReturned(const IRPosition &IRP, Attributor &A)
12938 : AAInvariantLoadPointerImpl(IRP, A) {}
12939
12940 void initialize(Attributor &A) override {
12941 const Function *F = getAssociatedFunction();
12942 assert(F && "no associated function for return from call");
12943
12944 if (!F->isDeclaration() && !F->isIntrinsic())
12945 return AAInvariantLoadPointerImpl::initialize(A);
12946
12947 const auto &CB = cast<CallBase>(Val&: getAnchorValue());
12948 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
12949 Call: &CB, /*MustPreserveNullness=*/false))
12950 return AAInvariantLoadPointerImpl::initialize(A);
12951
12952 if (F->onlyReadsMemory() && F->hasNoSync())
12953 return AAInvariantLoadPointerImpl::initialize(A);
12954
12955 // At this point, the function is opaque, so we conservatively assume
12956 // non-invariance.
12957 indicatePessimisticFixpoint();
12958 }
12959};
12960
12961struct AAInvariantLoadPointerArgument final : AAInvariantLoadPointerImpl {
12962 AAInvariantLoadPointerArgument(const IRPosition &IRP, Attributor &A)
12963 : AAInvariantLoadPointerImpl(IRP, A) {}
12964
12965 void initialize(Attributor &) override {
12966 const Function *F = getAssociatedFunction();
12967 assert(F && "no associated function for argument");
12968
12969 if (!isCallableCC(CC: F->getCallingConv())) {
12970 addKnownBits(Bits: IS_LOCALLY_CONSTRAINED);
12971 return;
12972 }
12973
12974 if (!F->hasLocalLinkage())
12975 removeAssumedBits(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12976 }
12977};
12978
12979struct AAInvariantLoadPointerCallSiteArgument final
12980 : AAInvariantLoadPointerImpl {
12981 AAInvariantLoadPointerCallSiteArgument(const IRPosition &IRP, Attributor &A)
12982 : AAInvariantLoadPointerImpl(IRP, A) {}
12983};
12984} // namespace
12985
12986/// ------------------------ Address Space ------------------------------------
12987namespace {
12988
12989template <typename InstType>
12990static bool makeChange(Attributor &A, InstType *MemInst, const Use &U,
12991 Value *OriginalValue, PointerType *NewPtrTy,
12992 bool UseOriginalValue) {
12993 if (U.getOperandNo() != InstType::getPointerOperandIndex())
12994 return false;
12995
12996 if (MemInst->isVolatile()) {
12997 auto *TTI = A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(
12998 *MemInst->getFunction());
12999 unsigned NewAS = NewPtrTy->getPointerAddressSpace();
13000 if (!TTI || !TTI->hasVolatileVariant(MemInst, NewAS))
13001 return false;
13002 }
13003
13004 if (UseOriginalValue) {
13005 A.changeUseAfterManifest(U&: const_cast<Use &>(U), NV&: *OriginalValue);
13006 return true;
13007 }
13008
13009 Instruction *CastInst = new AddrSpaceCastInst(OriginalValue, NewPtrTy);
13010 CastInst->insertBefore(MemInst->getIterator());
13011 A.changeUseAfterManifest(U&: const_cast<Use &>(U), NV&: *CastInst);
13012 return true;
13013}
13014
13015struct AAAddressSpaceImpl : public AAAddressSpace {
13016 AAAddressSpaceImpl(const IRPosition &IRP, Attributor &A)
13017 : AAAddressSpace(IRP, A) {}
13018
13019 uint32_t getAddressSpace() const override {
13020 assert(isValidState() && "the AA is invalid");
13021 return AssumedAddressSpace;
13022 }
13023
13024 /// See AbstractAttribute::initialize(...).
13025 void initialize(Attributor &A) override {
13026 assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
13027 "Associated value is not a pointer");
13028
13029 if (!A.getInfoCache().getFlatAddressSpace().has_value()) {
13030 indicatePessimisticFixpoint();
13031 return;
13032 }
13033
13034 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13035 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13036 if (AS != FlatAS) {
13037 [[maybe_unused]] bool R = takeAddressSpace(AS);
13038 assert(R && "The take should happen");
13039 indicateOptimisticFixpoint();
13040 }
13041 }
13042
13043 ChangeStatus updateImpl(Attributor &A) override {
13044 uint32_t OldAddressSpace = AssumedAddressSpace;
13045 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13046
13047 auto CheckAddressSpace = [&](Value &Obj) {
13048 // Ignore undef.
13049 if (isa<UndefValue>(Val: &Obj))
13050 return true;
13051
13052 // If the object already has a non-flat address space, we simply take it.
13053 unsigned ObjAS = Obj.getType()->getPointerAddressSpace();
13054 if (ObjAS != FlatAS)
13055 return takeAddressSpace(AS: ObjAS);
13056
13057 // At this point, we know Obj is in the flat address space. For a final
13058 // attempt, we want to use getAssumedAddrSpace, but first we must get the
13059 // associated function, if possible.
13060 Function *F = nullptr;
13061 if (auto *Arg = dyn_cast<Argument>(Val: &Obj))
13062 F = Arg->getParent();
13063 else if (auto *I = dyn_cast<Instruction>(Val: &Obj))
13064 F = I->getFunction();
13065
13066 // Use getAssumedAddrSpace if the associated function exists.
13067 if (F) {
13068 auto *TTI =
13069 A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(F: *F);
13070 unsigned AssumedAS = TTI->getAssumedAddrSpace(V: &Obj);
13071 if (AssumedAS != ~0U)
13072 return takeAddressSpace(AS: AssumedAS);
13073 }
13074
13075 // Now we can't do anything else but to take the flat AS.
13076 return takeAddressSpace(AS: FlatAS);
13077 };
13078
13079 auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(IRP: getIRPosition(), QueryingAA: this,
13080 DepClass: DepClassTy::REQUIRED);
13081 if (!AUO->forallUnderlyingObjects(Pred: CheckAddressSpace))
13082 return indicatePessimisticFixpoint();
13083
13084 return OldAddressSpace == AssumedAddressSpace ? ChangeStatus::UNCHANGED
13085 : ChangeStatus::CHANGED;
13086 }
13087
13088 /// See AbstractAttribute::manifest(...).
13089 ChangeStatus manifest(Attributor &A) override {
13090 unsigned NewAS = getAddressSpace();
13091
13092 if (NewAS == InvalidAddressSpace ||
13093 NewAS == getAssociatedType()->getPointerAddressSpace())
13094 return ChangeStatus::UNCHANGED;
13095
13096 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13097
13098 Value *AssociatedValue = &getAssociatedValue();
13099 Value *OriginalValue = peelAddrspacecast(V: AssociatedValue, FlatAS);
13100
13101 PointerType *NewPtrTy =
13102 PointerType::get(C&: getAssociatedType()->getContext(), AddressSpace: NewAS);
13103 bool UseOriginalValue =
13104 OriginalValue->getType()->getPointerAddressSpace() == NewAS;
13105
13106 bool Changed = false;
13107
13108 auto Pred = [&](const Use &U, bool &) {
13109 if (U.get() != AssociatedValue)
13110 return true;
13111 auto *Inst = dyn_cast<Instruction>(Val: U.getUser());
13112 if (!Inst)
13113 return true;
13114 // This is a WA to make sure we only change uses from the corresponding
13115 // CGSCC if the AA is run on CGSCC instead of the entire module.
13116 if (!A.isRunOn(Fn: Inst->getFunction()))
13117 return true;
13118 if (auto *LI = dyn_cast<LoadInst>(Val: Inst)) {
13119 Changed |=
13120 makeChange(A, MemInst: LI, U, OriginalValue, NewPtrTy, UseOriginalValue);
13121 } else if (auto *SI = dyn_cast<StoreInst>(Val: Inst)) {
13122 Changed |=
13123 makeChange(A, MemInst: SI, U, OriginalValue, NewPtrTy, UseOriginalValue);
13124 } else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: Inst)) {
13125 Changed |=
13126 makeChange(A, MemInst: RMW, U, OriginalValue, NewPtrTy, UseOriginalValue);
13127 } else if (auto *CmpX = dyn_cast<AtomicCmpXchgInst>(Val: Inst)) {
13128 Changed |=
13129 makeChange(A, MemInst: CmpX, U, OriginalValue, NewPtrTy, UseOriginalValue);
13130 }
13131 return true;
13132 };
13133
13134 // It doesn't matter if we can't check all uses as we can simply
13135 // conservatively ignore those that can not be visited.
13136 (void)A.checkForAllUses(Pred, QueryingAA: *this, V: getAssociatedValue(),
13137 /* CheckBBLivenessOnly */ true);
13138
13139 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
13140 }
13141
13142 /// See AbstractAttribute::getAsStr().
13143 const std::string getAsStr(Attributor *A) const override {
13144 if (!isValidState())
13145 return "addrspace(<invalid>)";
13146 return "addrspace(" +
13147 (AssumedAddressSpace == InvalidAddressSpace
13148 ? "none"
13149 : std::to_string(val: AssumedAddressSpace)) +
13150 ")";
13151 }
13152
13153private:
13154 uint32_t AssumedAddressSpace = InvalidAddressSpace;
13155
13156 bool takeAddressSpace(uint32_t AS) {
13157 if (AssumedAddressSpace == InvalidAddressSpace) {
13158 AssumedAddressSpace = AS;
13159 return true;
13160 }
13161 return AssumedAddressSpace == AS;
13162 }
13163
13164 static Value *peelAddrspacecast(Value *V, unsigned FlatAS) {
13165 if (auto *I = dyn_cast<AddrSpaceCastInst>(Val: V)) {
13166 assert(I->getSrcAddressSpace() != FlatAS &&
13167 "there should not be flat AS -> non-flat AS");
13168 return I->getPointerOperand();
13169 }
13170 if (auto *C = dyn_cast<ConstantExpr>(Val: V))
13171 if (C->getOpcode() == Instruction::AddrSpaceCast) {
13172 assert(C->getOperand(0)->getType()->getPointerAddressSpace() !=
13173 FlatAS &&
13174 "there should not be flat AS -> non-flat AS X");
13175 return C->getOperand(i_nocapture: 0);
13176 }
13177 return V;
13178 }
13179};
13180
13181struct AAAddressSpaceFloating final : AAAddressSpaceImpl {
13182 AAAddressSpaceFloating(const IRPosition &IRP, Attributor &A)
13183 : AAAddressSpaceImpl(IRP, A) {}
13184
13185 void trackStatistics() const override {
13186 STATS_DECLTRACK_FLOATING_ATTR(addrspace);
13187 }
13188};
13189
13190struct AAAddressSpaceReturned final : AAAddressSpaceImpl {
13191 AAAddressSpaceReturned(const IRPosition &IRP, Attributor &A)
13192 : AAAddressSpaceImpl(IRP, A) {}
13193
13194 /// See AbstractAttribute::initialize(...).
13195 void initialize(Attributor &A) override {
13196 // TODO: we don't rewrite function argument for now because it will need to
13197 // rewrite the function signature and all call sites.
13198 (void)indicatePessimisticFixpoint();
13199 }
13200
13201 void trackStatistics() const override {
13202 STATS_DECLTRACK_FNRET_ATTR(addrspace);
13203 }
13204};
13205
13206struct AAAddressSpaceCallSiteReturned final : AAAddressSpaceImpl {
13207 AAAddressSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
13208 : AAAddressSpaceImpl(IRP, A) {}
13209
13210 void trackStatistics() const override {
13211 STATS_DECLTRACK_CSRET_ATTR(addrspace);
13212 }
13213};
13214
13215struct AAAddressSpaceArgument final : AAAddressSpaceImpl {
13216 AAAddressSpaceArgument(const IRPosition &IRP, Attributor &A)
13217 : AAAddressSpaceImpl(IRP, A) {}
13218
13219 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(addrspace); }
13220};
13221
13222struct AAAddressSpaceCallSiteArgument final : AAAddressSpaceImpl {
13223 AAAddressSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
13224 : AAAddressSpaceImpl(IRP, A) {}
13225
13226 /// See AbstractAttribute::initialize(...).
13227 void initialize(Attributor &A) override {
13228 // TODO: we don't rewrite call site argument for now because it will need to
13229 // rewrite the function signature of the callee.
13230 (void)indicatePessimisticFixpoint();
13231 }
13232
13233 void trackStatistics() const override {
13234 STATS_DECLTRACK_CSARG_ATTR(addrspace);
13235 }
13236};
13237} // namespace
13238
13239/// ------------------------ No Alias Address Space ---------------------------
13240// This attribute assumes flat address space can alias all other address space
13241
13242// TODO: this is similar to AAAddressSpace, most of the code should be merged.
13243// But merging it created failing cased on gateway test that cannot be
13244// reproduced locally. So should open a separated PR to handle the merge of
13245// AANoAliasAddrSpace and AAAddressSpace attribute
13246
13247namespace {
13248struct AANoAliasAddrSpaceImpl : public AANoAliasAddrSpace {
13249 AANoAliasAddrSpaceImpl(const IRPosition &IRP, Attributor &A)
13250 : AANoAliasAddrSpace(IRP, A) {}
13251
13252 void initialize(Attributor &A) override {
13253 assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
13254 "Associated value is not a pointer");
13255
13256 resetASRanges(A);
13257
13258 std::optional<unsigned> FlatAS = A.getInfoCache().getFlatAddressSpace();
13259 if (!FlatAS.has_value()) {
13260 indicatePessimisticFixpoint();
13261 return;
13262 }
13263
13264 removeAS(AS: *FlatAS);
13265
13266 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13267 if (AS != *FlatAS) {
13268 removeAS(AS);
13269 indicateOptimisticFixpoint();
13270 }
13271 }
13272
13273 ChangeStatus updateImpl(Attributor &A) override {
13274 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13275 uint32_t OldAssumed = getAssumed();
13276
13277 auto CheckAddressSpace = [&](Value &Obj) {
13278 if (isa<PoisonValue>(Val: &Obj))
13279 return true;
13280
13281 unsigned AS = Obj.getType()->getPointerAddressSpace();
13282 if (AS == FlatAS)
13283 return false;
13284
13285 removeAS(AS: Obj.getType()->getPointerAddressSpace());
13286 return true;
13287 };
13288
13289 const AAUnderlyingObjects *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(
13290 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED);
13291 if (!AUO->forallUnderlyingObjects(Pred: CheckAddressSpace))
13292 return indicatePessimisticFixpoint();
13293
13294 return OldAssumed == getAssumed() ? ChangeStatus::UNCHANGED
13295 : ChangeStatus::CHANGED;
13296 }
13297
13298 /// See AbstractAttribute::manifest(...).
13299 ChangeStatus manifest(Attributor &A) override {
13300 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13301
13302 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13303 if (AS != FlatAS || Map.empty())
13304 return ChangeStatus::UNCHANGED;
13305
13306 LLVMContext &Ctx = getAssociatedValue().getContext();
13307 MDNode *NoAliasASNode = nullptr;
13308 MDBuilder MDB(Ctx);
13309 // Has to use iterator to get the range info.
13310 for (RangeMap::const_iterator I = Map.begin(); I != Map.end(); I++) {
13311 if (!I.value())
13312 continue;
13313 unsigned Upper = I.stop();
13314 unsigned Lower = I.start();
13315 if (!NoAliasASNode) {
13316 NoAliasASNode = MDB.createRange(Lo: APInt(32, Lower), Hi: APInt(32, Upper + 1));
13317 continue;
13318 }
13319 MDNode *ASRange = MDB.createRange(Lo: APInt(32, Lower), Hi: APInt(32, Upper + 1));
13320 NoAliasASNode = MDNode::getMostGenericRange(A: NoAliasASNode, B: ASRange);
13321 }
13322
13323 Value *AssociatedValue = &getAssociatedValue();
13324 bool Changed = false;
13325
13326 auto AddNoAliasAttr = [&](const Use &U, bool &) {
13327 if (U.get() != AssociatedValue)
13328 return true;
13329 Instruction *Inst = dyn_cast<Instruction>(Val: U.getUser());
13330 if (!Inst || Inst->hasMetadata(KindID: LLVMContext::MD_noalias_addrspace))
13331 return true;
13332 if (!isa<LoadInst>(Val: Inst) && !isa<StoreInst>(Val: Inst) &&
13333 !isa<AtomicCmpXchgInst>(Val: Inst) && !isa<AtomicRMWInst>(Val: Inst))
13334 return true;
13335 if (!A.isRunOn(Fn: Inst->getFunction()))
13336 return true;
13337 Inst->setMetadata(KindID: LLVMContext::MD_noalias_addrspace, Node: NoAliasASNode);
13338 Changed = true;
13339 return true;
13340 };
13341 (void)A.checkForAllUses(Pred: AddNoAliasAttr, QueryingAA: *this, V: *AssociatedValue,
13342 /*CheckBBLivenessOnly=*/true);
13343 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
13344 }
13345
13346 /// See AbstractAttribute::getAsStr().
13347 const std::string getAsStr(Attributor *A) const override {
13348 if (!isValidState())
13349 return "<invalid>";
13350 std::string Str;
13351 raw_string_ostream OS(Str);
13352 OS << "CanNotBeAddrSpace(";
13353 for (RangeMap::const_iterator I = Map.begin(); I != Map.end(); I++) {
13354 unsigned Upper = I.stop();
13355 unsigned Lower = I.start();
13356 OS << ' ' << '[' << Upper << ',' << Lower + 1 << ')';
13357 }
13358 OS << " )";
13359 return OS.str();
13360 }
13361
13362private:
13363 void removeAS(unsigned AS) {
13364 RangeMap::iterator I = Map.find(x: AS);
13365
13366 if (I != Map.end()) {
13367 unsigned Upper = I.stop();
13368 unsigned Lower = I.start();
13369 I.erase();
13370 if (Upper == Lower)
13371 return;
13372 if (AS != ~((unsigned)0) && AS + 1 <= Upper)
13373 Map.insert(a: AS + 1, b: Upper, /*what ever this variable name is=*/y: true);
13374 if (AS != 0 && Lower <= AS - 1)
13375 Map.insert(a: Lower, b: AS - 1, y: true);
13376 }
13377 }
13378
13379 void resetASRanges(Attributor &A) {
13380 Map.clear();
13381 Map.insert(a: 0, b: A.getInfoCache().getMaxAddrSpace(), y: true);
13382 }
13383};
13384
13385struct AANoAliasAddrSpaceFloating final : AANoAliasAddrSpaceImpl {
13386 AANoAliasAddrSpaceFloating(const IRPosition &IRP, Attributor &A)
13387 : AANoAliasAddrSpaceImpl(IRP, A) {}
13388
13389 void trackStatistics() const override {
13390 STATS_DECLTRACK_FLOATING_ATTR(noaliasaddrspace);
13391 }
13392};
13393
13394struct AANoAliasAddrSpaceReturned final : AANoAliasAddrSpaceImpl {
13395 AANoAliasAddrSpaceReturned(const IRPosition &IRP, Attributor &A)
13396 : AANoAliasAddrSpaceImpl(IRP, A) {}
13397
13398 void trackStatistics() const override {
13399 STATS_DECLTRACK_FNRET_ATTR(noaliasaddrspace);
13400 }
13401};
13402
13403struct AANoAliasAddrSpaceCallSiteReturned final : AANoAliasAddrSpaceImpl {
13404 AANoAliasAddrSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
13405 : AANoAliasAddrSpaceImpl(IRP, A) {}
13406
13407 void trackStatistics() const override {
13408 STATS_DECLTRACK_CSRET_ATTR(noaliasaddrspace);
13409 }
13410};
13411
13412struct AANoAliasAddrSpaceArgument final : AANoAliasAddrSpaceImpl {
13413 AANoAliasAddrSpaceArgument(const IRPosition &IRP, Attributor &A)
13414 : AANoAliasAddrSpaceImpl(IRP, A) {}
13415
13416 void trackStatistics() const override {
13417 STATS_DECLTRACK_ARG_ATTR(noaliasaddrspace);
13418 }
13419};
13420
13421struct AANoAliasAddrSpaceCallSiteArgument final : AANoAliasAddrSpaceImpl {
13422 AANoAliasAddrSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
13423 : AANoAliasAddrSpaceImpl(IRP, A) {}
13424
13425 void trackStatistics() const override {
13426 STATS_DECLTRACK_CSARG_ATTR(noaliasaddrspace);
13427 }
13428};
13429} // namespace
13430/// ----------- Allocation Info ----------
13431namespace {
13432struct AAAllocationInfoImpl : public AAAllocationInfo {
13433 AAAllocationInfoImpl(const IRPosition &IRP, Attributor &A)
13434 : AAAllocationInfo(IRP, A) {}
13435
13436 std::optional<TypeSize> getAllocatedSize() const override {
13437 assert(isValidState() && "the AA is invalid");
13438 return AssumedAllocatedSize;
13439 }
13440
13441 std::optional<TypeSize> findInitialAllocationSize(Instruction *I,
13442 const DataLayout &DL) {
13443
13444 // TODO: implement case for malloc like instructions
13445 switch (I->getOpcode()) {
13446 case Instruction::Alloca: {
13447 AllocaInst *AI = cast<AllocaInst>(Val: I);
13448 return AI->getAllocationSize(DL);
13449 }
13450 default:
13451 return std::nullopt;
13452 }
13453 }
13454
13455 ChangeStatus updateImpl(Attributor &A) override {
13456
13457 const IRPosition &IRP = getIRPosition();
13458 Instruction *I = IRP.getCtxI();
13459
13460 // TODO: update check for malloc like calls
13461 if (!isa<AllocaInst>(Val: I))
13462 return indicatePessimisticFixpoint();
13463
13464 bool IsKnownNoCapture;
13465 if (!AA::hasAssumedIRAttr<Attribute::Captures>(
13466 A, QueryingAA: this, IRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture))
13467 return indicatePessimisticFixpoint();
13468
13469 const AAPointerInfo *PI =
13470 A.getOrCreateAAFor<AAPointerInfo>(IRP, QueryingAA: *this, DepClass: DepClassTy::REQUIRED);
13471
13472 if (!PI)
13473 return indicatePessimisticFixpoint();
13474
13475 if (!PI->getState().isValidState() || PI->reachesReturn())
13476 return indicatePessimisticFixpoint();
13477
13478 const DataLayout &DL = A.getDataLayout();
13479 const auto AllocationSize = findInitialAllocationSize(I, DL);
13480
13481 // If allocation size is nullopt, we give up.
13482 if (!AllocationSize)
13483 return indicatePessimisticFixpoint();
13484
13485 // For zero sized allocations, we give up.
13486 // Since we can't reduce further
13487 if (*AllocationSize == 0)
13488 return indicatePessimisticFixpoint();
13489
13490 int64_t BinSize = PI->numOffsetBins();
13491
13492 // TODO: implement for multiple bins
13493 if (BinSize > 1)
13494 return indicatePessimisticFixpoint();
13495
13496 if (BinSize == 0) {
13497 auto NewAllocationSize = std::make_optional<TypeSize>(args: 0, args: false);
13498 if (!changeAllocationSize(Size: NewAllocationSize))
13499 return ChangeStatus::UNCHANGED;
13500 return ChangeStatus::CHANGED;
13501 }
13502
13503 // TODO: refactor this to be part of multiple bin case
13504 const auto &It = PI->begin();
13505
13506 // TODO: handle if Offset is not zero
13507 if (It->first.Offset != 0)
13508 return indicatePessimisticFixpoint();
13509
13510 uint64_t SizeOfBin = It->first.Offset + It->first.Size;
13511
13512 if (SizeOfBin >= *AllocationSize)
13513 return indicatePessimisticFixpoint();
13514
13515 auto NewAllocationSize = std::make_optional<TypeSize>(args: SizeOfBin * 8, args: false);
13516
13517 if (!changeAllocationSize(Size: NewAllocationSize))
13518 return ChangeStatus::UNCHANGED;
13519
13520 return ChangeStatus::CHANGED;
13521 }
13522
13523 /// See AbstractAttribute::manifest(...).
13524 ChangeStatus manifest(Attributor &A) override {
13525
13526 assert(isValidState() &&
13527 "Manifest should only be called if the state is valid.");
13528
13529 Instruction *I = getIRPosition().getCtxI();
13530
13531 auto FixedAllocatedSizeInBits = getAllocatedSize()->getFixedValue();
13532
13533 unsigned long NumBytesToAllocate = (FixedAllocatedSizeInBits + 7) / 8;
13534
13535 switch (I->getOpcode()) {
13536 // TODO: add case for malloc like calls
13537 case Instruction::Alloca: {
13538
13539 AllocaInst *AI = cast<AllocaInst>(Val: I);
13540
13541 Type *CharType = Type::getInt8Ty(C&: I->getContext());
13542
13543 auto *NumBytesToValue =
13544 ConstantInt::get(Context&: I->getContext(), V: APInt(32, NumBytesToAllocate));
13545
13546 BasicBlock::iterator insertPt = AI->getIterator();
13547 insertPt = std::next(x: insertPt);
13548 AllocaInst *NewAllocaInst =
13549 new AllocaInst(CharType, AI->getAddressSpace(), NumBytesToValue,
13550 AI->getAlign(), AI->getName(), insertPt);
13551
13552 if (A.changeAfterManifest(IRP: IRPosition::inst(I: *AI), NV&: *NewAllocaInst))
13553 return ChangeStatus::CHANGED;
13554
13555 break;
13556 }
13557 default:
13558 break;
13559 }
13560
13561 return ChangeStatus::UNCHANGED;
13562 }
13563
13564 /// See AbstractAttribute::getAsStr().
13565 const std::string getAsStr(Attributor *A) const override {
13566 if (!isValidState())
13567 return "allocationinfo(<invalid>)";
13568 return "allocationinfo(" +
13569 (AssumedAllocatedSize == HasNoAllocationSize
13570 ? "none"
13571 : std::to_string(val: AssumedAllocatedSize->getFixedValue())) +
13572 ")";
13573 }
13574
13575private:
13576 std::optional<TypeSize> AssumedAllocatedSize = HasNoAllocationSize;
13577
13578 // Maintain the computed allocation size of the object.
13579 // Returns (bool) weather the size of the allocation was modified or not.
13580 bool changeAllocationSize(std::optional<TypeSize> Size) {
13581 if (AssumedAllocatedSize == HasNoAllocationSize ||
13582 AssumedAllocatedSize != Size) {
13583 AssumedAllocatedSize = Size;
13584 return true;
13585 }
13586 return false;
13587 }
13588};
13589
13590struct AAAllocationInfoFloating : AAAllocationInfoImpl {
13591 AAAllocationInfoFloating(const IRPosition &IRP, Attributor &A)
13592 : AAAllocationInfoImpl(IRP, A) {}
13593
13594 void trackStatistics() const override {
13595 STATS_DECLTRACK_FLOATING_ATTR(allocationinfo);
13596 }
13597};
13598
13599struct AAAllocationInfoReturned : AAAllocationInfoImpl {
13600 AAAllocationInfoReturned(const IRPosition &IRP, Attributor &A)
13601 : AAAllocationInfoImpl(IRP, A) {}
13602
13603 /// See AbstractAttribute::initialize(...).
13604 void initialize(Attributor &A) override {
13605 // TODO: we don't rewrite function argument for now because it will need to
13606 // rewrite the function signature and all call sites
13607 (void)indicatePessimisticFixpoint();
13608 }
13609
13610 void trackStatistics() const override {
13611 STATS_DECLTRACK_FNRET_ATTR(allocationinfo);
13612 }
13613};
13614
13615struct AAAllocationInfoCallSiteReturned : AAAllocationInfoImpl {
13616 AAAllocationInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
13617 : AAAllocationInfoImpl(IRP, A) {}
13618
13619 void trackStatistics() const override {
13620 STATS_DECLTRACK_CSRET_ATTR(allocationinfo);
13621 }
13622};
13623
13624struct AAAllocationInfoArgument : AAAllocationInfoImpl {
13625 AAAllocationInfoArgument(const IRPosition &IRP, Attributor &A)
13626 : AAAllocationInfoImpl(IRP, A) {}
13627
13628 void trackStatistics() const override {
13629 STATS_DECLTRACK_ARG_ATTR(allocationinfo);
13630 }
13631};
13632
13633struct AAAllocationInfoCallSiteArgument : AAAllocationInfoImpl {
13634 AAAllocationInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
13635 : AAAllocationInfoImpl(IRP, A) {}
13636
13637 /// See AbstractAttribute::initialize(...).
13638 void initialize(Attributor &A) override {
13639
13640 (void)indicatePessimisticFixpoint();
13641 }
13642
13643 void trackStatistics() const override {
13644 STATS_DECLTRACK_CSARG_ATTR(allocationinfo);
13645 }
13646};
13647} // namespace
13648
13649const char AANoUnwind::ID = 0;
13650const char AANoSync::ID = 0;
13651const char AANoFree::ID = 0;
13652const char AANonNull::ID = 0;
13653const char AAMustProgress::ID = 0;
13654const char AANoRecurse::ID = 0;
13655const char AANonConvergent::ID = 0;
13656const char AAWillReturn::ID = 0;
13657const char AAUndefinedBehavior::ID = 0;
13658const char AANoAlias::ID = 0;
13659const char AAIntraFnReachability::ID = 0;
13660const char AANoReturn::ID = 0;
13661const char AAIsDead::ID = 0;
13662const char AADereferenceable::ID = 0;
13663const char AAAlign::ID = 0;
13664const char AAInstanceInfo::ID = 0;
13665const char AANoCapture::ID = 0;
13666const char AAValueSimplify::ID = 0;
13667const char AAHeapToStack::ID = 0;
13668const char AAPrivatizablePtr::ID = 0;
13669const char AAMemoryBehavior::ID = 0;
13670const char AAMemoryLocation::ID = 0;
13671const char AAValueConstantRange::ID = 0;
13672const char AAPotentialConstantValues::ID = 0;
13673const char AAPotentialValues::ID = 0;
13674const char AANoUndef::ID = 0;
13675const char AANoFPClass::ID = 0;
13676const char AACallEdges::ID = 0;
13677const char AAInterFnReachability::ID = 0;
13678const char AAPointerInfo::ID = 0;
13679const char AAAssumptionInfo::ID = 0;
13680const char AAUnderlyingObjects::ID = 0;
13681const char AAInvariantLoadPointer::ID = 0;
13682const char AAAddressSpace::ID = 0;
13683const char AANoAliasAddrSpace::ID = 0;
13684const char AAAllocationInfo::ID = 0;
13685const char AAIndirectCallInfo::ID = 0;
13686const char AAGlobalValueInfo::ID = 0;
13687const char AADenormalFPMath::ID = 0;
13688
13689// Macro magic to create the static generator function for attributes that
13690// follow the naming scheme.
13691
13692#define SWITCH_PK_INV(CLASS, PK, POS_NAME) \
13693 case IRPosition::PK: \
13694 llvm_unreachable("Cannot create " #CLASS " for a " POS_NAME " position!");
13695
13696#define SWITCH_PK_CREATE(CLASS, IRP, PK, SUFFIX) \
13697 case IRPosition::PK: \
13698 AA = new (A.Allocator) CLASS##SUFFIX(IRP, A); \
13699 ++NumAAs; \
13700 break;
13701
13702#define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13703 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13704 CLASS *AA = nullptr; \
13705 switch (IRP.getPositionKind()) { \
13706 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13707 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
13708 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
13709 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13710 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
13711 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
13712 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13713 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13714 } \
13715 return *AA; \
13716 }
13717
13718#define CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13719 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13720 CLASS *AA = nullptr; \
13721 switch (IRP.getPositionKind()) { \
13722 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13723 SWITCH_PK_INV(CLASS, IRP_FUNCTION, "function") \
13724 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
13725 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13726 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13727 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
13728 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13729 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13730 } \
13731 return *AA; \
13732 }
13733
13734#define CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(POS, SUFFIX, CLASS) \
13735 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13736 CLASS *AA = nullptr; \
13737 switch (IRP.getPositionKind()) { \
13738 SWITCH_PK_CREATE(CLASS, IRP, POS, SUFFIX) \
13739 default: \
13740 llvm_unreachable("Cannot create " #CLASS " for position otherthan " #POS \
13741 " position!"); \
13742 } \
13743 return *AA; \
13744 }
13745
13746#define CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13747 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13748 CLASS *AA = nullptr; \
13749 switch (IRP.getPositionKind()) { \
13750 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13751 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13752 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13753 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13754 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13755 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
13756 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13757 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13758 } \
13759 return *AA; \
13760 }
13761
13762#define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13763 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13764 CLASS *AA = nullptr; \
13765 switch (IRP.getPositionKind()) { \
13766 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13767 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
13768 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
13769 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13770 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
13771 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
13772 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
13773 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13774 } \
13775 return *AA; \
13776 }
13777
13778#define CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13779 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13780 CLASS *AA = nullptr; \
13781 switch (IRP.getPositionKind()) { \
13782 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13783 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13784 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13785 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13786 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13787 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13788 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13789 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13790 } \
13791 return *AA; \
13792 }
13793
13794CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUnwind)
13795CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoSync)
13796CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoRecurse)
13797CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAWillReturn)
13798CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoReturn)
13799CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryLocation)
13800CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AACallEdges)
13801CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAssumptionInfo)
13802CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMustProgress)
13803
13804CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonNull)
13805CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAlias)
13806CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPrivatizablePtr)
13807CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADereferenceable)
13808CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAlign)
13809CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInstanceInfo)
13810CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoCapture)
13811CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueConstantRange)
13812CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialConstantValues)
13813CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialValues)
13814CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUndef)
13815CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFPClass)
13816CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPointerInfo)
13817CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInvariantLoadPointer)
13818CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAddressSpace)
13819CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAliasAddrSpace)
13820CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAllocationInfo)
13821
13822CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueSimplify)
13823CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIsDead)
13824CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFree)
13825CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUnderlyingObjects)
13826
13827CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_CALL_SITE, CallSite,
13828 AAIndirectCallInfo)
13829CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_FLOAT, Floating,
13830 AAGlobalValueInfo)
13831
13832CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAHeapToStack)
13833CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUndefinedBehavior)
13834CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonConvergent)
13835CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIntraFnReachability)
13836CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInterFnReachability)
13837CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADenormalFPMath)
13838
13839CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryBehavior)
13840
13841#undef CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION
13842#undef CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION
13843#undef CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION
13844#undef CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION
13845#undef CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION
13846#undef CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION
13847#undef SWITCH_PK_CREATE
13848#undef SWITCH_PK_INV
13849