1//===- AttributorAttributes.cpp - Attributes for Attributor deduction -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// See the Attributor.h file comment and the class descriptions in that file for
10// more information.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/IPO/Attributor.h"
15
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/DenseMapInfo.h"
19#include "llvm/ADT/MapVector.h"
20#include "llvm/ADT/SCCIterator.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/ADT/SetOperations.h"
23#include "llvm/ADT/SetVector.h"
24#include "llvm/ADT/SmallPtrSet.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/ADT/StringExtras.h"
28#include "llvm/Analysis/AliasAnalysis.h"
29#include "llvm/Analysis/AssumeBundleQueries.h"
30#include "llvm/Analysis/AssumptionCache.h"
31#include "llvm/Analysis/CaptureTracking.h"
32#include "llvm/Analysis/CycleAnalysis.h"
33#include "llvm/Analysis/InstructionSimplify.h"
34#include "llvm/Analysis/LazyValueInfo.h"
35#include "llvm/Analysis/MemoryBuiltins.h"
36#include "llvm/Analysis/ScalarEvolution.h"
37#include "llvm/Analysis/TargetTransformInfo.h"
38#include "llvm/Analysis/ValueTracking.h"
39#include "llvm/IR/Argument.h"
40#include "llvm/IR/Assumptions.h"
41#include "llvm/IR/Attributes.h"
42#include "llvm/IR/BasicBlock.h"
43#include "llvm/IR/Constant.h"
44#include "llvm/IR/Constants.h"
45#include "llvm/IR/DataLayout.h"
46#include "llvm/IR/DerivedTypes.h"
47#include "llvm/IR/GlobalValue.h"
48#include "llvm/IR/IRBuilder.h"
49#include "llvm/IR/InlineAsm.h"
50#include "llvm/IR/InstrTypes.h"
51#include "llvm/IR/Instruction.h"
52#include "llvm/IR/Instructions.h"
53#include "llvm/IR/IntrinsicInst.h"
54#include "llvm/IR/IntrinsicsAMDGPU.h"
55#include "llvm/IR/IntrinsicsNVPTX.h"
56#include "llvm/IR/LLVMContext.h"
57#include "llvm/IR/MDBuilder.h"
58#include "llvm/IR/NoFolder.h"
59#include "llvm/IR/Value.h"
60#include "llvm/IR/ValueHandle.h"
61#include "llvm/Support/Alignment.h"
62#include "llvm/Support/Casting.h"
63#include "llvm/Support/CommandLine.h"
64#include "llvm/Support/ErrorHandling.h"
65#include "llvm/Support/GraphWriter.h"
66#include "llvm/Support/InterleavedRange.h"
67#include "llvm/Support/KnownFPClass.h"
68#include "llvm/Support/MathExtras.h"
69#include "llvm/Support/TypeSize.h"
70#include "llvm/Support/raw_ostream.h"
71#include "llvm/Transforms/Utils/BasicBlockUtils.h"
72#include "llvm/Transforms/Utils/CallPromotionUtils.h"
73#include "llvm/Transforms/Utils/Local.h"
74#include "llvm/Transforms/Utils/ValueMapper.h"
75#include <cassert>
76#include <numeric>
77#include <optional>
78#include <string>
79
80using namespace llvm;
81
82#define DEBUG_TYPE "attributor"
83
84static cl::opt<bool> ManifestInternal(
85 "attributor-manifest-internal", cl::Hidden,
86 cl::desc("Manifest Attributor internal string attributes."),
87 cl::init(Val: false));
88
89static cl::opt<int> MaxHeapToStackSize("max-heap-to-stack-size", cl::init(Val: 128),
90 cl::Hidden);
91
92template <>
93unsigned llvm::PotentialConstantIntValuesState::MaxPotentialValues = 0;
94
95template <> unsigned llvm::PotentialLLVMValuesState::MaxPotentialValues = -1;
96
97static cl::opt<unsigned, true> MaxPotentialValues(
98 "attributor-max-potential-values", cl::Hidden,
99 cl::desc("Maximum number of potential values to be "
100 "tracked for each position."),
101 cl::location(L&: llvm::PotentialConstantIntValuesState::MaxPotentialValues),
102 cl::init(Val: 7));
103
104static cl::opt<int> MaxPotentialValuesIterations(
105 "attributor-max-potential-values-iterations", cl::Hidden,
106 cl::desc(
107 "Maximum number of iterations we keep dismantling potential values."),
108 cl::init(Val: 64));
109
110STATISTIC(NumAAs, "Number of abstract attributes created");
111STATISTIC(NumIndirectCallsPromoted, "Number of indirect calls promoted");
112
113// Some helper macros to deal with statistics tracking.
114//
115// Usage:
116// For simple IR attribute tracking overload trackStatistics in the abstract
117// attribute and choose the right STATS_DECLTRACK_********* macro,
118// e.g.,:
119// void trackStatistics() const override {
120// STATS_DECLTRACK_ARG_ATTR(returned)
121// }
122// If there is a single "increment" side one can use the macro
123// STATS_DECLTRACK with a custom message. If there are multiple increment
124// sides, STATS_DECL and STATS_TRACK can also be used separately.
125//
126#define BUILD_STAT_MSG_IR_ATTR(TYPE, NAME) \
127 ("Number of " #TYPE " marked '" #NAME "'")
128#define BUILD_STAT_NAME(NAME, TYPE) NumIR##TYPE##_##NAME
129#define STATS_DECL_(NAME, MSG) STATISTIC(NAME, MSG);
130#define STATS_DECL(NAME, TYPE, MSG) \
131 STATS_DECL_(BUILD_STAT_NAME(NAME, TYPE), MSG);
132#define STATS_TRACK(NAME, TYPE) ++(BUILD_STAT_NAME(NAME, TYPE));
133#define STATS_DECLTRACK(NAME, TYPE, MSG) \
134 {STATS_DECL(NAME, TYPE, MSG) STATS_TRACK(NAME, TYPE)}
135#define STATS_DECLTRACK_ARG_ATTR(NAME) \
136 STATS_DECLTRACK(NAME, Arguments, BUILD_STAT_MSG_IR_ATTR(arguments, NAME))
137#define STATS_DECLTRACK_CSARG_ATTR(NAME) \
138 STATS_DECLTRACK(NAME, CSArguments, \
139 BUILD_STAT_MSG_IR_ATTR(call site arguments, NAME))
140#define STATS_DECLTRACK_FN_ATTR(NAME) \
141 STATS_DECLTRACK(NAME, Function, BUILD_STAT_MSG_IR_ATTR(functions, NAME))
142#define STATS_DECLTRACK_CS_ATTR(NAME) \
143 STATS_DECLTRACK(NAME, CS, BUILD_STAT_MSG_IR_ATTR(call site, NAME))
144#define STATS_DECLTRACK_FNRET_ATTR(NAME) \
145 STATS_DECLTRACK(NAME, FunctionReturn, \
146 BUILD_STAT_MSG_IR_ATTR(function returns, NAME))
147#define STATS_DECLTRACK_CSRET_ATTR(NAME) \
148 STATS_DECLTRACK(NAME, CSReturn, \
149 BUILD_STAT_MSG_IR_ATTR(call site returns, NAME))
150#define STATS_DECLTRACK_FLOATING_ATTR(NAME) \
151 STATS_DECLTRACK(NAME, Floating, \
152 ("Number of floating values known to be '" #NAME "'"))
153
154// Specialization of the operator<< for abstract attributes subclasses. This
155// disambiguates situations where multiple operators are applicable.
156namespace llvm {
157#define PIPE_OPERATOR(CLASS) \
158 raw_ostream &operator<<(raw_ostream &OS, const CLASS &AA) { \
159 return OS << static_cast<const AbstractAttribute &>(AA); \
160 }
161
162PIPE_OPERATOR(AAIsDead)
163PIPE_OPERATOR(AANoUnwind)
164PIPE_OPERATOR(AANoSync)
165PIPE_OPERATOR(AANoRecurse)
166PIPE_OPERATOR(AANonConvergent)
167PIPE_OPERATOR(AAWillReturn)
168PIPE_OPERATOR(AANoReturn)
169PIPE_OPERATOR(AANonNull)
170PIPE_OPERATOR(AAMustProgress)
171PIPE_OPERATOR(AANoAlias)
172PIPE_OPERATOR(AADereferenceable)
173PIPE_OPERATOR(AAAlign)
174PIPE_OPERATOR(AAInstanceInfo)
175PIPE_OPERATOR(AANoCapture)
176PIPE_OPERATOR(AAValueSimplify)
177PIPE_OPERATOR(AANoFree)
178PIPE_OPERATOR(AAHeapToStack)
179PIPE_OPERATOR(AAIntraFnReachability)
180PIPE_OPERATOR(AAMemoryBehavior)
181PIPE_OPERATOR(AAMemoryLocation)
182PIPE_OPERATOR(AAValueConstantRange)
183PIPE_OPERATOR(AAPrivatizablePtr)
184PIPE_OPERATOR(AAUndefinedBehavior)
185PIPE_OPERATOR(AAPotentialConstantValues)
186PIPE_OPERATOR(AAPotentialValues)
187PIPE_OPERATOR(AANoUndef)
188PIPE_OPERATOR(AANoFPClass)
189PIPE_OPERATOR(AACallEdges)
190PIPE_OPERATOR(AAInterFnReachability)
191PIPE_OPERATOR(AAPointerInfo)
192PIPE_OPERATOR(AAAssumptionInfo)
193PIPE_OPERATOR(AAUnderlyingObjects)
194PIPE_OPERATOR(AAInvariantLoadPointer)
195PIPE_OPERATOR(AAAddressSpace)
196PIPE_OPERATOR(AANoAliasAddrSpace)
197PIPE_OPERATOR(AAAllocationInfo)
198PIPE_OPERATOR(AAIndirectCallInfo)
199PIPE_OPERATOR(AAGlobalValueInfo)
200PIPE_OPERATOR(AADenormalFPMath)
201
202#undef PIPE_OPERATOR
203
204template <>
205ChangeStatus clampStateAndIndicateChange<DerefState>(DerefState &S,
206 const DerefState &R) {
207 ChangeStatus CS0 =
208 clampStateAndIndicateChange(S&: S.DerefBytesState, R: R.DerefBytesState);
209 ChangeStatus CS1 = clampStateAndIndicateChange(S&: S.GlobalState, R: R.GlobalState);
210 return CS0 | CS1;
211}
212
213} // namespace llvm
214
215static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I,
216 bool HeaderOnly, Cycle **CPtr = nullptr) {
217 if (!CI)
218 return true;
219 auto *BB = I->getParent();
220 auto *C = CI->getCycle(Block: BB);
221 if (!C)
222 return false;
223 if (CPtr)
224 *CPtr = C;
225 return !HeaderOnly || BB == C->getHeader();
226}
227
228/// Checks if a type could have padding bytes.
229static bool isDenselyPacked(Type *Ty, const DataLayout &DL) {
230 // There is no size information, so be conservative.
231 if (!Ty->isSized())
232 return false;
233
234 // If the alloc size is not equal to the storage size, then there are padding
235 // bytes. For x86_fp80 on x86-64, size: 80 alloc size: 128.
236 if (DL.getTypeSizeInBits(Ty) != DL.getTypeAllocSizeInBits(Ty))
237 return false;
238
239 // FIXME: This isn't the right way to check for padding in vectors with
240 // non-byte-size elements.
241 if (VectorType *SeqTy = dyn_cast<VectorType>(Val: Ty))
242 return isDenselyPacked(Ty: SeqTy->getElementType(), DL);
243
244 // For array types, check for padding within members.
245 if (ArrayType *SeqTy = dyn_cast<ArrayType>(Val: Ty))
246 return isDenselyPacked(Ty: SeqTy->getElementType(), DL);
247
248 if (!isa<StructType>(Val: Ty))
249 return true;
250
251 // Check for padding within and between elements of a struct.
252 StructType *StructTy = cast<StructType>(Val: Ty);
253 const StructLayout *Layout = DL.getStructLayout(Ty: StructTy);
254 uint64_t StartPos = 0;
255 for (unsigned I = 0, E = StructTy->getNumElements(); I < E; ++I) {
256 Type *ElTy = StructTy->getElementType(N: I);
257 if (!isDenselyPacked(Ty: ElTy, DL))
258 return false;
259 if (StartPos != Layout->getElementOffsetInBits(Idx: I))
260 return false;
261 StartPos += DL.getTypeAllocSizeInBits(Ty: ElTy);
262 }
263
264 return true;
265}
266
267/// Get pointer operand of memory accessing instruction. If \p I is
268/// not a memory accessing instruction, return nullptr. If \p AllowVolatile,
269/// is set to false and the instruction is volatile, return nullptr.
270static const Value *getPointerOperand(const Instruction *I,
271 bool AllowVolatile) {
272 if (!AllowVolatile && I->isVolatile())
273 return nullptr;
274
275 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
276 return LI->getPointerOperand();
277 }
278
279 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
280 return SI->getPointerOperand();
281 }
282
283 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
284 return CXI->getPointerOperand();
285 }
286
287 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: I)) {
288 return RMWI->getPointerOperand();
289 }
290
291 return nullptr;
292}
293
294/// Helper function to create a pointer based on \p Ptr, and advanced by \p
295/// Offset bytes.
296static Value *constructPointer(Value *Ptr, int64_t Offset,
297 IRBuilder<NoFolder> &IRB) {
298 LLVM_DEBUG(dbgs() << "Construct pointer: " << *Ptr << " + " << Offset
299 << "-bytes\n");
300
301 if (Offset)
302 Ptr = IRB.CreatePtrAdd(Ptr, Offset: IRB.getInt64(C: Offset),
303 Name: Ptr->getName() + ".b" + Twine(Offset));
304 return Ptr;
305}
306
307static const Value *
308stripAndAccumulateOffsets(Attributor &A, const AbstractAttribute &QueryingAA,
309 const Value *Val, const DataLayout &DL, APInt &Offset,
310 bool GetMinOffset, bool AllowNonInbounds,
311 bool UseAssumed = false) {
312
313 auto AttributorAnalysis = [&](Value &V, APInt &ROffset) -> bool {
314 const IRPosition &Pos = IRPosition::value(V);
315 // Only track dependence if we are going to use the assumed info.
316 const AAValueConstantRange *ValueConstantRangeAA =
317 A.getAAFor<AAValueConstantRange>(QueryingAA, IRP: Pos,
318 DepClass: UseAssumed ? DepClassTy::OPTIONAL
319 : DepClassTy::NONE);
320 if (!ValueConstantRangeAA)
321 return false;
322 ConstantRange Range = UseAssumed ? ValueConstantRangeAA->getAssumed()
323 : ValueConstantRangeAA->getKnown();
324 if (Range.isFullSet())
325 return false;
326
327 // We can only use the lower part of the range because the upper part can
328 // be higher than what the value can really be.
329 if (GetMinOffset)
330 ROffset = Range.getSignedMin();
331 else
332 ROffset = Range.getSignedMax();
333 return true;
334 };
335
336 return Val->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds,
337 /* AllowInvariant */ AllowInvariantGroup: true,
338 ExternalAnalysis: AttributorAnalysis);
339}
340
341static const Value *
342getMinimalBaseOfPointer(Attributor &A, const AbstractAttribute &QueryingAA,
343 const Value *Ptr, int64_t &BytesOffset,
344 const DataLayout &DL, bool AllowNonInbounds = false) {
345 APInt OffsetAPInt(DL.getIndexTypeSizeInBits(Ty: Ptr->getType()), 0);
346 const Value *Base =
347 stripAndAccumulateOffsets(A, QueryingAA, Val: Ptr, DL, Offset&: OffsetAPInt,
348 /* GetMinOffset */ true, AllowNonInbounds);
349
350 BytesOffset = OffsetAPInt.getSExtValue();
351 return Base;
352}
353
354/// Clamp the information known for all returned values of a function
355/// (identified by \p QueryingAA) into \p S.
356template <typename AAType, typename StateType = typename AAType::StateType,
357 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
358 bool RecurseForSelectAndPHI = true>
359static void clampReturnedValueStates(
360 Attributor &A, const AAType &QueryingAA, StateType &S,
361 const IRPosition::CallBaseContext *CBContext = nullptr) {
362 LLVM_DEBUG(dbgs() << "[Attributor] Clamp return value states for "
363 << QueryingAA << " into " << S << "\n");
364
365 assert((QueryingAA.getIRPosition().getPositionKind() ==
366 IRPosition::IRP_RETURNED ||
367 QueryingAA.getIRPosition().getPositionKind() ==
368 IRPosition::IRP_CALL_SITE_RETURNED) &&
369 "Can only clamp returned value states for a function returned or call "
370 "site returned position!");
371
372 // Use an optional state as there might not be any return values and we want
373 // to join (IntegerState::operator&) the state of all there are.
374 std::optional<StateType> T;
375
376 // Callback for each possibly returned value.
377 auto CheckReturnValue = [&](Value &RV) -> bool {
378 const IRPosition &RVPos = IRPosition::value(V: RV, CBContext);
379 // If possible, use the hasAssumedIRAttr interface.
380 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
381 bool IsKnown;
382 return AA::hasAssumedIRAttr<IRAttributeKind>(
383 A, &QueryingAA, RVPos, DepClassTy::REQUIRED, IsKnown);
384 }
385
386 const AAType *AA =
387 A.getAAFor<AAType>(QueryingAA, RVPos, DepClassTy::REQUIRED);
388 if (!AA)
389 return false;
390 LLVM_DEBUG(dbgs() << "[Attributor] RV: " << RV
391 << " AA: " << AA->getAsStr(&A) << " @ " << RVPos << "\n");
392 const StateType &AAS = AA->getState();
393 if (!T)
394 T = StateType::getBestState(AAS);
395 *T &= AAS;
396 LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " RV State: " << T
397 << "\n");
398 return T->isValidState();
399 };
400
401 if (!A.checkForAllReturnedValues(Pred: CheckReturnValue, QueryingAA,
402 S: AA::ValueScope::Intraprocedural,
403 RecurseForSelectAndPHI))
404 S.indicatePessimisticFixpoint();
405 else if (T)
406 S ^= *T;
407}
408
409namespace {
410/// Helper class for generic deduction: return value -> returned position.
411template <typename AAType, typename BaseType,
412 typename StateType = typename BaseType::StateType,
413 bool PropagateCallBaseContext = false,
414 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
415 bool RecurseForSelectAndPHI = true>
416struct AAReturnedFromReturnedValues : public BaseType {
417 AAReturnedFromReturnedValues(const IRPosition &IRP, Attributor &A)
418 : BaseType(IRP, A) {}
419
420 /// See AbstractAttribute::updateImpl(...).
421 ChangeStatus updateImpl(Attributor &A) override {
422 StateType S(StateType::getBestState(this->getState()));
423 clampReturnedValueStates<AAType, StateType, IRAttributeKind,
424 RecurseForSelectAndPHI>(
425 A, *this, S,
426 PropagateCallBaseContext ? this->getCallBaseContext() : nullptr);
427 // TODO: If we know we visited all returned values, thus no are assumed
428 // dead, we can take the known information from the state T.
429 return clampStateAndIndicateChange<StateType>(this->getState(), S);
430 }
431};
432
433/// Clamp the information known at all call sites for a given argument
434/// (identified by \p QueryingAA) into \p S.
435template <typename AAType, typename StateType = typename AAType::StateType,
436 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
437static void clampCallSiteArgumentStates(Attributor &A, const AAType &QueryingAA,
438 StateType &S) {
439 LLVM_DEBUG(dbgs() << "[Attributor] Clamp call site argument states for "
440 << QueryingAA << " into " << S << "\n");
441
442 assert(QueryingAA.getIRPosition().getPositionKind() ==
443 IRPosition::IRP_ARGUMENT &&
444 "Can only clamp call site argument states for an argument position!");
445
446 // Use an optional state as there might not be any return values and we want
447 // to join (IntegerState::operator&) the state of all there are.
448 std::optional<StateType> T;
449
450 // The argument number which is also the call site argument number.
451 unsigned ArgNo = QueryingAA.getIRPosition().getCallSiteArgNo();
452
453 auto CallSiteCheck = [&](AbstractCallSite ACS) {
454 const IRPosition &ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
455 // Check if a coresponding argument was found or if it is on not associated
456 // (which can happen for callback calls).
457 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
458 return false;
459
460 // If possible, use the hasAssumedIRAttr interface.
461 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
462 bool IsKnown;
463 return AA::hasAssumedIRAttr<IRAttributeKind>(
464 A, &QueryingAA, ACSArgPos, DepClassTy::REQUIRED, IsKnown);
465 }
466
467 const AAType *AA =
468 A.getAAFor<AAType>(QueryingAA, ACSArgPos, DepClassTy::REQUIRED);
469 if (!AA)
470 return false;
471 LLVM_DEBUG(dbgs() << "[Attributor] ACS: " << *ACS.getInstruction()
472 << " AA: " << AA->getAsStr(&A) << " @" << ACSArgPos
473 << "\n");
474 const StateType &AAS = AA->getState();
475 if (!T)
476 T = StateType::getBestState(AAS);
477 *T &= AAS;
478 LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " CSA State: " << T
479 << "\n");
480 return T->isValidState();
481 };
482
483 bool UsedAssumedInformation = false;
484 if (!A.checkForAllCallSites(CallSiteCheck, QueryingAA, true,
485 UsedAssumedInformation))
486 S.indicatePessimisticFixpoint();
487 else if (T)
488 S ^= *T;
489}
490
491/// This function is the bridge between argument position and the call base
492/// context.
493template <typename AAType, typename BaseType,
494 typename StateType = typename AAType::StateType,
495 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
496bool getArgumentStateFromCallBaseContext(Attributor &A,
497 BaseType &QueryingAttribute,
498 IRPosition &Pos, StateType &State) {
499 assert((Pos.getPositionKind() == IRPosition::IRP_ARGUMENT) &&
500 "Expected an 'argument' position !");
501 const CallBase *CBContext = Pos.getCallBaseContext();
502 if (!CBContext)
503 return false;
504
505 int ArgNo = Pos.getCallSiteArgNo();
506 assert(ArgNo >= 0 && "Invalid Arg No!");
507 const IRPosition CBArgPos = IRPosition::callsite_argument(CB: *CBContext, ArgNo);
508
509 // If possible, use the hasAssumedIRAttr interface.
510 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
511 bool IsKnown;
512 return AA::hasAssumedIRAttr<IRAttributeKind>(
513 A, &QueryingAttribute, CBArgPos, DepClassTy::REQUIRED, IsKnown);
514 }
515
516 const auto *AA =
517 A.getAAFor<AAType>(QueryingAttribute, CBArgPos, DepClassTy::REQUIRED);
518 if (!AA)
519 return false;
520 const StateType &CBArgumentState =
521 static_cast<const StateType &>(AA->getState());
522
523 LLVM_DEBUG(dbgs() << "[Attributor] Briding Call site context to argument"
524 << "Position:" << Pos << "CB Arg state:" << CBArgumentState
525 << "\n");
526
527 // NOTE: If we want to do call site grouping it should happen here.
528 State ^= CBArgumentState;
529 return true;
530}
531
532/// Helper class for generic deduction: call site argument -> argument position.
533template <typename AAType, typename BaseType,
534 typename StateType = typename AAType::StateType,
535 bool BridgeCallBaseContext = false,
536 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
537struct AAArgumentFromCallSiteArguments : public BaseType {
538 AAArgumentFromCallSiteArguments(const IRPosition &IRP, Attributor &A)
539 : BaseType(IRP, A) {}
540
541 /// See AbstractAttribute::updateImpl(...).
542 ChangeStatus updateImpl(Attributor &A) override {
543 StateType S = StateType::getBestState(this->getState());
544
545 if (BridgeCallBaseContext) {
546 bool Success =
547 getArgumentStateFromCallBaseContext<AAType, BaseType, StateType,
548 IRAttributeKind>(
549 A, *this, this->getIRPosition(), S);
550 if (Success)
551 return clampStateAndIndicateChange<StateType>(this->getState(), S);
552 }
553 clampCallSiteArgumentStates<AAType, StateType, IRAttributeKind>(A, *this,
554 S);
555
556 // TODO: If we know we visited all incoming values, thus no are assumed
557 // dead, we can take the known information from the state T.
558 return clampStateAndIndicateChange<StateType>(this->getState(), S);
559 }
560};
561
562/// Helper class for generic replication: function returned -> cs returned.
563template <typename AAType, typename BaseType,
564 typename StateType = typename BaseType::StateType,
565 bool IntroduceCallBaseContext = false,
566 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
567struct AACalleeToCallSite : public BaseType {
568 AACalleeToCallSite(const IRPosition &IRP, Attributor &A) : BaseType(IRP, A) {}
569
570 /// See AbstractAttribute::updateImpl(...).
571 ChangeStatus updateImpl(Attributor &A) override {
572 auto IRPKind = this->getIRPosition().getPositionKind();
573 assert((IRPKind == IRPosition::IRP_CALL_SITE_RETURNED ||
574 IRPKind == IRPosition::IRP_CALL_SITE) &&
575 "Can only wrap function returned positions for call site "
576 "returned positions!");
577 auto &S = this->getState();
578
579 CallBase &CB = cast<CallBase>(this->getAnchorValue());
580 if (IntroduceCallBaseContext)
581 LLVM_DEBUG(dbgs() << "[Attributor] Introducing call base context:" << CB
582 << "\n");
583
584 ChangeStatus Changed = ChangeStatus::UNCHANGED;
585 auto CalleePred = [&](ArrayRef<const Function *> Callees) {
586 for (const Function *Callee : Callees) {
587 IRPosition FnPos =
588 IRPKind == llvm::IRPosition::IRP_CALL_SITE_RETURNED
589 ? IRPosition::returned(F: *Callee,
590 CBContext: IntroduceCallBaseContext ? &CB : nullptr)
591 : IRPosition::function(
592 F: *Callee, CBContext: IntroduceCallBaseContext ? &CB : nullptr);
593 // If possible, use the hasAssumedIRAttr interface.
594 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
595 bool IsKnown;
596 if (!AA::hasAssumedIRAttr<IRAttributeKind>(
597 A, this, FnPos, DepClassTy::REQUIRED, IsKnown))
598 return false;
599 continue;
600 }
601
602 const AAType *AA =
603 A.getAAFor<AAType>(*this, FnPos, DepClassTy::REQUIRED);
604 if (!AA)
605 return false;
606 Changed |= clampStateAndIndicateChange(S, AA->getState());
607 if (S.isAtFixpoint())
608 return S.isValidState();
609 }
610 return true;
611 };
612 if (!A.checkForAllCallees(Pred: CalleePred, QueryingAA: *this, CB))
613 return S.indicatePessimisticFixpoint();
614 return Changed;
615 }
616};
617
618/// Helper function to accumulate uses.
619template <class AAType, typename StateType = typename AAType::StateType>
620static void followUsesInContext(AAType &AA, Attributor &A,
621 MustBeExecutedContextExplorer &Explorer,
622 const Instruction *CtxI,
623 SetVector<const Use *> &Uses,
624 StateType &State) {
625 auto EIt = Explorer.begin(PP: CtxI), EEnd = Explorer.end(CtxI);
626 for (unsigned u = 0; u < Uses.size(); ++u) {
627 const Use *U = Uses[u];
628 if (const Instruction *UserI = dyn_cast<Instruction>(Val: U->getUser())) {
629 bool Found = Explorer.findInContextOf(I: UserI, EIt, EEnd);
630 if (Found && AA.followUseInMBEC(A, U, UserI, State))
631 Uses.insert_range(R: llvm::make_pointer_range(Range: UserI->uses()));
632 }
633 }
634}
635
636/// Use the must-be-executed-context around \p I to add information into \p S.
637/// The AAType class is required to have `followUseInMBEC` method with the
638/// following signature and behaviour:
639///
640/// bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I)
641/// U - Underlying use.
642/// I - The user of the \p U.
643/// Returns true if the value should be tracked transitively.
644///
645template <class AAType, typename StateType = typename AAType::StateType>
646static void followUsesInMBEC(AAType &AA, Attributor &A, StateType &S,
647 Instruction &CtxI) {
648 const Value &Val = AA.getIRPosition().getAssociatedValue();
649 if (isa<ConstantData>(Val))
650 return;
651
652 MustBeExecutedContextExplorer *Explorer =
653 A.getInfoCache().getMustBeExecutedContextExplorer();
654 if (!Explorer)
655 return;
656
657 // Container for (transitive) uses of the associated value.
658 SetVector<const Use *> Uses;
659 for (const Use &U : Val.uses())
660 Uses.insert(X: &U);
661
662 followUsesInContext<AAType>(AA, A, *Explorer, &CtxI, Uses, S);
663
664 if (S.isAtFixpoint())
665 return;
666
667 SmallVector<const BranchInst *, 4> BrInsts;
668 auto Pred = [&](const Instruction *I) {
669 if (const BranchInst *Br = dyn_cast<BranchInst>(Val: I))
670 if (Br->isConditional())
671 BrInsts.push_back(Elt: Br);
672 return true;
673 };
674
675 // Here, accumulate conditional branch instructions in the context. We
676 // explore the child paths and collect the known states. The disjunction of
677 // those states can be merged to its own state. Let ParentState_i be a state
678 // to indicate the known information for an i-th branch instruction in the
679 // context. ChildStates are created for its successors respectively.
680 //
681 // ParentS_1 = ChildS_{1, 1} /\ ChildS_{1, 2} /\ ... /\ ChildS_{1, n_1}
682 // ParentS_2 = ChildS_{2, 1} /\ ChildS_{2, 2} /\ ... /\ ChildS_{2, n_2}
683 // ...
684 // ParentS_m = ChildS_{m, 1} /\ ChildS_{m, 2} /\ ... /\ ChildS_{m, n_m}
685 //
686 // Known State |= ParentS_1 \/ ParentS_2 \/... \/ ParentS_m
687 //
688 // FIXME: Currently, recursive branches are not handled. For example, we
689 // can't deduce that ptr must be dereferenced in below function.
690 //
691 // void f(int a, int c, int *ptr) {
692 // if(a)
693 // if (b) {
694 // *ptr = 0;
695 // } else {
696 // *ptr = 1;
697 // }
698 // else {
699 // if (b) {
700 // *ptr = 0;
701 // } else {
702 // *ptr = 1;
703 // }
704 // }
705 // }
706
707 Explorer->checkForAllContext(PP: &CtxI, Pred);
708 for (const BranchInst *Br : BrInsts) {
709 StateType ParentState;
710
711 // The known state of the parent state is a conjunction of children's
712 // known states so it is initialized with a best state.
713 ParentState.indicateOptimisticFixpoint();
714
715 for (const BasicBlock *BB : Br->successors()) {
716 StateType ChildState;
717
718 size_t BeforeSize = Uses.size();
719 followUsesInContext(AA, A, *Explorer, &BB->front(), Uses, ChildState);
720
721 // Erase uses which only appear in the child.
722 for (auto It = Uses.begin() + BeforeSize; It != Uses.end();)
723 It = Uses.erase(I: It);
724
725 ParentState &= ChildState;
726 }
727
728 // Use only known state.
729 S += ParentState;
730 }
731}
732} // namespace
733
734/// ------------------------ PointerInfo ---------------------------------------
735
736namespace llvm {
737namespace AA {
738namespace PointerInfo {
739
740struct State;
741
742} // namespace PointerInfo
743} // namespace AA
744
745/// Helper for AA::PointerInfo::Access DenseMap/Set usage.
746template <>
747struct DenseMapInfo<AAPointerInfo::Access> : DenseMapInfo<Instruction *> {
748 using Access = AAPointerInfo::Access;
749 static inline Access getEmptyKey();
750 static inline Access getTombstoneKey();
751 static unsigned getHashValue(const Access &A);
752 static bool isEqual(const Access &LHS, const Access &RHS);
753};
754
755/// Helper that allows RangeTy as a key in a DenseMap.
756template <> struct DenseMapInfo<AA::RangeTy> {
757 static inline AA::RangeTy getEmptyKey() {
758 auto EmptyKey = DenseMapInfo<int64_t>::getEmptyKey();
759 return AA::RangeTy{EmptyKey, EmptyKey};
760 }
761
762 static inline AA::RangeTy getTombstoneKey() {
763 auto TombstoneKey = DenseMapInfo<int64_t>::getTombstoneKey();
764 return AA::RangeTy{TombstoneKey, TombstoneKey};
765 }
766
767 static unsigned getHashValue(const AA::RangeTy &Range) {
768 return detail::combineHashValue(
769 a: DenseMapInfo<int64_t>::getHashValue(Val: Range.Offset),
770 b: DenseMapInfo<int64_t>::getHashValue(Val: Range.Size));
771 }
772
773 static bool isEqual(const AA::RangeTy &A, const AA::RangeTy B) {
774 return A == B;
775 }
776};
777
778/// Helper for AA::PointerInfo::Access DenseMap/Set usage ignoring everythign
779/// but the instruction
780struct AccessAsInstructionInfo : DenseMapInfo<Instruction *> {
781 using Base = DenseMapInfo<Instruction *>;
782 using Access = AAPointerInfo::Access;
783 static inline Access getEmptyKey();
784 static inline Access getTombstoneKey();
785 static unsigned getHashValue(const Access &A);
786 static bool isEqual(const Access &LHS, const Access &RHS);
787};
788
789} // namespace llvm
790
791/// A type to track pointer/struct usage and accesses for AAPointerInfo.
792struct AA::PointerInfo::State : public AbstractState {
793 /// Return the best possible representable state.
794 static State getBestState(const State &SIS) { return State(); }
795
796 /// Return the worst possible representable state.
797 static State getWorstState(const State &SIS) {
798 State R;
799 R.indicatePessimisticFixpoint();
800 return R;
801 }
802
803 State() = default;
804 State(State &&SIS) = default;
805
806 const State &getAssumed() const { return *this; }
807
808 /// See AbstractState::isValidState().
809 bool isValidState() const override { return BS.isValidState(); }
810
811 /// See AbstractState::isAtFixpoint().
812 bool isAtFixpoint() const override { return BS.isAtFixpoint(); }
813
814 /// See AbstractState::indicateOptimisticFixpoint().
815 ChangeStatus indicateOptimisticFixpoint() override {
816 BS.indicateOptimisticFixpoint();
817 return ChangeStatus::UNCHANGED;
818 }
819
820 /// See AbstractState::indicatePessimisticFixpoint().
821 ChangeStatus indicatePessimisticFixpoint() override {
822 BS.indicatePessimisticFixpoint();
823 return ChangeStatus::CHANGED;
824 }
825
826 State &operator=(const State &R) {
827 if (this == &R)
828 return *this;
829 BS = R.BS;
830 AccessList = R.AccessList;
831 OffsetBins = R.OffsetBins;
832 RemoteIMap = R.RemoteIMap;
833 ReturnedOffsets = R.ReturnedOffsets;
834 return *this;
835 }
836
837 State &operator=(State &&R) {
838 if (this == &R)
839 return *this;
840 std::swap(a&: BS, b&: R.BS);
841 std::swap(LHS&: AccessList, RHS&: R.AccessList);
842 std::swap(a&: OffsetBins, b&: R.OffsetBins);
843 std::swap(a&: RemoteIMap, b&: R.RemoteIMap);
844 std::swap(a&: ReturnedOffsets, b&: R.ReturnedOffsets);
845 return *this;
846 }
847
848 /// Add a new Access to the state at offset \p Offset and with size \p Size.
849 /// The access is associated with \p I, writes \p Content (if anything), and
850 /// is of kind \p Kind. If an Access already exists for the same \p I and same
851 /// \p RemoteI, the two are combined, potentially losing information about
852 /// offset and size. The resulting access must now be moved from its original
853 /// OffsetBin to the bin for its new offset.
854 ///
855 /// \Returns CHANGED, if the state changed, UNCHANGED otherwise.
856 ChangeStatus addAccess(Attributor &A, const AAPointerInfo::RangeList &Ranges,
857 Instruction &I, std::optional<Value *> Content,
858 AAPointerInfo::AccessKind Kind, Type *Ty,
859 Instruction *RemoteI = nullptr);
860
861 AAPointerInfo::const_bin_iterator begin() const { return OffsetBins.begin(); }
862 AAPointerInfo::const_bin_iterator end() const { return OffsetBins.end(); }
863 int64_t numOffsetBins() const { return OffsetBins.size(); }
864
865 const AAPointerInfo::Access &getAccess(unsigned Index) const {
866 return AccessList[Index];
867 }
868
869protected:
870 // Every memory instruction results in an Access object. We maintain a list of
871 // all Access objects that we own, along with the following maps:
872 //
873 // - OffsetBins: RangeTy -> { Access }
874 // - RemoteIMap: RemoteI x LocalI -> Access
875 //
876 // A RemoteI is any instruction that accesses memory. RemoteI is different
877 // from LocalI if and only if LocalI is a call; then RemoteI is some
878 // instruction in the callgraph starting from LocalI. Multiple paths in the
879 // callgraph from LocalI to RemoteI may produce multiple accesses, but these
880 // are all combined into a single Access object. This may result in loss of
881 // information in RangeTy in the Access object.
882 SmallVector<AAPointerInfo::Access> AccessList;
883 AAPointerInfo::OffsetBinsTy OffsetBins;
884 DenseMap<const Instruction *, SmallVector<unsigned>> RemoteIMap;
885
886 /// Flag to determine if the underlying pointer is reaching a return statement
887 /// in the associated function or not. Returns in other functions cause
888 /// invalidation.
889 AAPointerInfo::OffsetInfo ReturnedOffsets;
890
891 /// See AAPointerInfo::forallInterferingAccesses.
892 template <typename F>
893 bool forallInterferingAccesses(AA::RangeTy Range, F CB) const {
894 if (!isValidState() || !ReturnedOffsets.isUnassigned())
895 return false;
896
897 for (const auto &It : OffsetBins) {
898 AA::RangeTy ItRange = It.getFirst();
899 if (!Range.mayOverlap(Range: ItRange))
900 continue;
901 bool IsExact = Range == ItRange && !Range.offsetOrSizeAreUnknown();
902 for (auto Index : It.getSecond()) {
903 auto &Access = AccessList[Index];
904 if (!CB(Access, IsExact))
905 return false;
906 }
907 }
908 return true;
909 }
910
911 /// See AAPointerInfo::forallInterferingAccesses.
912 template <typename F>
913 bool forallInterferingAccesses(Instruction &I, F CB,
914 AA::RangeTy &Range) const {
915 if (!isValidState() || !ReturnedOffsets.isUnassigned())
916 return false;
917
918 auto LocalList = RemoteIMap.find(Val: &I);
919 if (LocalList == RemoteIMap.end()) {
920 return true;
921 }
922
923 for (unsigned Index : LocalList->getSecond()) {
924 for (auto &R : AccessList[Index]) {
925 Range &= R;
926 if (Range.offsetAndSizeAreUnknown())
927 break;
928 }
929 }
930 return forallInterferingAccesses(Range, CB);
931 }
932
933private:
934 /// State to track fixpoint and validity.
935 BooleanState BS;
936};
937
938ChangeStatus AA::PointerInfo::State::addAccess(
939 Attributor &A, const AAPointerInfo::RangeList &Ranges, Instruction &I,
940 std::optional<Value *> Content, AAPointerInfo::AccessKind Kind, Type *Ty,
941 Instruction *RemoteI) {
942 RemoteI = RemoteI ? RemoteI : &I;
943
944 // Check if we have an access for this instruction, if not, simply add it.
945 auto &LocalList = RemoteIMap[RemoteI];
946 bool AccExists = false;
947 unsigned AccIndex = AccessList.size();
948 for (auto Index : LocalList) {
949 auto &A = AccessList[Index];
950 if (A.getLocalInst() == &I) {
951 AccExists = true;
952 AccIndex = Index;
953 break;
954 }
955 }
956
957 auto AddToBins = [&](const AAPointerInfo::RangeList &ToAdd) {
958 LLVM_DEBUG(if (ToAdd.size()) dbgs()
959 << "[AAPointerInfo] Inserting access in new offset bins\n";);
960
961 for (auto Key : ToAdd) {
962 LLVM_DEBUG(dbgs() << " key " << Key << "\n");
963 OffsetBins[Key].insert(V: AccIndex);
964 }
965 };
966
967 if (!AccExists) {
968 AccessList.emplace_back(Args: &I, Args&: RemoteI, Args: Ranges, Args&: Content, Args&: Kind, Args&: Ty);
969 assert((AccessList.size() == AccIndex + 1) &&
970 "New Access should have been at AccIndex");
971 LocalList.push_back(Elt: AccIndex);
972 AddToBins(AccessList[AccIndex].getRanges());
973 return ChangeStatus::CHANGED;
974 }
975
976 // Combine the new Access with the existing Access, and then update the
977 // mapping in the offset bins.
978 AAPointerInfo::Access Acc(&I, RemoteI, Ranges, Content, Kind, Ty);
979 auto &Current = AccessList[AccIndex];
980 auto Before = Current;
981 Current &= Acc;
982 if (Current == Before)
983 return ChangeStatus::UNCHANGED;
984
985 auto &ExistingRanges = Before.getRanges();
986 auto &NewRanges = Current.getRanges();
987
988 // Ranges that are in the old access but not the new access need to be removed
989 // from the offset bins.
990 AAPointerInfo::RangeList ToRemove;
991 AAPointerInfo::RangeList::set_difference(L: ExistingRanges, R: NewRanges, D&: ToRemove);
992 LLVM_DEBUG(if (ToRemove.size()) dbgs()
993 << "[AAPointerInfo] Removing access from old offset bins\n";);
994
995 for (auto Key : ToRemove) {
996 LLVM_DEBUG(dbgs() << " key " << Key << "\n");
997 assert(OffsetBins.count(Key) && "Existing Access must be in some bin.");
998 auto &Bin = OffsetBins[Key];
999 assert(Bin.count(AccIndex) &&
1000 "Expected bin to actually contain the Access.");
1001 Bin.erase(V: AccIndex);
1002 }
1003
1004 // Ranges that are in the new access but not the old access need to be added
1005 // to the offset bins.
1006 AAPointerInfo::RangeList ToAdd;
1007 AAPointerInfo::RangeList::set_difference(L: NewRanges, R: ExistingRanges, D&: ToAdd);
1008 AddToBins(ToAdd);
1009 return ChangeStatus::CHANGED;
1010}
1011
1012namespace {
1013
1014#ifndef NDEBUG
1015static raw_ostream &operator<<(raw_ostream &OS,
1016 const AAPointerInfo::OffsetInfo &OI) {
1017 OS << llvm::interleaved_array(OI);
1018 return OS;
1019}
1020#endif // NDEBUG
1021
1022struct AAPointerInfoImpl
1023 : public StateWrapper<AA::PointerInfo::State, AAPointerInfo> {
1024 using BaseTy = StateWrapper<AA::PointerInfo::State, AAPointerInfo>;
1025 AAPointerInfoImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
1026
1027 /// See AbstractAttribute::getAsStr().
1028 const std::string getAsStr(Attributor *A) const override {
1029 return std::string("PointerInfo ") +
1030 (isValidState() ? (std::string("#") +
1031 std::to_string(val: OffsetBins.size()) + " bins")
1032 : "<invalid>") +
1033 (reachesReturn()
1034 ? (" (returned:" +
1035 join(R: map_range(C: ReturnedOffsets,
1036 F: [](int64_t O) { return std::to_string(val: O); }),
1037 Separator: ", ") +
1038 ")")
1039 : "");
1040 }
1041
1042 /// See AbstractAttribute::manifest(...).
1043 ChangeStatus manifest(Attributor &A) override {
1044 return AAPointerInfo::manifest(A);
1045 }
1046
1047 const_bin_iterator begin() const override { return State::begin(); }
1048 const_bin_iterator end() const override { return State::end(); }
1049 int64_t numOffsetBins() const override { return State::numOffsetBins(); }
1050 bool reachesReturn() const override {
1051 return !ReturnedOffsets.isUnassigned();
1052 }
1053 void addReturnedOffsetsTo(OffsetInfo &OI) const override {
1054 if (ReturnedOffsets.isUnknown()) {
1055 OI.setUnknown();
1056 return;
1057 }
1058
1059 OffsetInfo MergedOI;
1060 for (auto Offset : ReturnedOffsets) {
1061 OffsetInfo TmpOI = OI;
1062 TmpOI.addToAll(Inc: Offset);
1063 MergedOI.merge(R: TmpOI);
1064 }
1065 OI = std::move(MergedOI);
1066 }
1067
1068 ChangeStatus setReachesReturn(const OffsetInfo &ReachedReturnedOffsets) {
1069 if (ReturnedOffsets.isUnknown())
1070 return ChangeStatus::UNCHANGED;
1071 if (ReachedReturnedOffsets.isUnknown()) {
1072 ReturnedOffsets.setUnknown();
1073 return ChangeStatus::CHANGED;
1074 }
1075 if (ReturnedOffsets.merge(R: ReachedReturnedOffsets))
1076 return ChangeStatus::CHANGED;
1077 return ChangeStatus::UNCHANGED;
1078 }
1079
1080 bool forallInterferingAccesses(
1081 AA::RangeTy Range,
1082 function_ref<bool(const AAPointerInfo::Access &, bool)> CB)
1083 const override {
1084 return State::forallInterferingAccesses(Range, CB);
1085 }
1086
1087 bool forallInterferingAccesses(
1088 Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I,
1089 bool FindInterferingWrites, bool FindInterferingReads,
1090 function_ref<bool(const Access &, bool)> UserCB, bool &HasBeenWrittenTo,
1091 AA::RangeTy &Range,
1092 function_ref<bool(const Access &)> SkipCB) const override {
1093 HasBeenWrittenTo = false;
1094
1095 SmallPtrSet<const Access *, 8> DominatingWrites;
1096 SmallVector<std::pair<const Access *, bool>, 8> InterferingAccesses;
1097
1098 Function &Scope = *I.getFunction();
1099 bool IsKnownNoSync;
1100 bool IsAssumedNoSync = AA::hasAssumedIRAttr<Attribute::NoSync>(
1101 A, QueryingAA: &QueryingAA, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL,
1102 IsKnown&: IsKnownNoSync);
1103 const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
1104 IRP: IRPosition::function(F: Scope), QueryingAA: &QueryingAA, DepClass: DepClassTy::NONE);
1105 bool AllInSameNoSyncFn = IsAssumedNoSync;
1106 bool InstIsExecutedByInitialThreadOnly =
1107 ExecDomainAA && ExecDomainAA->isExecutedByInitialThreadOnly(I);
1108
1109 // If the function is not ending in aligned barriers, we need the stores to
1110 // be in aligned barriers. The load being in one is not sufficient since the
1111 // store might be executed by a thread that disappears after, causing the
1112 // aligned barrier guarding the load to unblock and the load to read a value
1113 // that has no CFG path to the load.
1114 bool InstIsExecutedInAlignedRegion =
1115 FindInterferingReads && ExecDomainAA &&
1116 ExecDomainAA->isExecutedInAlignedRegion(A, I);
1117
1118 if (InstIsExecutedInAlignedRegion || InstIsExecutedByInitialThreadOnly)
1119 A.recordDependence(FromAA: *ExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1120
1121 InformationCache &InfoCache = A.getInfoCache();
1122 bool IsThreadLocalObj =
1123 AA::isAssumedThreadLocalObject(A, Obj&: getAssociatedValue(), QueryingAA: *this);
1124
1125 // Helper to determine if we need to consider threading, which we cannot
1126 // right now. However, if the function is (assumed) nosync or the thread
1127 // executing all instructions is the main thread only we can ignore
1128 // threading. Also, thread-local objects do not require threading reasoning.
1129 // Finally, we can ignore threading if either access is executed in an
1130 // aligned region.
1131 auto CanIgnoreThreadingForInst = [&](const Instruction &I) -> bool {
1132 if (IsThreadLocalObj || AllInSameNoSyncFn)
1133 return true;
1134 const auto *FnExecDomainAA =
1135 I.getFunction() == &Scope
1136 ? ExecDomainAA
1137 : A.lookupAAFor<AAExecutionDomain>(
1138 IRP: IRPosition::function(F: *I.getFunction()), QueryingAA: &QueryingAA,
1139 DepClass: DepClassTy::NONE);
1140 if (!FnExecDomainAA)
1141 return false;
1142 if (InstIsExecutedInAlignedRegion ||
1143 (FindInterferingWrites &&
1144 FnExecDomainAA->isExecutedInAlignedRegion(A, I))) {
1145 A.recordDependence(FromAA: *FnExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1146 return true;
1147 }
1148 if (InstIsExecutedByInitialThreadOnly &&
1149 FnExecDomainAA->isExecutedByInitialThreadOnly(I)) {
1150 A.recordDependence(FromAA: *FnExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1151 return true;
1152 }
1153 return false;
1154 };
1155
1156 // Helper to determine if the access is executed by the same thread as the
1157 // given instruction, for now it is sufficient to avoid any potential
1158 // threading effects as we cannot deal with them anyway.
1159 auto CanIgnoreThreading = [&](const Access &Acc) -> bool {
1160 return CanIgnoreThreadingForInst(*Acc.getRemoteInst()) ||
1161 (Acc.getRemoteInst() != Acc.getLocalInst() &&
1162 CanIgnoreThreadingForInst(*Acc.getLocalInst()));
1163 };
1164
1165 // TODO: Use inter-procedural reachability and dominance.
1166 bool IsKnownNoRecurse;
1167 AA::hasAssumedIRAttr<Attribute::NoRecurse>(
1168 A, QueryingAA: this, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL,
1169 IsKnown&: IsKnownNoRecurse);
1170
1171 // TODO: Use reaching kernels from AAKernelInfo (or move it to
1172 // AAExecutionDomain) such that we allow scopes other than kernels as long
1173 // as the reaching kernels are disjoint.
1174 bool InstInKernel = A.getInfoCache().isKernel(F: Scope);
1175 bool ObjHasKernelLifetime = false;
1176 const bool UseDominanceReasoning =
1177 FindInterferingWrites && IsKnownNoRecurse;
1178 const DominatorTree *DT =
1179 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: Scope);
1180
1181 // Helper to check if a value has "kernel lifetime", that is it will not
1182 // outlive a GPU kernel. This is true for shared, constant, and local
1183 // globals on AMD and NVIDIA GPUs.
1184 auto HasKernelLifetime = [&](Value *V, Module &M) {
1185 if (!AA::isGPU(M))
1186 return false;
1187 switch (AA::GPUAddressSpace(V->getType()->getPointerAddressSpace())) {
1188 case AA::GPUAddressSpace::Shared:
1189 case AA::GPUAddressSpace::Constant:
1190 case AA::GPUAddressSpace::Local:
1191 return true;
1192 default:
1193 return false;
1194 };
1195 };
1196
1197 // The IsLiveInCalleeCB will be used by the AA::isPotentiallyReachable query
1198 // to determine if we should look at reachability from the callee. For
1199 // certain pointers we know the lifetime and we do not have to step into the
1200 // callee to determine reachability as the pointer would be dead in the
1201 // callee. See the conditional initialization below.
1202 std::function<bool(const Function &)> IsLiveInCalleeCB;
1203
1204 if (auto *AI = dyn_cast<AllocaInst>(Val: &getAssociatedValue())) {
1205 // If the alloca containing function is not recursive the alloca
1206 // must be dead in the callee.
1207 const Function *AIFn = AI->getFunction();
1208 ObjHasKernelLifetime = A.getInfoCache().isKernel(F: *AIFn);
1209 bool IsKnownNoRecurse;
1210 if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
1211 A, QueryingAA: this, IRP: IRPosition::function(F: *AIFn), DepClass: DepClassTy::OPTIONAL,
1212 IsKnown&: IsKnownNoRecurse)) {
1213 IsLiveInCalleeCB = [AIFn](const Function &Fn) { return AIFn != &Fn; };
1214 }
1215 } else if (auto *GV = dyn_cast<GlobalValue>(Val: &getAssociatedValue())) {
1216 // If the global has kernel lifetime we can stop if we reach a kernel
1217 // as it is "dead" in the (unknown) callees.
1218 ObjHasKernelLifetime = HasKernelLifetime(GV, *GV->getParent());
1219 if (ObjHasKernelLifetime)
1220 IsLiveInCalleeCB = [&A](const Function &Fn) {
1221 return !A.getInfoCache().isKernel(F: Fn);
1222 };
1223 }
1224
1225 // Set of accesses/instructions that will overwrite the result and are
1226 // therefore blockers in the reachability traversal.
1227 AA::InstExclusionSetTy ExclusionSet;
1228
1229 auto AccessCB = [&](const Access &Acc, bool Exact) {
1230 Function *AccScope = Acc.getRemoteInst()->getFunction();
1231 bool AccInSameScope = AccScope == &Scope;
1232
1233 // If the object has kernel lifetime we can ignore accesses only reachable
1234 // by other kernels. For now we only skip accesses *in* other kernels.
1235 if (InstInKernel && ObjHasKernelLifetime && !AccInSameScope &&
1236 A.getInfoCache().isKernel(F: *AccScope))
1237 return true;
1238
1239 if (Exact && Acc.isMustAccess() && Acc.getRemoteInst() != &I) {
1240 if (Acc.isWrite() || (isa<LoadInst>(Val: I) && Acc.isWriteOrAssumption()))
1241 ExclusionSet.insert(Ptr: Acc.getRemoteInst());
1242 }
1243
1244 if ((!FindInterferingWrites || !Acc.isWriteOrAssumption()) &&
1245 (!FindInterferingReads || !Acc.isRead()))
1246 return true;
1247
1248 bool Dominates = FindInterferingWrites && DT && Exact &&
1249 Acc.isMustAccess() && AccInSameScope &&
1250 DT->dominates(Def: Acc.getRemoteInst(), User: &I);
1251 if (Dominates)
1252 DominatingWrites.insert(Ptr: &Acc);
1253
1254 // Track if all interesting accesses are in the same `nosync` function as
1255 // the given instruction.
1256 AllInSameNoSyncFn &= Acc.getRemoteInst()->getFunction() == &Scope;
1257
1258 InterferingAccesses.push_back(Elt: {&Acc, Exact});
1259 return true;
1260 };
1261 if (!State::forallInterferingAccesses(I, CB: AccessCB, Range))
1262 return false;
1263
1264 HasBeenWrittenTo = !DominatingWrites.empty();
1265
1266 // Dominating writes form a chain, find the least/lowest member.
1267 Instruction *LeastDominatingWriteInst = nullptr;
1268 for (const Access *Acc : DominatingWrites) {
1269 if (!LeastDominatingWriteInst) {
1270 LeastDominatingWriteInst = Acc->getRemoteInst();
1271 } else if (DT->dominates(Def: LeastDominatingWriteInst,
1272 User: Acc->getRemoteInst())) {
1273 LeastDominatingWriteInst = Acc->getRemoteInst();
1274 }
1275 }
1276
1277 // Helper to determine if we can skip a specific write access.
1278 auto CanSkipAccess = [&](const Access &Acc, bool Exact) {
1279 if (SkipCB && SkipCB(Acc))
1280 return true;
1281 if (!CanIgnoreThreading(Acc))
1282 return false;
1283
1284 // Check read (RAW) dependences and write (WAR) dependences as necessary.
1285 // If we successfully excluded all effects we are interested in, the
1286 // access can be skipped.
1287 bool ReadChecked = !FindInterferingReads;
1288 bool WriteChecked = !FindInterferingWrites;
1289
1290 // If the instruction cannot reach the access, the former does not
1291 // interfere with what the access reads.
1292 if (!ReadChecked) {
1293 if (!AA::isPotentiallyReachable(A, FromI: I, ToI: *Acc.getRemoteInst(), QueryingAA,
1294 ExclusionSet: &ExclusionSet, GoBackwardsCB: IsLiveInCalleeCB))
1295 ReadChecked = true;
1296 }
1297 // If the instruction cannot be reach from the access, the latter does not
1298 // interfere with what the instruction reads.
1299 if (!WriteChecked) {
1300 if (!AA::isPotentiallyReachable(A, FromI: *Acc.getRemoteInst(), ToI: I, QueryingAA,
1301 ExclusionSet: &ExclusionSet, GoBackwardsCB: IsLiveInCalleeCB))
1302 WriteChecked = true;
1303 }
1304
1305 // If we still might be affected by the write of the access but there are
1306 // dominating writes in the function of the instruction
1307 // (HasBeenWrittenTo), we can try to reason that the access is overwritten
1308 // by them. This would have happend above if they are all in the same
1309 // function, so we only check the inter-procedural case. Effectively, we
1310 // want to show that there is no call after the dominting write that might
1311 // reach the access, and when it returns reach the instruction with the
1312 // updated value. To this end, we iterate all call sites, check if they
1313 // might reach the instruction without going through another access
1314 // (ExclusionSet) and at the same time might reach the access. However,
1315 // that is all part of AAInterFnReachability.
1316 if (!WriteChecked && HasBeenWrittenTo &&
1317 Acc.getRemoteInst()->getFunction() != &Scope) {
1318
1319 const auto *FnReachabilityAA = A.getAAFor<AAInterFnReachability>(
1320 QueryingAA, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL);
1321 if (FnReachabilityAA) {
1322 // Without going backwards in the call tree, can we reach the access
1323 // from the least dominating write. Do not allow to pass the
1324 // instruction itself either.
1325 bool Inserted = ExclusionSet.insert(Ptr: &I).second;
1326
1327 if (!FnReachabilityAA->instructionCanReach(
1328 A, Inst: *LeastDominatingWriteInst,
1329 Fn: *Acc.getRemoteInst()->getFunction(), ExclusionSet: &ExclusionSet))
1330 WriteChecked = true;
1331
1332 if (Inserted)
1333 ExclusionSet.erase(Ptr: &I);
1334 }
1335 }
1336
1337 if (ReadChecked && WriteChecked)
1338 return true;
1339
1340 if (!DT || !UseDominanceReasoning)
1341 return false;
1342 if (!DominatingWrites.count(Ptr: &Acc))
1343 return false;
1344 return LeastDominatingWriteInst != Acc.getRemoteInst();
1345 };
1346
1347 // Run the user callback on all accesses we cannot skip and return if
1348 // that succeeded for all or not.
1349 for (auto &It : InterferingAccesses) {
1350 if ((!AllInSameNoSyncFn && !IsThreadLocalObj && !ExecDomainAA) ||
1351 !CanSkipAccess(*It.first, It.second)) {
1352 if (!UserCB(*It.first, It.second))
1353 return false;
1354 }
1355 }
1356 return true;
1357 }
1358
1359 ChangeStatus translateAndAddStateFromCallee(Attributor &A,
1360 const AAPointerInfo &OtherAA,
1361 CallBase &CB) {
1362 using namespace AA::PointerInfo;
1363 if (!OtherAA.getState().isValidState() || !isValidState())
1364 return indicatePessimisticFixpoint();
1365
1366 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1367 const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1368 bool IsByval = OtherAAImpl.getAssociatedArgument()->hasByValAttr();
1369 Changed |= setReachesReturn(OtherAAImpl.ReturnedOffsets);
1370
1371 // Combine the accesses bin by bin.
1372 const auto &State = OtherAAImpl.getState();
1373 for (const auto &It : State) {
1374 for (auto Index : It.getSecond()) {
1375 const auto &RAcc = State.getAccess(Index);
1376 if (IsByval && !RAcc.isRead())
1377 continue;
1378 bool UsedAssumedInformation = false;
1379 AccessKind AK = RAcc.getKind();
1380 auto Content = A.translateArgumentToCallSiteContent(
1381 V: RAcc.getContent(), CB, AA: *this, UsedAssumedInformation);
1382 AK = AccessKind(AK & (IsByval ? AccessKind::AK_R : AccessKind::AK_RW));
1383 AK = AccessKind(AK | (RAcc.isMayAccess() ? AK_MAY : AK_MUST));
1384
1385 Changed |= addAccess(A, Ranges: RAcc.getRanges(), I&: CB, Content, Kind: AK,
1386 Ty: RAcc.getType(), RemoteI: RAcc.getRemoteInst());
1387 }
1388 }
1389 return Changed;
1390 }
1391
1392 ChangeStatus translateAndAddState(Attributor &A, const AAPointerInfo &OtherAA,
1393 const OffsetInfo &Offsets, CallBase &CB,
1394 bool IsMustAcc) {
1395 using namespace AA::PointerInfo;
1396 if (!OtherAA.getState().isValidState() || !isValidState())
1397 return indicatePessimisticFixpoint();
1398
1399 const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1400
1401 // Combine the accesses bin by bin.
1402 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1403 const auto &State = OtherAAImpl.getState();
1404 for (const auto &It : State) {
1405 for (auto Index : It.getSecond()) {
1406 const auto &RAcc = State.getAccess(Index);
1407 if (!IsMustAcc && RAcc.isAssumption())
1408 continue;
1409 for (auto Offset : Offsets) {
1410 auto NewRanges = Offset == AA::RangeTy::Unknown
1411 ? AA::RangeTy::getUnknown()
1412 : RAcc.getRanges();
1413 if (!NewRanges.isUnknown()) {
1414 NewRanges.addToAllOffsets(Inc: Offset);
1415 }
1416 AccessKind AK = RAcc.getKind();
1417 if (!IsMustAcc)
1418 AK = AccessKind((AK & ~AK_MUST) | AK_MAY);
1419 Changed |= addAccess(A, Ranges: NewRanges, I&: CB, Content: RAcc.getContent(), Kind: AK,
1420 Ty: RAcc.getType(), RemoteI: RAcc.getRemoteInst());
1421 }
1422 }
1423 }
1424 return Changed;
1425 }
1426
1427 /// Statistic tracking for all AAPointerInfo implementations.
1428 /// See AbstractAttribute::trackStatistics().
1429 void trackPointerInfoStatistics(const IRPosition &IRP) const {}
1430
1431 /// Dump the state into \p O.
1432 void dumpState(raw_ostream &O) {
1433 for (auto &It : OffsetBins) {
1434 O << "[" << It.first.Offset << "-" << It.first.Offset + It.first.Size
1435 << "] : " << It.getSecond().size() << "\n";
1436 for (auto AccIndex : It.getSecond()) {
1437 auto &Acc = AccessList[AccIndex];
1438 O << " - " << Acc.getKind() << " - " << *Acc.getLocalInst() << "\n";
1439 if (Acc.getLocalInst() != Acc.getRemoteInst())
1440 O << " --> " << *Acc.getRemoteInst()
1441 << "\n";
1442 if (!Acc.isWrittenValueYetUndetermined()) {
1443 if (isa_and_nonnull<Function>(Val: Acc.getWrittenValue()))
1444 O << " - c: func " << Acc.getWrittenValue()->getName()
1445 << "\n";
1446 else if (Acc.getWrittenValue())
1447 O << " - c: " << *Acc.getWrittenValue() << "\n";
1448 else
1449 O << " - c: <unknown>\n";
1450 }
1451 }
1452 }
1453 }
1454};
1455
1456struct AAPointerInfoFloating : public AAPointerInfoImpl {
1457 using AccessKind = AAPointerInfo::AccessKind;
1458 AAPointerInfoFloating(const IRPosition &IRP, Attributor &A)
1459 : AAPointerInfoImpl(IRP, A) {}
1460
1461 /// Deal with an access and signal if it was handled successfully.
1462 bool handleAccess(Attributor &A, Instruction &I,
1463 std::optional<Value *> Content, AccessKind Kind,
1464 OffsetInfo::VecTy &Offsets, ChangeStatus &Changed,
1465 Type &Ty) {
1466 using namespace AA::PointerInfo;
1467 auto Size = AA::RangeTy::Unknown;
1468 const DataLayout &DL = A.getDataLayout();
1469 TypeSize AccessSize = DL.getTypeStoreSize(Ty: &Ty);
1470 if (!AccessSize.isScalable())
1471 Size = AccessSize.getFixedValue();
1472
1473 // Make a strictly ascending list of offsets as required by addAccess()
1474 SmallVector<int64_t> OffsetsSorted(Offsets.begin(), Offsets.end());
1475 llvm::sort(C&: OffsetsSorted);
1476
1477 VectorType *VT = dyn_cast<VectorType>(Val: &Ty);
1478 if (!VT || VT->getElementCount().isScalable() ||
1479 !Content.value_or(u: nullptr) || !isa<Constant>(Val: *Content) ||
1480 (*Content)->getType() != VT ||
1481 DL.getTypeStoreSize(Ty: VT->getElementType()).isScalable()) {
1482 Changed =
1483 Changed | addAccess(A, Ranges: {OffsetsSorted, Size}, I, Content, Kind, Ty: &Ty);
1484 } else {
1485 // Handle vector stores with constant content element-wise.
1486 // TODO: We could look for the elements or create instructions
1487 // representing them.
1488 // TODO: We need to push the Content into the range abstraction
1489 // (AA::RangeTy) to allow different content values for different
1490 // ranges. ranges. Hence, support vectors storing different values.
1491 Type *ElementType = VT->getElementType();
1492 int64_t ElementSize = DL.getTypeStoreSize(Ty: ElementType).getFixedValue();
1493 auto *ConstContent = cast<Constant>(Val: *Content);
1494 Type *Int32Ty = Type::getInt32Ty(C&: ElementType->getContext());
1495 SmallVector<int64_t> ElementOffsets(Offsets.begin(), Offsets.end());
1496
1497 for (int i = 0, e = VT->getElementCount().getFixedValue(); i != e; ++i) {
1498 Value *ElementContent = ConstantExpr::getExtractElement(
1499 Vec: ConstContent, Idx: ConstantInt::get(Ty: Int32Ty, V: i));
1500
1501 // Add the element access.
1502 Changed = Changed | addAccess(A, Ranges: {ElementOffsets, ElementSize}, I,
1503 Content: ElementContent, Kind, Ty: ElementType);
1504
1505 // Advance the offsets for the next element.
1506 for (auto &ElementOffset : ElementOffsets)
1507 ElementOffset += ElementSize;
1508 }
1509 }
1510 return true;
1511 };
1512
1513 /// See AbstractAttribute::updateImpl(...).
1514 ChangeStatus updateImpl(Attributor &A) override;
1515
1516 /// If the indices to \p GEP can be traced to constants, incorporate all
1517 /// of these into \p UsrOI.
1518 ///
1519 /// \return true iff \p UsrOI is updated.
1520 bool collectConstantsForGEP(Attributor &A, const DataLayout &DL,
1521 OffsetInfo &UsrOI, const OffsetInfo &PtrOI,
1522 const GEPOperator *GEP);
1523
1524 /// See AbstractAttribute::trackStatistics()
1525 void trackStatistics() const override {
1526 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1527 }
1528};
1529
1530bool AAPointerInfoFloating::collectConstantsForGEP(Attributor &A,
1531 const DataLayout &DL,
1532 OffsetInfo &UsrOI,
1533 const OffsetInfo &PtrOI,
1534 const GEPOperator *GEP) {
1535 unsigned BitWidth = DL.getIndexTypeSizeInBits(Ty: GEP->getType());
1536 SmallMapVector<Value *, APInt, 4> VariableOffsets;
1537 APInt ConstantOffset(BitWidth, 0);
1538
1539 assert(!UsrOI.isUnknown() && !PtrOI.isUnknown() &&
1540 "Don't look for constant values if the offset has already been "
1541 "determined to be unknown.");
1542
1543 if (!GEP->collectOffset(DL, BitWidth, VariableOffsets, ConstantOffset)) {
1544 UsrOI.setUnknown();
1545 return true;
1546 }
1547
1548 LLVM_DEBUG(dbgs() << "[AAPointerInfo] GEP offset is "
1549 << (VariableOffsets.empty() ? "" : "not") << " constant "
1550 << *GEP << "\n");
1551
1552 auto Union = PtrOI;
1553 Union.addToAll(Inc: ConstantOffset.getSExtValue());
1554
1555 // Each VI in VariableOffsets has a set of potential constant values. Every
1556 // combination of elements, picked one each from these sets, is separately
1557 // added to the original set of offsets, thus resulting in more offsets.
1558 for (const auto &VI : VariableOffsets) {
1559 auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
1560 QueryingAA: *this, IRP: IRPosition::value(V: *VI.first), DepClass: DepClassTy::OPTIONAL);
1561 if (!PotentialConstantsAA || !PotentialConstantsAA->isValidState()) {
1562 UsrOI.setUnknown();
1563 return true;
1564 }
1565
1566 // UndefValue is treated as a zero, which leaves Union as is.
1567 if (PotentialConstantsAA->undefIsContained())
1568 continue;
1569
1570 // We need at least one constant in every set to compute an actual offset.
1571 // Otherwise, we end up pessimizing AAPointerInfo by respecting offsets that
1572 // don't actually exist. In other words, the absence of constant values
1573 // implies that the operation can be assumed dead for now.
1574 auto &AssumedSet = PotentialConstantsAA->getAssumedSet();
1575 if (AssumedSet.empty())
1576 return false;
1577
1578 OffsetInfo Product;
1579 for (const auto &ConstOffset : AssumedSet) {
1580 auto CopyPerOffset = Union;
1581 CopyPerOffset.addToAll(Inc: ConstOffset.getSExtValue() *
1582 VI.second.getZExtValue());
1583 Product.merge(R: CopyPerOffset);
1584 }
1585 Union = Product;
1586 }
1587
1588 UsrOI = std::move(Union);
1589 return true;
1590}
1591
1592ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
1593 using namespace AA::PointerInfo;
1594 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1595 const DataLayout &DL = A.getDataLayout();
1596 Value &AssociatedValue = getAssociatedValue();
1597
1598 DenseMap<Value *, OffsetInfo> OffsetInfoMap;
1599 OffsetInfoMap[&AssociatedValue].insert(Offset: 0);
1600
1601 auto HandlePassthroughUser = [&](Value *Usr, Value *CurPtr, bool &Follow) {
1602 // One does not simply walk into a map and assign a reference to a possibly
1603 // new location. That can cause an invalidation before the assignment
1604 // happens, like so:
1605 //
1606 // OffsetInfoMap[Usr] = OffsetInfoMap[CurPtr]; /* bad idea! */
1607 //
1608 // The RHS is a reference that may be invalidated by an insertion caused by
1609 // the LHS. So we ensure that the side-effect of the LHS happens first.
1610
1611 assert(OffsetInfoMap.contains(CurPtr) &&
1612 "CurPtr does not exist in the map!");
1613
1614 auto &UsrOI = OffsetInfoMap[Usr];
1615 auto &PtrOI = OffsetInfoMap[CurPtr];
1616 assert(!PtrOI.isUnassigned() &&
1617 "Cannot pass through if the input Ptr was not visited!");
1618 UsrOI.merge(R: PtrOI);
1619 Follow = true;
1620 return true;
1621 };
1622
1623 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
1624 Value *CurPtr = U.get();
1625 User *Usr = U.getUser();
1626 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Analyze " << *CurPtr << " in " << *Usr
1627 << "\n");
1628 assert(OffsetInfoMap.count(CurPtr) &&
1629 "The current pointer offset should have been seeded!");
1630 assert(!OffsetInfoMap[CurPtr].isUnassigned() &&
1631 "Current pointer should be assigned");
1632
1633 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: Usr)) {
1634 if (CE->isCast())
1635 return HandlePassthroughUser(Usr, CurPtr, Follow);
1636 if (!isa<GEPOperator>(Val: CE)) {
1637 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled constant user " << *CE
1638 << "\n");
1639 return false;
1640 }
1641 }
1642 if (auto *GEP = dyn_cast<GEPOperator>(Val: Usr)) {
1643 // Note the order here, the Usr access might change the map, CurPtr is
1644 // already in it though.
1645 auto &UsrOI = OffsetInfoMap[Usr];
1646 auto &PtrOI = OffsetInfoMap[CurPtr];
1647
1648 if (UsrOI.isUnknown())
1649 return true;
1650
1651 if (PtrOI.isUnknown()) {
1652 Follow = true;
1653 UsrOI.setUnknown();
1654 return true;
1655 }
1656
1657 Follow = collectConstantsForGEP(A, DL, UsrOI, PtrOI, GEP);
1658 return true;
1659 }
1660 if (isa<PtrToIntInst>(Val: Usr))
1661 return false;
1662 if (isa<CastInst>(Val: Usr) || isa<SelectInst>(Val: Usr))
1663 return HandlePassthroughUser(Usr, CurPtr, Follow);
1664 // Returns are allowed if they are in the associated functions. Users can
1665 // then check the call site return. Returns from other functions can't be
1666 // tracked and are cause for invalidation.
1667 if (auto *RI = dyn_cast<ReturnInst>(Val: Usr)) {
1668 if (RI->getFunction() == getAssociatedFunction()) {
1669 auto &PtrOI = OffsetInfoMap[CurPtr];
1670 Changed |= setReachesReturn(PtrOI);
1671 return true;
1672 }
1673 return false;
1674 }
1675
1676 // For PHIs we need to take care of the recurrence explicitly as the value
1677 // might change while we iterate through a loop. For now, we give up if
1678 // the PHI is not invariant.
1679 if (auto *PHI = dyn_cast<PHINode>(Val: Usr)) {
1680 // Note the order here, the Usr access might change the map, CurPtr is
1681 // already in it though.
1682 auto [PhiIt, IsFirstPHIUser] = OffsetInfoMap.try_emplace(Key: PHI);
1683 auto &UsrOI = PhiIt->second;
1684 auto &PtrOI = OffsetInfoMap[CurPtr];
1685
1686 // Check if the PHI operand has already an unknown offset as we can't
1687 // improve on that anymore.
1688 if (PtrOI.isUnknown()) {
1689 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand offset unknown "
1690 << *CurPtr << " in " << *PHI << "\n");
1691 Follow = !UsrOI.isUnknown();
1692 UsrOI.setUnknown();
1693 return true;
1694 }
1695
1696 // Check if the PHI is invariant (so far).
1697 if (UsrOI == PtrOI) {
1698 assert(!PtrOI.isUnassigned() &&
1699 "Cannot assign if the current Ptr was not visited!");
1700 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant (so far)");
1701 return true;
1702 }
1703
1704 // Check if the PHI operand can be traced back to AssociatedValue.
1705 APInt Offset(
1706 DL.getIndexSizeInBits(AS: CurPtr->getType()->getPointerAddressSpace()),
1707 0);
1708 Value *CurPtrBase = CurPtr->stripAndAccumulateConstantOffsets(
1709 DL, Offset, /* AllowNonInbounds */ true);
1710 auto It = OffsetInfoMap.find(Val: CurPtrBase);
1711 if (It == OffsetInfoMap.end()) {
1712 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand is too complex "
1713 << *CurPtr << " in " << *PHI
1714 << " (base: " << *CurPtrBase << ")\n");
1715 UsrOI.setUnknown();
1716 Follow = true;
1717 return true;
1718 }
1719
1720 // Check if the PHI operand is not dependent on the PHI itself. Every
1721 // recurrence is a cyclic net of PHIs in the data flow, and has an
1722 // equivalent Cycle in the control flow. One of those PHIs must be in the
1723 // header of that control flow Cycle. This is independent of the choice of
1724 // Cycles reported by CycleInfo. It is sufficient to check the PHIs in
1725 // every Cycle header; if such a node is marked unknown, this will
1726 // eventually propagate through the whole net of PHIs in the recurrence.
1727 const auto *CI =
1728 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
1729 F: *PHI->getFunction());
1730 if (mayBeInCycle(CI, I: cast<Instruction>(Val: Usr), /* HeaderOnly */ true)) {
1731 auto BaseOI = It->getSecond();
1732 BaseOI.addToAll(Inc: Offset.getZExtValue());
1733 if (IsFirstPHIUser || BaseOI == UsrOI) {
1734 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant " << *CurPtr
1735 << " in " << *Usr << "\n");
1736 return HandlePassthroughUser(Usr, CurPtr, Follow);
1737 }
1738
1739 LLVM_DEBUG(
1740 dbgs() << "[AAPointerInfo] PHI operand pointer offset mismatch "
1741 << *CurPtr << " in " << *PHI << "\n");
1742 UsrOI.setUnknown();
1743 Follow = true;
1744 return true;
1745 }
1746
1747 UsrOI.merge(R: PtrOI);
1748 Follow = true;
1749 return true;
1750 }
1751
1752 if (auto *LoadI = dyn_cast<LoadInst>(Val: Usr)) {
1753 // If the access is to a pointer that may or may not be the associated
1754 // value, e.g. due to a PHI, we cannot assume it will be read.
1755 AccessKind AK = AccessKind::AK_R;
1756 if (getUnderlyingObject(V: CurPtr) == &AssociatedValue)
1757 AK = AccessKind(AK | AccessKind::AK_MUST);
1758 else
1759 AK = AccessKind(AK | AccessKind::AK_MAY);
1760 if (!handleAccess(A, I&: *LoadI, /* Content */ nullptr, Kind: AK,
1761 Offsets&: OffsetInfoMap[CurPtr].Offsets, Changed,
1762 Ty&: *LoadI->getType()))
1763 return false;
1764
1765 auto IsAssumption = [](Instruction &I) {
1766 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
1767 return II->isAssumeLikeIntrinsic();
1768 return false;
1769 };
1770
1771 auto IsImpactedInRange = [&](Instruction *FromI, Instruction *ToI) {
1772 // Check if the assumption and the load are executed together without
1773 // memory modification.
1774 do {
1775 if (FromI->mayWriteToMemory() && !IsAssumption(*FromI))
1776 return true;
1777 FromI = FromI->getNextNode();
1778 } while (FromI && FromI != ToI);
1779 return false;
1780 };
1781
1782 BasicBlock *BB = LoadI->getParent();
1783 auto IsValidAssume = [&](IntrinsicInst &IntrI) {
1784 if (IntrI.getIntrinsicID() != Intrinsic::assume)
1785 return false;
1786 BasicBlock *IntrBB = IntrI.getParent();
1787 if (IntrI.getParent() == BB) {
1788 if (IsImpactedInRange(LoadI->getNextNode(), &IntrI))
1789 return false;
1790 } else {
1791 auto PredIt = pred_begin(BB: IntrBB);
1792 if (PredIt == pred_end(BB: IntrBB))
1793 return false;
1794 if ((*PredIt) != BB)
1795 return false;
1796 if (++PredIt != pred_end(BB: IntrBB))
1797 return false;
1798 for (auto *SuccBB : successors(BB)) {
1799 if (SuccBB == IntrBB)
1800 continue;
1801 if (isa<UnreachableInst>(Val: SuccBB->getTerminator()))
1802 continue;
1803 return false;
1804 }
1805 if (IsImpactedInRange(LoadI->getNextNode(), BB->getTerminator()))
1806 return false;
1807 if (IsImpactedInRange(&IntrBB->front(), &IntrI))
1808 return false;
1809 }
1810 return true;
1811 };
1812
1813 std::pair<Value *, IntrinsicInst *> Assumption;
1814 for (const Use &LoadU : LoadI->uses()) {
1815 if (auto *CmpI = dyn_cast<CmpInst>(Val: LoadU.getUser())) {
1816 if (!CmpI->isEquality() || !CmpI->isTrueWhenEqual())
1817 continue;
1818 for (const Use &CmpU : CmpI->uses()) {
1819 if (auto *IntrI = dyn_cast<IntrinsicInst>(Val: CmpU.getUser())) {
1820 if (!IsValidAssume(*IntrI))
1821 continue;
1822 int Idx = CmpI->getOperandUse(i: 0) == LoadU;
1823 Assumption = {CmpI->getOperand(i_nocapture: Idx), IntrI};
1824 break;
1825 }
1826 }
1827 }
1828 if (Assumption.first)
1829 break;
1830 }
1831
1832 // Check if we found an assumption associated with this load.
1833 if (!Assumption.first || !Assumption.second)
1834 return true;
1835
1836 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Assumption found "
1837 << *Assumption.second << ": " << *LoadI
1838 << " == " << *Assumption.first << "\n");
1839 bool UsedAssumedInformation = false;
1840 std::optional<Value *> Content = nullptr;
1841 if (Assumption.first)
1842 Content =
1843 A.getAssumedSimplified(V: *Assumption.first, AA: *this,
1844 UsedAssumedInformation, S: AA::Interprocedural);
1845 return handleAccess(
1846 A, I&: *Assumption.second, Content, Kind: AccessKind::AK_ASSUMPTION,
1847 Offsets&: OffsetInfoMap[CurPtr].Offsets, Changed, Ty&: *LoadI->getType());
1848 }
1849
1850 auto HandleStoreLike = [&](Instruction &I, Value *ValueOp, Type &ValueTy,
1851 ArrayRef<Value *> OtherOps, AccessKind AK) {
1852 for (auto *OtherOp : OtherOps) {
1853 if (OtherOp == CurPtr) {
1854 LLVM_DEBUG(
1855 dbgs()
1856 << "[AAPointerInfo] Escaping use in store like instruction " << I
1857 << "\n");
1858 return false;
1859 }
1860 }
1861
1862 // If the access is to a pointer that may or may not be the associated
1863 // value, e.g. due to a PHI, we cannot assume it will be written.
1864 if (getUnderlyingObject(V: CurPtr) == &AssociatedValue)
1865 AK = AccessKind(AK | AccessKind::AK_MUST);
1866 else
1867 AK = AccessKind(AK | AccessKind::AK_MAY);
1868 bool UsedAssumedInformation = false;
1869 std::optional<Value *> Content = nullptr;
1870 if (ValueOp)
1871 Content = A.getAssumedSimplified(
1872 V: *ValueOp, AA: *this, UsedAssumedInformation, S: AA::Interprocedural);
1873 return handleAccess(A, I, Content, Kind: AK, Offsets&: OffsetInfoMap[CurPtr].Offsets,
1874 Changed, Ty&: ValueTy);
1875 };
1876
1877 if (auto *StoreI = dyn_cast<StoreInst>(Val: Usr))
1878 return HandleStoreLike(*StoreI, StoreI->getValueOperand(),
1879 *StoreI->getValueOperand()->getType(),
1880 {StoreI->getValueOperand()}, AccessKind::AK_W);
1881 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: Usr))
1882 return HandleStoreLike(*RMWI, nullptr, *RMWI->getValOperand()->getType(),
1883 {RMWI->getValOperand()}, AccessKind::AK_RW);
1884 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: Usr))
1885 return HandleStoreLike(
1886 *CXI, nullptr, *CXI->getNewValOperand()->getType(),
1887 {CXI->getCompareOperand(), CXI->getNewValOperand()},
1888 AccessKind::AK_RW);
1889
1890 if (auto *CB = dyn_cast<CallBase>(Val: Usr)) {
1891 if (CB->isLifetimeStartOrEnd())
1892 return true;
1893 const auto *TLI =
1894 A.getInfoCache().getTargetLibraryInfoForFunction(F: *CB->getFunction());
1895 if (getFreedOperand(CB, TLI) == U)
1896 return true;
1897 if (CB->isArgOperand(U: &U)) {
1898 unsigned ArgNo = CB->getArgOperandNo(U: &U);
1899 const auto *CSArgPI = A.getAAFor<AAPointerInfo>(
1900 QueryingAA: *this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
1901 DepClass: DepClassTy::REQUIRED);
1902 if (!CSArgPI)
1903 return false;
1904 bool IsArgMustAcc = (getUnderlyingObject(V: CurPtr) == &AssociatedValue);
1905 Changed = translateAndAddState(A, OtherAA: *CSArgPI, Offsets: OffsetInfoMap[CurPtr], CB&: *CB,
1906 IsMustAcc: IsArgMustAcc) |
1907 Changed;
1908 if (!CSArgPI->reachesReturn())
1909 return isValidState();
1910
1911 Function *Callee = CB->getCalledFunction();
1912 if (!Callee || Callee->arg_size() <= ArgNo)
1913 return false;
1914 bool UsedAssumedInformation = false;
1915 auto ReturnedValue = A.getAssumedSimplified(
1916 IRP: IRPosition::returned(F: *Callee), AA: *this, UsedAssumedInformation,
1917 S: AA::ValueScope::Intraprocedural);
1918 auto *ReturnedArg =
1919 dyn_cast_or_null<Argument>(Val: ReturnedValue.value_or(u: nullptr));
1920 auto *Arg = Callee->getArg(i: ArgNo);
1921 if (ReturnedArg && Arg != ReturnedArg)
1922 return true;
1923 bool IsRetMustAcc = IsArgMustAcc && (ReturnedArg == Arg);
1924 const auto *CSRetPI = A.getAAFor<AAPointerInfo>(
1925 QueryingAA: *this, IRP: IRPosition::callsite_returned(CB: *CB), DepClass: DepClassTy::REQUIRED);
1926 if (!CSRetPI)
1927 return false;
1928 OffsetInfo OI = OffsetInfoMap[CurPtr];
1929 CSArgPI->addReturnedOffsetsTo(OI);
1930 Changed =
1931 translateAndAddState(A, OtherAA: *CSRetPI, Offsets: OI, CB&: *CB, IsMustAcc: IsRetMustAcc) | Changed;
1932 return isValidState();
1933 }
1934 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Call user not handled " << *CB
1935 << "\n");
1936 return false;
1937 }
1938
1939 LLVM_DEBUG(dbgs() << "[AAPointerInfo] User not handled " << *Usr << "\n");
1940 return false;
1941 };
1942 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
1943 assert(OffsetInfoMap.count(OldU) && "Old use should be known already!");
1944 assert(!OffsetInfoMap[OldU].isUnassigned() && "Old use should be assinged");
1945 if (OffsetInfoMap.count(Val: NewU)) {
1946 LLVM_DEBUG({
1947 if (!(OffsetInfoMap[NewU] == OffsetInfoMap[OldU])) {
1948 dbgs() << "[AAPointerInfo] Equivalent use callback failed: "
1949 << OffsetInfoMap[NewU] << " vs " << OffsetInfoMap[OldU]
1950 << "\n";
1951 }
1952 });
1953 return OffsetInfoMap[NewU] == OffsetInfoMap[OldU];
1954 }
1955 bool Unused;
1956 return HandlePassthroughUser(NewU.get(), OldU.get(), Unused);
1957 };
1958 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: AssociatedValue,
1959 /* CheckBBLivenessOnly */ true, LivenessDepClass: DepClassTy::OPTIONAL,
1960 /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
1961 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Check for all uses failed, abort!\n");
1962 return indicatePessimisticFixpoint();
1963 }
1964
1965 LLVM_DEBUG({
1966 dbgs() << "Accesses by bin after update:\n";
1967 dumpState(dbgs());
1968 });
1969
1970 return Changed;
1971}
1972
1973struct AAPointerInfoReturned final : AAPointerInfoImpl {
1974 AAPointerInfoReturned(const IRPosition &IRP, Attributor &A)
1975 : AAPointerInfoImpl(IRP, A) {}
1976
1977 /// See AbstractAttribute::updateImpl(...).
1978 ChangeStatus updateImpl(Attributor &A) override {
1979 return indicatePessimisticFixpoint();
1980 }
1981
1982 /// See AbstractAttribute::trackStatistics()
1983 void trackStatistics() const override {
1984 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1985 }
1986};
1987
1988struct AAPointerInfoArgument final : AAPointerInfoFloating {
1989 AAPointerInfoArgument(const IRPosition &IRP, Attributor &A)
1990 : AAPointerInfoFloating(IRP, A) {}
1991
1992 /// See AbstractAttribute::trackStatistics()
1993 void trackStatistics() const override {
1994 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1995 }
1996};
1997
1998struct AAPointerInfoCallSiteArgument final : AAPointerInfoFloating {
1999 AAPointerInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
2000 : AAPointerInfoFloating(IRP, A) {}
2001
2002 /// See AbstractAttribute::updateImpl(...).
2003 ChangeStatus updateImpl(Attributor &A) override {
2004 using namespace AA::PointerInfo;
2005 // We handle memory intrinsics explicitly, at least the first (=
2006 // destination) and second (=source) arguments as we know how they are
2007 // accessed.
2008 if (auto *MI = dyn_cast_or_null<MemIntrinsic>(Val: getCtxI())) {
2009 int64_t LengthVal = AA::RangeTy::Unknown;
2010 if (auto Length = MI->getLengthInBytes())
2011 LengthVal = Length->getSExtValue();
2012 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
2013 ChangeStatus Changed = ChangeStatus::UNCHANGED;
2014 if (ArgNo > 1) {
2015 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled memory intrinsic "
2016 << *MI << "\n");
2017 return indicatePessimisticFixpoint();
2018 } else {
2019 auto Kind =
2020 ArgNo == 0 ? AccessKind::AK_MUST_WRITE : AccessKind::AK_MUST_READ;
2021 Changed =
2022 Changed | addAccess(A, Ranges: {0, LengthVal}, I&: *MI, Content: nullptr, Kind, Ty: nullptr);
2023 }
2024 LLVM_DEBUG({
2025 dbgs() << "Accesses by bin after update:\n";
2026 dumpState(dbgs());
2027 });
2028
2029 return Changed;
2030 }
2031
2032 // TODO: Once we have call site specific value information we can provide
2033 // call site specific liveness information and then it makes
2034 // sense to specialize attributes for call sites arguments instead of
2035 // redirecting requests to the callee argument.
2036 Argument *Arg = getAssociatedArgument();
2037 if (Arg) {
2038 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
2039 auto *ArgAA =
2040 A.getAAFor<AAPointerInfo>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
2041 if (ArgAA && ArgAA->getState().isValidState())
2042 return translateAndAddStateFromCallee(A, OtherAA: *ArgAA,
2043 CB&: *cast<CallBase>(Val: getCtxI()));
2044 if (!Arg->getParent()->isDeclaration())
2045 return indicatePessimisticFixpoint();
2046 }
2047
2048 bool IsKnownNoCapture;
2049 if (!AA::hasAssumedIRAttr<Attribute::Captures>(
2050 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture))
2051 return indicatePessimisticFixpoint();
2052
2053 bool IsKnown = false;
2054 if (AA::isAssumedReadNone(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
2055 return ChangeStatus::UNCHANGED;
2056 bool ReadOnly = AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown);
2057 auto Kind =
2058 ReadOnly ? AccessKind::AK_MAY_READ : AccessKind::AK_MAY_READ_WRITE;
2059 return addAccess(A, Ranges: AA::RangeTy::getUnknown(), I&: *getCtxI(), Content: nullptr, Kind,
2060 Ty: nullptr);
2061 }
2062
2063 /// See AbstractAttribute::trackStatistics()
2064 void trackStatistics() const override {
2065 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
2066 }
2067};
2068
2069struct AAPointerInfoCallSiteReturned final : AAPointerInfoFloating {
2070 AAPointerInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
2071 : AAPointerInfoFloating(IRP, A) {}
2072
2073 /// See AbstractAttribute::trackStatistics()
2074 void trackStatistics() const override {
2075 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
2076 }
2077};
2078} // namespace
2079
2080/// -----------------------NoUnwind Function Attribute--------------------------
2081
2082namespace {
2083struct AANoUnwindImpl : AANoUnwind {
2084 AANoUnwindImpl(const IRPosition &IRP, Attributor &A) : AANoUnwind(IRP, A) {}
2085
2086 /// See AbstractAttribute::initialize(...).
2087 void initialize(Attributor &A) override {
2088 bool IsKnown;
2089 assert(!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
2090 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2091 (void)IsKnown;
2092 }
2093
2094 const std::string getAsStr(Attributor *A) const override {
2095 return getAssumed() ? "nounwind" : "may-unwind";
2096 }
2097
2098 /// See AbstractAttribute::updateImpl(...).
2099 ChangeStatus updateImpl(Attributor &A) override {
2100 auto Opcodes = {
2101 (unsigned)Instruction::Invoke, (unsigned)Instruction::CallBr,
2102 (unsigned)Instruction::Call, (unsigned)Instruction::CleanupRet,
2103 (unsigned)Instruction::CatchSwitch, (unsigned)Instruction::Resume};
2104
2105 auto CheckForNoUnwind = [&](Instruction &I) {
2106 if (!I.mayThrow(/* IncludePhaseOneUnwind */ true))
2107 return true;
2108
2109 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
2110 bool IsKnownNoUnwind;
2111 return AA::hasAssumedIRAttr<Attribute::NoUnwind>(
2112 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::REQUIRED,
2113 IsKnown&: IsKnownNoUnwind);
2114 }
2115 return false;
2116 };
2117
2118 bool UsedAssumedInformation = false;
2119 if (!A.checkForAllInstructions(Pred: CheckForNoUnwind, QueryingAA: *this, Opcodes,
2120 UsedAssumedInformation))
2121 return indicatePessimisticFixpoint();
2122
2123 return ChangeStatus::UNCHANGED;
2124 }
2125};
2126
2127struct AANoUnwindFunction final : public AANoUnwindImpl {
2128 AANoUnwindFunction(const IRPosition &IRP, Attributor &A)
2129 : AANoUnwindImpl(IRP, A) {}
2130
2131 /// See AbstractAttribute::trackStatistics()
2132 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nounwind) }
2133};
2134
2135/// NoUnwind attribute deduction for a call sites.
2136struct AANoUnwindCallSite final
2137 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl> {
2138 AANoUnwindCallSite(const IRPosition &IRP, Attributor &A)
2139 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl>(IRP, A) {}
2140
2141 /// See AbstractAttribute::trackStatistics()
2142 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nounwind); }
2143};
2144} // namespace
2145
2146/// ------------------------ NoSync Function Attribute -------------------------
2147
2148bool AANoSync::isAlignedBarrier(const CallBase &CB, bool ExecutedAligned) {
2149 switch (CB.getIntrinsicID()) {
2150 case Intrinsic::nvvm_barrier_cta_sync_aligned_all:
2151 case Intrinsic::nvvm_barrier_cta_sync_aligned_count:
2152 case Intrinsic::nvvm_barrier_cta_red_and_aligned_all:
2153 case Intrinsic::nvvm_barrier_cta_red_and_aligned_count:
2154 case Intrinsic::nvvm_barrier_cta_red_or_aligned_all:
2155 case Intrinsic::nvvm_barrier_cta_red_or_aligned_count:
2156 case Intrinsic::nvvm_barrier_cta_red_popc_aligned_all:
2157 case Intrinsic::nvvm_barrier_cta_red_popc_aligned_count:
2158 return true;
2159 case Intrinsic::amdgcn_s_barrier:
2160 if (ExecutedAligned)
2161 return true;
2162 break;
2163 default:
2164 break;
2165 }
2166 return hasAssumption(CB, AssumptionStr: KnownAssumptionString("ompx_aligned_barrier"));
2167}
2168
2169bool AANoSync::isNonRelaxedAtomic(const Instruction *I) {
2170 if (!I->isAtomic())
2171 return false;
2172
2173 if (auto *FI = dyn_cast<FenceInst>(Val: I))
2174 // All legal orderings for fence are stronger than monotonic.
2175 return FI->getSyncScopeID() != SyncScope::SingleThread;
2176 if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
2177 // Unordered is not a legal ordering for cmpxchg.
2178 return (AI->getSuccessOrdering() != AtomicOrdering::Monotonic ||
2179 AI->getFailureOrdering() != AtomicOrdering::Monotonic);
2180 }
2181
2182 AtomicOrdering Ordering;
2183 switch (I->getOpcode()) {
2184 case Instruction::AtomicRMW:
2185 Ordering = cast<AtomicRMWInst>(Val: I)->getOrdering();
2186 break;
2187 case Instruction::Store:
2188 Ordering = cast<StoreInst>(Val: I)->getOrdering();
2189 break;
2190 case Instruction::Load:
2191 Ordering = cast<LoadInst>(Val: I)->getOrdering();
2192 break;
2193 default:
2194 llvm_unreachable(
2195 "New atomic operations need to be known in the attributor.");
2196 }
2197
2198 return (Ordering != AtomicOrdering::Unordered &&
2199 Ordering != AtomicOrdering::Monotonic);
2200}
2201
2202/// Return true if this intrinsic is nosync. This is only used for intrinsics
2203/// which would be nosync except that they have a volatile flag. All other
2204/// intrinsics are simply annotated with the nosync attribute in Intrinsics.td.
2205bool AANoSync::isNoSyncIntrinsic(const Instruction *I) {
2206 if (auto *MI = dyn_cast<MemIntrinsic>(Val: I))
2207 return !MI->isVolatile();
2208 return false;
2209}
2210
2211namespace {
2212struct AANoSyncImpl : AANoSync {
2213 AANoSyncImpl(const IRPosition &IRP, Attributor &A) : AANoSync(IRP, A) {}
2214
2215 /// See AbstractAttribute::initialize(...).
2216 void initialize(Attributor &A) override {
2217 bool IsKnown;
2218 assert(!AA::hasAssumedIRAttr<Attribute::NoSync>(A, nullptr, getIRPosition(),
2219 DepClassTy::NONE, IsKnown));
2220 (void)IsKnown;
2221 }
2222
2223 const std::string getAsStr(Attributor *A) const override {
2224 return getAssumed() ? "nosync" : "may-sync";
2225 }
2226
2227 /// See AbstractAttribute::updateImpl(...).
2228 ChangeStatus updateImpl(Attributor &A) override;
2229};
2230
2231ChangeStatus AANoSyncImpl::updateImpl(Attributor &A) {
2232
2233 auto CheckRWInstForNoSync = [&](Instruction &I) {
2234 return AA::isNoSyncInst(A, I, QueryingAA: *this);
2235 };
2236
2237 auto CheckForNoSync = [&](Instruction &I) {
2238 // At this point we handled all read/write effects and they are all
2239 // nosync, so they can be skipped.
2240 if (I.mayReadOrWriteMemory())
2241 return true;
2242
2243 bool IsKnown;
2244 CallBase &CB = cast<CallBase>(Val&: I);
2245 if (AA::hasAssumedIRAttr<Attribute::NoSync>(
2246 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::OPTIONAL,
2247 IsKnown))
2248 return true;
2249
2250 // non-convergent and readnone imply nosync.
2251 return !CB.isConvergent();
2252 };
2253
2254 bool UsedAssumedInformation = false;
2255 if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInstForNoSync, QueryingAA&: *this,
2256 UsedAssumedInformation) ||
2257 !A.checkForAllCallLikeInstructions(Pred: CheckForNoSync, QueryingAA: *this,
2258 UsedAssumedInformation))
2259 return indicatePessimisticFixpoint();
2260
2261 return ChangeStatus::UNCHANGED;
2262}
2263
2264struct AANoSyncFunction final : public AANoSyncImpl {
2265 AANoSyncFunction(const IRPosition &IRP, Attributor &A)
2266 : AANoSyncImpl(IRP, A) {}
2267
2268 /// See AbstractAttribute::trackStatistics()
2269 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nosync) }
2270};
2271
2272/// NoSync attribute deduction for a call sites.
2273struct AANoSyncCallSite final : AACalleeToCallSite<AANoSync, AANoSyncImpl> {
2274 AANoSyncCallSite(const IRPosition &IRP, Attributor &A)
2275 : AACalleeToCallSite<AANoSync, AANoSyncImpl>(IRP, A) {}
2276
2277 /// See AbstractAttribute::trackStatistics()
2278 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nosync); }
2279};
2280} // namespace
2281
2282/// ------------------------ No-Free Attributes ----------------------------
2283
2284namespace {
2285struct AANoFreeImpl : public AANoFree {
2286 AANoFreeImpl(const IRPosition &IRP, Attributor &A) : AANoFree(IRP, A) {}
2287
2288 /// See AbstractAttribute::initialize(...).
2289 void initialize(Attributor &A) override {
2290 bool IsKnown;
2291 assert(!AA::hasAssumedIRAttr<Attribute::NoFree>(A, nullptr, getIRPosition(),
2292 DepClassTy::NONE, IsKnown));
2293 (void)IsKnown;
2294 }
2295
2296 /// See AbstractAttribute::updateImpl(...).
2297 ChangeStatus updateImpl(Attributor &A) override {
2298 auto CheckForNoFree = [&](Instruction &I) {
2299 bool IsKnown;
2300 return AA::hasAssumedIRAttr<Attribute::NoFree>(
2301 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB: cast<CallBase>(Val&: I)),
2302 DepClass: DepClassTy::REQUIRED, IsKnown);
2303 };
2304
2305 bool UsedAssumedInformation = false;
2306 if (!A.checkForAllCallLikeInstructions(Pred: CheckForNoFree, QueryingAA: *this,
2307 UsedAssumedInformation))
2308 return indicatePessimisticFixpoint();
2309 return ChangeStatus::UNCHANGED;
2310 }
2311
2312 /// See AbstractAttribute::getAsStr().
2313 const std::string getAsStr(Attributor *A) const override {
2314 return getAssumed() ? "nofree" : "may-free";
2315 }
2316};
2317
2318struct AANoFreeFunction final : public AANoFreeImpl {
2319 AANoFreeFunction(const IRPosition &IRP, Attributor &A)
2320 : AANoFreeImpl(IRP, A) {}
2321
2322 /// See AbstractAttribute::trackStatistics()
2323 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nofree) }
2324};
2325
2326/// NoFree attribute deduction for a call sites.
2327struct AANoFreeCallSite final : AACalleeToCallSite<AANoFree, AANoFreeImpl> {
2328 AANoFreeCallSite(const IRPosition &IRP, Attributor &A)
2329 : AACalleeToCallSite<AANoFree, AANoFreeImpl>(IRP, A) {}
2330
2331 /// See AbstractAttribute::trackStatistics()
2332 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nofree); }
2333};
2334
2335/// NoFree attribute for floating values.
2336struct AANoFreeFloating : AANoFreeImpl {
2337 AANoFreeFloating(const IRPosition &IRP, Attributor &A)
2338 : AANoFreeImpl(IRP, A) {}
2339
2340 /// See AbstractAttribute::trackStatistics()
2341 void trackStatistics() const override{STATS_DECLTRACK_FLOATING_ATTR(nofree)}
2342
2343 /// See Abstract Attribute::updateImpl(...).
2344 ChangeStatus updateImpl(Attributor &A) override {
2345 const IRPosition &IRP = getIRPosition();
2346
2347 bool IsKnown;
2348 if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, QueryingAA: this,
2349 IRP: IRPosition::function_scope(IRP),
2350 DepClass: DepClassTy::OPTIONAL, IsKnown))
2351 return ChangeStatus::UNCHANGED;
2352
2353 Value &AssociatedValue = getIRPosition().getAssociatedValue();
2354 auto Pred = [&](const Use &U, bool &Follow) -> bool {
2355 Instruction *UserI = cast<Instruction>(Val: U.getUser());
2356 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
2357 if (CB->isBundleOperand(U: &U))
2358 return false;
2359 if (!CB->isArgOperand(U: &U))
2360 return true;
2361 unsigned ArgNo = CB->getArgOperandNo(U: &U);
2362
2363 bool IsKnown;
2364 return AA::hasAssumedIRAttr<Attribute::NoFree>(
2365 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
2366 DepClass: DepClassTy::REQUIRED, IsKnown);
2367 }
2368
2369 if (isa<GetElementPtrInst>(Val: UserI) || isa<PHINode>(Val: UserI) ||
2370 isa<SelectInst>(Val: UserI)) {
2371 Follow = true;
2372 return true;
2373 }
2374 if (isa<StoreInst>(Val: UserI) || isa<LoadInst>(Val: UserI))
2375 return true;
2376
2377 if (isa<ReturnInst>(Val: UserI) && getIRPosition().isArgumentPosition())
2378 return true;
2379
2380 // Unknown user.
2381 return false;
2382 };
2383 if (!A.checkForAllUses(Pred, QueryingAA: *this, V: AssociatedValue))
2384 return indicatePessimisticFixpoint();
2385
2386 return ChangeStatus::UNCHANGED;
2387 }
2388};
2389
2390/// NoFree attribute for a call site argument.
2391struct AANoFreeArgument final : AANoFreeFloating {
2392 AANoFreeArgument(const IRPosition &IRP, Attributor &A)
2393 : AANoFreeFloating(IRP, A) {}
2394
2395 /// See AbstractAttribute::trackStatistics()
2396 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofree) }
2397};
2398
2399/// NoFree attribute for call site arguments.
2400struct AANoFreeCallSiteArgument final : AANoFreeFloating {
2401 AANoFreeCallSiteArgument(const IRPosition &IRP, Attributor &A)
2402 : AANoFreeFloating(IRP, A) {}
2403
2404 /// See AbstractAttribute::updateImpl(...).
2405 ChangeStatus updateImpl(Attributor &A) override {
2406 // TODO: Once we have call site specific value information we can provide
2407 // call site specific liveness information and then it makes
2408 // sense to specialize attributes for call sites arguments instead of
2409 // redirecting requests to the callee argument.
2410 Argument *Arg = getAssociatedArgument();
2411 if (!Arg)
2412 return indicatePessimisticFixpoint();
2413 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
2414 bool IsKnown;
2415 if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, QueryingAA: this, IRP: ArgPos,
2416 DepClass: DepClassTy::REQUIRED, IsKnown))
2417 return ChangeStatus::UNCHANGED;
2418 return indicatePessimisticFixpoint();
2419 }
2420
2421 /// See AbstractAttribute::trackStatistics()
2422 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nofree) };
2423};
2424
2425/// NoFree attribute for function return value.
2426struct AANoFreeReturned final : AANoFreeFloating {
2427 AANoFreeReturned(const IRPosition &IRP, Attributor &A)
2428 : AANoFreeFloating(IRP, A) {
2429 llvm_unreachable("NoFree is not applicable to function returns!");
2430 }
2431
2432 /// See AbstractAttribute::initialize(...).
2433 void initialize(Attributor &A) override {
2434 llvm_unreachable("NoFree is not applicable to function returns!");
2435 }
2436
2437 /// See AbstractAttribute::updateImpl(...).
2438 ChangeStatus updateImpl(Attributor &A) override {
2439 llvm_unreachable("NoFree is not applicable to function returns!");
2440 }
2441
2442 /// See AbstractAttribute::trackStatistics()
2443 void trackStatistics() const override {}
2444};
2445
2446/// NoFree attribute deduction for a call site return value.
2447struct AANoFreeCallSiteReturned final : AANoFreeFloating {
2448 AANoFreeCallSiteReturned(const IRPosition &IRP, Attributor &A)
2449 : AANoFreeFloating(IRP, A) {}
2450
2451 ChangeStatus manifest(Attributor &A) override {
2452 return ChangeStatus::UNCHANGED;
2453 }
2454 /// See AbstractAttribute::trackStatistics()
2455 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nofree) }
2456};
2457} // namespace
2458
2459/// ------------------------ NonNull Argument Attribute ------------------------
2460
2461bool AANonNull::isImpliedByIR(Attributor &A, const IRPosition &IRP,
2462 Attribute::AttrKind ImpliedAttributeKind,
2463 bool IgnoreSubsumingPositions) {
2464 SmallVector<Attribute::AttrKind, 2> AttrKinds;
2465 AttrKinds.push_back(Elt: Attribute::NonNull);
2466 if (!NullPointerIsDefined(F: IRP.getAnchorScope(),
2467 AS: IRP.getAssociatedType()->getPointerAddressSpace()))
2468 AttrKinds.push_back(Elt: Attribute::Dereferenceable);
2469 if (A.hasAttr(IRP, AKs: AttrKinds, IgnoreSubsumingPositions, ImpliedAttributeKind: Attribute::NonNull))
2470 return true;
2471
2472 DominatorTree *DT = nullptr;
2473 AssumptionCache *AC = nullptr;
2474 InformationCache &InfoCache = A.getInfoCache();
2475 if (const Function *Fn = IRP.getAnchorScope()) {
2476 if (!Fn->isDeclaration()) {
2477 DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *Fn);
2478 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *Fn);
2479 }
2480 }
2481
2482 SmallVector<AA::ValueAndContext> Worklist;
2483 if (IRP.getPositionKind() != IRP_RETURNED) {
2484 Worklist.push_back(Elt: {IRP.getAssociatedValue(), IRP.getCtxI()});
2485 } else {
2486 bool UsedAssumedInformation = false;
2487 if (!A.checkForAllInstructions(
2488 Pred: [&](Instruction &I) {
2489 Worklist.push_back(Elt: {*cast<ReturnInst>(Val&: I).getReturnValue(), &I});
2490 return true;
2491 },
2492 Fn: IRP.getAssociatedFunction(), QueryingAA: nullptr, Opcodes: {Instruction::Ret},
2493 UsedAssumedInformation, CheckBBLivenessOnly: false, /*CheckPotentiallyDead=*/true))
2494 return false;
2495 }
2496
2497 if (llvm::any_of(Range&: Worklist, P: [&](AA::ValueAndContext VAC) {
2498 return !isKnownNonZero(
2499 V: VAC.getValue(),
2500 Q: SimplifyQuery(A.getDataLayout(), DT, AC, VAC.getCtxI()));
2501 }))
2502 return false;
2503
2504 A.manifestAttrs(IRP, DeducedAttrs: {Attribute::get(Context&: IRP.getAnchorValue().getContext(),
2505 Kind: Attribute::NonNull)});
2506 return true;
2507}
2508
2509namespace {
2510static int64_t getKnownNonNullAndDerefBytesForUse(
2511 Attributor &A, const AbstractAttribute &QueryingAA, Value &AssociatedValue,
2512 const Use *U, const Instruction *I, bool &IsNonNull, bool &TrackUse) {
2513 TrackUse = false;
2514
2515 const Value *UseV = U->get();
2516 if (!UseV->getType()->isPointerTy())
2517 return 0;
2518
2519 // We need to follow common pointer manipulation uses to the accesses they
2520 // feed into. We can try to be smart to avoid looking through things we do not
2521 // like for now, e.g., non-inbounds GEPs.
2522 if (isa<CastInst>(Val: I)) {
2523 TrackUse = true;
2524 return 0;
2525 }
2526
2527 if (isa<GetElementPtrInst>(Val: I)) {
2528 TrackUse = true;
2529 return 0;
2530 }
2531
2532 Type *PtrTy = UseV->getType();
2533 const Function *F = I->getFunction();
2534 bool NullPointerIsDefined =
2535 F ? llvm::NullPointerIsDefined(F, AS: PtrTy->getPointerAddressSpace()) : true;
2536 const DataLayout &DL = A.getInfoCache().getDL();
2537 if (const auto *CB = dyn_cast<CallBase>(Val: I)) {
2538 if (CB->isBundleOperand(U)) {
2539 if (RetainedKnowledge RK = getKnowledgeFromUse(
2540 U, AttrKinds: {Attribute::NonNull, Attribute::Dereferenceable})) {
2541 IsNonNull |=
2542 (RK.AttrKind == Attribute::NonNull || !NullPointerIsDefined);
2543 return RK.ArgValue;
2544 }
2545 return 0;
2546 }
2547
2548 if (CB->isCallee(U)) {
2549 IsNonNull |= !NullPointerIsDefined;
2550 return 0;
2551 }
2552
2553 unsigned ArgNo = CB->getArgOperandNo(U);
2554 IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
2555 // As long as we only use known information there is no need to track
2556 // dependences here.
2557 bool IsKnownNonNull;
2558 AA::hasAssumedIRAttr<Attribute::NonNull>(A, QueryingAA: &QueryingAA, IRP,
2559 DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
2560 IsNonNull |= IsKnownNonNull;
2561 auto *DerefAA =
2562 A.getAAFor<AADereferenceable>(QueryingAA, IRP, DepClass: DepClassTy::NONE);
2563 return DerefAA ? DerefAA->getKnownDereferenceableBytes() : 0;
2564 }
2565
2566 std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(Inst: I);
2567 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() ||
2568 Loc->Size.isScalable() || I->isVolatile())
2569 return 0;
2570
2571 int64_t Offset;
2572 const Value *Base =
2573 getMinimalBaseOfPointer(A, QueryingAA, Ptr: Loc->Ptr, BytesOffset&: Offset, DL);
2574 if (Base && Base == &AssociatedValue) {
2575 int64_t DerefBytes = Loc->Size.getValue() + Offset;
2576 IsNonNull |= !NullPointerIsDefined;
2577 return std::max(a: int64_t(0), b: DerefBytes);
2578 }
2579
2580 /// Corner case when an offset is 0.
2581 Base = GetPointerBaseWithConstantOffset(Ptr: Loc->Ptr, Offset, DL,
2582 /*AllowNonInbounds*/ true);
2583 if (Base && Base == &AssociatedValue && Offset == 0) {
2584 int64_t DerefBytes = Loc->Size.getValue();
2585 IsNonNull |= !NullPointerIsDefined;
2586 return std::max(a: int64_t(0), b: DerefBytes);
2587 }
2588
2589 return 0;
2590}
2591
2592struct AANonNullImpl : AANonNull {
2593 AANonNullImpl(const IRPosition &IRP, Attributor &A) : AANonNull(IRP, A) {}
2594
2595 /// See AbstractAttribute::initialize(...).
2596 void initialize(Attributor &A) override {
2597 Value &V = *getAssociatedValue().stripPointerCasts();
2598 if (isa<ConstantPointerNull>(Val: V)) {
2599 indicatePessimisticFixpoint();
2600 return;
2601 }
2602
2603 if (Instruction *CtxI = getCtxI())
2604 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
2605 }
2606
2607 /// See followUsesInMBEC
2608 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
2609 AANonNull::StateType &State) {
2610 bool IsNonNull = false;
2611 bool TrackUse = false;
2612 getKnownNonNullAndDerefBytesForUse(A, QueryingAA: *this, AssociatedValue&: getAssociatedValue(), U, I,
2613 IsNonNull, TrackUse);
2614 State.setKnown(IsNonNull);
2615 return TrackUse;
2616 }
2617
2618 /// See AbstractAttribute::getAsStr().
2619 const std::string getAsStr(Attributor *A) const override {
2620 return getAssumed() ? "nonnull" : "may-null";
2621 }
2622};
2623
2624/// NonNull attribute for a floating value.
2625struct AANonNullFloating : public AANonNullImpl {
2626 AANonNullFloating(const IRPosition &IRP, Attributor &A)
2627 : AANonNullImpl(IRP, A) {}
2628
2629 /// See AbstractAttribute::updateImpl(...).
2630 ChangeStatus updateImpl(Attributor &A) override {
2631 auto CheckIRP = [&](const IRPosition &IRP) {
2632 bool IsKnownNonNull;
2633 return AA::hasAssumedIRAttr<Attribute::NonNull>(
2634 A, QueryingAA: *this, IRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNonNull);
2635 };
2636
2637 bool Stripped;
2638 bool UsedAssumedInformation = false;
2639 Value *AssociatedValue = &getAssociatedValue();
2640 SmallVector<AA::ValueAndContext> Values;
2641 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
2642 S: AA::AnyScope, UsedAssumedInformation))
2643 Stripped = false;
2644 else
2645 Stripped =
2646 Values.size() != 1 || Values.front().getValue() != AssociatedValue;
2647
2648 if (!Stripped) {
2649 bool IsKnown;
2650 if (auto *PHI = dyn_cast<PHINode>(Val: AssociatedValue))
2651 if (llvm::all_of(Range: PHI->incoming_values(), P: [&](Value *Op) {
2652 return AA::hasAssumedIRAttr<Attribute::NonNull>(
2653 A, QueryingAA: this, IRP: IRPosition::value(V: *Op), DepClass: DepClassTy::OPTIONAL,
2654 IsKnown);
2655 }))
2656 return ChangeStatus::UNCHANGED;
2657 if (auto *Select = dyn_cast<SelectInst>(Val: AssociatedValue))
2658 if (AA::hasAssumedIRAttr<Attribute::NonNull>(
2659 A, QueryingAA: this, IRP: IRPosition::value(V: *Select->getFalseValue()),
2660 DepClass: DepClassTy::OPTIONAL, IsKnown) &&
2661 AA::hasAssumedIRAttr<Attribute::NonNull>(
2662 A, QueryingAA: this, IRP: IRPosition::value(V: *Select->getTrueValue()),
2663 DepClass: DepClassTy::OPTIONAL, IsKnown))
2664 return ChangeStatus::UNCHANGED;
2665
2666 // If we haven't stripped anything we might still be able to use a
2667 // different AA, but only if the IRP changes. Effectively when we
2668 // interpret this not as a call site value but as a floating/argument
2669 // value.
2670 const IRPosition AVIRP = IRPosition::value(V: *AssociatedValue);
2671 if (AVIRP == getIRPosition() || !CheckIRP(AVIRP))
2672 return indicatePessimisticFixpoint();
2673 return ChangeStatus::UNCHANGED;
2674 }
2675
2676 for (const auto &VAC : Values)
2677 if (!CheckIRP(IRPosition::value(V: *VAC.getValue())))
2678 return indicatePessimisticFixpoint();
2679
2680 return ChangeStatus::UNCHANGED;
2681 }
2682
2683 /// See AbstractAttribute::trackStatistics()
2684 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2685};
2686
2687/// NonNull attribute for function return value.
2688struct AANonNullReturned final
2689 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2690 false, AANonNull::IRAttributeKind, false> {
2691 AANonNullReturned(const IRPosition &IRP, Attributor &A)
2692 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2693 false, Attribute::NonNull, false>(IRP, A) {
2694 }
2695
2696 /// See AbstractAttribute::getAsStr().
2697 const std::string getAsStr(Attributor *A) const override {
2698 return getAssumed() ? "nonnull" : "may-null";
2699 }
2700
2701 /// See AbstractAttribute::trackStatistics()
2702 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2703};
2704
2705/// NonNull attribute for function argument.
2706struct AANonNullArgument final
2707 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl> {
2708 AANonNullArgument(const IRPosition &IRP, Attributor &A)
2709 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl>(IRP, A) {}
2710
2711 /// See AbstractAttribute::trackStatistics()
2712 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nonnull) }
2713};
2714
2715struct AANonNullCallSiteArgument final : AANonNullFloating {
2716 AANonNullCallSiteArgument(const IRPosition &IRP, Attributor &A)
2717 : AANonNullFloating(IRP, A) {}
2718
2719 /// See AbstractAttribute::trackStatistics()
2720 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nonnull) }
2721};
2722
2723/// NonNull attribute for a call site return position.
2724struct AANonNullCallSiteReturned final
2725 : AACalleeToCallSite<AANonNull, AANonNullImpl> {
2726 AANonNullCallSiteReturned(const IRPosition &IRP, Attributor &A)
2727 : AACalleeToCallSite<AANonNull, AANonNullImpl>(IRP, A) {}
2728
2729 /// See AbstractAttribute::trackStatistics()
2730 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nonnull) }
2731};
2732} // namespace
2733
2734/// ------------------------ Must-Progress Attributes --------------------------
2735namespace {
2736struct AAMustProgressImpl : public AAMustProgress {
2737 AAMustProgressImpl(const IRPosition &IRP, Attributor &A)
2738 : AAMustProgress(IRP, A) {}
2739
2740 /// See AbstractAttribute::initialize(...).
2741 void initialize(Attributor &A) override {
2742 bool IsKnown;
2743 assert(!AA::hasAssumedIRAttr<Attribute::MustProgress>(
2744 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2745 (void)IsKnown;
2746 }
2747
2748 /// See AbstractAttribute::getAsStr()
2749 const std::string getAsStr(Attributor *A) const override {
2750 return getAssumed() ? "mustprogress" : "may-not-progress";
2751 }
2752};
2753
2754struct AAMustProgressFunction final : AAMustProgressImpl {
2755 AAMustProgressFunction(const IRPosition &IRP, Attributor &A)
2756 : AAMustProgressImpl(IRP, A) {}
2757
2758 /// See AbstractAttribute::updateImpl(...).
2759 ChangeStatus updateImpl(Attributor &A) override {
2760 bool IsKnown;
2761 if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
2762 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown)) {
2763 if (IsKnown)
2764 return indicateOptimisticFixpoint();
2765 return ChangeStatus::UNCHANGED;
2766 }
2767
2768 auto CheckForMustProgress = [&](AbstractCallSite ACS) {
2769 IRPosition IPos = IRPosition::callsite_function(CB: *ACS.getInstruction());
2770 bool IsKnownMustProgress;
2771 return AA::hasAssumedIRAttr<Attribute::MustProgress>(
2772 A, QueryingAA: this, IRP: IPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownMustProgress,
2773 /* IgnoreSubsumingPositions */ true);
2774 };
2775
2776 bool AllCallSitesKnown = true;
2777 if (!A.checkForAllCallSites(Pred: CheckForMustProgress, QueryingAA: *this,
2778 /* RequireAllCallSites */ true,
2779 UsedAssumedInformation&: AllCallSitesKnown))
2780 return indicatePessimisticFixpoint();
2781
2782 return ChangeStatus::UNCHANGED;
2783 }
2784
2785 /// See AbstractAttribute::trackStatistics()
2786 void trackStatistics() const override {
2787 STATS_DECLTRACK_FN_ATTR(mustprogress)
2788 }
2789};
2790
2791/// MustProgress attribute deduction for a call sites.
2792struct AAMustProgressCallSite final : AAMustProgressImpl {
2793 AAMustProgressCallSite(const IRPosition &IRP, Attributor &A)
2794 : AAMustProgressImpl(IRP, A) {}
2795
2796 /// See AbstractAttribute::updateImpl(...).
2797 ChangeStatus updateImpl(Attributor &A) override {
2798 // TODO: Once we have call site specific value information we can provide
2799 // call site specific liveness information and then it makes
2800 // sense to specialize attributes for call sites arguments instead of
2801 // redirecting requests to the callee argument.
2802 const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
2803 bool IsKnownMustProgress;
2804 if (!AA::hasAssumedIRAttr<Attribute::MustProgress>(
2805 A, QueryingAA: this, IRP: FnPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownMustProgress))
2806 return indicatePessimisticFixpoint();
2807 return ChangeStatus::UNCHANGED;
2808 }
2809
2810 /// See AbstractAttribute::trackStatistics()
2811 void trackStatistics() const override {
2812 STATS_DECLTRACK_CS_ATTR(mustprogress);
2813 }
2814};
2815} // namespace
2816
2817/// ------------------------ No-Recurse Attributes ----------------------------
2818
2819namespace {
2820struct AANoRecurseImpl : public AANoRecurse {
2821 AANoRecurseImpl(const IRPosition &IRP, Attributor &A) : AANoRecurse(IRP, A) {}
2822
2823 /// See AbstractAttribute::initialize(...).
2824 void initialize(Attributor &A) override {
2825 bool IsKnown;
2826 assert(!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
2827 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2828 (void)IsKnown;
2829 }
2830
2831 /// See AbstractAttribute::getAsStr()
2832 const std::string getAsStr(Attributor *A) const override {
2833 return getAssumed() ? "norecurse" : "may-recurse";
2834 }
2835};
2836
2837struct AANoRecurseFunction final : AANoRecurseImpl {
2838 AANoRecurseFunction(const IRPosition &IRP, Attributor &A)
2839 : AANoRecurseImpl(IRP, A) {}
2840
2841 /// See AbstractAttribute::updateImpl(...).
2842 ChangeStatus updateImpl(Attributor &A) override {
2843
2844 // If all live call sites are known to be no-recurse, we are as well.
2845 auto CallSitePred = [&](AbstractCallSite ACS) {
2846 bool IsKnownNoRecurse;
2847 if (!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
2848 A, QueryingAA: this,
2849 IRP: IRPosition::function(F: *ACS.getInstruction()->getFunction()),
2850 DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoRecurse))
2851 return false;
2852 return IsKnownNoRecurse;
2853 };
2854 bool UsedAssumedInformation = false;
2855 if (A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this, RequireAllCallSites: true,
2856 UsedAssumedInformation)) {
2857 // If we know all call sites and all are known no-recurse, we are done.
2858 // If all known call sites, which might not be all that exist, are known
2859 // to be no-recurse, we are not done but we can continue to assume
2860 // no-recurse. If one of the call sites we have not visited will become
2861 // live, another update is triggered.
2862 if (!UsedAssumedInformation)
2863 indicateOptimisticFixpoint();
2864 return ChangeStatus::UNCHANGED;
2865 }
2866
2867 const AAInterFnReachability *EdgeReachability =
2868 A.getAAFor<AAInterFnReachability>(QueryingAA: *this, IRP: getIRPosition(),
2869 DepClass: DepClassTy::REQUIRED);
2870 if (EdgeReachability && EdgeReachability->canReach(A, Fn: *getAnchorScope()))
2871 return indicatePessimisticFixpoint();
2872 return ChangeStatus::UNCHANGED;
2873 }
2874
2875 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(norecurse) }
2876};
2877
2878/// NoRecurse attribute deduction for a call sites.
2879struct AANoRecurseCallSite final
2880 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl> {
2881 AANoRecurseCallSite(const IRPosition &IRP, Attributor &A)
2882 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl>(IRP, A) {}
2883
2884 /// See AbstractAttribute::trackStatistics()
2885 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(norecurse); }
2886};
2887} // namespace
2888
2889/// ------------------------ No-Convergent Attribute --------------------------
2890
2891namespace {
2892struct AANonConvergentImpl : public AANonConvergent {
2893 AANonConvergentImpl(const IRPosition &IRP, Attributor &A)
2894 : AANonConvergent(IRP, A) {}
2895
2896 /// See AbstractAttribute::getAsStr()
2897 const std::string getAsStr(Attributor *A) const override {
2898 return getAssumed() ? "non-convergent" : "may-be-convergent";
2899 }
2900};
2901
2902struct AANonConvergentFunction final : AANonConvergentImpl {
2903 AANonConvergentFunction(const IRPosition &IRP, Attributor &A)
2904 : AANonConvergentImpl(IRP, A) {}
2905
2906 /// See AbstractAttribute::updateImpl(...).
2907 ChangeStatus updateImpl(Attributor &A) override {
2908 // If all function calls are known to not be convergent, we are not
2909 // convergent.
2910 auto CalleeIsNotConvergent = [&](Instruction &Inst) {
2911 CallBase &CB = cast<CallBase>(Val&: Inst);
2912 auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand());
2913 if (!Callee || Callee->isIntrinsic()) {
2914 return false;
2915 }
2916 if (Callee->isDeclaration()) {
2917 return !Callee->hasFnAttribute(Kind: Attribute::Convergent);
2918 }
2919 const auto *ConvergentAA = A.getAAFor<AANonConvergent>(
2920 QueryingAA: *this, IRP: IRPosition::function(F: *Callee), DepClass: DepClassTy::REQUIRED);
2921 return ConvergentAA && ConvergentAA->isAssumedNotConvergent();
2922 };
2923
2924 bool UsedAssumedInformation = false;
2925 if (!A.checkForAllCallLikeInstructions(Pred: CalleeIsNotConvergent, QueryingAA: *this,
2926 UsedAssumedInformation)) {
2927 return indicatePessimisticFixpoint();
2928 }
2929 return ChangeStatus::UNCHANGED;
2930 }
2931
2932 ChangeStatus manifest(Attributor &A) override {
2933 if (isKnownNotConvergent() &&
2934 A.hasAttr(IRP: getIRPosition(), AKs: Attribute::Convergent)) {
2935 A.removeAttrs(IRP: getIRPosition(), AttrKinds: {Attribute::Convergent});
2936 return ChangeStatus::CHANGED;
2937 }
2938 return ChangeStatus::UNCHANGED;
2939 }
2940
2941 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(convergent) }
2942};
2943} // namespace
2944
2945/// -------------------- Undefined-Behavior Attributes ------------------------
2946
2947namespace {
2948struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior {
2949 AAUndefinedBehaviorImpl(const IRPosition &IRP, Attributor &A)
2950 : AAUndefinedBehavior(IRP, A) {}
2951
2952 /// See AbstractAttribute::updateImpl(...).
2953 // through a pointer (i.e. also branches etc.)
2954 ChangeStatus updateImpl(Attributor &A) override {
2955 const size_t UBPrevSize = KnownUBInsts.size();
2956 const size_t NoUBPrevSize = AssumedNoUBInsts.size();
2957
2958 auto InspectMemAccessInstForUB = [&](Instruction &I) {
2959 // Lang ref now states volatile store is not UB, let's skip them.
2960 if (I.isVolatile() && I.mayWriteToMemory())
2961 return true;
2962
2963 // Skip instructions that are already saved.
2964 if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Ptr: &I))
2965 return true;
2966
2967 // If we reach here, we know we have an instruction
2968 // that accesses memory through a pointer operand,
2969 // for which getPointerOperand() should give it to us.
2970 Value *PtrOp =
2971 const_cast<Value *>(getPointerOperand(I: &I, /* AllowVolatile */ true));
2972 assert(PtrOp &&
2973 "Expected pointer operand of memory accessing instruction");
2974
2975 // Either we stopped and the appropriate action was taken,
2976 // or we got back a simplified value to continue.
2977 std::optional<Value *> SimplifiedPtrOp =
2978 stopOnUndefOrAssumed(A, V: PtrOp, I: &I);
2979 if (!SimplifiedPtrOp || !*SimplifiedPtrOp)
2980 return true;
2981 const Value *PtrOpVal = *SimplifiedPtrOp;
2982
2983 // A memory access through a pointer is considered UB
2984 // only if the pointer has constant null value.
2985 // TODO: Expand it to not only check constant values.
2986 if (!isa<ConstantPointerNull>(Val: PtrOpVal)) {
2987 AssumedNoUBInsts.insert(Ptr: &I);
2988 return true;
2989 }
2990 const Type *PtrTy = PtrOpVal->getType();
2991
2992 // Because we only consider instructions inside functions,
2993 // assume that a parent function exists.
2994 const Function *F = I.getFunction();
2995
2996 // A memory access using constant null pointer is only considered UB
2997 // if null pointer is _not_ defined for the target platform.
2998 if (llvm::NullPointerIsDefined(F, AS: PtrTy->getPointerAddressSpace()))
2999 AssumedNoUBInsts.insert(Ptr: &I);
3000 else
3001 KnownUBInsts.insert(Ptr: &I);
3002 return true;
3003 };
3004
3005 auto InspectBrInstForUB = [&](Instruction &I) {
3006 // A conditional branch instruction is considered UB if it has `undef`
3007 // condition.
3008
3009 // Skip instructions that are already saved.
3010 if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Ptr: &I))
3011 return true;
3012
3013 // We know we have a branch instruction.
3014 auto *BrInst = cast<BranchInst>(Val: &I);
3015
3016 // Unconditional branches are never considered UB.
3017 if (BrInst->isUnconditional())
3018 return true;
3019
3020 // Either we stopped and the appropriate action was taken,
3021 // or we got back a simplified value to continue.
3022 std::optional<Value *> SimplifiedCond =
3023 stopOnUndefOrAssumed(A, V: BrInst->getCondition(), I: BrInst);
3024 if (!SimplifiedCond || !*SimplifiedCond)
3025 return true;
3026 AssumedNoUBInsts.insert(Ptr: &I);
3027 return true;
3028 };
3029
3030 auto InspectCallSiteForUB = [&](Instruction &I) {
3031 // Check whether a callsite always cause UB or not
3032
3033 // Skip instructions that are already saved.
3034 if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Ptr: &I))
3035 return true;
3036
3037 // Check nonnull and noundef argument attribute violation for each
3038 // callsite.
3039 CallBase &CB = cast<CallBase>(Val&: I);
3040 auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand());
3041 if (!Callee)
3042 return true;
3043 for (unsigned idx = 0; idx < CB.arg_size(); idx++) {
3044 // If current argument is known to be simplified to null pointer and the
3045 // corresponding argument position is known to have nonnull attribute,
3046 // the argument is poison. Furthermore, if the argument is poison and
3047 // the position is known to have noundef attriubte, this callsite is
3048 // considered UB.
3049 if (idx >= Callee->arg_size())
3050 break;
3051 Value *ArgVal = CB.getArgOperand(i: idx);
3052 if (!ArgVal)
3053 continue;
3054 // Here, we handle three cases.
3055 // (1) Not having a value means it is dead. (we can replace the value
3056 // with undef)
3057 // (2) Simplified to undef. The argument violate noundef attriubte.
3058 // (3) Simplified to null pointer where known to be nonnull.
3059 // The argument is a poison value and violate noundef attribute.
3060 IRPosition CalleeArgumentIRP = IRPosition::callsite_argument(CB, ArgNo: idx);
3061 bool IsKnownNoUndef;
3062 AA::hasAssumedIRAttr<Attribute::NoUndef>(
3063 A, QueryingAA: this, IRP: CalleeArgumentIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoUndef);
3064 if (!IsKnownNoUndef)
3065 continue;
3066 bool UsedAssumedInformation = false;
3067 std::optional<Value *> SimplifiedVal =
3068 A.getAssumedSimplified(IRP: IRPosition::value(V: *ArgVal), AA: *this,
3069 UsedAssumedInformation, S: AA::Interprocedural);
3070 if (UsedAssumedInformation)
3071 continue;
3072 if (SimplifiedVal && !*SimplifiedVal)
3073 return true;
3074 if (!SimplifiedVal || isa<UndefValue>(Val: **SimplifiedVal)) {
3075 KnownUBInsts.insert(Ptr: &I);
3076 continue;
3077 }
3078 if (!ArgVal->getType()->isPointerTy() ||
3079 !isa<ConstantPointerNull>(Val: **SimplifiedVal))
3080 continue;
3081 bool IsKnownNonNull;
3082 AA::hasAssumedIRAttr<Attribute::NonNull>(
3083 A, QueryingAA: this, IRP: CalleeArgumentIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
3084 if (IsKnownNonNull)
3085 KnownUBInsts.insert(Ptr: &I);
3086 }
3087 return true;
3088 };
3089
3090 auto InspectReturnInstForUB = [&](Instruction &I) {
3091 auto &RI = cast<ReturnInst>(Val&: I);
3092 // Either we stopped and the appropriate action was taken,
3093 // or we got back a simplified return value to continue.
3094 std::optional<Value *> SimplifiedRetValue =
3095 stopOnUndefOrAssumed(A, V: RI.getReturnValue(), I: &I);
3096 if (!SimplifiedRetValue || !*SimplifiedRetValue)
3097 return true;
3098
3099 // Check if a return instruction always cause UB or not
3100 // Note: It is guaranteed that the returned position of the anchor
3101 // scope has noundef attribute when this is called.
3102 // We also ensure the return position is not "assumed dead"
3103 // because the returned value was then potentially simplified to
3104 // `undef` in AAReturnedValues without removing the `noundef`
3105 // attribute yet.
3106
3107 // When the returned position has noundef attriubte, UB occurs in the
3108 // following cases.
3109 // (1) Returned value is known to be undef.
3110 // (2) The value is known to be a null pointer and the returned
3111 // position has nonnull attribute (because the returned value is
3112 // poison).
3113 if (isa<ConstantPointerNull>(Val: *SimplifiedRetValue)) {
3114 bool IsKnownNonNull;
3115 AA::hasAssumedIRAttr<Attribute::NonNull>(
3116 A, QueryingAA: this, IRP: IRPosition::returned(F: *getAnchorScope()), DepClass: DepClassTy::NONE,
3117 IsKnown&: IsKnownNonNull);
3118 if (IsKnownNonNull)
3119 KnownUBInsts.insert(Ptr: &I);
3120 }
3121
3122 return true;
3123 };
3124
3125 bool UsedAssumedInformation = false;
3126 A.checkForAllInstructions(Pred: InspectMemAccessInstForUB, QueryingAA: *this,
3127 Opcodes: {Instruction::Load, Instruction::Store,
3128 Instruction::AtomicCmpXchg,
3129 Instruction::AtomicRMW},
3130 UsedAssumedInformation,
3131 /* CheckBBLivenessOnly */ true);
3132 A.checkForAllInstructions(Pred: InspectBrInstForUB, QueryingAA: *this, Opcodes: {Instruction::Br},
3133 UsedAssumedInformation,
3134 /* CheckBBLivenessOnly */ true);
3135 A.checkForAllCallLikeInstructions(Pred: InspectCallSiteForUB, QueryingAA: *this,
3136 UsedAssumedInformation);
3137
3138 // If the returned position of the anchor scope has noundef attriubte, check
3139 // all returned instructions.
3140 if (!getAnchorScope()->getReturnType()->isVoidTy()) {
3141 const IRPosition &ReturnIRP = IRPosition::returned(F: *getAnchorScope());
3142 if (!A.isAssumedDead(IRP: ReturnIRP, QueryingAA: this, FnLivenessAA: nullptr, UsedAssumedInformation)) {
3143 bool IsKnownNoUndef;
3144 AA::hasAssumedIRAttr<Attribute::NoUndef>(
3145 A, QueryingAA: this, IRP: ReturnIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoUndef);
3146 if (IsKnownNoUndef)
3147 A.checkForAllInstructions(Pred: InspectReturnInstForUB, QueryingAA: *this,
3148 Opcodes: {Instruction::Ret}, UsedAssumedInformation,
3149 /* CheckBBLivenessOnly */ true);
3150 }
3151 }
3152
3153 if (NoUBPrevSize != AssumedNoUBInsts.size() ||
3154 UBPrevSize != KnownUBInsts.size())
3155 return ChangeStatus::CHANGED;
3156 return ChangeStatus::UNCHANGED;
3157 }
3158
3159 bool isKnownToCauseUB(Instruction *I) const override {
3160 return KnownUBInsts.count(Ptr: I);
3161 }
3162
3163 bool isAssumedToCauseUB(Instruction *I) const override {
3164 // In simple words, if an instruction is not in the assumed to _not_
3165 // cause UB, then it is assumed UB (that includes those
3166 // in the KnownUBInsts set). The rest is boilerplate
3167 // is to ensure that it is one of the instructions we test
3168 // for UB.
3169
3170 switch (I->getOpcode()) {
3171 case Instruction::Load:
3172 case Instruction::Store:
3173 case Instruction::AtomicCmpXchg:
3174 case Instruction::AtomicRMW:
3175 return !AssumedNoUBInsts.count(Ptr: I);
3176 case Instruction::Br: {
3177 auto *BrInst = cast<BranchInst>(Val: I);
3178 if (BrInst->isUnconditional())
3179 return false;
3180 return !AssumedNoUBInsts.count(Ptr: I);
3181 } break;
3182 default:
3183 return false;
3184 }
3185 return false;
3186 }
3187
3188 ChangeStatus manifest(Attributor &A) override {
3189 if (KnownUBInsts.empty())
3190 return ChangeStatus::UNCHANGED;
3191 for (Instruction *I : KnownUBInsts)
3192 A.changeToUnreachableAfterManifest(I);
3193 return ChangeStatus::CHANGED;
3194 }
3195
3196 /// See AbstractAttribute::getAsStr()
3197 const std::string getAsStr(Attributor *A) const override {
3198 return getAssumed() ? "undefined-behavior" : "no-ub";
3199 }
3200
3201 /// Note: The correctness of this analysis depends on the fact that the
3202 /// following 2 sets will stop changing after some point.
3203 /// "Change" here means that their size changes.
3204 /// The size of each set is monotonically increasing
3205 /// (we only add items to them) and it is upper bounded by the number of
3206 /// instructions in the processed function (we can never save more
3207 /// elements in either set than this number). Hence, at some point,
3208 /// they will stop increasing.
3209 /// Consequently, at some point, both sets will have stopped
3210 /// changing, effectively making the analysis reach a fixpoint.
3211
3212 /// Note: These 2 sets are disjoint and an instruction can be considered
3213 /// one of 3 things:
3214 /// 1) Known to cause UB (AAUndefinedBehavior could prove it) and put it in
3215 /// the KnownUBInsts set.
3216 /// 2) Assumed to cause UB (in every updateImpl, AAUndefinedBehavior
3217 /// has a reason to assume it).
3218 /// 3) Assumed to not cause UB. very other instruction - AAUndefinedBehavior
3219 /// could not find a reason to assume or prove that it can cause UB,
3220 /// hence it assumes it doesn't. We have a set for these instructions
3221 /// so that we don't reprocess them in every update.
3222 /// Note however that instructions in this set may cause UB.
3223
3224protected:
3225 /// A set of all live instructions _known_ to cause UB.
3226 SmallPtrSet<Instruction *, 8> KnownUBInsts;
3227
3228private:
3229 /// A set of all the (live) instructions that are assumed to _not_ cause UB.
3230 SmallPtrSet<Instruction *, 8> AssumedNoUBInsts;
3231
3232 // Should be called on updates in which if we're processing an instruction
3233 // \p I that depends on a value \p V, one of the following has to happen:
3234 // - If the value is assumed, then stop.
3235 // - If the value is known but undef, then consider it UB.
3236 // - Otherwise, do specific processing with the simplified value.
3237 // We return std::nullopt in the first 2 cases to signify that an appropriate
3238 // action was taken and the caller should stop.
3239 // Otherwise, we return the simplified value that the caller should
3240 // use for specific processing.
3241 std::optional<Value *> stopOnUndefOrAssumed(Attributor &A, Value *V,
3242 Instruction *I) {
3243 bool UsedAssumedInformation = false;
3244 std::optional<Value *> SimplifiedV =
3245 A.getAssumedSimplified(IRP: IRPosition::value(V: *V), AA: *this,
3246 UsedAssumedInformation, S: AA::Interprocedural);
3247 if (!UsedAssumedInformation) {
3248 // Don't depend on assumed values.
3249 if (!SimplifiedV) {
3250 // If it is known (which we tested above) but it doesn't have a value,
3251 // then we can assume `undef` and hence the instruction is UB.
3252 KnownUBInsts.insert(Ptr: I);
3253 return std::nullopt;
3254 }
3255 if (!*SimplifiedV)
3256 return nullptr;
3257 V = *SimplifiedV;
3258 }
3259 if (isa<UndefValue>(Val: V)) {
3260 KnownUBInsts.insert(Ptr: I);
3261 return std::nullopt;
3262 }
3263 return V;
3264 }
3265};
3266
3267struct AAUndefinedBehaviorFunction final : AAUndefinedBehaviorImpl {
3268 AAUndefinedBehaviorFunction(const IRPosition &IRP, Attributor &A)
3269 : AAUndefinedBehaviorImpl(IRP, A) {}
3270
3271 /// See AbstractAttribute::trackStatistics()
3272 void trackStatistics() const override {
3273 STATS_DECL(UndefinedBehaviorInstruction, Instruction,
3274 "Number of instructions known to have UB");
3275 BUILD_STAT_NAME(UndefinedBehaviorInstruction, Instruction) +=
3276 KnownUBInsts.size();
3277 }
3278};
3279} // namespace
3280
3281/// ------------------------ Will-Return Attributes ----------------------------
3282
3283namespace {
3284// Helper function that checks whether a function has any cycle which we don't
3285// know if it is bounded or not.
3286// Loops with maximum trip count are considered bounded, any other cycle not.
3287static bool mayContainUnboundedCycle(Function &F, Attributor &A) {
3288 ScalarEvolution *SE =
3289 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(F);
3290 LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F);
3291 // If either SCEV or LoopInfo is not available for the function then we assume
3292 // any cycle to be unbounded cycle.
3293 // We use scc_iterator which uses Tarjan algorithm to find all the maximal
3294 // SCCs.To detect if there's a cycle, we only need to find the maximal ones.
3295 if (!SE || !LI) {
3296 for (scc_iterator<Function *> SCCI = scc_begin(G: &F); !SCCI.isAtEnd(); ++SCCI)
3297 if (SCCI.hasCycle())
3298 return true;
3299 return false;
3300 }
3301
3302 // If there's irreducible control, the function may contain non-loop cycles.
3303 if (mayContainIrreducibleControl(F, LI))
3304 return true;
3305
3306 // Any loop that does not have a max trip count is considered unbounded cycle.
3307 for (auto *L : LI->getLoopsInPreorder()) {
3308 if (!SE->getSmallConstantMaxTripCount(L))
3309 return true;
3310 }
3311 return false;
3312}
3313
3314struct AAWillReturnImpl : public AAWillReturn {
3315 AAWillReturnImpl(const IRPosition &IRP, Attributor &A)
3316 : AAWillReturn(IRP, A) {}
3317
3318 /// See AbstractAttribute::initialize(...).
3319 void initialize(Attributor &A) override {
3320 bool IsKnown;
3321 assert(!AA::hasAssumedIRAttr<Attribute::WillReturn>(
3322 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
3323 (void)IsKnown;
3324 }
3325
3326 /// Check for `mustprogress` and `readonly` as they imply `willreturn`.
3327 bool isImpliedByMustprogressAndReadonly(Attributor &A, bool KnownOnly) {
3328 if (!A.hasAttr(IRP: getIRPosition(), AKs: {Attribute::MustProgress}))
3329 return false;
3330
3331 bool IsKnown;
3332 if (AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
3333 return IsKnown || !KnownOnly;
3334 return false;
3335 }
3336
3337 /// See AbstractAttribute::updateImpl(...).
3338 ChangeStatus updateImpl(Attributor &A) override {
3339 if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3340 return ChangeStatus::UNCHANGED;
3341
3342 auto CheckForWillReturn = [&](Instruction &I) {
3343 IRPosition IPos = IRPosition::callsite_function(CB: cast<CallBase>(Val&: I));
3344 bool IsKnown;
3345 if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
3346 A, QueryingAA: this, IRP: IPos, DepClass: DepClassTy::REQUIRED, IsKnown)) {
3347 if (IsKnown)
3348 return true;
3349 } else {
3350 return false;
3351 }
3352 bool IsKnownNoRecurse;
3353 return AA::hasAssumedIRAttr<Attribute::NoRecurse>(
3354 A, QueryingAA: this, IRP: IPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoRecurse);
3355 };
3356
3357 bool UsedAssumedInformation = false;
3358 if (!A.checkForAllCallLikeInstructions(Pred: CheckForWillReturn, QueryingAA: *this,
3359 UsedAssumedInformation))
3360 return indicatePessimisticFixpoint();
3361
3362 return ChangeStatus::UNCHANGED;
3363 }
3364
3365 /// See AbstractAttribute::getAsStr()
3366 const std::string getAsStr(Attributor *A) const override {
3367 return getAssumed() ? "willreturn" : "may-noreturn";
3368 }
3369};
3370
3371struct AAWillReturnFunction final : AAWillReturnImpl {
3372 AAWillReturnFunction(const IRPosition &IRP, Attributor &A)
3373 : AAWillReturnImpl(IRP, A) {}
3374
3375 /// See AbstractAttribute::initialize(...).
3376 void initialize(Attributor &A) override {
3377 AAWillReturnImpl::initialize(A);
3378
3379 Function *F = getAnchorScope();
3380 assert(F && "Did expect an anchor function");
3381 if (F->isDeclaration() || mayContainUnboundedCycle(F&: *F, A))
3382 indicatePessimisticFixpoint();
3383 }
3384
3385 /// See AbstractAttribute::trackStatistics()
3386 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(willreturn) }
3387};
3388
3389/// WillReturn attribute deduction for a call sites.
3390struct AAWillReturnCallSite final
3391 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl> {
3392 AAWillReturnCallSite(const IRPosition &IRP, Attributor &A)
3393 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl>(IRP, A) {}
3394
3395 /// See AbstractAttribute::updateImpl(...).
3396 ChangeStatus updateImpl(Attributor &A) override {
3397 if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3398 return ChangeStatus::UNCHANGED;
3399
3400 return AACalleeToCallSite::updateImpl(A);
3401 }
3402
3403 /// See AbstractAttribute::trackStatistics()
3404 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(willreturn); }
3405};
3406} // namespace
3407
3408/// -------------------AAIntraFnReachability Attribute--------------------------
3409
3410/// All information associated with a reachability query. This boilerplate code
3411/// is used by both AAIntraFnReachability and AAInterFnReachability, with
3412/// different \p ToTy values.
3413template <typename ToTy> struct ReachabilityQueryInfo {
3414 enum class Reachable {
3415 No,
3416 Yes,
3417 };
3418
3419 /// Start here,
3420 const Instruction *From = nullptr;
3421 /// reach this place,
3422 const ToTy *To = nullptr;
3423 /// without going through any of these instructions,
3424 const AA::InstExclusionSetTy *ExclusionSet = nullptr;
3425 /// and remember if it worked:
3426 Reachable Result = Reachable::No;
3427
3428 /// Precomputed hash for this RQI.
3429 unsigned Hash = 0;
3430
3431 unsigned computeHashValue() const {
3432 assert(Hash == 0 && "Computed hash twice!");
3433 using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;
3434 using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;
3435 return const_cast<ReachabilityQueryInfo<ToTy> *>(this)->Hash =
3436 detail::combineHashValue(a: PairDMI ::getHashValue({From, To}),
3437 b: InstSetDMI::getHashValue(BES: ExclusionSet));
3438 }
3439
3440 ReachabilityQueryInfo(const Instruction *From, const ToTy *To)
3441 : From(From), To(To) {}
3442
3443 /// Constructor replacement to ensure unique and stable sets are used for the
3444 /// cache.
3445 ReachabilityQueryInfo(Attributor &A, const Instruction &From, const ToTy &To,
3446 const AA::InstExclusionSetTy *ES, bool MakeUnique)
3447 : From(&From), To(&To), ExclusionSet(ES) {
3448
3449 if (!ES || ES->empty()) {
3450 ExclusionSet = nullptr;
3451 } else if (MakeUnique) {
3452 ExclusionSet = A.getInfoCache().getOrCreateUniqueBlockExecutionSet(BES: ES);
3453 }
3454 }
3455
3456 ReachabilityQueryInfo(const ReachabilityQueryInfo &RQI)
3457 : From(RQI.From), To(RQI.To), ExclusionSet(RQI.ExclusionSet) {}
3458};
3459
3460namespace llvm {
3461template <typename ToTy> struct DenseMapInfo<ReachabilityQueryInfo<ToTy> *> {
3462 using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;
3463 using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;
3464
3465 static ReachabilityQueryInfo<ToTy> EmptyKey;
3466 static ReachabilityQueryInfo<ToTy> TombstoneKey;
3467
3468 static inline ReachabilityQueryInfo<ToTy> *getEmptyKey() { return &EmptyKey; }
3469 static inline ReachabilityQueryInfo<ToTy> *getTombstoneKey() {
3470 return &TombstoneKey;
3471 }
3472 static unsigned getHashValue(const ReachabilityQueryInfo<ToTy> *RQI) {
3473 return RQI->Hash ? RQI->Hash : RQI->computeHashValue();
3474 }
3475 static bool isEqual(const ReachabilityQueryInfo<ToTy> *LHS,
3476 const ReachabilityQueryInfo<ToTy> *RHS) {
3477 if (!PairDMI::isEqual({LHS->From, LHS->To}, {RHS->From, RHS->To}))
3478 return false;
3479 return InstSetDMI::isEqual(LHS: LHS->ExclusionSet, RHS: RHS->ExclusionSet);
3480 }
3481};
3482
3483#define DefineKeys(ToTy) \
3484 template <> \
3485 ReachabilityQueryInfo<ToTy> \
3486 DenseMapInfo<ReachabilityQueryInfo<ToTy> *>::EmptyKey = \
3487 ReachabilityQueryInfo<ToTy>( \
3488 DenseMapInfo<const Instruction *>::getEmptyKey(), \
3489 DenseMapInfo<const ToTy *>::getEmptyKey()); \
3490 template <> \
3491 ReachabilityQueryInfo<ToTy> \
3492 DenseMapInfo<ReachabilityQueryInfo<ToTy> *>::TombstoneKey = \
3493 ReachabilityQueryInfo<ToTy>( \
3494 DenseMapInfo<const Instruction *>::getTombstoneKey(), \
3495 DenseMapInfo<const ToTy *>::getTombstoneKey());
3496
3497DefineKeys(Instruction) DefineKeys(Function)
3498#undef DefineKeys
3499
3500} // namespace llvm
3501
3502namespace {
3503
3504template <typename BaseTy, typename ToTy>
3505struct CachedReachabilityAA : public BaseTy {
3506 using RQITy = ReachabilityQueryInfo<ToTy>;
3507
3508 CachedReachabilityAA(const IRPosition &IRP, Attributor &A) : BaseTy(IRP, A) {}
3509
3510 /// See AbstractAttribute::isQueryAA.
3511 bool isQueryAA() const override { return true; }
3512
3513 /// See AbstractAttribute::updateImpl(...).
3514 ChangeStatus updateImpl(Attributor &A) override {
3515 ChangeStatus Changed = ChangeStatus::UNCHANGED;
3516 for (unsigned u = 0, e = QueryVector.size(); u < e; ++u) {
3517 RQITy *RQI = QueryVector[u];
3518 if (RQI->Result == RQITy::Reachable::No &&
3519 isReachableImpl(A, RQI&: *RQI, /*IsTemporaryRQI=*/false))
3520 Changed = ChangeStatus::CHANGED;
3521 }
3522 return Changed;
3523 }
3524
3525 virtual bool isReachableImpl(Attributor &A, RQITy &RQI,
3526 bool IsTemporaryRQI) = 0;
3527
3528 bool rememberResult(Attributor &A, typename RQITy::Reachable Result,
3529 RQITy &RQI, bool UsedExclusionSet, bool IsTemporaryRQI) {
3530 RQI.Result = Result;
3531
3532 // Remove the temporary RQI from the cache.
3533 if (IsTemporaryRQI)
3534 QueryCache.erase(&RQI);
3535
3536 // Insert a plain RQI (w/o exclusion set) if that makes sense. Two options:
3537 // 1) If it is reachable, it doesn't matter if we have an exclusion set for
3538 // this query. 2) We did not use the exclusion set, potentially because
3539 // there is none.
3540 if (Result == RQITy::Reachable::Yes || !UsedExclusionSet) {
3541 RQITy PlainRQI(RQI.From, RQI.To);
3542 if (!QueryCache.count(&PlainRQI)) {
3543 RQITy *RQIPtr = new (A.Allocator) RQITy(RQI.From, RQI.To);
3544 RQIPtr->Result = Result;
3545 QueryVector.push_back(RQIPtr);
3546 QueryCache.insert(RQIPtr);
3547 }
3548 }
3549
3550 // Check if we need to insert a new permanent RQI with the exclusion set.
3551 if (IsTemporaryRQI && Result != RQITy::Reachable::Yes && UsedExclusionSet) {
3552 assert((!RQI.ExclusionSet || !RQI.ExclusionSet->empty()) &&
3553 "Did not expect empty set!");
3554 RQITy *RQIPtr = new (A.Allocator)
3555 RQITy(A, *RQI.From, *RQI.To, RQI.ExclusionSet, true);
3556 assert(RQIPtr->Result == RQITy::Reachable::No && "Already reachable?");
3557 RQIPtr->Result = Result;
3558 assert(!QueryCache.count(RQIPtr));
3559 QueryVector.push_back(RQIPtr);
3560 QueryCache.insert(RQIPtr);
3561 }
3562
3563 if (Result == RQITy::Reachable::No && IsTemporaryRQI)
3564 A.registerForUpdate(AA&: *this);
3565 return Result == RQITy::Reachable::Yes;
3566 }
3567
3568 const std::string getAsStr(Attributor *A) const override {
3569 // TODO: Return the number of reachable queries.
3570 return "#queries(" + std::to_string(QueryVector.size()) + ")";
3571 }
3572
3573 bool checkQueryCache(Attributor &A, RQITy &StackRQI,
3574 typename RQITy::Reachable &Result) {
3575 if (!this->getState().isValidState()) {
3576 Result = RQITy::Reachable::Yes;
3577 return true;
3578 }
3579
3580 // If we have an exclusion set we might be able to find our answer by
3581 // ignoring it first.
3582 if (StackRQI.ExclusionSet) {
3583 RQITy PlainRQI(StackRQI.From, StackRQI.To);
3584 auto It = QueryCache.find(&PlainRQI);
3585 if (It != QueryCache.end() && (*It)->Result == RQITy::Reachable::No) {
3586 Result = RQITy::Reachable::No;
3587 return true;
3588 }
3589 }
3590
3591 auto It = QueryCache.find(&StackRQI);
3592 if (It != QueryCache.end()) {
3593 Result = (*It)->Result;
3594 return true;
3595 }
3596
3597 // Insert a temporary for recursive queries. We will replace it with a
3598 // permanent entry later.
3599 QueryCache.insert(&StackRQI);
3600 return false;
3601 }
3602
3603private:
3604 SmallVector<RQITy *> QueryVector;
3605 DenseSet<RQITy *> QueryCache;
3606};
3607
3608struct AAIntraFnReachabilityFunction final
3609 : public CachedReachabilityAA<AAIntraFnReachability, Instruction> {
3610 using Base = CachedReachabilityAA<AAIntraFnReachability, Instruction>;
3611 AAIntraFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
3612 : Base(IRP, A) {
3613 DT = A.getInfoCache().getAnalysisResultForFunction<DominatorTreeAnalysis>(
3614 F: *IRP.getAssociatedFunction());
3615 }
3616
3617 bool isAssumedReachable(
3618 Attributor &A, const Instruction &From, const Instruction &To,
3619 const AA::InstExclusionSetTy *ExclusionSet) const override {
3620 auto *NonConstThis = const_cast<AAIntraFnReachabilityFunction *>(this);
3621 if (&From == &To)
3622 return true;
3623
3624 RQITy StackRQI(A, From, To, ExclusionSet, false);
3625 RQITy::Reachable Result;
3626 if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
3627 return NonConstThis->isReachableImpl(A, RQI&: StackRQI,
3628 /*IsTemporaryRQI=*/true);
3629 return Result == RQITy::Reachable::Yes;
3630 }
3631
3632 ChangeStatus updateImpl(Attributor &A) override {
3633 // We only depend on liveness. DeadEdges is all we care about, check if any
3634 // of them changed.
3635 auto *LivenessAA =
3636 A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL);
3637 if (LivenessAA &&
3638 llvm::all_of(Range&: DeadEdges,
3639 P: [&](const auto &DeadEdge) {
3640 return LivenessAA->isEdgeDead(From: DeadEdge.first,
3641 To: DeadEdge.second);
3642 }) &&
3643 llvm::all_of(Range&: DeadBlocks, P: [&](const BasicBlock *BB) {
3644 return LivenessAA->isAssumedDead(BB);
3645 })) {
3646 return ChangeStatus::UNCHANGED;
3647 }
3648 DeadEdges.clear();
3649 DeadBlocks.clear();
3650 return Base::updateImpl(A);
3651 }
3652
3653 bool isReachableImpl(Attributor &A, RQITy &RQI,
3654 bool IsTemporaryRQI) override {
3655 const Instruction *Origin = RQI.From;
3656 bool UsedExclusionSet = false;
3657
3658 auto WillReachInBlock = [&](const Instruction &From, const Instruction &To,
3659 const AA::InstExclusionSetTy *ExclusionSet) {
3660 const Instruction *IP = &From;
3661 while (IP && IP != &To) {
3662 if (ExclusionSet && IP != Origin && ExclusionSet->count(Ptr: IP)) {
3663 UsedExclusionSet = true;
3664 break;
3665 }
3666 IP = IP->getNextNode();
3667 }
3668 return IP == &To;
3669 };
3670
3671 const BasicBlock *FromBB = RQI.From->getParent();
3672 const BasicBlock *ToBB = RQI.To->getParent();
3673 assert(FromBB->getParent() == ToBB->getParent() &&
3674 "Not an intra-procedural query!");
3675
3676 // Check intra-block reachability, however, other reaching paths are still
3677 // possible.
3678 if (FromBB == ToBB &&
3679 WillReachInBlock(*RQI.From, *RQI.To, RQI.ExclusionSet))
3680 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3681 IsTemporaryRQI);
3682
3683 // Check if reaching the ToBB block is sufficient or if even that would not
3684 // ensure reaching the target. In the latter case we are done.
3685 if (!WillReachInBlock(ToBB->front(), *RQI.To, RQI.ExclusionSet))
3686 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3687 IsTemporaryRQI);
3688
3689 const Function *Fn = FromBB->getParent();
3690 SmallPtrSet<const BasicBlock *, 16> ExclusionBlocks;
3691 if (RQI.ExclusionSet)
3692 for (auto *I : *RQI.ExclusionSet)
3693 if (I->getFunction() == Fn)
3694 ExclusionBlocks.insert(Ptr: I->getParent());
3695
3696 // Check if we make it out of the FromBB block at all.
3697 if (ExclusionBlocks.count(Ptr: FromBB) &&
3698 !WillReachInBlock(*RQI.From, *FromBB->getTerminator(),
3699 RQI.ExclusionSet))
3700 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet: true, IsTemporaryRQI);
3701
3702 auto *LivenessAA =
3703 A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL);
3704 if (LivenessAA && LivenessAA->isAssumedDead(BB: ToBB)) {
3705 DeadBlocks.insert(V: ToBB);
3706 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3707 IsTemporaryRQI);
3708 }
3709
3710 SmallPtrSet<const BasicBlock *, 16> Visited;
3711 SmallVector<const BasicBlock *, 16> Worklist;
3712 Worklist.push_back(Elt: FromBB);
3713
3714 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> LocalDeadEdges;
3715 while (!Worklist.empty()) {
3716 const BasicBlock *BB = Worklist.pop_back_val();
3717 if (!Visited.insert(Ptr: BB).second)
3718 continue;
3719 for (const BasicBlock *SuccBB : successors(BB)) {
3720 if (LivenessAA && LivenessAA->isEdgeDead(From: BB, To: SuccBB)) {
3721 LocalDeadEdges.insert(V: {BB, SuccBB});
3722 continue;
3723 }
3724 // We checked before if we just need to reach the ToBB block.
3725 if (SuccBB == ToBB)
3726 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3727 IsTemporaryRQI);
3728 if (DT && ExclusionBlocks.empty() && DT->dominates(A: BB, B: ToBB))
3729 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3730 IsTemporaryRQI);
3731
3732 if (ExclusionBlocks.count(Ptr: SuccBB)) {
3733 UsedExclusionSet = true;
3734 continue;
3735 }
3736 Worklist.push_back(Elt: SuccBB);
3737 }
3738 }
3739
3740 DeadEdges.insert_range(R&: LocalDeadEdges);
3741 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3742 IsTemporaryRQI);
3743 }
3744
3745 /// See AbstractAttribute::trackStatistics()
3746 void trackStatistics() const override {}
3747
3748private:
3749 // Set of assumed dead blocks we used in the last query. If any changes we
3750 // update the state.
3751 DenseSet<const BasicBlock *> DeadBlocks;
3752
3753 // Set of assumed dead edges we used in the last query. If any changes we
3754 // update the state.
3755 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> DeadEdges;
3756
3757 /// The dominator tree of the function to short-circuit reasoning.
3758 const DominatorTree *DT = nullptr;
3759};
3760} // namespace
3761
3762/// ------------------------ NoAlias Argument Attribute ------------------------
3763
3764bool AANoAlias::isImpliedByIR(Attributor &A, const IRPosition &IRP,
3765 Attribute::AttrKind ImpliedAttributeKind,
3766 bool IgnoreSubsumingPositions) {
3767 assert(ImpliedAttributeKind == Attribute::NoAlias &&
3768 "Unexpected attribute kind");
3769 Value *Val = &IRP.getAssociatedValue();
3770 if (IRP.getPositionKind() != IRP_CALL_SITE_ARGUMENT) {
3771 if (isa<AllocaInst>(Val))
3772 return true;
3773 } else {
3774 IgnoreSubsumingPositions = true;
3775 }
3776
3777 if (isa<UndefValue>(Val))
3778 return true;
3779
3780 if (isa<ConstantPointerNull>(Val) &&
3781 !NullPointerIsDefined(F: IRP.getAnchorScope(),
3782 AS: Val->getType()->getPointerAddressSpace()))
3783 return true;
3784
3785 if (A.hasAttr(IRP, AKs: {Attribute::ByVal, Attribute::NoAlias},
3786 IgnoreSubsumingPositions, ImpliedAttributeKind: Attribute::NoAlias))
3787 return true;
3788
3789 return false;
3790}
3791
3792namespace {
3793struct AANoAliasImpl : AANoAlias {
3794 AANoAliasImpl(const IRPosition &IRP, Attributor &A) : AANoAlias(IRP, A) {
3795 assert(getAssociatedType()->isPointerTy() &&
3796 "Noalias is a pointer attribute");
3797 }
3798
3799 const std::string getAsStr(Attributor *A) const override {
3800 return getAssumed() ? "noalias" : "may-alias";
3801 }
3802};
3803
3804/// NoAlias attribute for a floating value.
3805struct AANoAliasFloating final : AANoAliasImpl {
3806 AANoAliasFloating(const IRPosition &IRP, Attributor &A)
3807 : AANoAliasImpl(IRP, A) {}
3808
3809 /// See AbstractAttribute::updateImpl(...).
3810 ChangeStatus updateImpl(Attributor &A) override {
3811 // TODO: Implement this.
3812 return indicatePessimisticFixpoint();
3813 }
3814
3815 /// See AbstractAttribute::trackStatistics()
3816 void trackStatistics() const override {
3817 STATS_DECLTRACK_FLOATING_ATTR(noalias)
3818 }
3819};
3820
3821/// NoAlias attribute for an argument.
3822struct AANoAliasArgument final
3823 : AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl> {
3824 using Base = AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl>;
3825 AANoAliasArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
3826
3827 /// See AbstractAttribute::update(...).
3828 ChangeStatus updateImpl(Attributor &A) override {
3829 // We have to make sure no-alias on the argument does not break
3830 // synchronization when this is a callback argument, see also [1] below.
3831 // If synchronization cannot be affected, we delegate to the base updateImpl
3832 // function, otherwise we give up for now.
3833
3834 // If the function is no-sync, no-alias cannot break synchronization.
3835 bool IsKnownNoSycn;
3836 if (AA::hasAssumedIRAttr<Attribute::NoSync>(
3837 A, QueryingAA: this, IRP: IRPosition::function_scope(IRP: getIRPosition()),
3838 DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoSycn))
3839 return Base::updateImpl(A);
3840
3841 // If the argument is read-only, no-alias cannot break synchronization.
3842 bool IsKnown;
3843 if (AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
3844 return Base::updateImpl(A);
3845
3846 // If the argument is never passed through callbacks, no-alias cannot break
3847 // synchronization.
3848 bool UsedAssumedInformation = false;
3849 if (A.checkForAllCallSites(
3850 Pred: [](AbstractCallSite ACS) { return !ACS.isCallbackCall(); }, QueryingAA: *this,
3851 RequireAllCallSites: true, UsedAssumedInformation))
3852 return Base::updateImpl(A);
3853
3854 // TODO: add no-alias but make sure it doesn't break synchronization by
3855 // introducing fake uses. See:
3856 // [1] Compiler Optimizations for OpenMP, J. Doerfert and H. Finkel,
3857 // International Workshop on OpenMP 2018,
3858 // http://compilers.cs.uni-saarland.de/people/doerfert/par_opt18.pdf
3859
3860 return indicatePessimisticFixpoint();
3861 }
3862
3863 /// See AbstractAttribute::trackStatistics()
3864 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noalias) }
3865};
3866
3867struct AANoAliasCallSiteArgument final : AANoAliasImpl {
3868 AANoAliasCallSiteArgument(const IRPosition &IRP, Attributor &A)
3869 : AANoAliasImpl(IRP, A) {}
3870
3871 /// Determine if the underlying value may alias with the call site argument
3872 /// \p OtherArgNo of \p ICS (= the underlying call site).
3873 bool mayAliasWithArgument(Attributor &A, AAResults *&AAR,
3874 const AAMemoryBehavior &MemBehaviorAA,
3875 const CallBase &CB, unsigned OtherArgNo) {
3876 // We do not need to worry about aliasing with the underlying IRP.
3877 if (this->getCalleeArgNo() == (int)OtherArgNo)
3878 return false;
3879
3880 // If it is not a pointer or pointer vector we do not alias.
3881 const Value *ArgOp = CB.getArgOperand(i: OtherArgNo);
3882 if (!ArgOp->getType()->isPtrOrPtrVectorTy())
3883 return false;
3884
3885 auto *CBArgMemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
3886 QueryingAA: *this, IRP: IRPosition::callsite_argument(CB, ArgNo: OtherArgNo), DepClass: DepClassTy::NONE);
3887
3888 // If the argument is readnone, there is no read-write aliasing.
3889 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadNone()) {
3890 A.recordDependence(FromAA: *CBArgMemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3891 return false;
3892 }
3893
3894 // If the argument is readonly and the underlying value is readonly, there
3895 // is no read-write aliasing.
3896 bool IsReadOnly = MemBehaviorAA.isAssumedReadOnly();
3897 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadOnly() &&
3898 IsReadOnly) {
3899 A.recordDependence(FromAA: MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3900 A.recordDependence(FromAA: *CBArgMemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3901 return false;
3902 }
3903
3904 // We have to utilize actual alias analysis queries so we need the object.
3905 if (!AAR)
3906 AAR = A.getInfoCache().getAnalysisResultForFunction<AAManager>(
3907 F: *getAnchorScope());
3908
3909 // Try to rule it out at the call site.
3910 bool IsAliasing = !AAR || !AAR->isNoAlias(V1: &getAssociatedValue(), V2: ArgOp);
3911 LLVM_DEBUG(dbgs() << "[NoAliasCSArg] Check alias between "
3912 "callsite arguments: "
3913 << getAssociatedValue() << " " << *ArgOp << " => "
3914 << (IsAliasing ? "" : "no-") << "alias \n");
3915
3916 return IsAliasing;
3917 }
3918
3919 bool isKnownNoAliasDueToNoAliasPreservation(
3920 Attributor &A, AAResults *&AAR, const AAMemoryBehavior &MemBehaviorAA) {
3921 // We can deduce "noalias" if the following conditions hold.
3922 // (i) Associated value is assumed to be noalias in the definition.
3923 // (ii) Associated value is assumed to be no-capture in all the uses
3924 // possibly executed before this callsite.
3925 // (iii) There is no other pointer argument which could alias with the
3926 // value.
3927
3928 const IRPosition &VIRP = IRPosition::value(V: getAssociatedValue());
3929 const Function *ScopeFn = VIRP.getAnchorScope();
3930 // Check whether the value is captured in the scope using AANoCapture.
3931 // Look at CFG and check only uses possibly executed before this
3932 // callsite.
3933 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
3934 Instruction *UserI = cast<Instruction>(Val: U.getUser());
3935
3936 // If UserI is the curr instruction and there is a single potential use of
3937 // the value in UserI we allow the use.
3938 // TODO: We should inspect the operands and allow those that cannot alias
3939 // with the value.
3940 if (UserI == getCtxI() && UserI->getNumOperands() == 1)
3941 return true;
3942
3943 if (ScopeFn) {
3944 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
3945 if (CB->isArgOperand(U: &U)) {
3946
3947 unsigned ArgNo = CB->getArgOperandNo(U: &U);
3948
3949 bool IsKnownNoCapture;
3950 if (AA::hasAssumedIRAttr<Attribute::Captures>(
3951 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
3952 DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture))
3953 return true;
3954 }
3955 }
3956
3957 if (!AA::isPotentiallyReachable(
3958 A, FromI: *UserI, ToI: *getCtxI(), QueryingAA: *this, /* ExclusionSet */ nullptr,
3959 GoBackwardsCB: [ScopeFn](const Function &Fn) { return &Fn != ScopeFn; }))
3960 return true;
3961 }
3962
3963 // TODO: We should track the capturing uses in AANoCapture but the problem
3964 // is CGSCC runs. For those we would need to "allow" AANoCapture for
3965 // a value in the module slice.
3966 // TODO(captures): Make this more precise.
3967 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
3968 if (capturesNothing(CC: CI))
3969 return true;
3970 if (CI.isPassthrough()) {
3971 Follow = true;
3972 return true;
3973 }
3974 LLVM_DEBUG(dbgs() << "[AANoAliasCSArg] Unknown user: " << *UserI << "\n");
3975 return false;
3976 };
3977
3978 bool IsKnownNoCapture;
3979 const AANoCapture *NoCaptureAA = nullptr;
3980 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
3981 A, QueryingAA: this, IRP: VIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false, AAPtr: &NoCaptureAA);
3982 if (!IsAssumedNoCapture &&
3983 (!NoCaptureAA || !NoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
3984 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: getAssociatedValue())) {
3985 LLVM_DEBUG(
3986 dbgs() << "[AANoAliasCSArg] " << getAssociatedValue()
3987 << " cannot be noalias as it is potentially captured\n");
3988 return false;
3989 }
3990 }
3991 if (NoCaptureAA)
3992 A.recordDependence(FromAA: *NoCaptureAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3993
3994 // Check there is no other pointer argument which could alias with the
3995 // value passed at this call site.
3996 // TODO: AbstractCallSite
3997 const auto &CB = cast<CallBase>(Val&: getAnchorValue());
3998 for (unsigned OtherArgNo = 0; OtherArgNo < CB.arg_size(); OtherArgNo++)
3999 if (mayAliasWithArgument(A, AAR, MemBehaviorAA, CB, OtherArgNo))
4000 return false;
4001
4002 return true;
4003 }
4004
4005 /// See AbstractAttribute::updateImpl(...).
4006 ChangeStatus updateImpl(Attributor &A) override {
4007 // If the argument is readnone we are done as there are no accesses via the
4008 // argument.
4009 auto *MemBehaviorAA =
4010 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::NONE);
4011 if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
4012 A.recordDependence(FromAA: *MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
4013 return ChangeStatus::UNCHANGED;
4014 }
4015
4016 bool IsKnownNoAlias;
4017 const IRPosition &VIRP = IRPosition::value(V: getAssociatedValue());
4018 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
4019 A, QueryingAA: this, IRP: VIRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoAlias)) {
4020 LLVM_DEBUG(dbgs() << "[AANoAlias] " << getAssociatedValue()
4021 << " is not no-alias at the definition\n");
4022 return indicatePessimisticFixpoint();
4023 }
4024
4025 AAResults *AAR = nullptr;
4026 if (MemBehaviorAA &&
4027 isKnownNoAliasDueToNoAliasPreservation(A, AAR, MemBehaviorAA: *MemBehaviorAA)) {
4028 LLVM_DEBUG(
4029 dbgs() << "[AANoAlias] No-Alias deduced via no-alias preservation\n");
4030 return ChangeStatus::UNCHANGED;
4031 }
4032
4033 return indicatePessimisticFixpoint();
4034 }
4035
4036 /// See AbstractAttribute::trackStatistics()
4037 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noalias) }
4038};
4039
4040/// NoAlias attribute for function return value.
4041struct AANoAliasReturned final : AANoAliasImpl {
4042 AANoAliasReturned(const IRPosition &IRP, Attributor &A)
4043 : AANoAliasImpl(IRP, A) {}
4044
4045 /// See AbstractAttribute::updateImpl(...).
4046 ChangeStatus updateImpl(Attributor &A) override {
4047
4048 auto CheckReturnValue = [&](Value &RV) -> bool {
4049 if (Constant *C = dyn_cast<Constant>(Val: &RV))
4050 if (C->isNullValue() || isa<UndefValue>(Val: C))
4051 return true;
4052
4053 /// For now, we can only deduce noalias if we have call sites.
4054 /// FIXME: add more support.
4055 if (!isa<CallBase>(Val: &RV))
4056 return false;
4057
4058 const IRPosition &RVPos = IRPosition::value(V: RV);
4059 bool IsKnownNoAlias;
4060 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
4061 A, QueryingAA: this, IRP: RVPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoAlias))
4062 return false;
4063
4064 bool IsKnownNoCapture;
4065 const AANoCapture *NoCaptureAA = nullptr;
4066 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
4067 A, QueryingAA: this, IRP: RVPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
4068 AAPtr: &NoCaptureAA);
4069 return IsAssumedNoCapture ||
4070 (NoCaptureAA && NoCaptureAA->isAssumedNoCaptureMaybeReturned());
4071 };
4072
4073 if (!A.checkForAllReturnedValues(Pred: CheckReturnValue, QueryingAA: *this))
4074 return indicatePessimisticFixpoint();
4075
4076 return ChangeStatus::UNCHANGED;
4077 }
4078
4079 /// See AbstractAttribute::trackStatistics()
4080 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noalias) }
4081};
4082
4083/// NoAlias attribute deduction for a call site return value.
4084struct AANoAliasCallSiteReturned final
4085 : AACalleeToCallSite<AANoAlias, AANoAliasImpl> {
4086 AANoAliasCallSiteReturned(const IRPosition &IRP, Attributor &A)
4087 : AACalleeToCallSite<AANoAlias, AANoAliasImpl>(IRP, A) {}
4088
4089 /// See AbstractAttribute::trackStatistics()
4090 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noalias); }
4091};
4092} // namespace
4093
4094/// -------------------AAIsDead Function Attribute-----------------------
4095
4096namespace {
4097struct AAIsDeadValueImpl : public AAIsDead {
4098 AAIsDeadValueImpl(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4099
4100 /// See AAIsDead::isAssumedDead().
4101 bool isAssumedDead() const override { return isAssumed(BitsEncoding: IS_DEAD); }
4102
4103 /// See AAIsDead::isKnownDead().
4104 bool isKnownDead() const override { return isKnown(BitsEncoding: IS_DEAD); }
4105
4106 /// See AAIsDead::isAssumedDead(BasicBlock *).
4107 bool isAssumedDead(const BasicBlock *BB) const override { return false; }
4108
4109 /// See AAIsDead::isKnownDead(BasicBlock *).
4110 bool isKnownDead(const BasicBlock *BB) const override { return false; }
4111
4112 /// See AAIsDead::isAssumedDead(Instruction *I).
4113 bool isAssumedDead(const Instruction *I) const override {
4114 return I == getCtxI() && isAssumedDead();
4115 }
4116
4117 /// See AAIsDead::isKnownDead(Instruction *I).
4118 bool isKnownDead(const Instruction *I) const override {
4119 return isAssumedDead(I) && isKnownDead();
4120 }
4121
4122 /// See AbstractAttribute::getAsStr().
4123 const std::string getAsStr(Attributor *A) const override {
4124 return isAssumedDead() ? "assumed-dead" : "assumed-live";
4125 }
4126
4127 /// Check if all uses are assumed dead.
4128 bool areAllUsesAssumedDead(Attributor &A, Value &V) {
4129 // Callers might not check the type, void has no uses.
4130 if (V.getType()->isVoidTy() || V.use_empty())
4131 return true;
4132
4133 // If we replace a value with a constant there are no uses left afterwards.
4134 if (!isa<Constant>(Val: V)) {
4135 if (auto *I = dyn_cast<Instruction>(Val: &V))
4136 if (!A.isRunOn(Fn&: *I->getFunction()))
4137 return false;
4138 bool UsedAssumedInformation = false;
4139 std::optional<Constant *> C =
4140 A.getAssumedConstant(V, AA: *this, UsedAssumedInformation);
4141 if (!C || *C)
4142 return true;
4143 }
4144
4145 auto UsePred = [&](const Use &U, bool &Follow) { return false; };
4146 // Explicitly set the dependence class to required because we want a long
4147 // chain of N dependent instructions to be considered live as soon as one is
4148 // without going through N update cycles. This is not required for
4149 // correctness.
4150 return A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V, /* CheckBBLivenessOnly */ false,
4151 LivenessDepClass: DepClassTy::REQUIRED,
4152 /* IgnoreDroppableUses */ false);
4153 }
4154
4155 /// Determine if \p I is assumed to be side-effect free.
4156 bool isAssumedSideEffectFree(Attributor &A, Instruction *I) {
4157 if (!I || wouldInstructionBeTriviallyDead(I))
4158 return true;
4159
4160 auto *CB = dyn_cast<CallBase>(Val: I);
4161 if (!CB || isa<IntrinsicInst>(Val: CB))
4162 return false;
4163
4164 const IRPosition &CallIRP = IRPosition::callsite_function(CB: *CB);
4165
4166 bool IsKnownNoUnwind;
4167 if (!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
4168 A, QueryingAA: this, IRP: CallIRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoUnwind))
4169 return false;
4170
4171 bool IsKnown;
4172 return AA::isAssumedReadOnly(A, IRP: CallIRP, QueryingAA: *this, IsKnown);
4173 }
4174};
4175
4176struct AAIsDeadFloating : public AAIsDeadValueImpl {
4177 AAIsDeadFloating(const IRPosition &IRP, Attributor &A)
4178 : AAIsDeadValueImpl(IRP, A) {}
4179
4180 /// See AbstractAttribute::initialize(...).
4181 void initialize(Attributor &A) override {
4182 AAIsDeadValueImpl::initialize(A);
4183
4184 if (isa<UndefValue>(Val: getAssociatedValue())) {
4185 indicatePessimisticFixpoint();
4186 return;
4187 }
4188
4189 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4190 if (!isAssumedSideEffectFree(A, I)) {
4191 if (!isa_and_nonnull<StoreInst>(Val: I) && !isa_and_nonnull<FenceInst>(Val: I))
4192 indicatePessimisticFixpoint();
4193 else
4194 removeAssumedBits(BitsEncoding: HAS_NO_EFFECT);
4195 }
4196 }
4197
4198 bool isDeadFence(Attributor &A, FenceInst &FI) {
4199 const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
4200 IRP: IRPosition::function(F: *FI.getFunction()), QueryingAA: *this, DepClass: DepClassTy::NONE);
4201 if (!ExecDomainAA || !ExecDomainAA->isNoOpFence(FI))
4202 return false;
4203 A.recordDependence(FromAA: *ExecDomainAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
4204 return true;
4205 }
4206
4207 bool isDeadStore(Attributor &A, StoreInst &SI,
4208 SmallSetVector<Instruction *, 8> *AssumeOnlyInst = nullptr) {
4209 // Lang ref now states volatile store is not UB/dead, let's skip them.
4210 if (SI.isVolatile())
4211 return false;
4212
4213 // If we are collecting assumes to be deleted we are in the manifest stage.
4214 // It's problematic to collect the potential copies again now so we use the
4215 // cached ones.
4216 bool UsedAssumedInformation = false;
4217 if (!AssumeOnlyInst) {
4218 PotentialCopies.clear();
4219 if (!AA::getPotentialCopiesOfStoredValue(A, SI, PotentialCopies, QueryingAA: *this,
4220 UsedAssumedInformation)) {
4221 LLVM_DEBUG(
4222 dbgs()
4223 << "[AAIsDead] Could not determine potential copies of store!\n");
4224 return false;
4225 }
4226 }
4227 LLVM_DEBUG(dbgs() << "[AAIsDead] Store has " << PotentialCopies.size()
4228 << " potential copies.\n");
4229
4230 InformationCache &InfoCache = A.getInfoCache();
4231 return llvm::all_of(Range&: PotentialCopies, P: [&](Value *V) {
4232 if (A.isAssumedDead(IRP: IRPosition::value(V: *V), QueryingAA: this, FnLivenessAA: nullptr,
4233 UsedAssumedInformation))
4234 return true;
4235 if (auto *LI = dyn_cast<LoadInst>(Val: V)) {
4236 if (llvm::all_of(Range: LI->uses(), P: [&](const Use &U) {
4237 auto &UserI = cast<Instruction>(Val&: *U.getUser());
4238 if (InfoCache.isOnlyUsedByAssume(I: UserI)) {
4239 if (AssumeOnlyInst)
4240 AssumeOnlyInst->insert(X: &UserI);
4241 return true;
4242 }
4243 return A.isAssumedDead(U, QueryingAA: this, FnLivenessAA: nullptr, UsedAssumedInformation);
4244 })) {
4245 return true;
4246 }
4247 }
4248 LLVM_DEBUG(dbgs() << "[AAIsDead] Potential copy " << *V
4249 << " is assumed live!\n");
4250 return false;
4251 });
4252 }
4253
4254 /// See AbstractAttribute::getAsStr().
4255 const std::string getAsStr(Attributor *A) const override {
4256 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4257 if (isa_and_nonnull<StoreInst>(Val: I))
4258 if (isValidState())
4259 return "assumed-dead-store";
4260 if (isa_and_nonnull<FenceInst>(Val: I))
4261 if (isValidState())
4262 return "assumed-dead-fence";
4263 return AAIsDeadValueImpl::getAsStr(A);
4264 }
4265
4266 /// See AbstractAttribute::updateImpl(...).
4267 ChangeStatus updateImpl(Attributor &A) override {
4268 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4269 if (auto *SI = dyn_cast_or_null<StoreInst>(Val: I)) {
4270 if (!isDeadStore(A, SI&: *SI))
4271 return indicatePessimisticFixpoint();
4272 } else if (auto *FI = dyn_cast_or_null<FenceInst>(Val: I)) {
4273 if (!isDeadFence(A, FI&: *FI))
4274 return indicatePessimisticFixpoint();
4275 } else {
4276 if (!isAssumedSideEffectFree(A, I))
4277 return indicatePessimisticFixpoint();
4278 if (!areAllUsesAssumedDead(A, V&: getAssociatedValue()))
4279 return indicatePessimisticFixpoint();
4280 }
4281 return ChangeStatus::UNCHANGED;
4282 }
4283
4284 bool isRemovableStore() const override {
4285 return isAssumed(BitsEncoding: IS_REMOVABLE) && isa<StoreInst>(Val: &getAssociatedValue());
4286 }
4287
4288 /// See AbstractAttribute::manifest(...).
4289 ChangeStatus manifest(Attributor &A) override {
4290 Value &V = getAssociatedValue();
4291 if (auto *I = dyn_cast<Instruction>(Val: &V)) {
4292 // If we get here we basically know the users are all dead. We check if
4293 // isAssumedSideEffectFree returns true here again because it might not be
4294 // the case and only the users are dead but the instruction (=call) is
4295 // still needed.
4296 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
4297 SmallSetVector<Instruction *, 8> AssumeOnlyInst;
4298 bool IsDead = isDeadStore(A, SI&: *SI, AssumeOnlyInst: &AssumeOnlyInst);
4299 (void)IsDead;
4300 assert(IsDead && "Store was assumed to be dead!");
4301 A.deleteAfterManifest(I&: *I);
4302 for (size_t i = 0; i < AssumeOnlyInst.size(); ++i) {
4303 Instruction *AOI = AssumeOnlyInst[i];
4304 for (auto *Usr : AOI->users())
4305 AssumeOnlyInst.insert(X: cast<Instruction>(Val: Usr));
4306 A.deleteAfterManifest(I&: *AOI);
4307 }
4308 return ChangeStatus::CHANGED;
4309 }
4310 if (auto *FI = dyn_cast<FenceInst>(Val: I)) {
4311 assert(isDeadFence(A, *FI));
4312 A.deleteAfterManifest(I&: *FI);
4313 return ChangeStatus::CHANGED;
4314 }
4315 if (isAssumedSideEffectFree(A, I) && !isa<InvokeInst>(Val: I)) {
4316 A.deleteAfterManifest(I&: *I);
4317 return ChangeStatus::CHANGED;
4318 }
4319 }
4320 return ChangeStatus::UNCHANGED;
4321 }
4322
4323 /// See AbstractAttribute::trackStatistics()
4324 void trackStatistics() const override {
4325 STATS_DECLTRACK_FLOATING_ATTR(IsDead)
4326 }
4327
4328private:
4329 // The potential copies of a dead store, used for deletion during manifest.
4330 SmallSetVector<Value *, 4> PotentialCopies;
4331};
4332
4333struct AAIsDeadArgument : public AAIsDeadFloating {
4334 AAIsDeadArgument(const IRPosition &IRP, Attributor &A)
4335 : AAIsDeadFloating(IRP, A) {}
4336
4337 /// See AbstractAttribute::manifest(...).
4338 ChangeStatus manifest(Attributor &A) override {
4339 Argument &Arg = *getAssociatedArgument();
4340 if (A.isValidFunctionSignatureRewrite(Arg, /* ReplacementTypes */ {}))
4341 if (A.registerFunctionSignatureRewrite(
4342 Arg, /* ReplacementTypes */ {},
4343 CalleeRepairCB: Attributor::ArgumentReplacementInfo::CalleeRepairCBTy{},
4344 ACSRepairCB: Attributor::ArgumentReplacementInfo::ACSRepairCBTy{})) {
4345 return ChangeStatus::CHANGED;
4346 }
4347 return ChangeStatus::UNCHANGED;
4348 }
4349
4350 /// See AbstractAttribute::trackStatistics()
4351 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(IsDead) }
4352};
4353
4354struct AAIsDeadCallSiteArgument : public AAIsDeadValueImpl {
4355 AAIsDeadCallSiteArgument(const IRPosition &IRP, Attributor &A)
4356 : AAIsDeadValueImpl(IRP, A) {}
4357
4358 /// See AbstractAttribute::initialize(...).
4359 void initialize(Attributor &A) override {
4360 AAIsDeadValueImpl::initialize(A);
4361 if (isa<UndefValue>(Val: getAssociatedValue()))
4362 indicatePessimisticFixpoint();
4363 }
4364
4365 /// See AbstractAttribute::updateImpl(...).
4366 ChangeStatus updateImpl(Attributor &A) override {
4367 // TODO: Once we have call site specific value information we can provide
4368 // call site specific liveness information and then it makes
4369 // sense to specialize attributes for call sites arguments instead of
4370 // redirecting requests to the callee argument.
4371 Argument *Arg = getAssociatedArgument();
4372 if (!Arg)
4373 return indicatePessimisticFixpoint();
4374 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
4375 auto *ArgAA = A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
4376 if (!ArgAA)
4377 return indicatePessimisticFixpoint();
4378 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
4379 }
4380
4381 /// See AbstractAttribute::manifest(...).
4382 ChangeStatus manifest(Attributor &A) override {
4383 CallBase &CB = cast<CallBase>(Val&: getAnchorValue());
4384 Use &U = CB.getArgOperandUse(i: getCallSiteArgNo());
4385 assert(!isa<UndefValue>(U.get()) &&
4386 "Expected undef values to be filtered out!");
4387 UndefValue &UV = *UndefValue::get(T: U->getType());
4388 if (A.changeUseAfterManifest(U, NV&: UV))
4389 return ChangeStatus::CHANGED;
4390 return ChangeStatus::UNCHANGED;
4391 }
4392
4393 /// See AbstractAttribute::trackStatistics()
4394 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(IsDead) }
4395};
4396
4397struct AAIsDeadCallSiteReturned : public AAIsDeadFloating {
4398 AAIsDeadCallSiteReturned(const IRPosition &IRP, Attributor &A)
4399 : AAIsDeadFloating(IRP, A) {}
4400
4401 /// See AAIsDead::isAssumedDead().
4402 bool isAssumedDead() const override {
4403 return AAIsDeadFloating::isAssumedDead() && IsAssumedSideEffectFree;
4404 }
4405
4406 /// See AbstractAttribute::initialize(...).
4407 void initialize(Attributor &A) override {
4408 AAIsDeadFloating::initialize(A);
4409 if (isa<UndefValue>(Val: getAssociatedValue())) {
4410 indicatePessimisticFixpoint();
4411 return;
4412 }
4413
4414 // We track this separately as a secondary state.
4415 IsAssumedSideEffectFree = isAssumedSideEffectFree(A, I: getCtxI());
4416 }
4417
4418 /// See AbstractAttribute::updateImpl(...).
4419 ChangeStatus updateImpl(Attributor &A) override {
4420 ChangeStatus Changed = ChangeStatus::UNCHANGED;
4421 if (IsAssumedSideEffectFree && !isAssumedSideEffectFree(A, I: getCtxI())) {
4422 IsAssumedSideEffectFree = false;
4423 Changed = ChangeStatus::CHANGED;
4424 }
4425 if (!areAllUsesAssumedDead(A, V&: getAssociatedValue()))
4426 return indicatePessimisticFixpoint();
4427 return Changed;
4428 }
4429
4430 /// See AbstractAttribute::trackStatistics()
4431 void trackStatistics() const override {
4432 if (IsAssumedSideEffectFree)
4433 STATS_DECLTRACK_CSRET_ATTR(IsDead)
4434 else
4435 STATS_DECLTRACK_CSRET_ATTR(UnusedResult)
4436 }
4437
4438 /// See AbstractAttribute::getAsStr().
4439 const std::string getAsStr(Attributor *A) const override {
4440 return isAssumedDead()
4441 ? "assumed-dead"
4442 : (getAssumed() ? "assumed-dead-users" : "assumed-live");
4443 }
4444
4445private:
4446 bool IsAssumedSideEffectFree = true;
4447};
4448
4449struct AAIsDeadReturned : public AAIsDeadValueImpl {
4450 AAIsDeadReturned(const IRPosition &IRP, Attributor &A)
4451 : AAIsDeadValueImpl(IRP, A) {}
4452
4453 /// See AbstractAttribute::updateImpl(...).
4454 ChangeStatus updateImpl(Attributor &A) override {
4455
4456 bool UsedAssumedInformation = false;
4457 A.checkForAllInstructions(Pred: [](Instruction &) { return true; }, QueryingAA: *this,
4458 Opcodes: {Instruction::Ret}, UsedAssumedInformation);
4459
4460 auto PredForCallSite = [&](AbstractCallSite ACS) {
4461 if (ACS.isCallbackCall() || !ACS.getInstruction())
4462 return false;
4463 return areAllUsesAssumedDead(A, V&: *ACS.getInstruction());
4464 };
4465
4466 if (!A.checkForAllCallSites(Pred: PredForCallSite, QueryingAA: *this, RequireAllCallSites: true,
4467 UsedAssumedInformation))
4468 return indicatePessimisticFixpoint();
4469
4470 return ChangeStatus::UNCHANGED;
4471 }
4472
4473 /// See AbstractAttribute::manifest(...).
4474 ChangeStatus manifest(Attributor &A) override {
4475 // TODO: Rewrite the signature to return void?
4476 bool AnyChange = false;
4477 UndefValue &UV = *UndefValue::get(T: getAssociatedFunction()->getReturnType());
4478 auto RetInstPred = [&](Instruction &I) {
4479 ReturnInst &RI = cast<ReturnInst>(Val&: I);
4480 if (!isa<UndefValue>(Val: RI.getReturnValue()))
4481 AnyChange |= A.changeUseAfterManifest(U&: RI.getOperandUse(i: 0), NV&: UV);
4482 return true;
4483 };
4484 bool UsedAssumedInformation = false;
4485 A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
4486 UsedAssumedInformation);
4487 return AnyChange ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
4488 }
4489
4490 /// See AbstractAttribute::trackStatistics()
4491 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(IsDead) }
4492};
4493
4494struct AAIsDeadFunction : public AAIsDead {
4495 AAIsDeadFunction(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4496
4497 /// See AbstractAttribute::initialize(...).
4498 void initialize(Attributor &A) override {
4499 Function *F = getAnchorScope();
4500 assert(F && "Did expect an anchor function");
4501 if (!isAssumedDeadInternalFunction(A)) {
4502 ToBeExploredFrom.insert(X: &F->getEntryBlock().front());
4503 assumeLive(A, BB: F->getEntryBlock());
4504 }
4505 }
4506
4507 bool isAssumedDeadInternalFunction(Attributor &A) {
4508 if (!getAnchorScope()->hasLocalLinkage())
4509 return false;
4510 bool UsedAssumedInformation = false;
4511 return A.checkForAllCallSites(Pred: [](AbstractCallSite) { return false; }, QueryingAA: *this,
4512 RequireAllCallSites: true, UsedAssumedInformation);
4513 }
4514
4515 /// See AbstractAttribute::getAsStr().
4516 const std::string getAsStr(Attributor *A) const override {
4517 return "Live[#BB " + std::to_string(val: AssumedLiveBlocks.size()) + "/" +
4518 std::to_string(val: getAnchorScope()->size()) + "][#TBEP " +
4519 std::to_string(val: ToBeExploredFrom.size()) + "][#KDE " +
4520 std::to_string(val: KnownDeadEnds.size()) + "]";
4521 }
4522
4523 /// See AbstractAttribute::manifest(...).
4524 ChangeStatus manifest(Attributor &A) override {
4525 assert(getState().isValidState() &&
4526 "Attempted to manifest an invalid state!");
4527
4528 ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
4529 Function &F = *getAnchorScope();
4530
4531 if (AssumedLiveBlocks.empty()) {
4532 A.deleteAfterManifest(F);
4533 return ChangeStatus::CHANGED;
4534 }
4535
4536 // Flag to determine if we can change an invoke to a call assuming the
4537 // callee is nounwind. This is not possible if the personality of the
4538 // function allows to catch asynchronous exceptions.
4539 bool Invoke2CallAllowed = !mayCatchAsynchronousExceptions(F);
4540
4541 KnownDeadEnds.set_union(ToBeExploredFrom);
4542 for (const Instruction *DeadEndI : KnownDeadEnds) {
4543 auto *CB = dyn_cast<CallBase>(Val: DeadEndI);
4544 if (!CB)
4545 continue;
4546 bool IsKnownNoReturn;
4547 bool MayReturn = !AA::hasAssumedIRAttr<Attribute::NoReturn>(
4548 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL,
4549 IsKnown&: IsKnownNoReturn);
4550 if (MayReturn && (!Invoke2CallAllowed || !isa<InvokeInst>(Val: CB)))
4551 continue;
4552
4553 if (auto *II = dyn_cast<InvokeInst>(Val: DeadEndI))
4554 A.registerInvokeWithDeadSuccessor(II&: const_cast<InvokeInst &>(*II));
4555 else
4556 A.changeToUnreachableAfterManifest(
4557 I: const_cast<Instruction *>(DeadEndI->getNextNode()));
4558 HasChanged = ChangeStatus::CHANGED;
4559 }
4560
4561 STATS_DECL(AAIsDead, BasicBlock, "Number of dead basic blocks deleted.");
4562 for (BasicBlock &BB : F)
4563 if (!AssumedLiveBlocks.count(V: &BB)) {
4564 A.deleteAfterManifest(BB);
4565 ++BUILD_STAT_NAME(AAIsDead, BasicBlock);
4566 HasChanged = ChangeStatus::CHANGED;
4567 }
4568
4569 return HasChanged;
4570 }
4571
4572 /// See AbstractAttribute::updateImpl(...).
4573 ChangeStatus updateImpl(Attributor &A) override;
4574
4575 bool isEdgeDead(const BasicBlock *From, const BasicBlock *To) const override {
4576 assert(From->getParent() == getAnchorScope() &&
4577 To->getParent() == getAnchorScope() &&
4578 "Used AAIsDead of the wrong function");
4579 return isValidState() && !AssumedLiveEdges.count(V: std::make_pair(x&: From, y&: To));
4580 }
4581
4582 /// See AbstractAttribute::trackStatistics()
4583 void trackStatistics() const override {}
4584
4585 /// Returns true if the function is assumed dead.
4586 bool isAssumedDead() const override { return false; }
4587
4588 /// See AAIsDead::isKnownDead().
4589 bool isKnownDead() const override { return false; }
4590
4591 /// See AAIsDead::isAssumedDead(BasicBlock *).
4592 bool isAssumedDead(const BasicBlock *BB) const override {
4593 assert(BB->getParent() == getAnchorScope() &&
4594 "BB must be in the same anchor scope function.");
4595
4596 if (!getAssumed())
4597 return false;
4598 return !AssumedLiveBlocks.count(V: BB);
4599 }
4600
4601 /// See AAIsDead::isKnownDead(BasicBlock *).
4602 bool isKnownDead(const BasicBlock *BB) const override {
4603 return getKnown() && isAssumedDead(BB);
4604 }
4605
4606 /// See AAIsDead::isAssumed(Instruction *I).
4607 bool isAssumedDead(const Instruction *I) const override {
4608 assert(I->getParent()->getParent() == getAnchorScope() &&
4609 "Instruction must be in the same anchor scope function.");
4610
4611 if (!getAssumed())
4612 return false;
4613
4614 // If it is not in AssumedLiveBlocks then it for sure dead.
4615 // Otherwise, it can still be after noreturn call in a live block.
4616 if (!AssumedLiveBlocks.count(V: I->getParent()))
4617 return true;
4618
4619 // If it is not after a liveness barrier it is live.
4620 const Instruction *PrevI = I->getPrevNode();
4621 while (PrevI) {
4622 if (KnownDeadEnds.count(key: PrevI) || ToBeExploredFrom.count(key: PrevI))
4623 return true;
4624 PrevI = PrevI->getPrevNode();
4625 }
4626 return false;
4627 }
4628
4629 /// See AAIsDead::isKnownDead(Instruction *I).
4630 bool isKnownDead(const Instruction *I) const override {
4631 return getKnown() && isAssumedDead(I);
4632 }
4633
4634 /// Assume \p BB is (partially) live now and indicate to the Attributor \p A
4635 /// that internal function called from \p BB should now be looked at.
4636 bool assumeLive(Attributor &A, const BasicBlock &BB) {
4637 if (!AssumedLiveBlocks.insert(V: &BB).second)
4638 return false;
4639
4640 // We assume that all of BB is (probably) live now and if there are calls to
4641 // internal functions we will assume that those are now live as well. This
4642 // is a performance optimization for blocks with calls to a lot of internal
4643 // functions. It can however cause dead functions to be treated as live.
4644 for (const Instruction &I : BB)
4645 if (const auto *CB = dyn_cast<CallBase>(Val: &I))
4646 if (auto *F = dyn_cast_if_present<Function>(Val: CB->getCalledOperand()))
4647 if (F->hasLocalLinkage())
4648 A.markLiveInternalFunction(F: *F);
4649 return true;
4650 }
4651
4652 /// Collection of instructions that need to be explored again, e.g., we
4653 /// did assume they do not transfer control to (one of their) successors.
4654 SmallSetVector<const Instruction *, 8> ToBeExploredFrom;
4655
4656 /// Collection of instructions that are known to not transfer control.
4657 SmallSetVector<const Instruction *, 8> KnownDeadEnds;
4658
4659 /// Collection of all assumed live edges
4660 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> AssumedLiveEdges;
4661
4662 /// Collection of all assumed live BasicBlocks.
4663 DenseSet<const BasicBlock *> AssumedLiveBlocks;
4664};
4665
4666static bool
4667identifyAliveSuccessors(Attributor &A, const CallBase &CB,
4668 AbstractAttribute &AA,
4669 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4670 const IRPosition &IPos = IRPosition::callsite_function(CB);
4671
4672 bool IsKnownNoReturn;
4673 if (AA::hasAssumedIRAttr<Attribute::NoReturn>(
4674 A, QueryingAA: &AA, IRP: IPos, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoReturn))
4675 return !IsKnownNoReturn;
4676 if (CB.isTerminator())
4677 AliveSuccessors.push_back(Elt: &CB.getSuccessor(Idx: 0)->front());
4678 else
4679 AliveSuccessors.push_back(Elt: CB.getNextNode());
4680 return false;
4681}
4682
4683static bool
4684identifyAliveSuccessors(Attributor &A, const InvokeInst &II,
4685 AbstractAttribute &AA,
4686 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4687 bool UsedAssumedInformation =
4688 identifyAliveSuccessors(A, CB: cast<CallBase>(Val: II), AA, AliveSuccessors);
4689
4690 // First, determine if we can change an invoke to a call assuming the
4691 // callee is nounwind. This is not possible if the personality of the
4692 // function allows to catch asynchronous exceptions.
4693 if (AAIsDeadFunction::mayCatchAsynchronousExceptions(F: *II.getFunction())) {
4694 AliveSuccessors.push_back(Elt: &II.getUnwindDest()->front());
4695 } else {
4696 const IRPosition &IPos = IRPosition::callsite_function(CB: II);
4697
4698 bool IsKnownNoUnwind;
4699 if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
4700 A, QueryingAA: &AA, IRP: IPos, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoUnwind)) {
4701 UsedAssumedInformation |= !IsKnownNoUnwind;
4702 } else {
4703 AliveSuccessors.push_back(Elt: &II.getUnwindDest()->front());
4704 }
4705 }
4706 return UsedAssumedInformation;
4707}
4708
4709static bool
4710identifyAliveSuccessors(Attributor &A, const BranchInst &BI,
4711 AbstractAttribute &AA,
4712 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4713 bool UsedAssumedInformation = false;
4714 if (BI.getNumSuccessors() == 1) {
4715 AliveSuccessors.push_back(Elt: &BI.getSuccessor(i: 0)->front());
4716 } else {
4717 std::optional<Constant *> C =
4718 A.getAssumedConstant(V: *BI.getCondition(), AA, UsedAssumedInformation);
4719 if (!C || isa_and_nonnull<UndefValue>(Val: *C)) {
4720 // No value yet, assume both edges are dead.
4721 } else if (isa_and_nonnull<ConstantInt>(Val: *C)) {
4722 const BasicBlock *SuccBB =
4723 BI.getSuccessor(i: 1 - cast<ConstantInt>(Val: *C)->getValue().getZExtValue());
4724 AliveSuccessors.push_back(Elt: &SuccBB->front());
4725 } else {
4726 AliveSuccessors.push_back(Elt: &BI.getSuccessor(i: 0)->front());
4727 AliveSuccessors.push_back(Elt: &BI.getSuccessor(i: 1)->front());
4728 UsedAssumedInformation = false;
4729 }
4730 }
4731 return UsedAssumedInformation;
4732}
4733
4734static bool
4735identifyAliveSuccessors(Attributor &A, const SwitchInst &SI,
4736 AbstractAttribute &AA,
4737 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4738 bool UsedAssumedInformation = false;
4739 SmallVector<AA::ValueAndContext> Values;
4740 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *SI.getCondition()), AA: &AA,
4741 Values, S: AA::AnyScope,
4742 UsedAssumedInformation)) {
4743 // Something went wrong, assume all successors are live.
4744 for (const BasicBlock *SuccBB : successors(BB: SI.getParent()))
4745 AliveSuccessors.push_back(Elt: &SuccBB->front());
4746 return false;
4747 }
4748
4749 if (Values.empty() ||
4750 (Values.size() == 1 &&
4751 isa_and_nonnull<UndefValue>(Val: Values.front().getValue()))) {
4752 // No valid value yet, assume all edges are dead.
4753 return UsedAssumedInformation;
4754 }
4755
4756 Type &Ty = *SI.getCondition()->getType();
4757 SmallPtrSet<ConstantInt *, 8> Constants;
4758 auto CheckForConstantInt = [&](Value *V) {
4759 if (auto *CI = dyn_cast_if_present<ConstantInt>(Val: AA::getWithType(V&: *V, Ty))) {
4760 Constants.insert(Ptr: CI);
4761 return true;
4762 }
4763 return false;
4764 };
4765
4766 if (!all_of(Range&: Values, P: [&](AA::ValueAndContext &VAC) {
4767 return CheckForConstantInt(VAC.getValue());
4768 })) {
4769 for (const BasicBlock *SuccBB : successors(BB: SI.getParent()))
4770 AliveSuccessors.push_back(Elt: &SuccBB->front());
4771 return UsedAssumedInformation;
4772 }
4773
4774 unsigned MatchedCases = 0;
4775 for (const auto &CaseIt : SI.cases()) {
4776 if (Constants.count(Ptr: CaseIt.getCaseValue())) {
4777 ++MatchedCases;
4778 AliveSuccessors.push_back(Elt: &CaseIt.getCaseSuccessor()->front());
4779 }
4780 }
4781
4782 // If all potential values have been matched, we will not visit the default
4783 // case.
4784 if (MatchedCases < Constants.size())
4785 AliveSuccessors.push_back(Elt: &SI.getDefaultDest()->front());
4786 return UsedAssumedInformation;
4787}
4788
4789ChangeStatus AAIsDeadFunction::updateImpl(Attributor &A) {
4790 ChangeStatus Change = ChangeStatus::UNCHANGED;
4791
4792 if (AssumedLiveBlocks.empty()) {
4793 if (isAssumedDeadInternalFunction(A))
4794 return ChangeStatus::UNCHANGED;
4795
4796 Function *F = getAnchorScope();
4797 ToBeExploredFrom.insert(X: &F->getEntryBlock().front());
4798 assumeLive(A, BB: F->getEntryBlock());
4799 Change = ChangeStatus::CHANGED;
4800 }
4801
4802 LLVM_DEBUG(dbgs() << "[AAIsDead] Live [" << AssumedLiveBlocks.size() << "/"
4803 << getAnchorScope()->size() << "] BBs and "
4804 << ToBeExploredFrom.size() << " exploration points and "
4805 << KnownDeadEnds.size() << " known dead ends\n");
4806
4807 // Copy and clear the list of instructions we need to explore from. It is
4808 // refilled with instructions the next update has to look at.
4809 SmallVector<const Instruction *, 8> Worklist(ToBeExploredFrom.begin(),
4810 ToBeExploredFrom.end());
4811 decltype(ToBeExploredFrom) NewToBeExploredFrom;
4812
4813 SmallVector<const Instruction *, 8> AliveSuccessors;
4814 while (!Worklist.empty()) {
4815 const Instruction *I = Worklist.pop_back_val();
4816 LLVM_DEBUG(dbgs() << "[AAIsDead] Exploration inst: " << *I << "\n");
4817
4818 // Fast forward for uninteresting instructions. We could look for UB here
4819 // though.
4820 while (!I->isTerminator() && !isa<CallBase>(Val: I))
4821 I = I->getNextNode();
4822
4823 AliveSuccessors.clear();
4824
4825 bool UsedAssumedInformation = false;
4826 switch (I->getOpcode()) {
4827 // TODO: look for (assumed) UB to backwards propagate "deadness".
4828 default:
4829 assert(I->isTerminator() &&
4830 "Expected non-terminators to be handled already!");
4831 for (const BasicBlock *SuccBB : successors(BB: I->getParent()))
4832 AliveSuccessors.push_back(Elt: &SuccBB->front());
4833 break;
4834 case Instruction::Call:
4835 UsedAssumedInformation = identifyAliveSuccessors(A, CB: cast<CallInst>(Val: *I),
4836 AA&: *this, AliveSuccessors);
4837 break;
4838 case Instruction::Invoke:
4839 UsedAssumedInformation = identifyAliveSuccessors(A, II: cast<InvokeInst>(Val: *I),
4840 AA&: *this, AliveSuccessors);
4841 break;
4842 case Instruction::Br:
4843 UsedAssumedInformation = identifyAliveSuccessors(A, BI: cast<BranchInst>(Val: *I),
4844 AA&: *this, AliveSuccessors);
4845 break;
4846 case Instruction::Switch:
4847 UsedAssumedInformation = identifyAliveSuccessors(A, SI: cast<SwitchInst>(Val: *I),
4848 AA&: *this, AliveSuccessors);
4849 break;
4850 }
4851
4852 if (UsedAssumedInformation) {
4853 NewToBeExploredFrom.insert(X: I);
4854 } else if (AliveSuccessors.empty() ||
4855 (I->isTerminator() &&
4856 AliveSuccessors.size() < I->getNumSuccessors())) {
4857 if (KnownDeadEnds.insert(X: I))
4858 Change = ChangeStatus::CHANGED;
4859 }
4860
4861 LLVM_DEBUG(dbgs() << "[AAIsDead] #AliveSuccessors: "
4862 << AliveSuccessors.size() << " UsedAssumedInformation: "
4863 << UsedAssumedInformation << "\n");
4864
4865 for (const Instruction *AliveSuccessor : AliveSuccessors) {
4866 if (!I->isTerminator()) {
4867 assert(AliveSuccessors.size() == 1 &&
4868 "Non-terminator expected to have a single successor!");
4869 Worklist.push_back(Elt: AliveSuccessor);
4870 } else {
4871 // record the assumed live edge
4872 auto Edge = std::make_pair(x: I->getParent(), y: AliveSuccessor->getParent());
4873 if (AssumedLiveEdges.insert(V: Edge).second)
4874 Change = ChangeStatus::CHANGED;
4875 if (assumeLive(A, BB: *AliveSuccessor->getParent()))
4876 Worklist.push_back(Elt: AliveSuccessor);
4877 }
4878 }
4879 }
4880
4881 // Check if the content of ToBeExploredFrom changed, ignore the order.
4882 if (NewToBeExploredFrom.size() != ToBeExploredFrom.size() ||
4883 llvm::any_of(Range&: NewToBeExploredFrom, P: [&](const Instruction *I) {
4884 return !ToBeExploredFrom.count(key: I);
4885 })) {
4886 Change = ChangeStatus::CHANGED;
4887 ToBeExploredFrom = std::move(NewToBeExploredFrom);
4888 }
4889
4890 // If we know everything is live there is no need to query for liveness.
4891 // Instead, indicating a pessimistic fixpoint will cause the state to be
4892 // "invalid" and all queries to be answered conservatively without lookups.
4893 // To be in this state we have to (1) finished the exploration and (3) not
4894 // discovered any non-trivial dead end and (2) not ruled unreachable code
4895 // dead.
4896 if (ToBeExploredFrom.empty() &&
4897 getAnchorScope()->size() == AssumedLiveBlocks.size() &&
4898 llvm::all_of(Range&: KnownDeadEnds, P: [](const Instruction *DeadEndI) {
4899 return DeadEndI->isTerminator() && DeadEndI->getNumSuccessors() == 0;
4900 }))
4901 return indicatePessimisticFixpoint();
4902 return Change;
4903}
4904
4905/// Liveness information for a call sites.
4906struct AAIsDeadCallSite final : AAIsDeadFunction {
4907 AAIsDeadCallSite(const IRPosition &IRP, Attributor &A)
4908 : AAIsDeadFunction(IRP, A) {}
4909
4910 /// See AbstractAttribute::initialize(...).
4911 void initialize(Attributor &A) override {
4912 // TODO: Once we have call site specific value information we can provide
4913 // call site specific liveness information and then it makes
4914 // sense to specialize attributes for call sites instead of
4915 // redirecting requests to the callee.
4916 llvm_unreachable("Abstract attributes for liveness are not "
4917 "supported for call sites yet!");
4918 }
4919
4920 /// See AbstractAttribute::updateImpl(...).
4921 ChangeStatus updateImpl(Attributor &A) override {
4922 return indicatePessimisticFixpoint();
4923 }
4924
4925 /// See AbstractAttribute::trackStatistics()
4926 void trackStatistics() const override {}
4927};
4928} // namespace
4929
4930/// -------------------- Dereferenceable Argument Attribute --------------------
4931
4932namespace {
4933struct AADereferenceableImpl : AADereferenceable {
4934 AADereferenceableImpl(const IRPosition &IRP, Attributor &A)
4935 : AADereferenceable(IRP, A) {}
4936 using StateType = DerefState;
4937
4938 /// See AbstractAttribute::initialize(...).
4939 void initialize(Attributor &A) override {
4940 Value &V = *getAssociatedValue().stripPointerCasts();
4941 SmallVector<Attribute, 4> Attrs;
4942 A.getAttrs(IRP: getIRPosition(),
4943 AKs: {Attribute::Dereferenceable, Attribute::DereferenceableOrNull},
4944 Attrs, /* IgnoreSubsumingPositions */ false);
4945 for (const Attribute &Attr : Attrs)
4946 takeKnownDerefBytesMaximum(Bytes: Attr.getValueAsInt());
4947
4948 // Ensure we initialize the non-null AA (if necessary).
4949 bool IsKnownNonNull;
4950 AA::hasAssumedIRAttr<Attribute::NonNull>(
4951 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNonNull);
4952
4953 bool CanBeNull, CanBeFreed;
4954 takeKnownDerefBytesMaximum(Bytes: V.getPointerDereferenceableBytes(
4955 DL: A.getDataLayout(), CanBeNull, CanBeFreed));
4956
4957 if (Instruction *CtxI = getCtxI())
4958 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
4959 }
4960
4961 /// See AbstractAttribute::getState()
4962 /// {
4963 StateType &getState() override { return *this; }
4964 const StateType &getState() const override { return *this; }
4965 /// }
4966
4967 /// Helper function for collecting accessed bytes in must-be-executed-context
4968 void addAccessedBytesForUse(Attributor &A, const Use *U, const Instruction *I,
4969 DerefState &State) {
4970 const Value *UseV = U->get();
4971 if (!UseV->getType()->isPointerTy())
4972 return;
4973
4974 std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(Inst: I);
4975 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() || I->isVolatile())
4976 return;
4977
4978 int64_t Offset;
4979 const Value *Base = GetPointerBaseWithConstantOffset(
4980 Ptr: Loc->Ptr, Offset, DL: A.getDataLayout(), /*AllowNonInbounds*/ true);
4981 if (Base && Base == &getAssociatedValue())
4982 State.addAccessedBytes(Offset, Size: Loc->Size.getValue());
4983 }
4984
4985 /// See followUsesInMBEC
4986 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
4987 AADereferenceable::StateType &State) {
4988 bool IsNonNull = false;
4989 bool TrackUse = false;
4990 int64_t DerefBytes = getKnownNonNullAndDerefBytesForUse(
4991 A, QueryingAA: *this, AssociatedValue&: getAssociatedValue(), U, I, IsNonNull, TrackUse);
4992 LLVM_DEBUG(dbgs() << "[AADereferenceable] Deref bytes: " << DerefBytes
4993 << " for instruction " << *I << "\n");
4994
4995 addAccessedBytesForUse(A, U, I, State);
4996 State.takeKnownDerefBytesMaximum(Bytes: DerefBytes);
4997 return TrackUse;
4998 }
4999
5000 /// See AbstractAttribute::manifest(...).
5001 ChangeStatus manifest(Attributor &A) override {
5002 ChangeStatus Change = AADereferenceable::manifest(A);
5003 bool IsKnownNonNull;
5004 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
5005 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
5006 if (IsAssumedNonNull &&
5007 A.hasAttr(IRP: getIRPosition(), AKs: Attribute::DereferenceableOrNull)) {
5008 A.removeAttrs(IRP: getIRPosition(), AttrKinds: {Attribute::DereferenceableOrNull});
5009 return ChangeStatus::CHANGED;
5010 }
5011 return Change;
5012 }
5013
5014 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5015 SmallVectorImpl<Attribute> &Attrs) const override {
5016 // TODO: Add *_globally support
5017 bool IsKnownNonNull;
5018 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
5019 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
5020 if (IsAssumedNonNull)
5021 Attrs.emplace_back(Args: Attribute::getWithDereferenceableBytes(
5022 Context&: Ctx, Bytes: getAssumedDereferenceableBytes()));
5023 else
5024 Attrs.emplace_back(Args: Attribute::getWithDereferenceableOrNullBytes(
5025 Context&: Ctx, Bytes: getAssumedDereferenceableBytes()));
5026 }
5027
5028 /// See AbstractAttribute::getAsStr().
5029 const std::string getAsStr(Attributor *A) const override {
5030 if (!getAssumedDereferenceableBytes())
5031 return "unknown-dereferenceable";
5032 bool IsKnownNonNull;
5033 bool IsAssumedNonNull = false;
5034 if (A)
5035 IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
5036 A&: *A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
5037 return std::string("dereferenceable") +
5038 (IsAssumedNonNull ? "" : "_or_null") +
5039 (isAssumedGlobal() ? "_globally" : "") + "<" +
5040 std::to_string(val: getKnownDereferenceableBytes()) + "-" +
5041 std::to_string(val: getAssumedDereferenceableBytes()) + ">" +
5042 (!A ? " [non-null is unknown]" : "");
5043 }
5044};
5045
5046/// Dereferenceable attribute for a floating value.
5047struct AADereferenceableFloating : AADereferenceableImpl {
5048 AADereferenceableFloating(const IRPosition &IRP, Attributor &A)
5049 : AADereferenceableImpl(IRP, A) {}
5050
5051 /// See AbstractAttribute::updateImpl(...).
5052 ChangeStatus updateImpl(Attributor &A) override {
5053 bool Stripped;
5054 bool UsedAssumedInformation = false;
5055 SmallVector<AA::ValueAndContext> Values;
5056 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
5057 S: AA::AnyScope, UsedAssumedInformation)) {
5058 Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
5059 Stripped = false;
5060 } else {
5061 Stripped = Values.size() != 1 ||
5062 Values.front().getValue() != &getAssociatedValue();
5063 }
5064
5065 const DataLayout &DL = A.getDataLayout();
5066 DerefState T;
5067
5068 auto VisitValueCB = [&](const Value &V) -> bool {
5069 unsigned IdxWidth =
5070 DL.getIndexSizeInBits(AS: V.getType()->getPointerAddressSpace());
5071 APInt Offset(IdxWidth, 0);
5072 const Value *Base = stripAndAccumulateOffsets(
5073 A, QueryingAA: *this, Val: &V, DL, Offset, /* GetMinOffset */ false,
5074 /* AllowNonInbounds */ true);
5075
5076 const auto *AA = A.getAAFor<AADereferenceable>(
5077 QueryingAA: *this, IRP: IRPosition::value(V: *Base), DepClass: DepClassTy::REQUIRED);
5078 int64_t DerefBytes = 0;
5079 if (!AA || (!Stripped && this == AA)) {
5080 // Use IR information if we did not strip anything.
5081 // TODO: track globally.
5082 bool CanBeNull, CanBeFreed;
5083 DerefBytes =
5084 Base->getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed);
5085 T.GlobalState.indicatePessimisticFixpoint();
5086 } else {
5087 const DerefState &DS = AA->getState();
5088 DerefBytes = DS.DerefBytesState.getAssumed();
5089 T.GlobalState &= DS.GlobalState;
5090 }
5091
5092 // For now we do not try to "increase" dereferenceability due to negative
5093 // indices as we first have to come up with code to deal with loops and
5094 // for overflows of the dereferenceable bytes.
5095 int64_t OffsetSExt = Offset.getSExtValue();
5096 if (OffsetSExt < 0)
5097 OffsetSExt = 0;
5098
5099 T.takeAssumedDerefBytesMinimum(
5100 Bytes: std::max(a: int64_t(0), b: DerefBytes - OffsetSExt));
5101
5102 if (this == AA) {
5103 if (!Stripped) {
5104 // If nothing was stripped IR information is all we got.
5105 T.takeKnownDerefBytesMaximum(
5106 Bytes: std::max(a: int64_t(0), b: DerefBytes - OffsetSExt));
5107 T.indicatePessimisticFixpoint();
5108 } else if (OffsetSExt > 0) {
5109 // If something was stripped but there is circular reasoning we look
5110 // for the offset. If it is positive we basically decrease the
5111 // dereferenceable bytes in a circular loop now, which will simply
5112 // drive them down to the known value in a very slow way which we
5113 // can accelerate.
5114 T.indicatePessimisticFixpoint();
5115 }
5116 }
5117
5118 return T.isValidState();
5119 };
5120
5121 for (const auto &VAC : Values)
5122 if (!VisitValueCB(*VAC.getValue()))
5123 return indicatePessimisticFixpoint();
5124
5125 return clampStateAndIndicateChange(S&: getState(), R: T);
5126 }
5127
5128 /// See AbstractAttribute::trackStatistics()
5129 void trackStatistics() const override {
5130 STATS_DECLTRACK_FLOATING_ATTR(dereferenceable)
5131 }
5132};
5133
5134/// Dereferenceable attribute for a return value.
5135struct AADereferenceableReturned final
5136 : AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl> {
5137 using Base =
5138 AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl>;
5139 AADereferenceableReturned(const IRPosition &IRP, Attributor &A)
5140 : Base(IRP, A) {}
5141
5142 /// See AbstractAttribute::trackStatistics()
5143 void trackStatistics() const override {
5144 STATS_DECLTRACK_FNRET_ATTR(dereferenceable)
5145 }
5146};
5147
5148/// Dereferenceable attribute for an argument
5149struct AADereferenceableArgument final
5150 : AAArgumentFromCallSiteArguments<AADereferenceable,
5151 AADereferenceableImpl> {
5152 using Base =
5153 AAArgumentFromCallSiteArguments<AADereferenceable, AADereferenceableImpl>;
5154 AADereferenceableArgument(const IRPosition &IRP, Attributor &A)
5155 : Base(IRP, A) {}
5156
5157 /// See AbstractAttribute::trackStatistics()
5158 void trackStatistics() const override {
5159 STATS_DECLTRACK_ARG_ATTR(dereferenceable)
5160 }
5161};
5162
5163/// Dereferenceable attribute for a call site argument.
5164struct AADereferenceableCallSiteArgument final : AADereferenceableFloating {
5165 AADereferenceableCallSiteArgument(const IRPosition &IRP, Attributor &A)
5166 : AADereferenceableFloating(IRP, A) {}
5167
5168 /// See AbstractAttribute::trackStatistics()
5169 void trackStatistics() const override {
5170 STATS_DECLTRACK_CSARG_ATTR(dereferenceable)
5171 }
5172};
5173
5174/// Dereferenceable attribute deduction for a call site return value.
5175struct AADereferenceableCallSiteReturned final
5176 : AACalleeToCallSite<AADereferenceable, AADereferenceableImpl> {
5177 using Base = AACalleeToCallSite<AADereferenceable, AADereferenceableImpl>;
5178 AADereferenceableCallSiteReturned(const IRPosition &IRP, Attributor &A)
5179 : Base(IRP, A) {}
5180
5181 /// See AbstractAttribute::trackStatistics()
5182 void trackStatistics() const override {
5183 STATS_DECLTRACK_CS_ATTR(dereferenceable);
5184 }
5185};
5186} // namespace
5187
5188// ------------------------ Align Argument Attribute ------------------------
5189
5190namespace {
5191
5192static unsigned getKnownAlignForUse(Attributor &A, AAAlign &QueryingAA,
5193 Value &AssociatedValue, const Use *U,
5194 const Instruction *I, bool &TrackUse) {
5195 // We need to follow common pointer manipulation uses to the accesses they
5196 // feed into.
5197 if (isa<CastInst>(Val: I)) {
5198 // Follow all but ptr2int casts.
5199 TrackUse = !isa<PtrToIntInst>(Val: I);
5200 return 0;
5201 }
5202 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: I)) {
5203 if (GEP->hasAllConstantIndices())
5204 TrackUse = true;
5205 return 0;
5206 }
5207 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I))
5208 switch (II->getIntrinsicID()) {
5209 case Intrinsic::ptrmask: {
5210 // Is it appropriate to pull attribute in initialization?
5211 const auto *ConstVals = A.getAAFor<AAPotentialConstantValues>(
5212 QueryingAA, IRP: IRPosition::value(V: *II->getOperand(i_nocapture: 1)), DepClass: DepClassTy::NONE);
5213 const auto *AlignAA = A.getAAFor<AAAlign>(
5214 QueryingAA, IRP: IRPosition::value(V: *II), DepClass: DepClassTy::NONE);
5215 if (ConstVals && ConstVals->isValidState() && ConstVals->isAtFixpoint()) {
5216 unsigned ShiftValue = std::min(a: ConstVals->getAssumedMinTrailingZeros(),
5217 b: Value::MaxAlignmentExponent);
5218 Align ConstAlign(UINT64_C(1) << ShiftValue);
5219 if (ConstAlign >= AlignAA->getKnownAlign())
5220 return Align(1).value();
5221 }
5222 if (AlignAA)
5223 return AlignAA->getKnownAlign().value();
5224 break;
5225 }
5226 case Intrinsic::amdgcn_make_buffer_rsrc: {
5227 const auto *AlignAA = A.getAAFor<AAAlign>(
5228 QueryingAA, IRP: IRPosition::value(V: *II), DepClass: DepClassTy::NONE);
5229 if (AlignAA)
5230 return AlignAA->getKnownAlign().value();
5231 break;
5232 }
5233 default:
5234 break;
5235 }
5236
5237 MaybeAlign MA;
5238 if (const auto *CB = dyn_cast<CallBase>(Val: I)) {
5239 if (CB->isBundleOperand(U) || CB->isCallee(U))
5240 return 0;
5241
5242 unsigned ArgNo = CB->getArgOperandNo(U);
5243 IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
5244 // As long as we only use known information there is no need to track
5245 // dependences here.
5246 auto *AlignAA = A.getAAFor<AAAlign>(QueryingAA, IRP, DepClass: DepClassTy::NONE);
5247 if (AlignAA)
5248 MA = MaybeAlign(AlignAA->getKnownAlign());
5249 }
5250
5251 const DataLayout &DL = A.getDataLayout();
5252 const Value *UseV = U->get();
5253 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
5254 if (SI->getPointerOperand() == UseV)
5255 MA = SI->getAlign();
5256 } else if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
5257 if (LI->getPointerOperand() == UseV)
5258 MA = LI->getAlign();
5259 } else if (auto *AI = dyn_cast<AtomicRMWInst>(Val: I)) {
5260 if (AI->getPointerOperand() == UseV)
5261 MA = AI->getAlign();
5262 } else if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
5263 if (AI->getPointerOperand() == UseV)
5264 MA = AI->getAlign();
5265 }
5266
5267 if (!MA || *MA <= QueryingAA.getKnownAlign())
5268 return 0;
5269
5270 unsigned Alignment = MA->value();
5271 int64_t Offset;
5272
5273 if (const Value *Base = GetPointerBaseWithConstantOffset(Ptr: UseV, Offset, DL)) {
5274 if (Base == &AssociatedValue) {
5275 // BasePointerAddr + Offset = Alignment * Q for some integer Q.
5276 // So we can say that the maximum power of two which is a divisor of
5277 // gcd(Offset, Alignment) is an alignment.
5278
5279 uint32_t gcd = std::gcd(m: uint32_t(abs(x: (int32_t)Offset)), n: Alignment);
5280 Alignment = llvm::bit_floor(Value: gcd);
5281 }
5282 }
5283
5284 return Alignment;
5285}
5286
5287struct AAAlignImpl : AAAlign {
5288 AAAlignImpl(const IRPosition &IRP, Attributor &A) : AAAlign(IRP, A) {}
5289
5290 /// See AbstractAttribute::initialize(...).
5291 void initialize(Attributor &A) override {
5292 SmallVector<Attribute, 4> Attrs;
5293 A.getAttrs(IRP: getIRPosition(), AKs: {Attribute::Alignment}, Attrs);
5294 for (const Attribute &Attr : Attrs)
5295 takeKnownMaximum(Value: Attr.getValueAsInt());
5296
5297 Value &V = *getAssociatedValue().stripPointerCasts();
5298 takeKnownMaximum(Value: V.getPointerAlignment(DL: A.getDataLayout()).value());
5299
5300 if (Instruction *CtxI = getCtxI())
5301 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
5302 }
5303
5304 /// See AbstractAttribute::manifest(...).
5305 ChangeStatus manifest(Attributor &A) override {
5306 ChangeStatus InstrChanged = ChangeStatus::UNCHANGED;
5307
5308 // Check for users that allow alignment annotations.
5309 Value &AssociatedValue = getAssociatedValue();
5310 if (isa<ConstantData>(Val: AssociatedValue))
5311 return ChangeStatus::UNCHANGED;
5312
5313 for (const Use &U : AssociatedValue.uses()) {
5314 if (auto *SI = dyn_cast<StoreInst>(Val: U.getUser())) {
5315 if (SI->getPointerOperand() == &AssociatedValue)
5316 if (SI->getAlign() < getAssumedAlign()) {
5317 STATS_DECLTRACK(AAAlign, Store,
5318 "Number of times alignment added to a store");
5319 SI->setAlignment(getAssumedAlign());
5320 InstrChanged = ChangeStatus::CHANGED;
5321 }
5322 } else if (auto *LI = dyn_cast<LoadInst>(Val: U.getUser())) {
5323 if (LI->getPointerOperand() == &AssociatedValue)
5324 if (LI->getAlign() < getAssumedAlign()) {
5325 LI->setAlignment(getAssumedAlign());
5326 STATS_DECLTRACK(AAAlign, Load,
5327 "Number of times alignment added to a load");
5328 InstrChanged = ChangeStatus::CHANGED;
5329 }
5330 } else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: U.getUser())) {
5331 if (RMW->getPointerOperand() == &AssociatedValue) {
5332 if (RMW->getAlign() < getAssumedAlign()) {
5333 STATS_DECLTRACK(AAAlign, AtomicRMW,
5334 "Number of times alignment added to atomicrmw");
5335
5336 RMW->setAlignment(getAssumedAlign());
5337 InstrChanged = ChangeStatus::CHANGED;
5338 }
5339 }
5340 } else if (auto *CAS = dyn_cast<AtomicCmpXchgInst>(Val: U.getUser())) {
5341 if (CAS->getPointerOperand() == &AssociatedValue) {
5342 if (CAS->getAlign() < getAssumedAlign()) {
5343 STATS_DECLTRACK(AAAlign, AtomicCmpXchg,
5344 "Number of times alignment added to cmpxchg");
5345 CAS->setAlignment(getAssumedAlign());
5346 InstrChanged = ChangeStatus::CHANGED;
5347 }
5348 }
5349 }
5350 }
5351
5352 ChangeStatus Changed = AAAlign::manifest(A);
5353
5354 Align InheritAlign =
5355 getAssociatedValue().getPointerAlignment(DL: A.getDataLayout());
5356 if (InheritAlign >= getAssumedAlign())
5357 return InstrChanged;
5358 return Changed | InstrChanged;
5359 }
5360
5361 // TODO: Provide a helper to determine the implied ABI alignment and check in
5362 // the existing manifest method and a new one for AAAlignImpl that value
5363 // to avoid making the alignment explicit if it did not improve.
5364
5365 /// See AbstractAttribute::getDeducedAttributes
5366 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5367 SmallVectorImpl<Attribute> &Attrs) const override {
5368 if (getAssumedAlign() > 1)
5369 Attrs.emplace_back(
5370 Args: Attribute::getWithAlignment(Context&: Ctx, Alignment: Align(getAssumedAlign())));
5371 }
5372
5373 /// See followUsesInMBEC
5374 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
5375 AAAlign::StateType &State) {
5376 bool TrackUse = false;
5377
5378 unsigned int KnownAlign =
5379 getKnownAlignForUse(A, QueryingAA&: *this, AssociatedValue&: getAssociatedValue(), U, I, TrackUse);
5380 State.takeKnownMaximum(Value: KnownAlign);
5381
5382 return TrackUse;
5383 }
5384
5385 /// See AbstractAttribute::getAsStr().
5386 const std::string getAsStr(Attributor *A) const override {
5387 return "align<" + std::to_string(val: getKnownAlign().value()) + "-" +
5388 std::to_string(val: getAssumedAlign().value()) + ">";
5389 }
5390};
5391
5392/// Align attribute for a floating value.
5393struct AAAlignFloating : AAAlignImpl {
5394 AAAlignFloating(const IRPosition &IRP, Attributor &A) : AAAlignImpl(IRP, A) {}
5395
5396 /// See AbstractAttribute::updateImpl(...).
5397 ChangeStatus updateImpl(Attributor &A) override {
5398 const DataLayout &DL = A.getDataLayout();
5399
5400 bool Stripped;
5401 bool UsedAssumedInformation = false;
5402 SmallVector<AA::ValueAndContext> Values;
5403 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
5404 S: AA::AnyScope, UsedAssumedInformation)) {
5405 Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
5406 Stripped = false;
5407 } else {
5408 Stripped = Values.size() != 1 ||
5409 Values.front().getValue() != &getAssociatedValue();
5410 }
5411
5412 StateType T;
5413 auto VisitValueCB = [&](Value &V) -> bool {
5414 if (isa<UndefValue>(Val: V) || isa<ConstantPointerNull>(Val: V))
5415 return true;
5416 const auto *AA = A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V),
5417 DepClass: DepClassTy::REQUIRED);
5418 if (!AA || (!Stripped && this == AA)) {
5419 int64_t Offset;
5420 unsigned Alignment = 1;
5421 if (const Value *Base =
5422 GetPointerBaseWithConstantOffset(Ptr: &V, Offset, DL)) {
5423 // TODO: Use AAAlign for the base too.
5424 Align PA = Base->getPointerAlignment(DL);
5425 // BasePointerAddr + Offset = Alignment * Q for some integer Q.
5426 // So we can say that the maximum power of two which is a divisor of
5427 // gcd(Offset, Alignment) is an alignment.
5428
5429 uint32_t gcd =
5430 std::gcd(m: uint32_t(abs(x: (int32_t)Offset)), n: uint32_t(PA.value()));
5431 Alignment = llvm::bit_floor(Value: gcd);
5432 } else {
5433 Alignment = V.getPointerAlignment(DL).value();
5434 }
5435 // Use only IR information if we did not strip anything.
5436 T.takeKnownMaximum(Value: Alignment);
5437 T.indicatePessimisticFixpoint();
5438 } else {
5439 // Use abstract attribute information.
5440 const AAAlign::StateType &DS = AA->getState();
5441 T ^= DS;
5442 }
5443 return T.isValidState();
5444 };
5445
5446 for (const auto &VAC : Values) {
5447 if (!VisitValueCB(*VAC.getValue()))
5448 return indicatePessimisticFixpoint();
5449 }
5450
5451 // TODO: If we know we visited all incoming values, thus no are assumed
5452 // dead, we can take the known information from the state T.
5453 return clampStateAndIndicateChange(S&: getState(), R: T);
5454 }
5455
5456 /// See AbstractAttribute::trackStatistics()
5457 void trackStatistics() const override { STATS_DECLTRACK_FLOATING_ATTR(align) }
5458};
5459
5460/// Align attribute for function return value.
5461struct AAAlignReturned final
5462 : AAReturnedFromReturnedValues<AAAlign, AAAlignImpl> {
5463 using Base = AAReturnedFromReturnedValues<AAAlign, AAAlignImpl>;
5464 AAAlignReturned(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5465
5466 /// See AbstractAttribute::trackStatistics()
5467 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(aligned) }
5468};
5469
5470/// Align attribute for function argument.
5471struct AAAlignArgument final
5472 : AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl> {
5473 using Base = AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl>;
5474 AAAlignArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5475
5476 /// See AbstractAttribute::manifest(...).
5477 ChangeStatus manifest(Attributor &A) override {
5478 // If the associated argument is involved in a must-tail call we give up
5479 // because we would need to keep the argument alignments of caller and
5480 // callee in-sync. Just does not seem worth the trouble right now.
5481 if (A.getInfoCache().isInvolvedInMustTailCall(Arg: *getAssociatedArgument()))
5482 return ChangeStatus::UNCHANGED;
5483 return Base::manifest(A);
5484 }
5485
5486 /// See AbstractAttribute::trackStatistics()
5487 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(aligned) }
5488};
5489
5490struct AAAlignCallSiteArgument final : AAAlignFloating {
5491 AAAlignCallSiteArgument(const IRPosition &IRP, Attributor &A)
5492 : AAAlignFloating(IRP, A) {}
5493
5494 /// See AbstractAttribute::manifest(...).
5495 ChangeStatus manifest(Attributor &A) override {
5496 // If the associated argument is involved in a must-tail call we give up
5497 // because we would need to keep the argument alignments of caller and
5498 // callee in-sync. Just does not seem worth the trouble right now.
5499 if (Argument *Arg = getAssociatedArgument())
5500 if (A.getInfoCache().isInvolvedInMustTailCall(Arg: *Arg))
5501 return ChangeStatus::UNCHANGED;
5502 ChangeStatus Changed = AAAlignImpl::manifest(A);
5503 Align InheritAlign =
5504 getAssociatedValue().getPointerAlignment(DL: A.getDataLayout());
5505 if (InheritAlign >= getAssumedAlign())
5506 Changed = ChangeStatus::UNCHANGED;
5507 return Changed;
5508 }
5509
5510 /// See AbstractAttribute::updateImpl(Attributor &A).
5511 ChangeStatus updateImpl(Attributor &A) override {
5512 ChangeStatus Changed = AAAlignFloating::updateImpl(A);
5513 if (Argument *Arg = getAssociatedArgument()) {
5514 // We only take known information from the argument
5515 // so we do not need to track a dependence.
5516 const auto *ArgAlignAA = A.getAAFor<AAAlign>(
5517 QueryingAA: *this, IRP: IRPosition::argument(Arg: *Arg), DepClass: DepClassTy::NONE);
5518 if (ArgAlignAA)
5519 takeKnownMaximum(Value: ArgAlignAA->getKnownAlign().value());
5520 }
5521 return Changed;
5522 }
5523
5524 /// See AbstractAttribute::trackStatistics()
5525 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(aligned) }
5526};
5527
5528/// Align attribute deduction for a call site return value.
5529struct AAAlignCallSiteReturned final
5530 : AACalleeToCallSite<AAAlign, AAAlignImpl> {
5531 using Base = AACalleeToCallSite<AAAlign, AAAlignImpl>;
5532 AAAlignCallSiteReturned(const IRPosition &IRP, Attributor &A)
5533 : Base(IRP, A) {}
5534
5535 ChangeStatus updateImpl(Attributor &A) override {
5536 Instruction *I = getIRPosition().getCtxI();
5537 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
5538 switch (II->getIntrinsicID()) {
5539 case Intrinsic::ptrmask: {
5540 Align Alignment;
5541 bool Valid = false;
5542
5543 const auto *ConstVals = A.getAAFor<AAPotentialConstantValues>(
5544 QueryingAA: *this, IRP: IRPosition::value(V: *II->getOperand(i_nocapture: 1)), DepClass: DepClassTy::REQUIRED);
5545 if (ConstVals && ConstVals->isValidState()) {
5546 unsigned ShiftValue =
5547 std::min(a: ConstVals->getAssumedMinTrailingZeros(),
5548 b: Value::MaxAlignmentExponent);
5549 Alignment = Align(UINT64_C(1) << ShiftValue);
5550 Valid = true;
5551 }
5552
5553 const auto *AlignAA =
5554 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *(II->getOperand(i_nocapture: 0))),
5555 DepClass: DepClassTy::REQUIRED);
5556 if (AlignAA) {
5557 Alignment = std::max(a: AlignAA->getAssumedAlign(), b: Alignment);
5558 Valid = true;
5559 }
5560
5561 if (Valid)
5562 return clampStateAndIndicateChange<StateType>(
5563 S&: this->getState(),
5564 R: std::min(a: this->getAssumedAlign(), b: Alignment).value());
5565 break;
5566 }
5567 // FIXME: Should introduce target specific sub-attributes and letting
5568 // getAAfor<AAAlign> lead to create sub-attribute to handle target
5569 // specific intrinsics.
5570 case Intrinsic::amdgcn_make_buffer_rsrc: {
5571 const auto *AlignAA =
5572 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *(II->getOperand(i_nocapture: 0))),
5573 DepClass: DepClassTy::REQUIRED);
5574 if (AlignAA)
5575 return clampStateAndIndicateChange<StateType>(
5576 S&: this->getState(), R: AlignAA->getAssumedAlign().value());
5577 break;
5578 }
5579 default:
5580 break;
5581 }
5582 }
5583 return Base::updateImpl(A);
5584 };
5585 /// See AbstractAttribute::trackStatistics()
5586 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(align); }
5587};
5588} // namespace
5589
5590/// ------------------ Function No-Return Attribute ----------------------------
5591namespace {
5592struct AANoReturnImpl : public AANoReturn {
5593 AANoReturnImpl(const IRPosition &IRP, Attributor &A) : AANoReturn(IRP, A) {}
5594
5595 /// See AbstractAttribute::initialize(...).
5596 void initialize(Attributor &A) override {
5597 bool IsKnown;
5598 assert(!AA::hasAssumedIRAttr<Attribute::NoReturn>(
5599 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5600 (void)IsKnown;
5601 }
5602
5603 /// See AbstractAttribute::getAsStr().
5604 const std::string getAsStr(Attributor *A) const override {
5605 return getAssumed() ? "noreturn" : "may-return";
5606 }
5607
5608 /// See AbstractAttribute::updateImpl(Attributor &A).
5609 ChangeStatus updateImpl(Attributor &A) override {
5610 auto CheckForNoReturn = [](Instruction &) { return false; };
5611 bool UsedAssumedInformation = false;
5612 if (!A.checkForAllInstructions(Pred: CheckForNoReturn, QueryingAA: *this,
5613 Opcodes: {(unsigned)Instruction::Ret},
5614 UsedAssumedInformation))
5615 return indicatePessimisticFixpoint();
5616 return ChangeStatus::UNCHANGED;
5617 }
5618};
5619
5620struct AANoReturnFunction final : AANoReturnImpl {
5621 AANoReturnFunction(const IRPosition &IRP, Attributor &A)
5622 : AANoReturnImpl(IRP, A) {}
5623
5624 /// See AbstractAttribute::trackStatistics()
5625 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(noreturn) }
5626};
5627
5628/// NoReturn attribute deduction for a call sites.
5629struct AANoReturnCallSite final
5630 : AACalleeToCallSite<AANoReturn, AANoReturnImpl> {
5631 AANoReturnCallSite(const IRPosition &IRP, Attributor &A)
5632 : AACalleeToCallSite<AANoReturn, AANoReturnImpl>(IRP, A) {}
5633
5634 /// See AbstractAttribute::trackStatistics()
5635 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(noreturn); }
5636};
5637} // namespace
5638
5639/// ----------------------- Instance Info ---------------------------------
5640
5641namespace {
5642/// A class to hold the state of for no-capture attributes.
5643struct AAInstanceInfoImpl : public AAInstanceInfo {
5644 AAInstanceInfoImpl(const IRPosition &IRP, Attributor &A)
5645 : AAInstanceInfo(IRP, A) {}
5646
5647 /// See AbstractAttribute::initialize(...).
5648 void initialize(Attributor &A) override {
5649 Value &V = getAssociatedValue();
5650 if (auto *C = dyn_cast<Constant>(Val: &V)) {
5651 if (C->isThreadDependent())
5652 indicatePessimisticFixpoint();
5653 else
5654 indicateOptimisticFixpoint();
5655 return;
5656 }
5657 if (auto *CB = dyn_cast<CallBase>(Val: &V))
5658 if (CB->arg_size() == 0 && !CB->mayHaveSideEffects() &&
5659 !CB->mayReadFromMemory()) {
5660 indicateOptimisticFixpoint();
5661 return;
5662 }
5663 if (auto *I = dyn_cast<Instruction>(Val: &V)) {
5664 const auto *CI =
5665 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
5666 F: *I->getFunction());
5667 if (mayBeInCycle(CI, I, /* HeaderOnly */ false)) {
5668 indicatePessimisticFixpoint();
5669 return;
5670 }
5671 }
5672 }
5673
5674 /// See AbstractAttribute::updateImpl(...).
5675 ChangeStatus updateImpl(Attributor &A) override {
5676 ChangeStatus Changed = ChangeStatus::UNCHANGED;
5677
5678 Value &V = getAssociatedValue();
5679 const Function *Scope = nullptr;
5680 if (auto *I = dyn_cast<Instruction>(Val: &V))
5681 Scope = I->getFunction();
5682 if (auto *A = dyn_cast<Argument>(Val: &V)) {
5683 Scope = A->getParent();
5684 if (!Scope->hasLocalLinkage())
5685 return Changed;
5686 }
5687 if (!Scope)
5688 return indicateOptimisticFixpoint();
5689
5690 bool IsKnownNoRecurse;
5691 if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
5692 A, QueryingAA: this, IRP: IRPosition::function(F: *Scope), DepClass: DepClassTy::OPTIONAL,
5693 IsKnown&: IsKnownNoRecurse))
5694 return Changed;
5695
5696 auto UsePred = [&](const Use &U, bool &Follow) {
5697 const Instruction *UserI = dyn_cast<Instruction>(Val: U.getUser());
5698 if (!UserI || isa<GetElementPtrInst>(Val: UserI) || isa<CastInst>(Val: UserI) ||
5699 isa<PHINode>(Val: UserI) || isa<SelectInst>(Val: UserI)) {
5700 Follow = true;
5701 return true;
5702 }
5703 if (isa<LoadInst>(Val: UserI) || isa<CmpInst>(Val: UserI) ||
5704 (isa<StoreInst>(Val: UserI) &&
5705 cast<StoreInst>(Val: UserI)->getValueOperand() != U.get()))
5706 return true;
5707 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
5708 // This check is not guaranteeing uniqueness but for now that we cannot
5709 // end up with two versions of \p U thinking it was one.
5710 auto *Callee = dyn_cast_if_present<Function>(Val: CB->getCalledOperand());
5711 if (!Callee || !Callee->hasLocalLinkage())
5712 return true;
5713 if (!CB->isArgOperand(U: &U))
5714 return false;
5715 const auto *ArgInstanceInfoAA = A.getAAFor<AAInstanceInfo>(
5716 QueryingAA: *this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U)),
5717 DepClass: DepClassTy::OPTIONAL);
5718 if (!ArgInstanceInfoAA ||
5719 !ArgInstanceInfoAA->isAssumedUniqueForAnalysis())
5720 return false;
5721 // If this call base might reach the scope again we might forward the
5722 // argument back here. This is very conservative.
5723 if (AA::isPotentiallyReachable(
5724 A, FromI: *CB, ToFn: *Scope, QueryingAA: *this, /* ExclusionSet */ nullptr,
5725 GoBackwardsCB: [Scope](const Function &Fn) { return &Fn != Scope; }))
5726 return false;
5727 return true;
5728 }
5729 return false;
5730 };
5731
5732 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
5733 if (auto *SI = dyn_cast<StoreInst>(Val: OldU.getUser())) {
5734 auto *Ptr = SI->getPointerOperand()->stripPointerCasts();
5735 if ((isa<AllocaInst>(Val: Ptr) || isNoAliasCall(V: Ptr)) &&
5736 AA::isDynamicallyUnique(A, QueryingAA: *this, V: *Ptr))
5737 return true;
5738 }
5739 return false;
5740 };
5741
5742 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V, /* CheckBBLivenessOnly */ true,
5743 LivenessDepClass: DepClassTy::OPTIONAL,
5744 /* IgnoreDroppableUses */ true, EquivalentUseCB))
5745 return indicatePessimisticFixpoint();
5746
5747 return Changed;
5748 }
5749
5750 /// See AbstractState::getAsStr().
5751 const std::string getAsStr(Attributor *A) const override {
5752 return isAssumedUniqueForAnalysis() ? "<unique [fAa]>" : "<unknown>";
5753 }
5754
5755 /// See AbstractAttribute::trackStatistics()
5756 void trackStatistics() const override {}
5757};
5758
5759/// InstanceInfo attribute for floating values.
5760struct AAInstanceInfoFloating : AAInstanceInfoImpl {
5761 AAInstanceInfoFloating(const IRPosition &IRP, Attributor &A)
5762 : AAInstanceInfoImpl(IRP, A) {}
5763};
5764
5765/// NoCapture attribute for function arguments.
5766struct AAInstanceInfoArgument final : AAInstanceInfoFloating {
5767 AAInstanceInfoArgument(const IRPosition &IRP, Attributor &A)
5768 : AAInstanceInfoFloating(IRP, A) {}
5769};
5770
5771/// InstanceInfo attribute for call site arguments.
5772struct AAInstanceInfoCallSiteArgument final : AAInstanceInfoImpl {
5773 AAInstanceInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
5774 : AAInstanceInfoImpl(IRP, A) {}
5775
5776 /// See AbstractAttribute::updateImpl(...).
5777 ChangeStatus updateImpl(Attributor &A) override {
5778 // TODO: Once we have call site specific value information we can provide
5779 // call site specific liveness information and then it makes
5780 // sense to specialize attributes for call sites arguments instead of
5781 // redirecting requests to the callee argument.
5782 Argument *Arg = getAssociatedArgument();
5783 if (!Arg)
5784 return indicatePessimisticFixpoint();
5785 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
5786 auto *ArgAA =
5787 A.getAAFor<AAInstanceInfo>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
5788 if (!ArgAA)
5789 return indicatePessimisticFixpoint();
5790 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
5791 }
5792};
5793
5794/// InstanceInfo attribute for function return value.
5795struct AAInstanceInfoReturned final : AAInstanceInfoImpl {
5796 AAInstanceInfoReturned(const IRPosition &IRP, Attributor &A)
5797 : AAInstanceInfoImpl(IRP, A) {
5798 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5799 }
5800
5801 /// See AbstractAttribute::initialize(...).
5802 void initialize(Attributor &A) override {
5803 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5804 }
5805
5806 /// See AbstractAttribute::updateImpl(...).
5807 ChangeStatus updateImpl(Attributor &A) override {
5808 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5809 }
5810};
5811
5812/// InstanceInfo attribute deduction for a call site return value.
5813struct AAInstanceInfoCallSiteReturned final : AAInstanceInfoFloating {
5814 AAInstanceInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
5815 : AAInstanceInfoFloating(IRP, A) {}
5816};
5817} // namespace
5818
5819/// ----------------------- Variable Capturing ---------------------------------
5820bool AANoCapture::isImpliedByIR(Attributor &A, const IRPosition &IRP,
5821 Attribute::AttrKind ImpliedAttributeKind,
5822 bool IgnoreSubsumingPositions) {
5823 assert(ImpliedAttributeKind == Attribute::Captures &&
5824 "Unexpected attribute kind");
5825 Value &V = IRP.getAssociatedValue();
5826 if (!isa<Constant>(Val: V) && !IRP.isArgumentPosition())
5827 return V.use_empty();
5828
5829 // You cannot "capture" null in the default address space.
5830 //
5831 // FIXME: This should use NullPointerIsDefined to account for the function
5832 // attribute.
5833 if (isa<UndefValue>(Val: V) || (isa<ConstantPointerNull>(Val: V) &&
5834 V.getType()->getPointerAddressSpace() == 0)) {
5835 return true;
5836 }
5837
5838 SmallVector<Attribute, 1> Attrs;
5839 A.getAttrs(IRP, AKs: {Attribute::Captures}, Attrs,
5840 /* IgnoreSubsumingPositions */ true);
5841 for (const Attribute &Attr : Attrs)
5842 if (capturesNothing(CC: Attr.getCaptureInfo()))
5843 return true;
5844
5845 if (IRP.getPositionKind() == IRP_CALL_SITE_ARGUMENT)
5846 if (Argument *Arg = IRP.getAssociatedArgument()) {
5847 SmallVector<Attribute, 1> Attrs;
5848 A.getAttrs(IRP: IRPosition::argument(Arg: *Arg),
5849 AKs: {Attribute::Captures, Attribute::ByVal}, Attrs,
5850 /* IgnoreSubsumingPositions */ true);
5851 bool ArgNoCapture = any_of(Range&: Attrs, P: [](Attribute Attr) {
5852 return Attr.getKindAsEnum() == Attribute::ByVal ||
5853 capturesNothing(CC: Attr.getCaptureInfo());
5854 });
5855 if (ArgNoCapture) {
5856 A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithCaptureInfo(
5857 Context&: V.getContext(), CI: CaptureInfo::none()));
5858 return true;
5859 }
5860 }
5861
5862 if (const Function *F = IRP.getAssociatedFunction()) {
5863 // Check what state the associated function can actually capture.
5864 AANoCapture::StateType State;
5865 determineFunctionCaptureCapabilities(IRP, F: *F, State);
5866 if (State.isKnown(BitsEncoding: NO_CAPTURE)) {
5867 A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithCaptureInfo(Context&: V.getContext(),
5868 CI: CaptureInfo::none()));
5869 return true;
5870 }
5871 }
5872
5873 return false;
5874}
5875
5876/// Set the NOT_CAPTURED_IN_MEM and NOT_CAPTURED_IN_RET bits in \p Known
5877/// depending on the ability of the function associated with \p IRP to capture
5878/// state in memory and through "returning/throwing", respectively.
5879void AANoCapture::determineFunctionCaptureCapabilities(const IRPosition &IRP,
5880 const Function &F,
5881 BitIntegerState &State) {
5882 // TODO: Once we have memory behavior attributes we should use them here.
5883
5884 // If we know we cannot communicate or write to memory, we do not care about
5885 // ptr2int anymore.
5886 bool ReadOnly = F.onlyReadsMemory();
5887 bool NoThrow = F.doesNotThrow();
5888 bool IsVoidReturn = F.getReturnType()->isVoidTy();
5889 if (ReadOnly && NoThrow && IsVoidReturn) {
5890 State.addKnownBits(Bits: NO_CAPTURE);
5891 return;
5892 }
5893
5894 // A function cannot capture state in memory if it only reads memory, it can
5895 // however return/throw state and the state might be influenced by the
5896 // pointer value, e.g., loading from a returned pointer might reveal a bit.
5897 if (ReadOnly)
5898 State.addKnownBits(Bits: NOT_CAPTURED_IN_MEM);
5899
5900 // A function cannot communicate state back if it does not through
5901 // exceptions and doesn not return values.
5902 if (NoThrow && IsVoidReturn)
5903 State.addKnownBits(Bits: NOT_CAPTURED_IN_RET);
5904
5905 // Check existing "returned" attributes.
5906 int ArgNo = IRP.getCalleeArgNo();
5907 if (!NoThrow || ArgNo < 0 ||
5908 !F.getAttributes().hasAttrSomewhere(Kind: Attribute::Returned))
5909 return;
5910
5911 for (unsigned U = 0, E = F.arg_size(); U < E; ++U)
5912 if (F.hasParamAttribute(ArgNo: U, Kind: Attribute::Returned)) {
5913 if (U == unsigned(ArgNo))
5914 State.removeAssumedBits(BitsEncoding: NOT_CAPTURED_IN_RET);
5915 else if (ReadOnly)
5916 State.addKnownBits(Bits: NO_CAPTURE);
5917 else
5918 State.addKnownBits(Bits: NOT_CAPTURED_IN_RET);
5919 break;
5920 }
5921}
5922
5923namespace {
5924/// A class to hold the state of for no-capture attributes.
5925struct AANoCaptureImpl : public AANoCapture {
5926 AANoCaptureImpl(const IRPosition &IRP, Attributor &A) : AANoCapture(IRP, A) {}
5927
5928 /// See AbstractAttribute::initialize(...).
5929 void initialize(Attributor &A) override {
5930 bool IsKnown;
5931 assert(!AA::hasAssumedIRAttr<Attribute::Captures>(
5932 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5933 (void)IsKnown;
5934 }
5935
5936 /// See AbstractAttribute::updateImpl(...).
5937 ChangeStatus updateImpl(Attributor &A) override;
5938
5939 /// see AbstractAttribute::isAssumedNoCaptureMaybeReturned(...).
5940 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5941 SmallVectorImpl<Attribute> &Attrs) const override {
5942 if (!isAssumedNoCaptureMaybeReturned())
5943 return;
5944
5945 if (isArgumentPosition()) {
5946 if (isAssumedNoCapture())
5947 Attrs.emplace_back(Args: Attribute::get(Context&: Ctx, Kind: Attribute::Captures));
5948 else if (ManifestInternal)
5949 Attrs.emplace_back(Args: Attribute::get(Context&: Ctx, Kind: "no-capture-maybe-returned"));
5950 }
5951 }
5952
5953 /// See AbstractState::getAsStr().
5954 const std::string getAsStr(Attributor *A) const override {
5955 if (isKnownNoCapture())
5956 return "known not-captured";
5957 if (isAssumedNoCapture())
5958 return "assumed not-captured";
5959 if (isKnownNoCaptureMaybeReturned())
5960 return "known not-captured-maybe-returned";
5961 if (isAssumedNoCaptureMaybeReturned())
5962 return "assumed not-captured-maybe-returned";
5963 return "assumed-captured";
5964 }
5965
5966 /// Check the use \p U and update \p State accordingly. Return true if we
5967 /// should continue to update the state.
5968 bool checkUse(Attributor &A, AANoCapture::StateType &State, const Use &U,
5969 bool &Follow) {
5970 Instruction *UInst = cast<Instruction>(Val: U.getUser());
5971 LLVM_DEBUG(dbgs() << "[AANoCapture] Check use: " << *U.get() << " in "
5972 << *UInst << "\n");
5973
5974 // Deal with ptr2int by following uses.
5975 if (isa<PtrToIntInst>(Val: UInst)) {
5976 LLVM_DEBUG(dbgs() << " - ptr2int assume the worst!\n");
5977 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
5978 /* Return */ CapturedInRet: true);
5979 }
5980
5981 // For stores we already checked if we can follow them, if they make it
5982 // here we give up.
5983 if (isa<StoreInst>(Val: UInst))
5984 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
5985 /* Return */ CapturedInRet: true);
5986
5987 // Explicitly catch return instructions.
5988 if (isa<ReturnInst>(Val: UInst)) {
5989 if (UInst->getFunction() == getAnchorScope())
5990 return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
5991 /* Return */ CapturedInRet: true);
5992 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
5993 /* Return */ CapturedInRet: true);
5994 }
5995
5996 // For now we only use special logic for call sites. However, the tracker
5997 // itself knows about a lot of other non-capturing cases already.
5998 auto *CB = dyn_cast<CallBase>(Val: UInst);
5999 if (!CB || !CB->isArgOperand(U: &U))
6000 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
6001 /* Return */ CapturedInRet: true);
6002
6003 unsigned ArgNo = CB->getArgOperandNo(U: &U);
6004 const IRPosition &CSArgPos = IRPosition::callsite_argument(CB: *CB, ArgNo);
6005 // If we have a abstract no-capture attribute for the argument we can use
6006 // it to justify a non-capture attribute here. This allows recursion!
6007 bool IsKnownNoCapture;
6008 const AANoCapture *ArgNoCaptureAA = nullptr;
6009 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
6010 A, QueryingAA: this, IRP: CSArgPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
6011 AAPtr: &ArgNoCaptureAA);
6012 if (IsAssumedNoCapture)
6013 return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
6014 /* Return */ CapturedInRet: false);
6015 if (ArgNoCaptureAA && ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned()) {
6016 Follow = true;
6017 return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
6018 /* Return */ CapturedInRet: false);
6019 }
6020
6021 // Lastly, we could not find a reason no-capture can be assumed so we don't.
6022 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
6023 /* Return */ CapturedInRet: true);
6024 }
6025
6026 /// Update \p State according to \p CapturedInMem, \p CapturedInInt, and
6027 /// \p CapturedInRet, then return true if we should continue updating the
6028 /// state.
6029 static bool isCapturedIn(AANoCapture::StateType &State, bool CapturedInMem,
6030 bool CapturedInInt, bool CapturedInRet) {
6031 LLVM_DEBUG(dbgs() << " - captures [Mem " << CapturedInMem << "|Int "
6032 << CapturedInInt << "|Ret " << CapturedInRet << "]\n");
6033 if (CapturedInMem)
6034 State.removeAssumedBits(BitsEncoding: AANoCapture::NOT_CAPTURED_IN_MEM);
6035 if (CapturedInInt)
6036 State.removeAssumedBits(BitsEncoding: AANoCapture::NOT_CAPTURED_IN_INT);
6037 if (CapturedInRet)
6038 State.removeAssumedBits(BitsEncoding: AANoCapture::NOT_CAPTURED_IN_RET);
6039 return State.isAssumed(BitsEncoding: AANoCapture::NO_CAPTURE_MAYBE_RETURNED);
6040 }
6041};
6042
6043ChangeStatus AANoCaptureImpl::updateImpl(Attributor &A) {
6044 const IRPosition &IRP = getIRPosition();
6045 Value *V = isArgumentPosition() ? IRP.getAssociatedArgument()
6046 : &IRP.getAssociatedValue();
6047 if (!V)
6048 return indicatePessimisticFixpoint();
6049
6050 const Function *F =
6051 isArgumentPosition() ? IRP.getAssociatedFunction() : IRP.getAnchorScope();
6052
6053 // TODO: Is the checkForAllUses below useful for constants?
6054 if (!F)
6055 return indicatePessimisticFixpoint();
6056
6057 AANoCapture::StateType T;
6058 const IRPosition &FnPos = IRPosition::function(F: *F);
6059
6060 // Readonly means we cannot capture through memory.
6061 bool IsKnown;
6062 if (AA::isAssumedReadOnly(A, IRP: FnPos, QueryingAA: *this, IsKnown)) {
6063 T.addKnownBits(Bits: NOT_CAPTURED_IN_MEM);
6064 if (IsKnown)
6065 addKnownBits(Bits: NOT_CAPTURED_IN_MEM);
6066 }
6067
6068 // Make sure all returned values are different than the underlying value.
6069 // TODO: we could do this in a more sophisticated way inside
6070 // AAReturnedValues, e.g., track all values that escape through returns
6071 // directly somehow.
6072 auto CheckReturnedArgs = [&](bool &UsedAssumedInformation) {
6073 SmallVector<AA::ValueAndContext> Values;
6074 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::returned(F: *F), AA: this, Values,
6075 S: AA::ValueScope::Intraprocedural,
6076 UsedAssumedInformation))
6077 return false;
6078 bool SeenConstant = false;
6079 for (const AA::ValueAndContext &VAC : Values) {
6080 if (isa<Constant>(Val: VAC.getValue())) {
6081 if (SeenConstant)
6082 return false;
6083 SeenConstant = true;
6084 } else if (!isa<Argument>(Val: VAC.getValue()) ||
6085 VAC.getValue() == getAssociatedArgument())
6086 return false;
6087 }
6088 return true;
6089 };
6090
6091 bool IsKnownNoUnwind;
6092 if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
6093 A, QueryingAA: this, IRP: FnPos, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoUnwind)) {
6094 bool IsVoidTy = F->getReturnType()->isVoidTy();
6095 bool UsedAssumedInformation = false;
6096 if (IsVoidTy || CheckReturnedArgs(UsedAssumedInformation)) {
6097 T.addKnownBits(Bits: NOT_CAPTURED_IN_RET);
6098 if (T.isKnown(BitsEncoding: NOT_CAPTURED_IN_MEM))
6099 return ChangeStatus::UNCHANGED;
6100 if (IsKnownNoUnwind && (IsVoidTy || !UsedAssumedInformation)) {
6101 addKnownBits(Bits: NOT_CAPTURED_IN_RET);
6102 if (isKnown(BitsEncoding: NOT_CAPTURED_IN_MEM))
6103 return indicateOptimisticFixpoint();
6104 }
6105 }
6106 }
6107
6108 auto UseCheck = [&](const Use &U, bool &Follow) -> bool {
6109 // TODO(captures): Make this more precise.
6110 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
6111 if (capturesNothing(CC: CI))
6112 return true;
6113 if (CI.isPassthrough()) {
6114 Follow = true;
6115 return true;
6116 }
6117 return checkUse(A, State&: T, U, Follow);
6118 };
6119
6120 if (!A.checkForAllUses(Pred: UseCheck, QueryingAA: *this, V: *V))
6121 return indicatePessimisticFixpoint();
6122
6123 AANoCapture::StateType &S = getState();
6124 auto Assumed = S.getAssumed();
6125 S.intersectAssumedBits(BitsEncoding: T.getAssumed());
6126 if (!isAssumedNoCaptureMaybeReturned())
6127 return indicatePessimisticFixpoint();
6128 return Assumed == S.getAssumed() ? ChangeStatus::UNCHANGED
6129 : ChangeStatus::CHANGED;
6130}
6131
6132/// NoCapture attribute for function arguments.
6133struct AANoCaptureArgument final : AANoCaptureImpl {
6134 AANoCaptureArgument(const IRPosition &IRP, Attributor &A)
6135 : AANoCaptureImpl(IRP, A) {}
6136
6137 /// See AbstractAttribute::trackStatistics()
6138 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nocapture) }
6139};
6140
6141/// NoCapture attribute for call site arguments.
6142struct AANoCaptureCallSiteArgument final : AANoCaptureImpl {
6143 AANoCaptureCallSiteArgument(const IRPosition &IRP, Attributor &A)
6144 : AANoCaptureImpl(IRP, A) {}
6145
6146 /// See AbstractAttribute::updateImpl(...).
6147 ChangeStatus updateImpl(Attributor &A) override {
6148 // TODO: Once we have call site specific value information we can provide
6149 // call site specific liveness information and then it makes
6150 // sense to specialize attributes for call sites arguments instead of
6151 // redirecting requests to the callee argument.
6152 Argument *Arg = getAssociatedArgument();
6153 if (!Arg)
6154 return indicatePessimisticFixpoint();
6155 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
6156 bool IsKnownNoCapture;
6157 const AANoCapture *ArgAA = nullptr;
6158 if (AA::hasAssumedIRAttr<Attribute::Captures>(
6159 A, QueryingAA: this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
6160 AAPtr: &ArgAA))
6161 return ChangeStatus::UNCHANGED;
6162 if (!ArgAA || !ArgAA->isAssumedNoCaptureMaybeReturned())
6163 return indicatePessimisticFixpoint();
6164 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
6165 }
6166
6167 /// See AbstractAttribute::trackStatistics()
6168 void trackStatistics() const override {
6169 STATS_DECLTRACK_CSARG_ATTR(nocapture)
6170 };
6171};
6172
6173/// NoCapture attribute for floating values.
6174struct AANoCaptureFloating final : AANoCaptureImpl {
6175 AANoCaptureFloating(const IRPosition &IRP, Attributor &A)
6176 : AANoCaptureImpl(IRP, A) {}
6177
6178 /// See AbstractAttribute::trackStatistics()
6179 void trackStatistics() const override {
6180 STATS_DECLTRACK_FLOATING_ATTR(nocapture)
6181 }
6182};
6183
6184/// NoCapture attribute for function return value.
6185struct AANoCaptureReturned final : AANoCaptureImpl {
6186 AANoCaptureReturned(const IRPosition &IRP, Attributor &A)
6187 : AANoCaptureImpl(IRP, A) {
6188 llvm_unreachable("NoCapture is not applicable to function returns!");
6189 }
6190
6191 /// See AbstractAttribute::initialize(...).
6192 void initialize(Attributor &A) override {
6193 llvm_unreachable("NoCapture is not applicable to function returns!");
6194 }
6195
6196 /// See AbstractAttribute::updateImpl(...).
6197 ChangeStatus updateImpl(Attributor &A) override {
6198 llvm_unreachable("NoCapture is not applicable to function returns!");
6199 }
6200
6201 /// See AbstractAttribute::trackStatistics()
6202 void trackStatistics() const override {}
6203};
6204
6205/// NoCapture attribute deduction for a call site return value.
6206struct AANoCaptureCallSiteReturned final : AANoCaptureImpl {
6207 AANoCaptureCallSiteReturned(const IRPosition &IRP, Attributor &A)
6208 : AANoCaptureImpl(IRP, A) {}
6209
6210 /// See AbstractAttribute::initialize(...).
6211 void initialize(Attributor &A) override {
6212 const Function *F = getAnchorScope();
6213 // Check what state the associated function can actually capture.
6214 determineFunctionCaptureCapabilities(IRP: getIRPosition(), F: *F, State&: *this);
6215 }
6216
6217 /// See AbstractAttribute::trackStatistics()
6218 void trackStatistics() const override {
6219 STATS_DECLTRACK_CSRET_ATTR(nocapture)
6220 }
6221};
6222} // namespace
6223
6224/// ------------------ Value Simplify Attribute ----------------------------
6225
6226bool ValueSimplifyStateType::unionAssumed(std::optional<Value *> Other) {
6227 // FIXME: Add a typecast support.
6228 SimplifiedAssociatedValue = AA::combineOptionalValuesInAAValueLatice(
6229 A: SimplifiedAssociatedValue, B: Other, Ty);
6230 if (SimplifiedAssociatedValue == std::optional<Value *>(nullptr))
6231 return false;
6232
6233 LLVM_DEBUG({
6234 if (SimplifiedAssociatedValue)
6235 dbgs() << "[ValueSimplify] is assumed to be "
6236 << **SimplifiedAssociatedValue << "\n";
6237 else
6238 dbgs() << "[ValueSimplify] is assumed to be <none>\n";
6239 });
6240 return true;
6241}
6242
6243namespace {
6244struct AAValueSimplifyImpl : AAValueSimplify {
6245 AAValueSimplifyImpl(const IRPosition &IRP, Attributor &A)
6246 : AAValueSimplify(IRP, A) {}
6247
6248 /// See AbstractAttribute::initialize(...).
6249 void initialize(Attributor &A) override {
6250 if (getAssociatedValue().getType()->isVoidTy())
6251 indicatePessimisticFixpoint();
6252 if (A.hasSimplificationCallback(IRP: getIRPosition()))
6253 indicatePessimisticFixpoint();
6254 }
6255
6256 /// See AbstractAttribute::getAsStr().
6257 const std::string getAsStr(Attributor *A) const override {
6258 LLVM_DEBUG({
6259 dbgs() << "SAV: " << (bool)SimplifiedAssociatedValue << " ";
6260 if (SimplifiedAssociatedValue && *SimplifiedAssociatedValue)
6261 dbgs() << "SAV: " << **SimplifiedAssociatedValue << " ";
6262 });
6263 return isValidState() ? (isAtFixpoint() ? "simplified" : "maybe-simple")
6264 : "not-simple";
6265 }
6266
6267 /// See AbstractAttribute::trackStatistics()
6268 void trackStatistics() const override {}
6269
6270 /// See AAValueSimplify::getAssumedSimplifiedValue()
6271 std::optional<Value *>
6272 getAssumedSimplifiedValue(Attributor &A) const override {
6273 return SimplifiedAssociatedValue;
6274 }
6275
6276 /// Ensure the return value is \p V with type \p Ty, if not possible return
6277 /// nullptr. If \p Check is true we will only verify such an operation would
6278 /// suceed and return a non-nullptr value if that is the case. No IR is
6279 /// generated or modified.
6280 static Value *ensureType(Attributor &A, Value &V, Type &Ty, Instruction *CtxI,
6281 bool Check) {
6282 if (auto *TypedV = AA::getWithType(V, Ty))
6283 return TypedV;
6284 if (CtxI && V.getType()->canLosslesslyBitCastTo(Ty: &Ty))
6285 return Check ? &V
6286 : BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6287 S: &V, Ty: &Ty, Name: "", InsertBefore: CtxI->getIterator());
6288 return nullptr;
6289 }
6290
6291 /// Reproduce \p I with type \p Ty or return nullptr if that is not posisble.
6292 /// If \p Check is true we will only verify such an operation would suceed and
6293 /// return a non-nullptr value if that is the case. No IR is generated or
6294 /// modified.
6295 static Value *reproduceInst(Attributor &A,
6296 const AbstractAttribute &QueryingAA,
6297 Instruction &I, Type &Ty, Instruction *CtxI,
6298 bool Check, ValueToValueMapTy &VMap) {
6299 assert(CtxI && "Cannot reproduce an instruction without context!");
6300 if (Check && (I.mayReadFromMemory() ||
6301 !isSafeToSpeculativelyExecute(I: &I, CtxI, /* DT */ AC: nullptr,
6302 /* TLI */ DT: nullptr)))
6303 return nullptr;
6304 for (Value *Op : I.operands()) {
6305 Value *NewOp = reproduceValue(A, QueryingAA, V&: *Op, Ty, CtxI, Check, VMap);
6306 if (!NewOp) {
6307 assert(Check && "Manifest of new value unexpectedly failed!");
6308 return nullptr;
6309 }
6310 if (!Check)
6311 VMap[Op] = NewOp;
6312 }
6313 if (Check)
6314 return &I;
6315
6316 Instruction *CloneI = I.clone();
6317 // TODO: Try to salvage debug information here.
6318 CloneI->setDebugLoc(DebugLoc());
6319 VMap[&I] = CloneI;
6320 CloneI->insertBefore(InsertPos: CtxI->getIterator());
6321 RemapInstruction(I: CloneI, VM&: VMap);
6322 return CloneI;
6323 }
6324
6325 /// Reproduce \p V with type \p Ty or return nullptr if that is not posisble.
6326 /// If \p Check is true we will only verify such an operation would suceed and
6327 /// return a non-nullptr value if that is the case. No IR is generated or
6328 /// modified.
6329 static Value *reproduceValue(Attributor &A,
6330 const AbstractAttribute &QueryingAA, Value &V,
6331 Type &Ty, Instruction *CtxI, bool Check,
6332 ValueToValueMapTy &VMap) {
6333 if (const auto &NewV = VMap.lookup(Val: &V))
6334 return NewV;
6335 bool UsedAssumedInformation = false;
6336 std::optional<Value *> SimpleV = A.getAssumedSimplified(
6337 V, AA: QueryingAA, UsedAssumedInformation, S: AA::Interprocedural);
6338 if (!SimpleV.has_value())
6339 return PoisonValue::get(T: &Ty);
6340 Value *EffectiveV = &V;
6341 if (*SimpleV)
6342 EffectiveV = *SimpleV;
6343 if (auto *C = dyn_cast<Constant>(Val: EffectiveV))
6344 return C;
6345 if (CtxI && AA::isValidAtPosition(VAC: AA::ValueAndContext(*EffectiveV, *CtxI),
6346 InfoCache&: A.getInfoCache()))
6347 return ensureType(A, V&: *EffectiveV, Ty, CtxI, Check);
6348 if (auto *I = dyn_cast<Instruction>(Val: EffectiveV))
6349 if (Value *NewV = reproduceInst(A, QueryingAA, I&: *I, Ty, CtxI, Check, VMap))
6350 return ensureType(A, V&: *NewV, Ty, CtxI, Check);
6351 return nullptr;
6352 }
6353
6354 /// Return a value we can use as replacement for the associated one, or
6355 /// nullptr if we don't have one that makes sense.
6356 Value *manifestReplacementValue(Attributor &A, Instruction *CtxI) const {
6357 Value *NewV = SimplifiedAssociatedValue
6358 ? *SimplifiedAssociatedValue
6359 : UndefValue::get(T: getAssociatedType());
6360 if (NewV && NewV != &getAssociatedValue()) {
6361 ValueToValueMapTy VMap;
6362 // First verify we can reprduce the value with the required type at the
6363 // context location before we actually start modifying the IR.
6364 if (reproduceValue(A, QueryingAA: *this, V&: *NewV, Ty&: *getAssociatedType(), CtxI,
6365 /* CheckOnly */ Check: true, VMap))
6366 return reproduceValue(A, QueryingAA: *this, V&: *NewV, Ty&: *getAssociatedType(), CtxI,
6367 /* CheckOnly */ Check: false, VMap);
6368 }
6369 return nullptr;
6370 }
6371
6372 /// Helper function for querying AAValueSimplify and updating candidate.
6373 /// \param IRP The value position we are trying to unify with SimplifiedValue
6374 bool checkAndUpdate(Attributor &A, const AbstractAttribute &QueryingAA,
6375 const IRPosition &IRP, bool Simplify = true) {
6376 bool UsedAssumedInformation = false;
6377 std::optional<Value *> QueryingValueSimplified = &IRP.getAssociatedValue();
6378 if (Simplify)
6379 QueryingValueSimplified = A.getAssumedSimplified(
6380 IRP, AA: QueryingAA, UsedAssumedInformation, S: AA::Interprocedural);
6381 return unionAssumed(Other: QueryingValueSimplified);
6382 }
6383
6384 /// Returns a candidate is found or not
6385 template <typename AAType> bool askSimplifiedValueFor(Attributor &A) {
6386 if (!getAssociatedValue().getType()->isIntegerTy())
6387 return false;
6388
6389 // This will also pass the call base context.
6390 const auto *AA =
6391 A.getAAFor<AAType>(*this, getIRPosition(), DepClassTy::NONE);
6392 if (!AA)
6393 return false;
6394
6395 std::optional<Constant *> COpt = AA->getAssumedConstant(A);
6396
6397 if (!COpt) {
6398 SimplifiedAssociatedValue = std::nullopt;
6399 A.recordDependence(FromAA: *AA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
6400 return true;
6401 }
6402 if (auto *C = *COpt) {
6403 SimplifiedAssociatedValue = C;
6404 A.recordDependence(FromAA: *AA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
6405 return true;
6406 }
6407 return false;
6408 }
6409
6410 bool askSimplifiedValueForOtherAAs(Attributor &A) {
6411 if (askSimplifiedValueFor<AAValueConstantRange>(A))
6412 return true;
6413 if (askSimplifiedValueFor<AAPotentialConstantValues>(A))
6414 return true;
6415 return false;
6416 }
6417
6418 /// See AbstractAttribute::manifest(...).
6419 ChangeStatus manifest(Attributor &A) override {
6420 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6421 for (auto &U : getAssociatedValue().uses()) {
6422 // Check if we need to adjust the insertion point to make sure the IR is
6423 // valid.
6424 Instruction *IP = dyn_cast<Instruction>(Val: U.getUser());
6425 if (auto *PHI = dyn_cast_or_null<PHINode>(Val: IP))
6426 IP = PHI->getIncomingBlock(U)->getTerminator();
6427 if (auto *NewV = manifestReplacementValue(A, CtxI: IP)) {
6428 LLVM_DEBUG(dbgs() << "[ValueSimplify] " << getAssociatedValue()
6429 << " -> " << *NewV << " :: " << *this << "\n");
6430 if (A.changeUseAfterManifest(U, NV&: *NewV))
6431 Changed = ChangeStatus::CHANGED;
6432 }
6433 }
6434
6435 return Changed | AAValueSimplify::manifest(A);
6436 }
6437
6438 /// See AbstractState::indicatePessimisticFixpoint(...).
6439 ChangeStatus indicatePessimisticFixpoint() override {
6440 SimplifiedAssociatedValue = &getAssociatedValue();
6441 return AAValueSimplify::indicatePessimisticFixpoint();
6442 }
6443};
6444
6445struct AAValueSimplifyArgument final : AAValueSimplifyImpl {
6446 AAValueSimplifyArgument(const IRPosition &IRP, Attributor &A)
6447 : AAValueSimplifyImpl(IRP, A) {}
6448
6449 void initialize(Attributor &A) override {
6450 AAValueSimplifyImpl::initialize(A);
6451 if (A.hasAttr(IRP: getIRPosition(),
6452 AKs: {Attribute::InAlloca, Attribute::Preallocated,
6453 Attribute::StructRet, Attribute::Nest, Attribute::ByVal},
6454 /* IgnoreSubsumingPositions */ true))
6455 indicatePessimisticFixpoint();
6456 }
6457
6458 /// See AbstractAttribute::updateImpl(...).
6459 ChangeStatus updateImpl(Attributor &A) override {
6460 // Byval is only replacable if it is readonly otherwise we would write into
6461 // the replaced value and not the copy that byval creates implicitly.
6462 Argument *Arg = getAssociatedArgument();
6463 if (Arg->hasByValAttr()) {
6464 // TODO: We probably need to verify synchronization is not an issue, e.g.,
6465 // there is no race by not copying a constant byval.
6466 bool IsKnown;
6467 if (!AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
6468 return indicatePessimisticFixpoint();
6469 }
6470
6471 auto Before = SimplifiedAssociatedValue;
6472
6473 auto PredForCallSite = [&](AbstractCallSite ACS) {
6474 const IRPosition &ACSArgPos =
6475 IRPosition::callsite_argument(ACS, ArgNo: getCallSiteArgNo());
6476 // Check if a coresponding argument was found or if it is on not
6477 // associated (which can happen for callback calls).
6478 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
6479 return false;
6480
6481 // Simplify the argument operand explicitly and check if the result is
6482 // valid in the current scope. This avoids refering to simplified values
6483 // in other functions, e.g., we don't want to say a an argument in a
6484 // static function is actually an argument in a different function.
6485 bool UsedAssumedInformation = false;
6486 std::optional<Constant *> SimpleArgOp =
6487 A.getAssumedConstant(IRP: ACSArgPos, AA: *this, UsedAssumedInformation);
6488 if (!SimpleArgOp)
6489 return true;
6490 if (!*SimpleArgOp)
6491 return false;
6492 if (!AA::isDynamicallyUnique(A, QueryingAA: *this, V: **SimpleArgOp))
6493 return false;
6494 return unionAssumed(Other: *SimpleArgOp);
6495 };
6496
6497 // Generate a answer specific to a call site context.
6498 bool Success;
6499 bool UsedAssumedInformation = false;
6500 if (hasCallBaseContext() &&
6501 getCallBaseContext()->getCalledOperand() == Arg->getParent())
6502 Success = PredForCallSite(
6503 AbstractCallSite(&getCallBaseContext()->getCalledOperandUse()));
6504 else
6505 Success = A.checkForAllCallSites(Pred: PredForCallSite, QueryingAA: *this, RequireAllCallSites: true,
6506 UsedAssumedInformation);
6507
6508 if (!Success)
6509 if (!askSimplifiedValueForOtherAAs(A))
6510 return indicatePessimisticFixpoint();
6511
6512 // If a candidate was found in this update, return CHANGED.
6513 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6514 : ChangeStatus ::CHANGED;
6515 }
6516
6517 /// See AbstractAttribute::trackStatistics()
6518 void trackStatistics() const override {
6519 STATS_DECLTRACK_ARG_ATTR(value_simplify)
6520 }
6521};
6522
6523struct AAValueSimplifyReturned : AAValueSimplifyImpl {
6524 AAValueSimplifyReturned(const IRPosition &IRP, Attributor &A)
6525 : AAValueSimplifyImpl(IRP, A) {}
6526
6527 /// See AAValueSimplify::getAssumedSimplifiedValue()
6528 std::optional<Value *>
6529 getAssumedSimplifiedValue(Attributor &A) const override {
6530 if (!isValidState())
6531 return nullptr;
6532 return SimplifiedAssociatedValue;
6533 }
6534
6535 /// See AbstractAttribute::updateImpl(...).
6536 ChangeStatus updateImpl(Attributor &A) override {
6537 auto Before = SimplifiedAssociatedValue;
6538
6539 auto ReturnInstCB = [&](Instruction &I) {
6540 auto &RI = cast<ReturnInst>(Val&: I);
6541 return checkAndUpdate(
6542 A, QueryingAA: *this,
6543 IRP: IRPosition::value(V: *RI.getReturnValue(), CBContext: getCallBaseContext()));
6544 };
6545
6546 bool UsedAssumedInformation = false;
6547 if (!A.checkForAllInstructions(Pred: ReturnInstCB, QueryingAA: *this, Opcodes: {Instruction::Ret},
6548 UsedAssumedInformation))
6549 if (!askSimplifiedValueForOtherAAs(A))
6550 return indicatePessimisticFixpoint();
6551
6552 // If a candidate was found in this update, return CHANGED.
6553 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6554 : ChangeStatus ::CHANGED;
6555 }
6556
6557 ChangeStatus manifest(Attributor &A) override {
6558 // We queried AAValueSimplify for the returned values so they will be
6559 // replaced if a simplified form was found. Nothing to do here.
6560 return ChangeStatus::UNCHANGED;
6561 }
6562
6563 /// See AbstractAttribute::trackStatistics()
6564 void trackStatistics() const override {
6565 STATS_DECLTRACK_FNRET_ATTR(value_simplify)
6566 }
6567};
6568
6569struct AAValueSimplifyFloating : AAValueSimplifyImpl {
6570 AAValueSimplifyFloating(const IRPosition &IRP, Attributor &A)
6571 : AAValueSimplifyImpl(IRP, A) {}
6572
6573 /// See AbstractAttribute::initialize(...).
6574 void initialize(Attributor &A) override {
6575 AAValueSimplifyImpl::initialize(A);
6576 Value &V = getAnchorValue();
6577
6578 // TODO: add other stuffs
6579 if (isa<Constant>(Val: V))
6580 indicatePessimisticFixpoint();
6581 }
6582
6583 /// See AbstractAttribute::updateImpl(...).
6584 ChangeStatus updateImpl(Attributor &A) override {
6585 auto Before = SimplifiedAssociatedValue;
6586 if (!askSimplifiedValueForOtherAAs(A))
6587 return indicatePessimisticFixpoint();
6588
6589 // If a candidate was found in this update, return CHANGED.
6590 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6591 : ChangeStatus ::CHANGED;
6592 }
6593
6594 /// See AbstractAttribute::trackStatistics()
6595 void trackStatistics() const override {
6596 STATS_DECLTRACK_FLOATING_ATTR(value_simplify)
6597 }
6598};
6599
6600struct AAValueSimplifyFunction : AAValueSimplifyImpl {
6601 AAValueSimplifyFunction(const IRPosition &IRP, Attributor &A)
6602 : AAValueSimplifyImpl(IRP, A) {}
6603
6604 /// See AbstractAttribute::initialize(...).
6605 void initialize(Attributor &A) override {
6606 SimplifiedAssociatedValue = nullptr;
6607 indicateOptimisticFixpoint();
6608 }
6609 /// See AbstractAttribute::initialize(...).
6610 ChangeStatus updateImpl(Attributor &A) override {
6611 llvm_unreachable(
6612 "AAValueSimplify(Function|CallSite)::updateImpl will not be called");
6613 }
6614 /// See AbstractAttribute::trackStatistics()
6615 void trackStatistics() const override {
6616 STATS_DECLTRACK_FN_ATTR(value_simplify)
6617 }
6618};
6619
6620struct AAValueSimplifyCallSite : AAValueSimplifyFunction {
6621 AAValueSimplifyCallSite(const IRPosition &IRP, Attributor &A)
6622 : AAValueSimplifyFunction(IRP, A) {}
6623 /// See AbstractAttribute::trackStatistics()
6624 void trackStatistics() const override {
6625 STATS_DECLTRACK_CS_ATTR(value_simplify)
6626 }
6627};
6628
6629struct AAValueSimplifyCallSiteReturned : AAValueSimplifyImpl {
6630 AAValueSimplifyCallSiteReturned(const IRPosition &IRP, Attributor &A)
6631 : AAValueSimplifyImpl(IRP, A) {}
6632
6633 void initialize(Attributor &A) override {
6634 AAValueSimplifyImpl::initialize(A);
6635 Function *Fn = getAssociatedFunction();
6636 assert(Fn && "Did expect an associted function");
6637 for (Argument &Arg : Fn->args()) {
6638 if (Arg.hasReturnedAttr()) {
6639 auto IRP = IRPosition::callsite_argument(CB: *cast<CallBase>(Val: getCtxI()),
6640 ArgNo: Arg.getArgNo());
6641 if (IRP.getPositionKind() == IRPosition::IRP_CALL_SITE_ARGUMENT &&
6642 checkAndUpdate(A, QueryingAA: *this, IRP))
6643 indicateOptimisticFixpoint();
6644 else
6645 indicatePessimisticFixpoint();
6646 return;
6647 }
6648 }
6649 }
6650
6651 /// See AbstractAttribute::updateImpl(...).
6652 ChangeStatus updateImpl(Attributor &A) override {
6653 return indicatePessimisticFixpoint();
6654 }
6655
6656 void trackStatistics() const override {
6657 STATS_DECLTRACK_CSRET_ATTR(value_simplify)
6658 }
6659};
6660
6661struct AAValueSimplifyCallSiteArgument : AAValueSimplifyFloating {
6662 AAValueSimplifyCallSiteArgument(const IRPosition &IRP, Attributor &A)
6663 : AAValueSimplifyFloating(IRP, A) {}
6664
6665 /// See AbstractAttribute::manifest(...).
6666 ChangeStatus manifest(Attributor &A) override {
6667 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6668 // TODO: We should avoid simplification duplication to begin with.
6669 auto *FloatAA = A.lookupAAFor<AAValueSimplify>(
6670 IRP: IRPosition::value(V: getAssociatedValue()), QueryingAA: this, DepClass: DepClassTy::NONE);
6671 if (FloatAA && FloatAA->getState().isValidState())
6672 return Changed;
6673
6674 if (auto *NewV = manifestReplacementValue(A, CtxI: getCtxI())) {
6675 Use &U = cast<CallBase>(Val: &getAnchorValue())
6676 ->getArgOperandUse(i: getCallSiteArgNo());
6677 if (A.changeUseAfterManifest(U, NV&: *NewV))
6678 Changed = ChangeStatus::CHANGED;
6679 }
6680
6681 return Changed | AAValueSimplify::manifest(A);
6682 }
6683
6684 void trackStatistics() const override {
6685 STATS_DECLTRACK_CSARG_ATTR(value_simplify)
6686 }
6687};
6688} // namespace
6689
6690/// ----------------------- Heap-To-Stack Conversion ---------------------------
6691namespace {
6692struct AAHeapToStackFunction final : public AAHeapToStack {
6693
6694 struct AllocationInfo {
6695 /// The call that allocates the memory.
6696 CallBase *const CB;
6697
6698 /// The library function id for the allocation.
6699 LibFunc LibraryFunctionId = NotLibFunc;
6700
6701 /// The status wrt. a rewrite.
6702 enum {
6703 STACK_DUE_TO_USE,
6704 STACK_DUE_TO_FREE,
6705 INVALID,
6706 } Status = STACK_DUE_TO_USE;
6707
6708 /// Flag to indicate if we encountered a use that might free this allocation
6709 /// but which is not in the deallocation infos.
6710 bool HasPotentiallyFreeingUnknownUses = false;
6711
6712 /// Flag to indicate that we should place the new alloca in the function
6713 /// entry block rather than where the call site (CB) is.
6714 bool MoveAllocaIntoEntry = true;
6715
6716 /// The set of free calls that use this allocation.
6717 SmallSetVector<CallBase *, 1> PotentialFreeCalls{};
6718 };
6719
6720 struct DeallocationInfo {
6721 /// The call that deallocates the memory.
6722 CallBase *const CB;
6723 /// The value freed by the call.
6724 Value *FreedOp;
6725
6726 /// Flag to indicate if we don't know all objects this deallocation might
6727 /// free.
6728 bool MightFreeUnknownObjects = false;
6729
6730 /// The set of allocation calls that are potentially freed.
6731 SmallSetVector<CallBase *, 1> PotentialAllocationCalls{};
6732 };
6733
6734 AAHeapToStackFunction(const IRPosition &IRP, Attributor &A)
6735 : AAHeapToStack(IRP, A) {}
6736
6737 ~AAHeapToStackFunction() override {
6738 // Ensure we call the destructor so we release any memory allocated in the
6739 // sets.
6740 for (auto &It : AllocationInfos)
6741 It.second->~AllocationInfo();
6742 for (auto &It : DeallocationInfos)
6743 It.second->~DeallocationInfo();
6744 }
6745
6746 void initialize(Attributor &A) override {
6747 AAHeapToStack::initialize(A);
6748
6749 const Function *F = getAnchorScope();
6750 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6751
6752 auto AllocationIdentifierCB = [&](Instruction &I) {
6753 CallBase *CB = dyn_cast<CallBase>(Val: &I);
6754 if (!CB)
6755 return true;
6756 if (Value *FreedOp = getFreedOperand(CB, TLI)) {
6757 DeallocationInfos[CB] = new (A.Allocator) DeallocationInfo{.CB: CB, .FreedOp: FreedOp};
6758 return true;
6759 }
6760 // To do heap to stack, we need to know that the allocation itself is
6761 // removable once uses are rewritten, and that we can initialize the
6762 // alloca to the same pattern as the original allocation result.
6763 if (isRemovableAlloc(V: CB, TLI)) {
6764 auto *I8Ty = Type::getInt8Ty(C&: CB->getParent()->getContext());
6765 if (nullptr != getInitialValueOfAllocation(V: CB, TLI, Ty: I8Ty)) {
6766 AllocationInfo *AI = new (A.Allocator) AllocationInfo{.CB: CB};
6767 AllocationInfos[CB] = AI;
6768 if (TLI)
6769 TLI->getLibFunc(CB: *CB, F&: AI->LibraryFunctionId);
6770 }
6771 }
6772 return true;
6773 };
6774
6775 bool UsedAssumedInformation = false;
6776 bool Success = A.checkForAllCallLikeInstructions(
6777 Pred: AllocationIdentifierCB, QueryingAA: *this, UsedAssumedInformation,
6778 /* CheckBBLivenessOnly */ false,
6779 /* CheckPotentiallyDead */ true);
6780 (void)Success;
6781 assert(Success && "Did not expect the call base visit callback to fail!");
6782
6783 Attributor::SimplifictionCallbackTy SCB =
6784 [](const IRPosition &, const AbstractAttribute *,
6785 bool &) -> std::optional<Value *> { return nullptr; };
6786 for (const auto &It : AllocationInfos)
6787 A.registerSimplificationCallback(IRP: IRPosition::callsite_returned(CB: *It.first),
6788 CB: SCB);
6789 for (const auto &It : DeallocationInfos)
6790 A.registerSimplificationCallback(IRP: IRPosition::callsite_returned(CB: *It.first),
6791 CB: SCB);
6792 }
6793
6794 const std::string getAsStr(Attributor *A) const override {
6795 unsigned NumH2SMallocs = 0, NumInvalidMallocs = 0;
6796 for (const auto &It : AllocationInfos) {
6797 if (It.second->Status == AllocationInfo::INVALID)
6798 ++NumInvalidMallocs;
6799 else
6800 ++NumH2SMallocs;
6801 }
6802 return "[H2S] Mallocs Good/Bad: " + std::to_string(val: NumH2SMallocs) + "/" +
6803 std::to_string(val: NumInvalidMallocs);
6804 }
6805
6806 /// See AbstractAttribute::trackStatistics().
6807 void trackStatistics() const override {
6808 STATS_DECL(
6809 MallocCalls, Function,
6810 "Number of malloc/calloc/aligned_alloc calls converted to allocas");
6811 for (const auto &It : AllocationInfos)
6812 if (It.second->Status != AllocationInfo::INVALID)
6813 ++BUILD_STAT_NAME(MallocCalls, Function);
6814 }
6815
6816 bool isAssumedHeapToStack(const CallBase &CB) const override {
6817 if (isValidState())
6818 if (AllocationInfo *AI =
6819 AllocationInfos.lookup(Key: const_cast<CallBase *>(&CB)))
6820 return AI->Status != AllocationInfo::INVALID;
6821 return false;
6822 }
6823
6824 bool isAssumedHeapToStackRemovedFree(CallBase &CB) const override {
6825 if (!isValidState())
6826 return false;
6827
6828 for (const auto &It : AllocationInfos) {
6829 AllocationInfo &AI = *It.second;
6830 if (AI.Status == AllocationInfo::INVALID)
6831 continue;
6832
6833 if (AI.PotentialFreeCalls.count(key: &CB))
6834 return true;
6835 }
6836
6837 return false;
6838 }
6839
6840 ChangeStatus manifest(Attributor &A) override {
6841 assert(getState().isValidState() &&
6842 "Attempted to manifest an invalid state!");
6843
6844 ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
6845 Function *F = getAnchorScope();
6846 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6847
6848 for (auto &It : AllocationInfos) {
6849 AllocationInfo &AI = *It.second;
6850 if (AI.Status == AllocationInfo::INVALID)
6851 continue;
6852
6853 for (CallBase *FreeCall : AI.PotentialFreeCalls) {
6854 LLVM_DEBUG(dbgs() << "H2S: Removing free call: " << *FreeCall << "\n");
6855 A.deleteAfterManifest(I&: *FreeCall);
6856 HasChanged = ChangeStatus::CHANGED;
6857 }
6858
6859 LLVM_DEBUG(dbgs() << "H2S: Removing malloc-like call: " << *AI.CB
6860 << "\n");
6861
6862 auto Remark = [&](OptimizationRemark OR) {
6863 LibFunc IsAllocShared;
6864 if (TLI->getLibFunc(CB: *AI.CB, F&: IsAllocShared))
6865 if (IsAllocShared == LibFunc___kmpc_alloc_shared)
6866 return OR << "Moving globalized variable to the stack.";
6867 return OR << "Moving memory allocation from the heap to the stack.";
6868 };
6869 if (AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared)
6870 A.emitRemark<OptimizationRemark>(I: AI.CB, RemarkName: "OMP110", RemarkCB&: Remark);
6871 else
6872 A.emitRemark<OptimizationRemark>(I: AI.CB, RemarkName: "HeapToStack", RemarkCB&: Remark);
6873
6874 const DataLayout &DL = A.getInfoCache().getDL();
6875 Value *Size;
6876 std::optional<APInt> SizeAPI = getSize(A, AA: *this, AI);
6877 if (SizeAPI) {
6878 Size = ConstantInt::get(Context&: AI.CB->getContext(), V: *SizeAPI);
6879 } else {
6880 LLVMContext &Ctx = AI.CB->getContext();
6881 ObjectSizeOpts Opts;
6882 ObjectSizeOffsetEvaluator Eval(DL, TLI, Ctx, Opts);
6883 SizeOffsetValue SizeOffsetPair = Eval.compute(V: AI.CB);
6884 assert(SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown() &&
6885 cast<ConstantInt>(SizeOffsetPair.Offset)->isZero());
6886 Size = SizeOffsetPair.Size;
6887 }
6888
6889 BasicBlock::iterator IP = AI.MoveAllocaIntoEntry
6890 ? F->getEntryBlock().begin()
6891 : AI.CB->getIterator();
6892
6893 Align Alignment(1);
6894 if (MaybeAlign RetAlign = AI.CB->getRetAlign())
6895 Alignment = std::max(a: Alignment, b: *RetAlign);
6896 if (Value *Align = getAllocAlignment(V: AI.CB, TLI)) {
6897 std::optional<APInt> AlignmentAPI = getAPInt(A, AA: *this, V&: *Align);
6898 assert(AlignmentAPI && AlignmentAPI->getZExtValue() > 0 &&
6899 "Expected an alignment during manifest!");
6900 Alignment =
6901 std::max(a: Alignment, b: assumeAligned(Value: AlignmentAPI->getZExtValue()));
6902 }
6903
6904 // TODO: Hoist the alloca towards the function entry.
6905 unsigned AS = DL.getAllocaAddrSpace();
6906 Instruction *Alloca =
6907 new AllocaInst(Type::getInt8Ty(C&: F->getContext()), AS, Size, Alignment,
6908 AI.CB->getName() + ".h2s", IP);
6909
6910 if (Alloca->getType() != AI.CB->getType())
6911 Alloca = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6912 S: Alloca, Ty: AI.CB->getType(), Name: "malloc_cast", InsertBefore: AI.CB->getIterator());
6913
6914 auto *I8Ty = Type::getInt8Ty(C&: F->getContext());
6915 auto *InitVal = getInitialValueOfAllocation(V: AI.CB, TLI, Ty: I8Ty);
6916 assert(InitVal &&
6917 "Must be able to materialize initial memory state of allocation");
6918
6919 A.changeAfterManifest(IRP: IRPosition::inst(I: *AI.CB), NV&: *Alloca);
6920
6921 if (auto *II = dyn_cast<InvokeInst>(Val: AI.CB)) {
6922 auto *NBB = II->getNormalDest();
6923 BranchInst::Create(IfTrue: NBB, InsertBefore: AI.CB->getParent());
6924 A.deleteAfterManifest(I&: *AI.CB);
6925 } else {
6926 A.deleteAfterManifest(I&: *AI.CB);
6927 }
6928
6929 // Initialize the alloca with the same value as used by the allocation
6930 // function. We can skip undef as the initial value of an alloc is
6931 // undef, and the memset would simply end up being DSEd.
6932 if (!isa<UndefValue>(Val: InitVal)) {
6933 IRBuilder<> Builder(Alloca->getNextNode());
6934 // TODO: Use alignment above if align!=1
6935 Builder.CreateMemSet(Ptr: Alloca, Val: InitVal, Size, Align: std::nullopt);
6936 }
6937 HasChanged = ChangeStatus::CHANGED;
6938 }
6939
6940 return HasChanged;
6941 }
6942
6943 std::optional<APInt> getAPInt(Attributor &A, const AbstractAttribute &AA,
6944 Value &V) {
6945 bool UsedAssumedInformation = false;
6946 std::optional<Constant *> SimpleV =
6947 A.getAssumedConstant(V, AA, UsedAssumedInformation);
6948 if (!SimpleV)
6949 return APInt(64, 0);
6950 if (auto *CI = dyn_cast_or_null<ConstantInt>(Val: *SimpleV))
6951 return CI->getValue();
6952 return std::nullopt;
6953 }
6954
6955 std::optional<APInt> getSize(Attributor &A, const AbstractAttribute &AA,
6956 AllocationInfo &AI) {
6957 auto Mapper = [&](const Value *V) -> const Value * {
6958 bool UsedAssumedInformation = false;
6959 if (std::optional<Constant *> SimpleV =
6960 A.getAssumedConstant(V: *V, AA, UsedAssumedInformation))
6961 if (*SimpleV)
6962 return *SimpleV;
6963 return V;
6964 };
6965
6966 const Function *F = getAnchorScope();
6967 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6968 return getAllocSize(CB: AI.CB, TLI, Mapper);
6969 }
6970
6971 /// Collection of all malloc-like calls in a function with associated
6972 /// information.
6973 MapVector<CallBase *, AllocationInfo *> AllocationInfos;
6974
6975 /// Collection of all free-like calls in a function with associated
6976 /// information.
6977 MapVector<CallBase *, DeallocationInfo *> DeallocationInfos;
6978
6979 ChangeStatus updateImpl(Attributor &A) override;
6980};
6981
6982ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) {
6983 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6984 const Function *F = getAnchorScope();
6985 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6986
6987 const auto *LivenessAA =
6988 A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: IRPosition::function(F: *F), DepClass: DepClassTy::NONE);
6989
6990 MustBeExecutedContextExplorer *Explorer =
6991 A.getInfoCache().getMustBeExecutedContextExplorer();
6992
6993 bool StackIsAccessibleByOtherThreads =
6994 A.getInfoCache().stackIsAccessibleByOtherThreads();
6995
6996 LoopInfo *LI =
6997 A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F: *F);
6998 std::optional<bool> MayContainIrreducibleControl;
6999 auto IsInLoop = [&](BasicBlock &BB) {
7000 if (&F->getEntryBlock() == &BB)
7001 return false;
7002 if (!MayContainIrreducibleControl.has_value())
7003 MayContainIrreducibleControl = mayContainIrreducibleControl(F: *F, LI);
7004 if (*MayContainIrreducibleControl)
7005 return true;
7006 if (!LI)
7007 return true;
7008 return LI->getLoopFor(BB: &BB) != nullptr;
7009 };
7010
7011 // Flag to ensure we update our deallocation information at most once per
7012 // updateImpl call and only if we use the free check reasoning.
7013 bool HasUpdatedFrees = false;
7014
7015 auto UpdateFrees = [&]() {
7016 HasUpdatedFrees = true;
7017
7018 for (auto &It : DeallocationInfos) {
7019 DeallocationInfo &DI = *It.second;
7020 // For now we cannot use deallocations that have unknown inputs, skip
7021 // them.
7022 if (DI.MightFreeUnknownObjects)
7023 continue;
7024
7025 // No need to analyze dead calls, ignore them instead.
7026 bool UsedAssumedInformation = false;
7027 if (A.isAssumedDead(I: *DI.CB, QueryingAA: this, LivenessAA, UsedAssumedInformation,
7028 /* CheckBBLivenessOnly */ true))
7029 continue;
7030
7031 // Use the non-optimistic version to get the freed object.
7032 Value *Obj = getUnderlyingObject(V: DI.FreedOp);
7033 if (!Obj) {
7034 LLVM_DEBUG(dbgs() << "[H2S] Unknown underlying object for free!\n");
7035 DI.MightFreeUnknownObjects = true;
7036 continue;
7037 }
7038
7039 // Free of null and undef can be ignored as no-ops (or UB in the latter
7040 // case).
7041 if (isa<ConstantPointerNull>(Val: Obj) || isa<UndefValue>(Val: Obj))
7042 continue;
7043
7044 CallBase *ObjCB = dyn_cast<CallBase>(Val: Obj);
7045 if (!ObjCB) {
7046 LLVM_DEBUG(dbgs() << "[H2S] Free of a non-call object: " << *Obj
7047 << "\n");
7048 DI.MightFreeUnknownObjects = true;
7049 continue;
7050 }
7051
7052 AllocationInfo *AI = AllocationInfos.lookup(Key: ObjCB);
7053 if (!AI) {
7054 LLVM_DEBUG(dbgs() << "[H2S] Free of a non-allocation object: " << *Obj
7055 << "\n");
7056 DI.MightFreeUnknownObjects = true;
7057 continue;
7058 }
7059
7060 DI.PotentialAllocationCalls.insert(X: ObjCB);
7061 }
7062 };
7063
7064 auto FreeCheck = [&](AllocationInfo &AI) {
7065 // If the stack is not accessible by other threads, the "must-free" logic
7066 // doesn't apply as the pointer could be shared and needs to be places in
7067 // "shareable" memory.
7068 if (!StackIsAccessibleByOtherThreads) {
7069 bool IsKnownNoSycn;
7070 if (!AA::hasAssumedIRAttr<Attribute::NoSync>(
7071 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoSycn)) {
7072 LLVM_DEBUG(
7073 dbgs() << "[H2S] found an escaping use, stack is not accessible by "
7074 "other threads and function is not nosync:\n");
7075 return false;
7076 }
7077 }
7078 if (!HasUpdatedFrees)
7079 UpdateFrees();
7080
7081 // TODO: Allow multi exit functions that have different free calls.
7082 if (AI.PotentialFreeCalls.size() != 1) {
7083 LLVM_DEBUG(dbgs() << "[H2S] did not find one free call but "
7084 << AI.PotentialFreeCalls.size() << "\n");
7085 return false;
7086 }
7087 CallBase *UniqueFree = *AI.PotentialFreeCalls.begin();
7088 DeallocationInfo *DI = DeallocationInfos.lookup(Key: UniqueFree);
7089 if (!DI) {
7090 LLVM_DEBUG(
7091 dbgs() << "[H2S] unique free call was not known as deallocation call "
7092 << *UniqueFree << "\n");
7093 return false;
7094 }
7095 if (DI->MightFreeUnknownObjects) {
7096 LLVM_DEBUG(
7097 dbgs() << "[H2S] unique free call might free unknown allocations\n");
7098 return false;
7099 }
7100 if (DI->PotentialAllocationCalls.empty())
7101 return true;
7102 if (DI->PotentialAllocationCalls.size() > 1) {
7103 LLVM_DEBUG(dbgs() << "[H2S] unique free call might free "
7104 << DI->PotentialAllocationCalls.size()
7105 << " different allocations\n");
7106 return false;
7107 }
7108 if (*DI->PotentialAllocationCalls.begin() != AI.CB) {
7109 LLVM_DEBUG(
7110 dbgs()
7111 << "[H2S] unique free call not known to free this allocation but "
7112 << **DI->PotentialAllocationCalls.begin() << "\n");
7113 return false;
7114 }
7115
7116 // __kmpc_alloc_shared and __kmpc_alloc_free are by construction matched.
7117 if (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared) {
7118 Instruction *CtxI = isa<InvokeInst>(Val: AI.CB) ? AI.CB : AI.CB->getNextNode();
7119 if (!Explorer || !Explorer->findInContextOf(I: UniqueFree, PP: CtxI)) {
7120 LLVM_DEBUG(dbgs() << "[H2S] unique free call might not be executed "
7121 "with the allocation "
7122 << *UniqueFree << "\n");
7123 return false;
7124 }
7125 }
7126 return true;
7127 };
7128
7129 auto UsesCheck = [&](AllocationInfo &AI) {
7130 bool ValidUsesOnly = true;
7131
7132 auto Pred = [&](const Use &U, bool &Follow) -> bool {
7133 Instruction *UserI = cast<Instruction>(Val: U.getUser());
7134 if (isa<LoadInst>(Val: UserI))
7135 return true;
7136 if (auto *SI = dyn_cast<StoreInst>(Val: UserI)) {
7137 if (SI->getValueOperand() == U.get()) {
7138 LLVM_DEBUG(dbgs()
7139 << "[H2S] escaping store to memory: " << *UserI << "\n");
7140 ValidUsesOnly = false;
7141 } else {
7142 // A store into the malloc'ed memory is fine.
7143 }
7144 return true;
7145 }
7146 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
7147 if (!CB->isArgOperand(U: &U) || CB->isLifetimeStartOrEnd())
7148 return true;
7149 if (DeallocationInfos.count(Key: CB)) {
7150 AI.PotentialFreeCalls.insert(X: CB);
7151 return true;
7152 }
7153
7154 unsigned ArgNo = CB->getArgOperandNo(U: &U);
7155 auto CBIRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
7156
7157 bool IsKnownNoCapture;
7158 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
7159 A, QueryingAA: this, IRP: CBIRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture);
7160
7161 // If a call site argument use is nofree, we are fine.
7162 bool IsKnownNoFree;
7163 bool IsAssumedNoFree = AA::hasAssumedIRAttr<Attribute::NoFree>(
7164 A, QueryingAA: this, IRP: CBIRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoFree);
7165
7166 if (!IsAssumedNoCapture ||
7167 (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared &&
7168 !IsAssumedNoFree)) {
7169 AI.HasPotentiallyFreeingUnknownUses |= !IsAssumedNoFree;
7170
7171 // Emit a missed remark if this is missed OpenMP globalization.
7172 auto Remark = [&](OptimizationRemarkMissed ORM) {
7173 return ORM
7174 << "Could not move globalized variable to the stack. "
7175 "Variable is potentially captured in call. Mark "
7176 "parameter as `__attribute__((noescape))` to override.";
7177 };
7178
7179 if (ValidUsesOnly &&
7180 AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared)
7181 A.emitRemark<OptimizationRemarkMissed>(I: CB, RemarkName: "OMP113", RemarkCB&: Remark);
7182
7183 LLVM_DEBUG(dbgs() << "[H2S] Bad user: " << *UserI << "\n");
7184 ValidUsesOnly = false;
7185 }
7186 return true;
7187 }
7188
7189 if (isa<GetElementPtrInst>(Val: UserI) || isa<BitCastInst>(Val: UserI) ||
7190 isa<PHINode>(Val: UserI) || isa<SelectInst>(Val: UserI)) {
7191 Follow = true;
7192 return true;
7193 }
7194 // Unknown user for which we can not track uses further (in a way that
7195 // makes sense).
7196 LLVM_DEBUG(dbgs() << "[H2S] Unknown user: " << *UserI << "\n");
7197 ValidUsesOnly = false;
7198 return true;
7199 };
7200 if (!A.checkForAllUses(Pred, QueryingAA: *this, V: *AI.CB, /* CheckBBLivenessOnly */ false,
7201 LivenessDepClass: DepClassTy::OPTIONAL, /* IgnoreDroppableUses */ true,
7202 EquivalentUseCB: [&](const Use &OldU, const Use &NewU) {
7203 auto *SI = dyn_cast<StoreInst>(Val: OldU.getUser());
7204 return !SI || StackIsAccessibleByOtherThreads ||
7205 AA::isAssumedThreadLocalObject(
7206 A, Obj&: *SI->getPointerOperand(), QueryingAA: *this);
7207 }))
7208 return false;
7209 return ValidUsesOnly;
7210 };
7211
7212 // The actual update starts here. We look at all allocations and depending on
7213 // their status perform the appropriate check(s).
7214 for (auto &It : AllocationInfos) {
7215 AllocationInfo &AI = *It.second;
7216 if (AI.Status == AllocationInfo::INVALID)
7217 continue;
7218
7219 if (Value *Align = getAllocAlignment(V: AI.CB, TLI)) {
7220 std::optional<APInt> APAlign = getAPInt(A, AA: *this, V&: *Align);
7221 if (!APAlign) {
7222 // Can't generate an alloca which respects the required alignment
7223 // on the allocation.
7224 LLVM_DEBUG(dbgs() << "[H2S] Unknown allocation alignment: " << *AI.CB
7225 << "\n");
7226 AI.Status = AllocationInfo::INVALID;
7227 Changed = ChangeStatus::CHANGED;
7228 continue;
7229 }
7230 if (APAlign->ugt(RHS: llvm::Value::MaximumAlignment) ||
7231 !APAlign->isPowerOf2()) {
7232 LLVM_DEBUG(dbgs() << "[H2S] Invalid allocation alignment: " << APAlign
7233 << "\n");
7234 AI.Status = AllocationInfo::INVALID;
7235 Changed = ChangeStatus::CHANGED;
7236 continue;
7237 }
7238 }
7239
7240 std::optional<APInt> Size = getSize(A, AA: *this, AI);
7241 if (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared &&
7242 MaxHeapToStackSize != -1) {
7243 if (!Size || Size->ugt(RHS: MaxHeapToStackSize)) {
7244 LLVM_DEBUG({
7245 if (!Size)
7246 dbgs() << "[H2S] Unknown allocation size: " << *AI.CB << "\n";
7247 else
7248 dbgs() << "[H2S] Allocation size too large: " << *AI.CB << " vs. "
7249 << MaxHeapToStackSize << "\n";
7250 });
7251
7252 AI.Status = AllocationInfo::INVALID;
7253 Changed = ChangeStatus::CHANGED;
7254 continue;
7255 }
7256 }
7257
7258 switch (AI.Status) {
7259 case AllocationInfo::STACK_DUE_TO_USE:
7260 if (UsesCheck(AI))
7261 break;
7262 AI.Status = AllocationInfo::STACK_DUE_TO_FREE;
7263 [[fallthrough]];
7264 case AllocationInfo::STACK_DUE_TO_FREE:
7265 if (FreeCheck(AI))
7266 break;
7267 AI.Status = AllocationInfo::INVALID;
7268 Changed = ChangeStatus::CHANGED;
7269 break;
7270 case AllocationInfo::INVALID:
7271 llvm_unreachable("Invalid allocations should never reach this point!");
7272 };
7273
7274 // Check if we still think we can move it into the entry block. If the
7275 // alloca comes from a converted __kmpc_alloc_shared then we can usually
7276 // ignore the potential compilations associated with loops.
7277 bool IsGlobalizedLocal =
7278 AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared;
7279 if (AI.MoveAllocaIntoEntry &&
7280 (!Size.has_value() ||
7281 (!IsGlobalizedLocal && IsInLoop(*AI.CB->getParent()))))
7282 AI.MoveAllocaIntoEntry = false;
7283 }
7284
7285 return Changed;
7286}
7287} // namespace
7288
7289/// ----------------------- Privatizable Pointers ------------------------------
7290namespace {
7291struct AAPrivatizablePtrImpl : public AAPrivatizablePtr {
7292 AAPrivatizablePtrImpl(const IRPosition &IRP, Attributor &A)
7293 : AAPrivatizablePtr(IRP, A), PrivatizableType(std::nullopt) {}
7294
7295 ChangeStatus indicatePessimisticFixpoint() override {
7296 AAPrivatizablePtr::indicatePessimisticFixpoint();
7297 PrivatizableType = nullptr;
7298 return ChangeStatus::CHANGED;
7299 }
7300
7301 /// Identify the type we can chose for a private copy of the underlying
7302 /// argument. std::nullopt means it is not clear yet, nullptr means there is
7303 /// none.
7304 virtual std::optional<Type *> identifyPrivatizableType(Attributor &A) = 0;
7305
7306 /// Return a privatizable type that encloses both T0 and T1.
7307 /// TODO: This is merely a stub for now as we should manage a mapping as well.
7308 std::optional<Type *> combineTypes(std::optional<Type *> T0,
7309 std::optional<Type *> T1) {
7310 if (!T0)
7311 return T1;
7312 if (!T1)
7313 return T0;
7314 if (T0 == T1)
7315 return T0;
7316 return nullptr;
7317 }
7318
7319 std::optional<Type *> getPrivatizableType() const override {
7320 return PrivatizableType;
7321 }
7322
7323 const std::string getAsStr(Attributor *A) const override {
7324 return isAssumedPrivatizablePtr() ? "[priv]" : "[no-priv]";
7325 }
7326
7327protected:
7328 std::optional<Type *> PrivatizableType;
7329};
7330
7331// TODO: Do this for call site arguments (probably also other values) as well.
7332
7333struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl {
7334 AAPrivatizablePtrArgument(const IRPosition &IRP, Attributor &A)
7335 : AAPrivatizablePtrImpl(IRP, A) {}
7336
7337 /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
7338 std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7339 // If this is a byval argument and we know all the call sites (so we can
7340 // rewrite them), there is no need to check them explicitly.
7341 bool UsedAssumedInformation = false;
7342 SmallVector<Attribute, 1> Attrs;
7343 A.getAttrs(IRP: getIRPosition(), AKs: {Attribute::ByVal}, Attrs,
7344 /* IgnoreSubsumingPositions */ true);
7345 if (!Attrs.empty() &&
7346 A.checkForAllCallSites(Pred: [](AbstractCallSite ACS) { return true; }, QueryingAA: *this,
7347 RequireAllCallSites: true, UsedAssumedInformation))
7348 return Attrs[0].getValueAsType();
7349
7350 std::optional<Type *> Ty;
7351 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
7352
7353 // Make sure the associated call site argument has the same type at all call
7354 // sites and it is an allocation we know is safe to privatize, for now that
7355 // means we only allow alloca instructions.
7356 // TODO: We can additionally analyze the accesses in the callee to create
7357 // the type from that information instead. That is a little more
7358 // involved and will be done in a follow up patch.
7359 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7360 IRPosition ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
7361 // Check if a coresponding argument was found or if it is one not
7362 // associated (which can happen for callback calls).
7363 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
7364 return false;
7365
7366 // Check that all call sites agree on a type.
7367 auto *PrivCSArgAA =
7368 A.getAAFor<AAPrivatizablePtr>(QueryingAA: *this, IRP: ACSArgPos, DepClass: DepClassTy::REQUIRED);
7369 if (!PrivCSArgAA)
7370 return false;
7371 std::optional<Type *> CSTy = PrivCSArgAA->getPrivatizableType();
7372
7373 LLVM_DEBUG({
7374 dbgs() << "[AAPrivatizablePtr] ACSPos: " << ACSArgPos << ", CSTy: ";
7375 if (CSTy && *CSTy)
7376 (*CSTy)->print(dbgs());
7377 else if (CSTy)
7378 dbgs() << "<nullptr>";
7379 else
7380 dbgs() << "<none>";
7381 });
7382
7383 Ty = combineTypes(T0: Ty, T1: CSTy);
7384
7385 LLVM_DEBUG({
7386 dbgs() << " : New Type: ";
7387 if (Ty && *Ty)
7388 (*Ty)->print(dbgs());
7389 else if (Ty)
7390 dbgs() << "<nullptr>";
7391 else
7392 dbgs() << "<none>";
7393 dbgs() << "\n";
7394 });
7395
7396 return !Ty || *Ty;
7397 };
7398
7399 if (!A.checkForAllCallSites(Pred: CallSiteCheck, QueryingAA: *this, RequireAllCallSites: true,
7400 UsedAssumedInformation))
7401 return nullptr;
7402 return Ty;
7403 }
7404
7405 /// See AbstractAttribute::updateImpl(...).
7406 ChangeStatus updateImpl(Attributor &A) override {
7407 PrivatizableType = identifyPrivatizableType(A);
7408 if (!PrivatizableType)
7409 return ChangeStatus::UNCHANGED;
7410 if (!*PrivatizableType)
7411 return indicatePessimisticFixpoint();
7412
7413 // The dependence is optional so we don't give up once we give up on the
7414 // alignment.
7415 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: getAssociatedValue()),
7416 DepClass: DepClassTy::OPTIONAL);
7417
7418 // Avoid arguments with padding for now.
7419 if (!A.hasAttr(IRP: getIRPosition(), AKs: Attribute::ByVal) &&
7420 !isDenselyPacked(Ty: *PrivatizableType, DL: A.getInfoCache().getDL())) {
7421 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Padding detected\n");
7422 return indicatePessimisticFixpoint();
7423 }
7424
7425 // Collect the types that will replace the privatizable type in the function
7426 // signature.
7427 SmallVector<Type *, 16> ReplacementTypes;
7428 identifyReplacementTypes(PrivType: *PrivatizableType, ReplacementTypes);
7429
7430 // Verify callee and caller agree on how the promoted argument would be
7431 // passed.
7432 Function &Fn = *getIRPosition().getAnchorScope();
7433 const auto *TTI =
7434 A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(F: Fn);
7435 if (!TTI) {
7436 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Missing TTI for function "
7437 << Fn.getName() << "\n");
7438 return indicatePessimisticFixpoint();
7439 }
7440
7441 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7442 CallBase *CB = ACS.getInstruction();
7443 return TTI->areTypesABICompatible(
7444 Caller: CB->getCaller(),
7445 Callee: dyn_cast_if_present<Function>(Val: CB->getCalledOperand()),
7446 Types: ReplacementTypes);
7447 };
7448 bool UsedAssumedInformation = false;
7449 if (!A.checkForAllCallSites(Pred: CallSiteCheck, QueryingAA: *this, RequireAllCallSites: true,
7450 UsedAssumedInformation)) {
7451 LLVM_DEBUG(
7452 dbgs() << "[AAPrivatizablePtr] ABI incompatibility detected for "
7453 << Fn.getName() << "\n");
7454 return indicatePessimisticFixpoint();
7455 }
7456
7457 // Register a rewrite of the argument.
7458 Argument *Arg = getAssociatedArgument();
7459 if (!A.isValidFunctionSignatureRewrite(Arg&: *Arg, ReplacementTypes)) {
7460 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Rewrite not valid\n");
7461 return indicatePessimisticFixpoint();
7462 }
7463
7464 unsigned ArgNo = Arg->getArgNo();
7465
7466 // Helper to check if for the given call site the associated argument is
7467 // passed to a callback where the privatization would be different.
7468 auto IsCompatiblePrivArgOfCallback = [&](CallBase &CB) {
7469 SmallVector<const Use *, 4> CallbackUses;
7470 AbstractCallSite::getCallbackUses(CB, CallbackUses);
7471 for (const Use *U : CallbackUses) {
7472 AbstractCallSite CBACS(U);
7473 assert(CBACS && CBACS.isCallbackCall());
7474 for (Argument &CBArg : CBACS.getCalledFunction()->args()) {
7475 int CBArgNo = CBACS.getCallArgOperandNo(Arg&: CBArg);
7476
7477 LLVM_DEBUG({
7478 dbgs()
7479 << "[AAPrivatizablePtr] Argument " << *Arg
7480 << "check if can be privatized in the context of its parent ("
7481 << Arg->getParent()->getName()
7482 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7483 "callback ("
7484 << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7485 << ")\n[AAPrivatizablePtr] " << CBArg << " : "
7486 << CBACS.getCallArgOperand(CBArg) << " vs "
7487 << CB.getArgOperand(ArgNo) << "\n"
7488 << "[AAPrivatizablePtr] " << CBArg << " : "
7489 << CBACS.getCallArgOperandNo(CBArg) << " vs " << ArgNo << "\n";
7490 });
7491
7492 if (CBArgNo != int(ArgNo))
7493 continue;
7494 const auto *CBArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7495 QueryingAA: *this, IRP: IRPosition::argument(Arg: CBArg), DepClass: DepClassTy::REQUIRED);
7496 if (CBArgPrivAA && CBArgPrivAA->isValidState()) {
7497 auto CBArgPrivTy = CBArgPrivAA->getPrivatizableType();
7498 if (!CBArgPrivTy)
7499 continue;
7500 if (*CBArgPrivTy == PrivatizableType)
7501 continue;
7502 }
7503
7504 LLVM_DEBUG({
7505 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7506 << " cannot be privatized in the context of its parent ("
7507 << Arg->getParent()->getName()
7508 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7509 "callback ("
7510 << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7511 << ").\n[AAPrivatizablePtr] for which the argument "
7512 "privatization is not compatible.\n";
7513 });
7514 return false;
7515 }
7516 }
7517 return true;
7518 };
7519
7520 // Helper to check if for the given call site the associated argument is
7521 // passed to a direct call where the privatization would be different.
7522 auto IsCompatiblePrivArgOfDirectCS = [&](AbstractCallSite ACS) {
7523 CallBase *DC = cast<CallBase>(Val: ACS.getInstruction());
7524 int DCArgNo = ACS.getCallArgOperandNo(ArgNo);
7525 assert(DCArgNo >= 0 && unsigned(DCArgNo) < DC->arg_size() &&
7526 "Expected a direct call operand for callback call operand");
7527
7528 Function *DCCallee =
7529 dyn_cast_if_present<Function>(Val: DC->getCalledOperand());
7530 LLVM_DEBUG({
7531 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7532 << " check if be privatized in the context of its parent ("
7533 << Arg->getParent()->getName()
7534 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7535 "direct call of ("
7536 << DCArgNo << "@" << DCCallee->getName() << ").\n";
7537 });
7538
7539 if (unsigned(DCArgNo) < DCCallee->arg_size()) {
7540 const auto *DCArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7541 QueryingAA: *this, IRP: IRPosition::argument(Arg: *DCCallee->getArg(i: DCArgNo)),
7542 DepClass: DepClassTy::REQUIRED);
7543 if (DCArgPrivAA && DCArgPrivAA->isValidState()) {
7544 auto DCArgPrivTy = DCArgPrivAA->getPrivatizableType();
7545 if (!DCArgPrivTy)
7546 return true;
7547 if (*DCArgPrivTy == PrivatizableType)
7548 return true;
7549 }
7550 }
7551
7552 LLVM_DEBUG({
7553 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7554 << " cannot be privatized in the context of its parent ("
7555 << Arg->getParent()->getName()
7556 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7557 "direct call of ("
7558 << ACS.getInstruction()->getCalledOperand()->getName()
7559 << ").\n[AAPrivatizablePtr] for which the argument "
7560 "privatization is not compatible.\n";
7561 });
7562 return false;
7563 };
7564
7565 // Helper to check if the associated argument is used at the given abstract
7566 // call site in a way that is incompatible with the privatization assumed
7567 // here.
7568 auto IsCompatiblePrivArgOfOtherCallSite = [&](AbstractCallSite ACS) {
7569 if (ACS.isDirectCall())
7570 return IsCompatiblePrivArgOfCallback(*ACS.getInstruction());
7571 if (ACS.isCallbackCall())
7572 return IsCompatiblePrivArgOfDirectCS(ACS);
7573 return false;
7574 };
7575
7576 if (!A.checkForAllCallSites(Pred: IsCompatiblePrivArgOfOtherCallSite, QueryingAA: *this, RequireAllCallSites: true,
7577 UsedAssumedInformation))
7578 return indicatePessimisticFixpoint();
7579
7580 return ChangeStatus::UNCHANGED;
7581 }
7582
7583 /// Given a type to private \p PrivType, collect the constituates (which are
7584 /// used) in \p ReplacementTypes.
7585 static void
7586 identifyReplacementTypes(Type *PrivType,
7587 SmallVectorImpl<Type *> &ReplacementTypes) {
7588 // TODO: For now we expand the privatization type to the fullest which can
7589 // lead to dead arguments that need to be removed later.
7590 assert(PrivType && "Expected privatizable type!");
7591
7592 // Traverse the type, extract constituate types on the outermost level.
7593 if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7594 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++)
7595 ReplacementTypes.push_back(Elt: PrivStructType->getElementType(N: u));
7596 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7597 ReplacementTypes.append(NumInputs: PrivArrayType->getNumElements(),
7598 Elt: PrivArrayType->getElementType());
7599 } else {
7600 ReplacementTypes.push_back(Elt: PrivType);
7601 }
7602 }
7603
7604 /// Initialize \p Base according to the type \p PrivType at position \p IP.
7605 /// The values needed are taken from the arguments of \p F starting at
7606 /// position \p ArgNo.
7607 static void createInitialization(Type *PrivType, Value &Base, Function &F,
7608 unsigned ArgNo, BasicBlock::iterator IP) {
7609 assert(PrivType && "Expected privatizable type!");
7610
7611 IRBuilder<NoFolder> IRB(IP->getParent(), IP);
7612 const DataLayout &DL = F.getDataLayout();
7613
7614 // Traverse the type, build GEPs and stores.
7615 if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7616 const StructLayout *PrivStructLayout = DL.getStructLayout(Ty: PrivStructType);
7617 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7618 Value *Ptr =
7619 constructPointer(Ptr: &Base, Offset: PrivStructLayout->getElementOffset(Idx: u), IRB);
7620 new StoreInst(F.getArg(i: ArgNo + u), Ptr, IP);
7621 }
7622 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7623 Type *PointeeTy = PrivArrayType->getElementType();
7624 uint64_t PointeeTySize = DL.getTypeStoreSize(Ty: PointeeTy);
7625 for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7626 Value *Ptr = constructPointer(Ptr: &Base, Offset: u * PointeeTySize, IRB);
7627 new StoreInst(F.getArg(i: ArgNo + u), Ptr, IP);
7628 }
7629 } else {
7630 new StoreInst(F.getArg(i: ArgNo), &Base, IP);
7631 }
7632 }
7633
7634 /// Extract values from \p Base according to the type \p PrivType at the
7635 /// call position \p ACS. The values are appended to \p ReplacementValues.
7636 void createReplacementValues(Align Alignment, Type *PrivType,
7637 AbstractCallSite ACS, Value *Base,
7638 SmallVectorImpl<Value *> &ReplacementValues) {
7639 assert(Base && "Expected base value!");
7640 assert(PrivType && "Expected privatizable type!");
7641 Instruction *IP = ACS.getInstruction();
7642
7643 IRBuilder<NoFolder> IRB(IP);
7644 const DataLayout &DL = IP->getDataLayout();
7645
7646 // Traverse the type, build GEPs and loads.
7647 if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7648 const StructLayout *PrivStructLayout = DL.getStructLayout(Ty: PrivStructType);
7649 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7650 Type *PointeeTy = PrivStructType->getElementType(N: u);
7651 Value *Ptr =
7652 constructPointer(Ptr: Base, Offset: PrivStructLayout->getElementOffset(Idx: u), IRB);
7653 LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7654 L->setAlignment(Alignment);
7655 ReplacementValues.push_back(Elt: L);
7656 }
7657 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7658 Type *PointeeTy = PrivArrayType->getElementType();
7659 uint64_t PointeeTySize = DL.getTypeStoreSize(Ty: PointeeTy);
7660 for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7661 Value *Ptr = constructPointer(Ptr: Base, Offset: u * PointeeTySize, IRB);
7662 LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7663 L->setAlignment(Alignment);
7664 ReplacementValues.push_back(Elt: L);
7665 }
7666 } else {
7667 LoadInst *L = new LoadInst(PrivType, Base, "", IP->getIterator());
7668 L->setAlignment(Alignment);
7669 ReplacementValues.push_back(Elt: L);
7670 }
7671 }
7672
7673 /// See AbstractAttribute::manifest(...)
7674 ChangeStatus manifest(Attributor &A) override {
7675 if (!PrivatizableType)
7676 return ChangeStatus::UNCHANGED;
7677 assert(*PrivatizableType && "Expected privatizable type!");
7678
7679 // Collect all tail calls in the function as we cannot allow new allocas to
7680 // escape into tail recursion.
7681 // TODO: Be smarter about new allocas escaping into tail calls.
7682 SmallVector<CallInst *, 16> TailCalls;
7683 bool UsedAssumedInformation = false;
7684 if (!A.checkForAllInstructions(
7685 Pred: [&](Instruction &I) {
7686 CallInst &CI = cast<CallInst>(Val&: I);
7687 if (CI.isTailCall())
7688 TailCalls.push_back(Elt: &CI);
7689 return true;
7690 },
7691 QueryingAA: *this, Opcodes: {Instruction::Call}, UsedAssumedInformation))
7692 return ChangeStatus::UNCHANGED;
7693
7694 Argument *Arg = getAssociatedArgument();
7695 // Query AAAlign attribute for alignment of associated argument to
7696 // determine the best alignment of loads.
7697 const auto *AlignAA =
7698 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *Arg), DepClass: DepClassTy::NONE);
7699
7700 // Callback to repair the associated function. A new alloca is placed at the
7701 // beginning and initialized with the values passed through arguments. The
7702 // new alloca replaces the use of the old pointer argument.
7703 Attributor::ArgumentReplacementInfo::CalleeRepairCBTy FnRepairCB =
7704 [=](const Attributor::ArgumentReplacementInfo &ARI,
7705 Function &ReplacementFn, Function::arg_iterator ArgIt) {
7706 BasicBlock &EntryBB = ReplacementFn.getEntryBlock();
7707 BasicBlock::iterator IP = EntryBB.getFirstInsertionPt();
7708 const DataLayout &DL = IP->getDataLayout();
7709 unsigned AS = DL.getAllocaAddrSpace();
7710 Instruction *AI = new AllocaInst(*PrivatizableType, AS,
7711 Arg->getName() + ".priv", IP);
7712 createInitialization(PrivType: *PrivatizableType, Base&: *AI, F&: ReplacementFn,
7713 ArgNo: ArgIt->getArgNo(), IP);
7714
7715 if (AI->getType() != Arg->getType())
7716 AI = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
7717 S: AI, Ty: Arg->getType(), Name: "", InsertBefore: IP);
7718 Arg->replaceAllUsesWith(V: AI);
7719
7720 for (CallInst *CI : TailCalls)
7721 CI->setTailCall(false);
7722 };
7723
7724 // Callback to repair a call site of the associated function. The elements
7725 // of the privatizable type are loaded prior to the call and passed to the
7726 // new function version.
7727 Attributor::ArgumentReplacementInfo::ACSRepairCBTy ACSRepairCB =
7728 [=](const Attributor::ArgumentReplacementInfo &ARI,
7729 AbstractCallSite ACS, SmallVectorImpl<Value *> &NewArgOperands) {
7730 // When no alignment is specified for the load instruction,
7731 // natural alignment is assumed.
7732 createReplacementValues(
7733 Alignment: AlignAA ? AlignAA->getAssumedAlign() : Align(0),
7734 PrivType: *PrivatizableType, ACS,
7735 Base: ACS.getCallArgOperand(ArgNo: ARI.getReplacedArg().getArgNo()),
7736 ReplacementValues&: NewArgOperands);
7737 };
7738
7739 // Collect the types that will replace the privatizable type in the function
7740 // signature.
7741 SmallVector<Type *, 16> ReplacementTypes;
7742 identifyReplacementTypes(PrivType: *PrivatizableType, ReplacementTypes);
7743
7744 // Register a rewrite of the argument.
7745 if (A.registerFunctionSignatureRewrite(Arg&: *Arg, ReplacementTypes,
7746 CalleeRepairCB: std::move(FnRepairCB),
7747 ACSRepairCB: std::move(ACSRepairCB)))
7748 return ChangeStatus::CHANGED;
7749 return ChangeStatus::UNCHANGED;
7750 }
7751
7752 /// See AbstractAttribute::trackStatistics()
7753 void trackStatistics() const override {
7754 STATS_DECLTRACK_ARG_ATTR(privatizable_ptr);
7755 }
7756};
7757
7758struct AAPrivatizablePtrFloating : public AAPrivatizablePtrImpl {
7759 AAPrivatizablePtrFloating(const IRPosition &IRP, Attributor &A)
7760 : AAPrivatizablePtrImpl(IRP, A) {}
7761
7762 /// See AbstractAttribute::initialize(...).
7763 void initialize(Attributor &A) override {
7764 // TODO: We can privatize more than arguments.
7765 indicatePessimisticFixpoint();
7766 }
7767
7768 ChangeStatus updateImpl(Attributor &A) override {
7769 llvm_unreachable("AAPrivatizablePtr(Floating|Returned|CallSiteReturned)::"
7770 "updateImpl will not be called");
7771 }
7772
7773 /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
7774 std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7775 Value *Obj = getUnderlyingObject(V: &getAssociatedValue());
7776 if (!Obj) {
7777 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] No underlying object found!\n");
7778 return nullptr;
7779 }
7780
7781 if (auto *AI = dyn_cast<AllocaInst>(Val: Obj))
7782 if (auto *CI = dyn_cast<ConstantInt>(Val: AI->getArraySize()))
7783 if (CI->isOne())
7784 return AI->getAllocatedType();
7785 if (auto *Arg = dyn_cast<Argument>(Val: Obj)) {
7786 auto *PrivArgAA = A.getAAFor<AAPrivatizablePtr>(
7787 QueryingAA: *this, IRP: IRPosition::argument(Arg: *Arg), DepClass: DepClassTy::REQUIRED);
7788 if (PrivArgAA && PrivArgAA->isAssumedPrivatizablePtr())
7789 return PrivArgAA->getPrivatizableType();
7790 }
7791
7792 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Underlying object neither valid "
7793 "alloca nor privatizable argument: "
7794 << *Obj << "!\n");
7795 return nullptr;
7796 }
7797
7798 /// See AbstractAttribute::trackStatistics()
7799 void trackStatistics() const override {
7800 STATS_DECLTRACK_FLOATING_ATTR(privatizable_ptr);
7801 }
7802};
7803
7804struct AAPrivatizablePtrCallSiteArgument final
7805 : public AAPrivatizablePtrFloating {
7806 AAPrivatizablePtrCallSiteArgument(const IRPosition &IRP, Attributor &A)
7807 : AAPrivatizablePtrFloating(IRP, A) {}
7808
7809 /// See AbstractAttribute::initialize(...).
7810 void initialize(Attributor &A) override {
7811 if (A.hasAttr(IRP: getIRPosition(), AKs: Attribute::ByVal))
7812 indicateOptimisticFixpoint();
7813 }
7814
7815 /// See AbstractAttribute::updateImpl(...).
7816 ChangeStatus updateImpl(Attributor &A) override {
7817 PrivatizableType = identifyPrivatizableType(A);
7818 if (!PrivatizableType)
7819 return ChangeStatus::UNCHANGED;
7820 if (!*PrivatizableType)
7821 return indicatePessimisticFixpoint();
7822
7823 const IRPosition &IRP = getIRPosition();
7824 bool IsKnownNoCapture;
7825 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
7826 A, QueryingAA: this, IRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture);
7827 if (!IsAssumedNoCapture) {
7828 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might be captured!\n");
7829 return indicatePessimisticFixpoint();
7830 }
7831
7832 bool IsKnownNoAlias;
7833 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
7834 A, QueryingAA: this, IRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoAlias)) {
7835 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might alias!\n");
7836 return indicatePessimisticFixpoint();
7837 }
7838
7839 bool IsKnown;
7840 if (!AA::isAssumedReadOnly(A, IRP, QueryingAA: *this, IsKnown)) {
7841 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer is written!\n");
7842 return indicatePessimisticFixpoint();
7843 }
7844
7845 return ChangeStatus::UNCHANGED;
7846 }
7847
7848 /// See AbstractAttribute::trackStatistics()
7849 void trackStatistics() const override {
7850 STATS_DECLTRACK_CSARG_ATTR(privatizable_ptr);
7851 }
7852};
7853
7854struct AAPrivatizablePtrCallSiteReturned final
7855 : public AAPrivatizablePtrFloating {
7856 AAPrivatizablePtrCallSiteReturned(const IRPosition &IRP, Attributor &A)
7857 : AAPrivatizablePtrFloating(IRP, A) {}
7858
7859 /// See AbstractAttribute::initialize(...).
7860 void initialize(Attributor &A) override {
7861 // TODO: We can privatize more than arguments.
7862 indicatePessimisticFixpoint();
7863 }
7864
7865 /// See AbstractAttribute::trackStatistics()
7866 void trackStatistics() const override {
7867 STATS_DECLTRACK_CSRET_ATTR(privatizable_ptr);
7868 }
7869};
7870
7871struct AAPrivatizablePtrReturned final : public AAPrivatizablePtrFloating {
7872 AAPrivatizablePtrReturned(const IRPosition &IRP, Attributor &A)
7873 : AAPrivatizablePtrFloating(IRP, A) {}
7874
7875 /// See AbstractAttribute::initialize(...).
7876 void initialize(Attributor &A) override {
7877 // TODO: We can privatize more than arguments.
7878 indicatePessimisticFixpoint();
7879 }
7880
7881 /// See AbstractAttribute::trackStatistics()
7882 void trackStatistics() const override {
7883 STATS_DECLTRACK_FNRET_ATTR(privatizable_ptr);
7884 }
7885};
7886} // namespace
7887
7888/// -------------------- Memory Behavior Attributes ----------------------------
7889/// Includes read-none, read-only, and write-only.
7890/// ----------------------------------------------------------------------------
7891namespace {
7892struct AAMemoryBehaviorImpl : public AAMemoryBehavior {
7893 AAMemoryBehaviorImpl(const IRPosition &IRP, Attributor &A)
7894 : AAMemoryBehavior(IRP, A) {}
7895
7896 /// See AbstractAttribute::initialize(...).
7897 void initialize(Attributor &A) override {
7898 intersectAssumedBits(BitsEncoding: BEST_STATE);
7899 getKnownStateFromValue(A, IRP: getIRPosition(), State&: getState());
7900 AAMemoryBehavior::initialize(A);
7901 }
7902
7903 /// Return the memory behavior information encoded in the IR for \p IRP.
7904 static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
7905 BitIntegerState &State,
7906 bool IgnoreSubsumingPositions = false) {
7907 SmallVector<Attribute, 2> Attrs;
7908 A.getAttrs(IRP, AKs: AttrKinds, Attrs, IgnoreSubsumingPositions);
7909 for (const Attribute &Attr : Attrs) {
7910 switch (Attr.getKindAsEnum()) {
7911 case Attribute::ReadNone:
7912 State.addKnownBits(Bits: NO_ACCESSES);
7913 break;
7914 case Attribute::ReadOnly:
7915 State.addKnownBits(Bits: NO_WRITES);
7916 break;
7917 case Attribute::WriteOnly:
7918 State.addKnownBits(Bits: NO_READS);
7919 break;
7920 default:
7921 llvm_unreachable("Unexpected attribute!");
7922 }
7923 }
7924
7925 if (auto *I = dyn_cast<Instruction>(Val: &IRP.getAnchorValue())) {
7926 if (!I->mayReadFromMemory())
7927 State.addKnownBits(Bits: NO_READS);
7928 if (!I->mayWriteToMemory())
7929 State.addKnownBits(Bits: NO_WRITES);
7930 }
7931 }
7932
7933 /// See AbstractAttribute::getDeducedAttributes(...).
7934 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
7935 SmallVectorImpl<Attribute> &Attrs) const override {
7936 assert(Attrs.size() == 0);
7937 if (isAssumedReadNone())
7938 Attrs.push_back(Elt: Attribute::get(Context&: Ctx, Kind: Attribute::ReadNone));
7939 else if (isAssumedReadOnly())
7940 Attrs.push_back(Elt: Attribute::get(Context&: Ctx, Kind: Attribute::ReadOnly));
7941 else if (isAssumedWriteOnly())
7942 Attrs.push_back(Elt: Attribute::get(Context&: Ctx, Kind: Attribute::WriteOnly));
7943 assert(Attrs.size() <= 1);
7944 }
7945
7946 /// See AbstractAttribute::manifest(...).
7947 ChangeStatus manifest(Attributor &A) override {
7948 const IRPosition &IRP = getIRPosition();
7949
7950 if (A.hasAttr(IRP, AKs: Attribute::ReadNone,
7951 /* IgnoreSubsumingPositions */ true))
7952 return ChangeStatus::UNCHANGED;
7953
7954 // Check if we would improve the existing attributes first.
7955 SmallVector<Attribute, 4> DeducedAttrs;
7956 getDeducedAttributes(A, Ctx&: IRP.getAnchorValue().getContext(), Attrs&: DeducedAttrs);
7957 if (llvm::all_of(Range&: DeducedAttrs, P: [&](const Attribute &Attr) {
7958 return A.hasAttr(IRP, AKs: Attr.getKindAsEnum(),
7959 /* IgnoreSubsumingPositions */ true);
7960 }))
7961 return ChangeStatus::UNCHANGED;
7962
7963 // Clear existing attributes.
7964 A.removeAttrs(IRP, AttrKinds);
7965 // Clear conflicting writable attribute.
7966 if (isAssumedReadOnly())
7967 A.removeAttrs(IRP, AttrKinds: Attribute::Writable);
7968
7969 // Use the generic manifest method.
7970 return IRAttribute::manifest(A);
7971 }
7972
7973 /// See AbstractState::getAsStr().
7974 const std::string getAsStr(Attributor *A) const override {
7975 if (isAssumedReadNone())
7976 return "readnone";
7977 if (isAssumedReadOnly())
7978 return "readonly";
7979 if (isAssumedWriteOnly())
7980 return "writeonly";
7981 return "may-read/write";
7982 }
7983
7984 /// The set of IR attributes AAMemoryBehavior deals with.
7985 static const Attribute::AttrKind AttrKinds[3];
7986};
7987
7988const Attribute::AttrKind AAMemoryBehaviorImpl::AttrKinds[] = {
7989 Attribute::ReadNone, Attribute::ReadOnly, Attribute::WriteOnly};
7990
7991/// Memory behavior attribute for a floating value.
7992struct AAMemoryBehaviorFloating : AAMemoryBehaviorImpl {
7993 AAMemoryBehaviorFloating(const IRPosition &IRP, Attributor &A)
7994 : AAMemoryBehaviorImpl(IRP, A) {}
7995
7996 /// See AbstractAttribute::updateImpl(...).
7997 ChangeStatus updateImpl(Attributor &A) override;
7998
7999 /// See AbstractAttribute::trackStatistics()
8000 void trackStatistics() const override {
8001 if (isAssumedReadNone())
8002 STATS_DECLTRACK_FLOATING_ATTR(readnone)
8003 else if (isAssumedReadOnly())
8004 STATS_DECLTRACK_FLOATING_ATTR(readonly)
8005 else if (isAssumedWriteOnly())
8006 STATS_DECLTRACK_FLOATING_ATTR(writeonly)
8007 }
8008
8009private:
8010 /// Return true if users of \p UserI might access the underlying
8011 /// variable/location described by \p U and should therefore be analyzed.
8012 bool followUsersOfUseIn(Attributor &A, const Use &U,
8013 const Instruction *UserI);
8014
8015 /// Update the state according to the effect of use \p U in \p UserI.
8016 void analyzeUseIn(Attributor &A, const Use &U, const Instruction *UserI);
8017};
8018
8019/// Memory behavior attribute for function argument.
8020struct AAMemoryBehaviorArgument : AAMemoryBehaviorFloating {
8021 AAMemoryBehaviorArgument(const IRPosition &IRP, Attributor &A)
8022 : AAMemoryBehaviorFloating(IRP, A) {}
8023
8024 /// See AbstractAttribute::initialize(...).
8025 void initialize(Attributor &A) override {
8026 intersectAssumedBits(BitsEncoding: BEST_STATE);
8027 const IRPosition &IRP = getIRPosition();
8028 // TODO: Make IgnoreSubsumingPositions a property of an IRAttribute so we
8029 // can query it when we use has/getAttr. That would allow us to reuse the
8030 // initialize of the base class here.
8031 bool HasByVal = A.hasAttr(IRP, AKs: {Attribute::ByVal},
8032 /* IgnoreSubsumingPositions */ true);
8033 getKnownStateFromValue(A, IRP, State&: getState(),
8034 /* IgnoreSubsumingPositions */ HasByVal);
8035 }
8036
8037 ChangeStatus manifest(Attributor &A) override {
8038 // TODO: Pointer arguments are not supported on vectors of pointers yet.
8039 if (!getAssociatedValue().getType()->isPointerTy())
8040 return ChangeStatus::UNCHANGED;
8041
8042 // TODO: From readattrs.ll: "inalloca parameters are always
8043 // considered written"
8044 if (A.hasAttr(IRP: getIRPosition(),
8045 AKs: {Attribute::InAlloca, Attribute::Preallocated})) {
8046 removeKnownBits(BitsEncoding: NO_WRITES);
8047 removeAssumedBits(BitsEncoding: NO_WRITES);
8048 }
8049 A.removeAttrs(IRP: getIRPosition(), AttrKinds);
8050 return AAMemoryBehaviorFloating::manifest(A);
8051 }
8052
8053 /// See AbstractAttribute::trackStatistics()
8054 void trackStatistics() const override {
8055 if (isAssumedReadNone())
8056 STATS_DECLTRACK_ARG_ATTR(readnone)
8057 else if (isAssumedReadOnly())
8058 STATS_DECLTRACK_ARG_ATTR(readonly)
8059 else if (isAssumedWriteOnly())
8060 STATS_DECLTRACK_ARG_ATTR(writeonly)
8061 }
8062};
8063
8064struct AAMemoryBehaviorCallSiteArgument final : AAMemoryBehaviorArgument {
8065 AAMemoryBehaviorCallSiteArgument(const IRPosition &IRP, Attributor &A)
8066 : AAMemoryBehaviorArgument(IRP, A) {}
8067
8068 /// See AbstractAttribute::initialize(...).
8069 void initialize(Attributor &A) override {
8070 // If we don't have an associated attribute this is either a variadic call
8071 // or an indirect call, either way, nothing to do here.
8072 Argument *Arg = getAssociatedArgument();
8073 if (!Arg) {
8074 indicatePessimisticFixpoint();
8075 return;
8076 }
8077 if (Arg->hasByValAttr()) {
8078 addKnownBits(Bits: NO_WRITES);
8079 removeKnownBits(BitsEncoding: NO_READS);
8080 removeAssumedBits(BitsEncoding: NO_READS);
8081 }
8082 AAMemoryBehaviorArgument::initialize(A);
8083 if (getAssociatedFunction()->isDeclaration())
8084 indicatePessimisticFixpoint();
8085 }
8086
8087 /// See AbstractAttribute::updateImpl(...).
8088 ChangeStatus updateImpl(Attributor &A) override {
8089 // TODO: Once we have call site specific value information we can provide
8090 // call site specific liveness liveness information and then it makes
8091 // sense to specialize attributes for call sites arguments instead of
8092 // redirecting requests to the callee argument.
8093 Argument *Arg = getAssociatedArgument();
8094 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
8095 auto *ArgAA =
8096 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
8097 if (!ArgAA)
8098 return indicatePessimisticFixpoint();
8099 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
8100 }
8101
8102 /// See AbstractAttribute::trackStatistics()
8103 void trackStatistics() const override {
8104 if (isAssumedReadNone())
8105 STATS_DECLTRACK_CSARG_ATTR(readnone)
8106 else if (isAssumedReadOnly())
8107 STATS_DECLTRACK_CSARG_ATTR(readonly)
8108 else if (isAssumedWriteOnly())
8109 STATS_DECLTRACK_CSARG_ATTR(writeonly)
8110 }
8111};
8112
8113/// Memory behavior attribute for a call site return position.
8114struct AAMemoryBehaviorCallSiteReturned final : AAMemoryBehaviorFloating {
8115 AAMemoryBehaviorCallSiteReturned(const IRPosition &IRP, Attributor &A)
8116 : AAMemoryBehaviorFloating(IRP, A) {}
8117
8118 /// See AbstractAttribute::initialize(...).
8119 void initialize(Attributor &A) override {
8120 AAMemoryBehaviorImpl::initialize(A);
8121 }
8122 /// See AbstractAttribute::manifest(...).
8123 ChangeStatus manifest(Attributor &A) override {
8124 // We do not annotate returned values.
8125 return ChangeStatus::UNCHANGED;
8126 }
8127
8128 /// See AbstractAttribute::trackStatistics()
8129 void trackStatistics() const override {}
8130};
8131
8132/// An AA to represent the memory behavior function attributes.
8133struct AAMemoryBehaviorFunction final : public AAMemoryBehaviorImpl {
8134 AAMemoryBehaviorFunction(const IRPosition &IRP, Attributor &A)
8135 : AAMemoryBehaviorImpl(IRP, A) {}
8136
8137 /// See AbstractAttribute::updateImpl(Attributor &A).
8138 ChangeStatus updateImpl(Attributor &A) override;
8139
8140 /// See AbstractAttribute::manifest(...).
8141 ChangeStatus manifest(Attributor &A) override {
8142 // TODO: It would be better to merge this with AAMemoryLocation, so that
8143 // we could determine read/write per location. This would also have the
8144 // benefit of only one place trying to manifest the memory attribute.
8145 Function &F = cast<Function>(Val&: getAnchorValue());
8146 MemoryEffects ME = MemoryEffects::unknown();
8147 if (isAssumedReadNone())
8148 ME = MemoryEffects::none();
8149 else if (isAssumedReadOnly())
8150 ME = MemoryEffects::readOnly();
8151 else if (isAssumedWriteOnly())
8152 ME = MemoryEffects::writeOnly();
8153
8154 A.removeAttrs(IRP: getIRPosition(), AttrKinds);
8155 // Clear conflicting writable attribute.
8156 if (ME.onlyReadsMemory())
8157 for (Argument &Arg : F.args())
8158 A.removeAttrs(IRP: IRPosition::argument(Arg), AttrKinds: Attribute::Writable);
8159 return A.manifestAttrs(IRP: getIRPosition(),
8160 DeducedAttrs: Attribute::getWithMemoryEffects(Context&: F.getContext(), ME));
8161 }
8162
8163 /// See AbstractAttribute::trackStatistics()
8164 void trackStatistics() const override {
8165 if (isAssumedReadNone())
8166 STATS_DECLTRACK_FN_ATTR(readnone)
8167 else if (isAssumedReadOnly())
8168 STATS_DECLTRACK_FN_ATTR(readonly)
8169 else if (isAssumedWriteOnly())
8170 STATS_DECLTRACK_FN_ATTR(writeonly)
8171 }
8172};
8173
8174/// AAMemoryBehavior attribute for call sites.
8175struct AAMemoryBehaviorCallSite final
8176 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl> {
8177 AAMemoryBehaviorCallSite(const IRPosition &IRP, Attributor &A)
8178 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl>(IRP, A) {}
8179
8180 /// See AbstractAttribute::manifest(...).
8181 ChangeStatus manifest(Attributor &A) override {
8182 // TODO: Deduplicate this with AAMemoryBehaviorFunction.
8183 CallBase &CB = cast<CallBase>(Val&: getAnchorValue());
8184 MemoryEffects ME = MemoryEffects::unknown();
8185 if (isAssumedReadNone())
8186 ME = MemoryEffects::none();
8187 else if (isAssumedReadOnly())
8188 ME = MemoryEffects::readOnly();
8189 else if (isAssumedWriteOnly())
8190 ME = MemoryEffects::writeOnly();
8191
8192 A.removeAttrs(IRP: getIRPosition(), AttrKinds);
8193 // Clear conflicting writable attribute.
8194 if (ME.onlyReadsMemory())
8195 for (Use &U : CB.args())
8196 A.removeAttrs(IRP: IRPosition::callsite_argument(CB, ArgNo: U.getOperandNo()),
8197 AttrKinds: Attribute::Writable);
8198 return A.manifestAttrs(
8199 IRP: getIRPosition(), DeducedAttrs: Attribute::getWithMemoryEffects(Context&: CB.getContext(), ME));
8200 }
8201
8202 /// See AbstractAttribute::trackStatistics()
8203 void trackStatistics() const override {
8204 if (isAssumedReadNone())
8205 STATS_DECLTRACK_CS_ATTR(readnone)
8206 else if (isAssumedReadOnly())
8207 STATS_DECLTRACK_CS_ATTR(readonly)
8208 else if (isAssumedWriteOnly())
8209 STATS_DECLTRACK_CS_ATTR(writeonly)
8210 }
8211};
8212
8213ChangeStatus AAMemoryBehaviorFunction::updateImpl(Attributor &A) {
8214
8215 // The current assumed state used to determine a change.
8216 auto AssumedState = getAssumed();
8217
8218 auto CheckRWInst = [&](Instruction &I) {
8219 // If the instruction has an own memory behavior state, use it to restrict
8220 // the local state. No further analysis is required as the other memory
8221 // state is as optimistic as it gets.
8222 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
8223 const auto *MemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
8224 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::REQUIRED);
8225 if (MemBehaviorAA) {
8226 intersectAssumedBits(BitsEncoding: MemBehaviorAA->getAssumed());
8227 return !isAtFixpoint();
8228 }
8229 }
8230
8231 // Remove access kind modifiers if necessary.
8232 if (I.mayReadFromMemory())
8233 removeAssumedBits(BitsEncoding: NO_READS);
8234 if (I.mayWriteToMemory())
8235 removeAssumedBits(BitsEncoding: NO_WRITES);
8236 return !isAtFixpoint();
8237 };
8238
8239 bool UsedAssumedInformation = false;
8240 if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInst, QueryingAA&: *this,
8241 UsedAssumedInformation))
8242 return indicatePessimisticFixpoint();
8243
8244 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8245 : ChangeStatus::UNCHANGED;
8246}
8247
8248ChangeStatus AAMemoryBehaviorFloating::updateImpl(Attributor &A) {
8249
8250 const IRPosition &IRP = getIRPosition();
8251 const IRPosition &FnPos = IRPosition::function_scope(IRP);
8252 AAMemoryBehavior::StateType &S = getState();
8253
8254 // First, check the function scope. We take the known information and we avoid
8255 // work if the assumed information implies the current assumed information for
8256 // this attribute. This is a valid for all but byval arguments.
8257 Argument *Arg = IRP.getAssociatedArgument();
8258 AAMemoryBehavior::base_t FnMemAssumedState =
8259 AAMemoryBehavior::StateType::getWorstState();
8260 if (!Arg || !Arg->hasByValAttr()) {
8261 const auto *FnMemAA =
8262 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::OPTIONAL);
8263 if (FnMemAA) {
8264 FnMemAssumedState = FnMemAA->getAssumed();
8265 S.addKnownBits(Bits: FnMemAA->getKnown());
8266 if ((S.getAssumed() & FnMemAA->getAssumed()) == S.getAssumed())
8267 return ChangeStatus::UNCHANGED;
8268 }
8269 }
8270
8271 // The current assumed state used to determine a change.
8272 auto AssumedState = S.getAssumed();
8273
8274 // Make sure the value is not captured (except through "return"), if
8275 // it is, any information derived would be irrelevant anyway as we cannot
8276 // check the potential aliases introduced by the capture. However, no need
8277 // to fall back to anythign less optimistic than the function state.
8278 bool IsKnownNoCapture;
8279 const AANoCapture *ArgNoCaptureAA = nullptr;
8280 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
8281 A, QueryingAA: this, IRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
8282 AAPtr: &ArgNoCaptureAA);
8283
8284 if (!IsAssumedNoCapture &&
8285 (!ArgNoCaptureAA || !ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
8286 S.intersectAssumedBits(BitsEncoding: FnMemAssumedState);
8287 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8288 : ChangeStatus::UNCHANGED;
8289 }
8290
8291 // Visit and expand uses until all are analyzed or a fixpoint is reached.
8292 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
8293 Instruction *UserI = cast<Instruction>(Val: U.getUser());
8294 LLVM_DEBUG(dbgs() << "[AAMemoryBehavior] Use: " << *U << " in " << *UserI
8295 << " \n");
8296
8297 // Droppable users, e.g., llvm::assume does not actually perform any action.
8298 if (UserI->isDroppable())
8299 return true;
8300
8301 // Check if the users of UserI should also be visited.
8302 Follow = followUsersOfUseIn(A, U, UserI);
8303
8304 // If UserI might touch memory we analyze the use in detail.
8305 if (UserI->mayReadOrWriteMemory())
8306 analyzeUseIn(A, U, UserI);
8307
8308 return !isAtFixpoint();
8309 };
8310
8311 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: getAssociatedValue()))
8312 return indicatePessimisticFixpoint();
8313
8314 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8315 : ChangeStatus::UNCHANGED;
8316}
8317
8318bool AAMemoryBehaviorFloating::followUsersOfUseIn(Attributor &A, const Use &U,
8319 const Instruction *UserI) {
8320 // The loaded value is unrelated to the pointer argument, no need to
8321 // follow the users of the load.
8322 if (isa<LoadInst>(Val: UserI) || isa<ReturnInst>(Val: UserI))
8323 return false;
8324
8325 // By default we follow all uses assuming UserI might leak information on U,
8326 // we have special handling for call sites operands though.
8327 const auto *CB = dyn_cast<CallBase>(Val: UserI);
8328 if (!CB || !CB->isArgOperand(U: &U))
8329 return true;
8330
8331 // If the use is a call argument known not to be captured, the users of
8332 // the call do not need to be visited because they have to be unrelated to
8333 // the input. Note that this check is not trivial even though we disallow
8334 // general capturing of the underlying argument. The reason is that the
8335 // call might the argument "through return", which we allow and for which we
8336 // need to check call users.
8337 if (U.get()->getType()->isPointerTy()) {
8338 unsigned ArgNo = CB->getArgOperandNo(U: &U);
8339 bool IsKnownNoCapture;
8340 return !AA::hasAssumedIRAttr<Attribute::Captures>(
8341 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
8342 DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture);
8343 }
8344
8345 return true;
8346}
8347
8348void AAMemoryBehaviorFloating::analyzeUseIn(Attributor &A, const Use &U,
8349 const Instruction *UserI) {
8350 assert(UserI->mayReadOrWriteMemory());
8351
8352 switch (UserI->getOpcode()) {
8353 default:
8354 // TODO: Handle all atomics and other side-effect operations we know of.
8355 break;
8356 case Instruction::Load:
8357 // Loads cause the NO_READS property to disappear.
8358 removeAssumedBits(BitsEncoding: NO_READS);
8359 return;
8360
8361 case Instruction::Store:
8362 // Stores cause the NO_WRITES property to disappear if the use is the
8363 // pointer operand. Note that while capturing was taken care of somewhere
8364 // else we need to deal with stores of the value that is not looked through.
8365 if (cast<StoreInst>(Val: UserI)->getPointerOperand() == U.get())
8366 removeAssumedBits(BitsEncoding: NO_WRITES);
8367 else
8368 indicatePessimisticFixpoint();
8369 return;
8370
8371 case Instruction::Call:
8372 case Instruction::CallBr:
8373 case Instruction::Invoke: {
8374 // For call sites we look at the argument memory behavior attribute (this
8375 // could be recursive!) in order to restrict our own state.
8376 const auto *CB = cast<CallBase>(Val: UserI);
8377
8378 // Give up on operand bundles.
8379 if (CB->isBundleOperand(U: &U)) {
8380 indicatePessimisticFixpoint();
8381 return;
8382 }
8383
8384 // Calling a function does read the function pointer, maybe write it if the
8385 // function is self-modifying.
8386 if (CB->isCallee(U: &U)) {
8387 removeAssumedBits(BitsEncoding: NO_READS);
8388 break;
8389 }
8390
8391 // Adjust the possible access behavior based on the information on the
8392 // argument.
8393 IRPosition Pos;
8394 if (U.get()->getType()->isPointerTy())
8395 Pos = IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U));
8396 else
8397 Pos = IRPosition::callsite_function(CB: *CB);
8398 const auto *MemBehaviorAA =
8399 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: Pos, DepClass: DepClassTy::OPTIONAL);
8400 if (!MemBehaviorAA)
8401 break;
8402 // "assumed" has at most the same bits as the MemBehaviorAA assumed
8403 // and at least "known".
8404 intersectAssumedBits(BitsEncoding: MemBehaviorAA->getAssumed());
8405 return;
8406 }
8407 };
8408
8409 // Generally, look at the "may-properties" and adjust the assumed state if we
8410 // did not trigger special handling before.
8411 if (UserI->mayReadFromMemory())
8412 removeAssumedBits(BitsEncoding: NO_READS);
8413 if (UserI->mayWriteToMemory())
8414 removeAssumedBits(BitsEncoding: NO_WRITES);
8415}
8416} // namespace
8417
8418/// -------------------- Memory Locations Attributes ---------------------------
8419/// Includes read-none, argmemonly, inaccessiblememonly,
8420/// inaccessiblememorargmemonly
8421/// ----------------------------------------------------------------------------
8422
8423std::string AAMemoryLocation::getMemoryLocationsAsStr(
8424 AAMemoryLocation::MemoryLocationsKind MLK) {
8425 if (0 == (MLK & AAMemoryLocation::NO_LOCATIONS))
8426 return "all memory";
8427 if (MLK == AAMemoryLocation::NO_LOCATIONS)
8428 return "no memory";
8429 std::string S = "memory:";
8430 if (0 == (MLK & AAMemoryLocation::NO_LOCAL_MEM))
8431 S += "stack,";
8432 if (0 == (MLK & AAMemoryLocation::NO_CONST_MEM))
8433 S += "constant,";
8434 if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_INTERNAL_MEM))
8435 S += "internal global,";
8436 if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_EXTERNAL_MEM))
8437 S += "external global,";
8438 if (0 == (MLK & AAMemoryLocation::NO_ARGUMENT_MEM))
8439 S += "argument,";
8440 if (0 == (MLK & AAMemoryLocation::NO_INACCESSIBLE_MEM))
8441 S += "inaccessible,";
8442 if (0 == (MLK & AAMemoryLocation::NO_MALLOCED_MEM))
8443 S += "malloced,";
8444 if (0 == (MLK & AAMemoryLocation::NO_UNKOWN_MEM))
8445 S += "unknown,";
8446 S.pop_back();
8447 return S;
8448}
8449
8450namespace {
8451struct AAMemoryLocationImpl : public AAMemoryLocation {
8452
8453 AAMemoryLocationImpl(const IRPosition &IRP, Attributor &A)
8454 : AAMemoryLocation(IRP, A), Allocator(A.Allocator) {
8455 AccessKind2Accesses.fill(u: nullptr);
8456 }
8457
8458 ~AAMemoryLocationImpl() override {
8459 // The AccessSets are allocated via a BumpPtrAllocator, we call
8460 // the destructor manually.
8461 for (AccessSet *AS : AccessKind2Accesses)
8462 if (AS)
8463 AS->~AccessSet();
8464 }
8465
8466 /// See AbstractAttribute::initialize(...).
8467 void initialize(Attributor &A) override {
8468 intersectAssumedBits(BitsEncoding: BEST_STATE);
8469 getKnownStateFromValue(A, IRP: getIRPosition(), State&: getState());
8470 AAMemoryLocation::initialize(A);
8471 }
8472
8473 /// Return the memory behavior information encoded in the IR for \p IRP.
8474 static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
8475 BitIntegerState &State,
8476 bool IgnoreSubsumingPositions = false) {
8477 // For internal functions we ignore `argmemonly` and
8478 // `inaccessiblememorargmemonly` as we might break it via interprocedural
8479 // constant propagation. It is unclear if this is the best way but it is
8480 // unlikely this will cause real performance problems. If we are deriving
8481 // attributes for the anchor function we even remove the attribute in
8482 // addition to ignoring it.
8483 // TODO: A better way to handle this would be to add ~NO_GLOBAL_MEM /
8484 // MemoryEffects::Other as a possible location.
8485 bool UseArgMemOnly = true;
8486 Function *AnchorFn = IRP.getAnchorScope();
8487 if (AnchorFn && A.isRunOn(Fn&: *AnchorFn))
8488 UseArgMemOnly = !AnchorFn->hasLocalLinkage();
8489
8490 SmallVector<Attribute, 2> Attrs;
8491 A.getAttrs(IRP, AKs: {Attribute::Memory}, Attrs, IgnoreSubsumingPositions);
8492 for (const Attribute &Attr : Attrs) {
8493 // TODO: We can map MemoryEffects to Attributor locations more precisely.
8494 MemoryEffects ME = Attr.getMemoryEffects();
8495 if (ME.doesNotAccessMemory()) {
8496 State.addKnownBits(Bits: NO_LOCAL_MEM | NO_CONST_MEM);
8497 continue;
8498 }
8499 if (ME.onlyAccessesInaccessibleMem()) {
8500 State.addKnownBits(Bits: inverseLocation(Loc: NO_INACCESSIBLE_MEM, AndLocalMem: true, AndConstMem: true));
8501 continue;
8502 }
8503 if (ME.onlyAccessesArgPointees()) {
8504 if (UseArgMemOnly)
8505 State.addKnownBits(Bits: inverseLocation(Loc: NO_ARGUMENT_MEM, AndLocalMem: true, AndConstMem: true));
8506 else {
8507 // Remove location information, only keep read/write info.
8508 ME = MemoryEffects(ME.getModRef());
8509 A.manifestAttrs(IRP,
8510 DeducedAttrs: Attribute::getWithMemoryEffects(
8511 Context&: IRP.getAnchorValue().getContext(), ME),
8512 /*ForceReplace*/ true);
8513 }
8514 continue;
8515 }
8516 if (ME.onlyAccessesInaccessibleOrArgMem()) {
8517 if (UseArgMemOnly)
8518 State.addKnownBits(Bits: inverseLocation(
8519 Loc: NO_INACCESSIBLE_MEM | NO_ARGUMENT_MEM, AndLocalMem: true, AndConstMem: true));
8520 else {
8521 // Remove location information, only keep read/write info.
8522 ME = MemoryEffects(ME.getModRef());
8523 A.manifestAttrs(IRP,
8524 DeducedAttrs: Attribute::getWithMemoryEffects(
8525 Context&: IRP.getAnchorValue().getContext(), ME),
8526 /*ForceReplace*/ true);
8527 }
8528 continue;
8529 }
8530 }
8531 }
8532
8533 /// See AbstractAttribute::getDeducedAttributes(...).
8534 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
8535 SmallVectorImpl<Attribute> &Attrs) const override {
8536 // TODO: We can map Attributor locations to MemoryEffects more precisely.
8537 assert(Attrs.size() == 0);
8538 if (getIRPosition().getPositionKind() == IRPosition::IRP_FUNCTION) {
8539 if (isAssumedReadNone())
8540 Attrs.push_back(
8541 Elt: Attribute::getWithMemoryEffects(Context&: Ctx, ME: MemoryEffects::none()));
8542 else if (isAssumedInaccessibleMemOnly())
8543 Attrs.push_back(Elt: Attribute::getWithMemoryEffects(
8544 Context&: Ctx, ME: MemoryEffects::inaccessibleMemOnly()));
8545 else if (isAssumedArgMemOnly())
8546 Attrs.push_back(
8547 Elt: Attribute::getWithMemoryEffects(Context&: Ctx, ME: MemoryEffects::argMemOnly()));
8548 else if (isAssumedInaccessibleOrArgMemOnly())
8549 Attrs.push_back(Elt: Attribute::getWithMemoryEffects(
8550 Context&: Ctx, ME: MemoryEffects::inaccessibleOrArgMemOnly()));
8551 }
8552 assert(Attrs.size() <= 1);
8553 }
8554
8555 /// See AbstractAttribute::manifest(...).
8556 ChangeStatus manifest(Attributor &A) override {
8557 // TODO: If AAMemoryLocation and AAMemoryBehavior are merged, we could
8558 // provide per-location modref information here.
8559 const IRPosition &IRP = getIRPosition();
8560
8561 SmallVector<Attribute, 1> DeducedAttrs;
8562 getDeducedAttributes(A, Ctx&: IRP.getAnchorValue().getContext(), Attrs&: DeducedAttrs);
8563 if (DeducedAttrs.size() != 1)
8564 return ChangeStatus::UNCHANGED;
8565 MemoryEffects ME = DeducedAttrs[0].getMemoryEffects();
8566
8567 return A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithMemoryEffects(
8568 Context&: IRP.getAnchorValue().getContext(), ME));
8569 }
8570
8571 /// See AAMemoryLocation::checkForAllAccessesToMemoryKind(...).
8572 bool checkForAllAccessesToMemoryKind(
8573 function_ref<bool(const Instruction *, const Value *, AccessKind,
8574 MemoryLocationsKind)>
8575 Pred,
8576 MemoryLocationsKind RequestedMLK) const override {
8577 if (!isValidState())
8578 return false;
8579
8580 MemoryLocationsKind AssumedMLK = getAssumedNotAccessedLocation();
8581 if (AssumedMLK == NO_LOCATIONS)
8582 return true;
8583
8584 unsigned Idx = 0;
8585 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS;
8586 CurMLK *= 2, ++Idx) {
8587 if (CurMLK & RequestedMLK)
8588 continue;
8589
8590 if (const AccessSet *Accesses = AccessKind2Accesses[Idx])
8591 for (const AccessInfo &AI : *Accesses)
8592 if (!Pred(AI.I, AI.Ptr, AI.Kind, CurMLK))
8593 return false;
8594 }
8595
8596 return true;
8597 }
8598
8599 ChangeStatus indicatePessimisticFixpoint() override {
8600 // If we give up and indicate a pessimistic fixpoint this instruction will
8601 // become an access for all potential access kinds:
8602 // TODO: Add pointers for argmemonly and globals to improve the results of
8603 // checkForAllAccessesToMemoryKind.
8604 bool Changed = false;
8605 MemoryLocationsKind KnownMLK = getKnown();
8606 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
8607 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2)
8608 if (!(CurMLK & KnownMLK))
8609 updateStateAndAccessesMap(State&: getState(), MLK: CurMLK, I, Ptr: nullptr, Changed,
8610 AK: getAccessKindFromInst(I));
8611 return AAMemoryLocation::indicatePessimisticFixpoint();
8612 }
8613
8614protected:
8615 /// Helper struct to tie together an instruction that has a read or write
8616 /// effect with the pointer it accesses (if any).
8617 struct AccessInfo {
8618
8619 /// The instruction that caused the access.
8620 const Instruction *I;
8621
8622 /// The base pointer that is accessed, or null if unknown.
8623 const Value *Ptr;
8624
8625 /// The kind of access (read/write/read+write).
8626 AccessKind Kind;
8627
8628 bool operator==(const AccessInfo &RHS) const {
8629 return I == RHS.I && Ptr == RHS.Ptr && Kind == RHS.Kind;
8630 }
8631 bool operator()(const AccessInfo &LHS, const AccessInfo &RHS) const {
8632 if (LHS.I != RHS.I)
8633 return LHS.I < RHS.I;
8634 if (LHS.Ptr != RHS.Ptr)
8635 return LHS.Ptr < RHS.Ptr;
8636 if (LHS.Kind != RHS.Kind)
8637 return LHS.Kind < RHS.Kind;
8638 return false;
8639 }
8640 };
8641
8642 /// Mapping from *single* memory location kinds, e.g., LOCAL_MEM with the
8643 /// value of NO_LOCAL_MEM, to the accesses encountered for this memory kind.
8644 using AccessSet = SmallSet<AccessInfo, 2, AccessInfo>;
8645 std::array<AccessSet *, llvm::ConstantLog2<VALID_STATE>()>
8646 AccessKind2Accesses;
8647
8648 /// Categorize the pointer arguments of CB that might access memory in
8649 /// AccessedLoc and update the state and access map accordingly.
8650 void
8651 categorizeArgumentPointerLocations(Attributor &A, CallBase &CB,
8652 AAMemoryLocation::StateType &AccessedLocs,
8653 bool &Changed);
8654
8655 /// Return the kind(s) of location that may be accessed by \p V.
8656 AAMemoryLocation::MemoryLocationsKind
8657 categorizeAccessedLocations(Attributor &A, Instruction &I, bool &Changed);
8658
8659 /// Return the access kind as determined by \p I.
8660 AccessKind getAccessKindFromInst(const Instruction *I) {
8661 AccessKind AK = READ_WRITE;
8662 if (I) {
8663 AK = I->mayReadFromMemory() ? READ : NONE;
8664 AK = AccessKind(AK | (I->mayWriteToMemory() ? WRITE : NONE));
8665 }
8666 return AK;
8667 }
8668
8669 /// Update the state \p State and the AccessKind2Accesses given that \p I is
8670 /// an access of kind \p AK to a \p MLK memory location with the access
8671 /// pointer \p Ptr.
8672 void updateStateAndAccessesMap(AAMemoryLocation::StateType &State,
8673 MemoryLocationsKind MLK, const Instruction *I,
8674 const Value *Ptr, bool &Changed,
8675 AccessKind AK = READ_WRITE) {
8676
8677 assert(isPowerOf2_32(MLK) && "Expected a single location set!");
8678 auto *&Accesses = AccessKind2Accesses[llvm::Log2_32(Value: MLK)];
8679 if (!Accesses)
8680 Accesses = new (Allocator) AccessSet();
8681 Changed |= Accesses->insert(V: AccessInfo{.I: I, .Ptr: Ptr, .Kind: AK}).second;
8682 if (MLK == NO_UNKOWN_MEM)
8683 MLK = NO_LOCATIONS;
8684 State.removeAssumedBits(BitsEncoding: MLK);
8685 }
8686
8687 /// Determine the underlying locations kinds for \p Ptr, e.g., globals or
8688 /// arguments, and update the state and access map accordingly.
8689 void categorizePtrValue(Attributor &A, const Instruction &I, const Value &Ptr,
8690 AAMemoryLocation::StateType &State, bool &Changed,
8691 unsigned AccessAS = 0);
8692
8693 /// Used to allocate access sets.
8694 BumpPtrAllocator &Allocator;
8695};
8696
8697void AAMemoryLocationImpl::categorizePtrValue(
8698 Attributor &A, const Instruction &I, const Value &Ptr,
8699 AAMemoryLocation::StateType &State, bool &Changed, unsigned AccessAS) {
8700 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize pointer locations for "
8701 << Ptr << " ["
8702 << getMemoryLocationsAsStr(State.getAssumed()) << "]\n");
8703
8704 auto Pred = [&](Value &Obj) {
8705 unsigned ObjectAS = Obj.getType()->getPointerAddressSpace();
8706 // TODO: recognize the TBAA used for constant accesses.
8707 MemoryLocationsKind MLK = NO_LOCATIONS;
8708
8709 // Filter accesses to constant (GPU) memory if we have an AS at the access
8710 // site or the object is known to actually have the associated AS.
8711 if ((AccessAS == (unsigned)AA::GPUAddressSpace::Constant ||
8712 (ObjectAS == (unsigned)AA::GPUAddressSpace::Constant &&
8713 isIdentifiedObject(V: &Obj))) &&
8714 AA::isGPU(M: *I.getModule()))
8715 return true;
8716
8717 if (isa<UndefValue>(Val: &Obj))
8718 return true;
8719 if (isa<Argument>(Val: &Obj)) {
8720 // TODO: For now we do not treat byval arguments as local copies performed
8721 // on the call edge, though, we should. To make that happen we need to
8722 // teach various passes, e.g., DSE, about the copy effect of a byval. That
8723 // would also allow us to mark functions only accessing byval arguments as
8724 // readnone again, arguably their accesses have no effect outside of the
8725 // function, like accesses to allocas.
8726 MLK = NO_ARGUMENT_MEM;
8727 } else if (auto *GV = dyn_cast<GlobalValue>(Val: &Obj)) {
8728 // Reading constant memory is not treated as a read "effect" by the
8729 // function attr pass so we won't neither. Constants defined by TBAA are
8730 // similar. (We know we do not write it because it is constant.)
8731 if (auto *GVar = dyn_cast<GlobalVariable>(Val: GV))
8732 if (GVar->isConstant())
8733 return true;
8734
8735 if (GV->hasLocalLinkage())
8736 MLK = NO_GLOBAL_INTERNAL_MEM;
8737 else
8738 MLK = NO_GLOBAL_EXTERNAL_MEM;
8739 } else if (isa<ConstantPointerNull>(Val: &Obj) &&
8740 (!NullPointerIsDefined(F: getAssociatedFunction(), AS: AccessAS) ||
8741 !NullPointerIsDefined(F: getAssociatedFunction(), AS: ObjectAS))) {
8742 return true;
8743 } else if (isa<AllocaInst>(Val: &Obj)) {
8744 MLK = NO_LOCAL_MEM;
8745 } else if (const auto *CB = dyn_cast<CallBase>(Val: &Obj)) {
8746 bool IsKnownNoAlias;
8747 if (AA::hasAssumedIRAttr<Attribute::NoAlias>(
8748 A, QueryingAA: this, IRP: IRPosition::callsite_returned(CB: *CB), DepClass: DepClassTy::OPTIONAL,
8749 IsKnown&: IsKnownNoAlias))
8750 MLK = NO_MALLOCED_MEM;
8751 else
8752 MLK = NO_UNKOWN_MEM;
8753 } else {
8754 MLK = NO_UNKOWN_MEM;
8755 }
8756
8757 assert(MLK != NO_LOCATIONS && "No location specified!");
8758 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Ptr value can be categorized: "
8759 << Obj << " -> " << getMemoryLocationsAsStr(MLK) << "\n");
8760 updateStateAndAccessesMap(State, MLK, I: &I, Ptr: &Obj, Changed,
8761 AK: getAccessKindFromInst(I: &I));
8762
8763 return true;
8764 };
8765
8766 const auto *AA = A.getAAFor<AAUnderlyingObjects>(
8767 QueryingAA: *this, IRP: IRPosition::value(V: Ptr), DepClass: DepClassTy::OPTIONAL);
8768 if (!AA || !AA->forallUnderlyingObjects(Pred, Scope: AA::Intraprocedural)) {
8769 LLVM_DEBUG(
8770 dbgs() << "[AAMemoryLocation] Pointer locations not categorized\n");
8771 updateStateAndAccessesMap(State, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr, Changed,
8772 AK: getAccessKindFromInst(I: &I));
8773 return;
8774 }
8775
8776 LLVM_DEBUG(
8777 dbgs() << "[AAMemoryLocation] Accessed locations with pointer locations: "
8778 << getMemoryLocationsAsStr(State.getAssumed()) << "\n");
8779}
8780
8781void AAMemoryLocationImpl::categorizeArgumentPointerLocations(
8782 Attributor &A, CallBase &CB, AAMemoryLocation::StateType &AccessedLocs,
8783 bool &Changed) {
8784 for (unsigned ArgNo = 0, E = CB.arg_size(); ArgNo < E; ++ArgNo) {
8785
8786 // Skip non-pointer arguments.
8787 const Value *ArgOp = CB.getArgOperand(i: ArgNo);
8788 if (!ArgOp->getType()->isPtrOrPtrVectorTy())
8789 continue;
8790
8791 // Skip readnone arguments.
8792 const IRPosition &ArgOpIRP = IRPosition::callsite_argument(CB, ArgNo);
8793 const auto *ArgOpMemLocationAA =
8794 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: ArgOpIRP, DepClass: DepClassTy::OPTIONAL);
8795
8796 if (ArgOpMemLocationAA && ArgOpMemLocationAA->isAssumedReadNone())
8797 continue;
8798
8799 // Categorize potentially accessed pointer arguments as if there was an
8800 // access instruction with them as pointer.
8801 categorizePtrValue(A, I: CB, Ptr: *ArgOp, State&: AccessedLocs, Changed);
8802 }
8803}
8804
8805AAMemoryLocation::MemoryLocationsKind
8806AAMemoryLocationImpl::categorizeAccessedLocations(Attributor &A, Instruction &I,
8807 bool &Changed) {
8808 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize accessed locations for "
8809 << I << "\n");
8810
8811 AAMemoryLocation::StateType AccessedLocs;
8812 AccessedLocs.intersectAssumedBits(BitsEncoding: NO_LOCATIONS);
8813
8814 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
8815
8816 // First check if we assume any memory is access is visible.
8817 const auto *CBMemLocationAA = A.getAAFor<AAMemoryLocation>(
8818 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL);
8819 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize call site: " << I
8820 << " [" << CBMemLocationAA << "]\n");
8821 if (!CBMemLocationAA) {
8822 updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr,
8823 Changed, AK: getAccessKindFromInst(I: &I));
8824 return NO_UNKOWN_MEM;
8825 }
8826
8827 if (CBMemLocationAA->isAssumedReadNone())
8828 return NO_LOCATIONS;
8829
8830 if (CBMemLocationAA->isAssumedInaccessibleMemOnly()) {
8831 updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_INACCESSIBLE_MEM, I: &I, Ptr: nullptr,
8832 Changed, AK: getAccessKindFromInst(I: &I));
8833 return AccessedLocs.getAssumed();
8834 }
8835
8836 uint32_t CBAssumedNotAccessedLocs =
8837 CBMemLocationAA->getAssumedNotAccessedLocation();
8838
8839 // Set the argmemonly and global bit as we handle them separately below.
8840 uint32_t CBAssumedNotAccessedLocsNoArgMem =
8841 CBAssumedNotAccessedLocs | NO_ARGUMENT_MEM | NO_GLOBAL_MEM;
8842
8843 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2) {
8844 if (CBAssumedNotAccessedLocsNoArgMem & CurMLK)
8845 continue;
8846 updateStateAndAccessesMap(State&: AccessedLocs, MLK: CurMLK, I: &I, Ptr: nullptr, Changed,
8847 AK: getAccessKindFromInst(I: &I));
8848 }
8849
8850 // Now handle global memory if it might be accessed. This is slightly tricky
8851 // as NO_GLOBAL_MEM has multiple bits set.
8852 bool HasGlobalAccesses = ((~CBAssumedNotAccessedLocs) & NO_GLOBAL_MEM);
8853 if (HasGlobalAccesses) {
8854 auto AccessPred = [&](const Instruction *, const Value *Ptr,
8855 AccessKind Kind, MemoryLocationsKind MLK) {
8856 updateStateAndAccessesMap(State&: AccessedLocs, MLK, I: &I, Ptr, Changed,
8857 AK: getAccessKindFromInst(I: &I));
8858 return true;
8859 };
8860 if (!CBMemLocationAA->checkForAllAccessesToMemoryKind(
8861 Pred: AccessPred, MLK: inverseLocation(Loc: NO_GLOBAL_MEM, AndLocalMem: false, AndConstMem: false)))
8862 return AccessedLocs.getWorstState();
8863 }
8864
8865 LLVM_DEBUG(
8866 dbgs() << "[AAMemoryLocation] Accessed state before argument handling: "
8867 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8868
8869 // Now handle argument memory if it might be accessed.
8870 bool HasArgAccesses = ((~CBAssumedNotAccessedLocs) & NO_ARGUMENT_MEM);
8871 if (HasArgAccesses)
8872 categorizeArgumentPointerLocations(A, CB&: *CB, AccessedLocs, Changed);
8873
8874 LLVM_DEBUG(
8875 dbgs() << "[AAMemoryLocation] Accessed state after argument handling: "
8876 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8877
8878 return AccessedLocs.getAssumed();
8879 }
8880
8881 if (const Value *Ptr = getPointerOperand(I: &I, /* AllowVolatile */ true)) {
8882 LLVM_DEBUG(
8883 dbgs() << "[AAMemoryLocation] Categorize memory access with pointer: "
8884 << I << " [" << *Ptr << "]\n");
8885 categorizePtrValue(A, I, Ptr: *Ptr, State&: AccessedLocs, Changed,
8886 AccessAS: Ptr->getType()->getPointerAddressSpace());
8887 return AccessedLocs.getAssumed();
8888 }
8889
8890 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Failed to categorize instruction: "
8891 << I << "\n");
8892 updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr, Changed,
8893 AK: getAccessKindFromInst(I: &I));
8894 return AccessedLocs.getAssumed();
8895}
8896
8897/// An AA to represent the memory behavior function attributes.
8898struct AAMemoryLocationFunction final : public AAMemoryLocationImpl {
8899 AAMemoryLocationFunction(const IRPosition &IRP, Attributor &A)
8900 : AAMemoryLocationImpl(IRP, A) {}
8901
8902 /// See AbstractAttribute::updateImpl(Attributor &A).
8903 ChangeStatus updateImpl(Attributor &A) override {
8904
8905 const auto *MemBehaviorAA =
8906 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::NONE);
8907 if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
8908 if (MemBehaviorAA->isKnownReadNone())
8909 return indicateOptimisticFixpoint();
8910 assert(isAssumedReadNone() &&
8911 "AAMemoryLocation was not read-none but AAMemoryBehavior was!");
8912 A.recordDependence(FromAA: *MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
8913 return ChangeStatus::UNCHANGED;
8914 }
8915
8916 // The current assumed state used to determine a change.
8917 auto AssumedState = getAssumed();
8918 bool Changed = false;
8919
8920 auto CheckRWInst = [&](Instruction &I) {
8921 MemoryLocationsKind MLK = categorizeAccessedLocations(A, I, Changed);
8922 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Accessed locations for " << I
8923 << ": " << getMemoryLocationsAsStr(MLK) << "\n");
8924 removeAssumedBits(BitsEncoding: inverseLocation(Loc: MLK, AndLocalMem: false, AndConstMem: false));
8925 // Stop once only the valid bit set in the *not assumed location*, thus
8926 // once we don't actually exclude any memory locations in the state.
8927 return getAssumedNotAccessedLocation() != VALID_STATE;
8928 };
8929
8930 bool UsedAssumedInformation = false;
8931 if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInst, QueryingAA&: *this,
8932 UsedAssumedInformation))
8933 return indicatePessimisticFixpoint();
8934
8935 Changed |= AssumedState != getAssumed();
8936 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
8937 }
8938
8939 /// See AbstractAttribute::trackStatistics()
8940 void trackStatistics() const override {
8941 if (isAssumedReadNone())
8942 STATS_DECLTRACK_FN_ATTR(readnone)
8943 else if (isAssumedArgMemOnly())
8944 STATS_DECLTRACK_FN_ATTR(argmemonly)
8945 else if (isAssumedInaccessibleMemOnly())
8946 STATS_DECLTRACK_FN_ATTR(inaccessiblememonly)
8947 else if (isAssumedInaccessibleOrArgMemOnly())
8948 STATS_DECLTRACK_FN_ATTR(inaccessiblememorargmemonly)
8949 }
8950};
8951
8952/// AAMemoryLocation attribute for call sites.
8953struct AAMemoryLocationCallSite final : AAMemoryLocationImpl {
8954 AAMemoryLocationCallSite(const IRPosition &IRP, Attributor &A)
8955 : AAMemoryLocationImpl(IRP, A) {}
8956
8957 /// See AbstractAttribute::updateImpl(...).
8958 ChangeStatus updateImpl(Attributor &A) override {
8959 // TODO: Once we have call site specific value information we can provide
8960 // call site specific liveness liveness information and then it makes
8961 // sense to specialize attributes for call sites arguments instead of
8962 // redirecting requests to the callee argument.
8963 Function *F = getAssociatedFunction();
8964 const IRPosition &FnPos = IRPosition::function(F: *F);
8965 auto *FnAA =
8966 A.getAAFor<AAMemoryLocation>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
8967 if (!FnAA)
8968 return indicatePessimisticFixpoint();
8969 bool Changed = false;
8970 auto AccessPred = [&](const Instruction *I, const Value *Ptr,
8971 AccessKind Kind, MemoryLocationsKind MLK) {
8972 updateStateAndAccessesMap(State&: getState(), MLK, I, Ptr, Changed,
8973 AK: getAccessKindFromInst(I));
8974 return true;
8975 };
8976 if (!FnAA->checkForAllAccessesToMemoryKind(Pred: AccessPred, MLK: ALL_LOCATIONS))
8977 return indicatePessimisticFixpoint();
8978 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
8979 }
8980
8981 /// See AbstractAttribute::trackStatistics()
8982 void trackStatistics() const override {
8983 if (isAssumedReadNone())
8984 STATS_DECLTRACK_CS_ATTR(readnone)
8985 }
8986};
8987} // namespace
8988
8989/// ------------------ denormal-fp-math Attribute -------------------------
8990
8991namespace {
8992struct AADenormalFPMathImpl : public AADenormalFPMath {
8993 AADenormalFPMathImpl(const IRPosition &IRP, Attributor &A)
8994 : AADenormalFPMath(IRP, A) {}
8995
8996 const std::string getAsStr(Attributor *A) const override {
8997 std::string Str("AADenormalFPMath[");
8998 raw_string_ostream OS(Str);
8999
9000 DenormalState Known = getKnown();
9001 if (Known.Mode.isValid())
9002 OS << "denormal-fp-math=" << Known.Mode;
9003 else
9004 OS << "invalid";
9005
9006 if (Known.ModeF32.isValid())
9007 OS << " denormal-fp-math-f32=" << Known.ModeF32;
9008 OS << ']';
9009 return Str;
9010 }
9011};
9012
9013struct AADenormalFPMathFunction final : AADenormalFPMathImpl {
9014 AADenormalFPMathFunction(const IRPosition &IRP, Attributor &A)
9015 : AADenormalFPMathImpl(IRP, A) {}
9016
9017 void initialize(Attributor &A) override {
9018 const Function *F = getAnchorScope();
9019 DenormalMode Mode = F->getDenormalModeRaw();
9020 DenormalMode ModeF32 = F->getDenormalModeF32Raw();
9021
9022 // TODO: Handling this here prevents handling the case where a callee has a
9023 // fixed denormal-fp-math with dynamic denormal-fp-math-f32, but called from
9024 // a function with a fully fixed mode.
9025 if (ModeF32 == DenormalMode::getInvalid())
9026 ModeF32 = Mode;
9027 Known = DenormalState{.Mode: Mode, .ModeF32: ModeF32};
9028 if (isModeFixed())
9029 indicateFixpoint();
9030 }
9031
9032 ChangeStatus updateImpl(Attributor &A) override {
9033 ChangeStatus Change = ChangeStatus::UNCHANGED;
9034
9035 auto CheckCallSite = [=, &Change, &A](AbstractCallSite CS) {
9036 Function *Caller = CS.getInstruction()->getFunction();
9037 LLVM_DEBUG(dbgs() << "[AADenormalFPMath] Call " << Caller->getName()
9038 << "->" << getAssociatedFunction()->getName() << '\n');
9039
9040 const auto *CallerInfo = A.getAAFor<AADenormalFPMath>(
9041 QueryingAA: *this, IRP: IRPosition::function(F: *Caller), DepClass: DepClassTy::REQUIRED);
9042 if (!CallerInfo)
9043 return false;
9044
9045 Change = Change | clampStateAndIndicateChange(S&: this->getState(),
9046 R: CallerInfo->getState());
9047 return true;
9048 };
9049
9050 bool AllCallSitesKnown = true;
9051 if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this, RequireAllCallSites: true, UsedAssumedInformation&: AllCallSitesKnown))
9052 return indicatePessimisticFixpoint();
9053
9054 if (Change == ChangeStatus::CHANGED && isModeFixed())
9055 indicateFixpoint();
9056 return Change;
9057 }
9058
9059 ChangeStatus manifest(Attributor &A) override {
9060 LLVMContext &Ctx = getAssociatedFunction()->getContext();
9061
9062 SmallVector<Attribute, 2> AttrToAdd;
9063 SmallVector<StringRef, 2> AttrToRemove;
9064 if (Known.Mode == DenormalMode::getDefault()) {
9065 AttrToRemove.push_back(Elt: "denormal-fp-math");
9066 } else {
9067 AttrToAdd.push_back(
9068 Elt: Attribute::get(Context&: Ctx, Kind: "denormal-fp-math", Val: Known.Mode.str()));
9069 }
9070
9071 if (Known.ModeF32 != Known.Mode) {
9072 AttrToAdd.push_back(
9073 Elt: Attribute::get(Context&: Ctx, Kind: "denormal-fp-math-f32", Val: Known.ModeF32.str()));
9074 } else {
9075 AttrToRemove.push_back(Elt: "denormal-fp-math-f32");
9076 }
9077
9078 auto &IRP = getIRPosition();
9079
9080 // TODO: There should be a combined add and remove API.
9081 return A.removeAttrs(IRP, Attrs: AttrToRemove) |
9082 A.manifestAttrs(IRP, DeducedAttrs: AttrToAdd, /*ForceReplace=*/true);
9083 }
9084
9085 void trackStatistics() const override {
9086 STATS_DECLTRACK_FN_ATTR(denormal_fp_math)
9087 }
9088};
9089} // namespace
9090
9091/// ------------------ Value Constant Range Attribute -------------------------
9092
9093namespace {
9094struct AAValueConstantRangeImpl : AAValueConstantRange {
9095 using StateType = IntegerRangeState;
9096 AAValueConstantRangeImpl(const IRPosition &IRP, Attributor &A)
9097 : AAValueConstantRange(IRP, A) {}
9098
9099 /// See AbstractAttribute::initialize(..).
9100 void initialize(Attributor &A) override {
9101 if (A.hasSimplificationCallback(IRP: getIRPosition())) {
9102 indicatePessimisticFixpoint();
9103 return;
9104 }
9105
9106 // Intersect a range given by SCEV.
9107 intersectKnown(R: getConstantRangeFromSCEV(A, I: getCtxI()));
9108
9109 // Intersect a range given by LVI.
9110 intersectKnown(R: getConstantRangeFromLVI(A, CtxI: getCtxI()));
9111 }
9112
9113 /// See AbstractAttribute::getAsStr().
9114 const std::string getAsStr(Attributor *A) const override {
9115 std::string Str;
9116 llvm::raw_string_ostream OS(Str);
9117 OS << "range(" << getBitWidth() << ")<";
9118 getKnown().print(OS);
9119 OS << " / ";
9120 getAssumed().print(OS);
9121 OS << ">";
9122 return Str;
9123 }
9124
9125 /// Helper function to get a SCEV expr for the associated value at program
9126 /// point \p I.
9127 const SCEV *getSCEV(Attributor &A, const Instruction *I = nullptr) const {
9128 if (!getAnchorScope())
9129 return nullptr;
9130
9131 ScalarEvolution *SE =
9132 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
9133 F: *getAnchorScope());
9134
9135 LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(
9136 F: *getAnchorScope());
9137
9138 if (!SE || !LI)
9139 return nullptr;
9140
9141 const SCEV *S = SE->getSCEV(V: &getAssociatedValue());
9142 if (!I)
9143 return S;
9144
9145 return SE->getSCEVAtScope(S, L: LI->getLoopFor(BB: I->getParent()));
9146 }
9147
9148 /// Helper function to get a range from SCEV for the associated value at
9149 /// program point \p I.
9150 ConstantRange getConstantRangeFromSCEV(Attributor &A,
9151 const Instruction *I = nullptr) const {
9152 if (!getAnchorScope())
9153 return getWorstState(BitWidth: getBitWidth());
9154
9155 ScalarEvolution *SE =
9156 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
9157 F: *getAnchorScope());
9158
9159 const SCEV *S = getSCEV(A, I);
9160 if (!SE || !S)
9161 return getWorstState(BitWidth: getBitWidth());
9162
9163 return SE->getUnsignedRange(S);
9164 }
9165
9166 /// Helper function to get a range from LVI for the associated value at
9167 /// program point \p I.
9168 ConstantRange
9169 getConstantRangeFromLVI(Attributor &A,
9170 const Instruction *CtxI = nullptr) const {
9171 if (!getAnchorScope())
9172 return getWorstState(BitWidth: getBitWidth());
9173
9174 LazyValueInfo *LVI =
9175 A.getInfoCache().getAnalysisResultForFunction<LazyValueAnalysis>(
9176 F: *getAnchorScope());
9177
9178 if (!LVI || !CtxI)
9179 return getWorstState(BitWidth: getBitWidth());
9180 return LVI->getConstantRange(V: &getAssociatedValue(),
9181 CxtI: const_cast<Instruction *>(CtxI),
9182 /*UndefAllowed*/ false);
9183 }
9184
9185 /// Return true if \p CtxI is valid for querying outside analyses.
9186 /// This basically makes sure we do not ask intra-procedural analysis
9187 /// about a context in the wrong function or a context that violates
9188 /// dominance assumptions they might have. The \p AllowAACtxI flag indicates
9189 /// if the original context of this AA is OK or should be considered invalid.
9190 bool isValidCtxInstructionForOutsideAnalysis(Attributor &A,
9191 const Instruction *CtxI,
9192 bool AllowAACtxI) const {
9193 if (!CtxI || (!AllowAACtxI && CtxI == getCtxI()))
9194 return false;
9195
9196 // Our context might be in a different function, neither intra-procedural
9197 // analysis (ScalarEvolution nor LazyValueInfo) can handle that.
9198 if (!AA::isValidInScope(V: getAssociatedValue(), Scope: CtxI->getFunction()))
9199 return false;
9200
9201 // If the context is not dominated by the value there are paths to the
9202 // context that do not define the value. This cannot be handled by
9203 // LazyValueInfo so we need to bail.
9204 if (auto *I = dyn_cast<Instruction>(Val: &getAssociatedValue())) {
9205 InformationCache &InfoCache = A.getInfoCache();
9206 const DominatorTree *DT =
9207 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(
9208 F: *I->getFunction());
9209 return DT && DT->dominates(Def: I, User: CtxI);
9210 }
9211
9212 return true;
9213 }
9214
9215 /// See AAValueConstantRange::getKnownConstantRange(..).
9216 ConstantRange
9217 getKnownConstantRange(Attributor &A,
9218 const Instruction *CtxI = nullptr) const override {
9219 if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
9220 /* AllowAACtxI */ false))
9221 return getKnown();
9222
9223 ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
9224 ConstantRange SCEVR = getConstantRangeFromSCEV(A, I: CtxI);
9225 return getKnown().intersectWith(CR: SCEVR).intersectWith(CR: LVIR);
9226 }
9227
9228 /// See AAValueConstantRange::getAssumedConstantRange(..).
9229 ConstantRange
9230 getAssumedConstantRange(Attributor &A,
9231 const Instruction *CtxI = nullptr) const override {
9232 // TODO: Make SCEV use Attributor assumption.
9233 // We may be able to bound a variable range via assumptions in
9234 // Attributor. ex.) If x is assumed to be in [1, 3] and y is known to
9235 // evolve to x^2 + x, then we can say that y is in [2, 12].
9236 if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
9237 /* AllowAACtxI */ false))
9238 return getAssumed();
9239
9240 ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
9241 ConstantRange SCEVR = getConstantRangeFromSCEV(A, I: CtxI);
9242 return getAssumed().intersectWith(CR: SCEVR).intersectWith(CR: LVIR);
9243 }
9244
9245 /// Helper function to create MDNode for range metadata.
9246 static MDNode *
9247 getMDNodeForConstantRange(Type *Ty, LLVMContext &Ctx,
9248 const ConstantRange &AssumedConstantRange) {
9249 Metadata *LowAndHigh[] = {ConstantAsMetadata::get(C: ConstantInt::get(
9250 Ty, V: AssumedConstantRange.getLower())),
9251 ConstantAsMetadata::get(C: ConstantInt::get(
9252 Ty, V: AssumedConstantRange.getUpper()))};
9253 return MDNode::get(Context&: Ctx, MDs: LowAndHigh);
9254 }
9255
9256 /// Return true if \p Assumed is included in ranges from instruction \p I.
9257 static bool isBetterRange(const ConstantRange &Assumed,
9258 const Instruction &I) {
9259 if (Assumed.isFullSet())
9260 return false;
9261
9262 std::optional<ConstantRange> Known;
9263
9264 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
9265 Known = CB->getRange();
9266 } else if (MDNode *KnownRanges = I.getMetadata(KindID: LLVMContext::MD_range)) {
9267 // If multiple ranges are annotated in IR, we give up to annotate assumed
9268 // range for now.
9269
9270 // TODO: If there exists a known range which containts assumed range, we
9271 // can say assumed range is better.
9272 if (KnownRanges->getNumOperands() > 2)
9273 return false;
9274
9275 ConstantInt *Lower =
9276 mdconst::extract<ConstantInt>(MD: KnownRanges->getOperand(I: 0));
9277 ConstantInt *Upper =
9278 mdconst::extract<ConstantInt>(MD: KnownRanges->getOperand(I: 1));
9279
9280 Known.emplace(args: Lower->getValue(), args: Upper->getValue());
9281 }
9282 return !Known || (*Known != Assumed && Known->contains(CR: Assumed));
9283 }
9284
9285 /// Helper function to set range metadata.
9286 static bool
9287 setRangeMetadataIfisBetterRange(Instruction *I,
9288 const ConstantRange &AssumedConstantRange) {
9289 if (isBetterRange(Assumed: AssumedConstantRange, I: *I)) {
9290 I->setMetadata(KindID: LLVMContext::MD_range,
9291 Node: getMDNodeForConstantRange(Ty: I->getType(), Ctx&: I->getContext(),
9292 AssumedConstantRange));
9293 return true;
9294 }
9295 return false;
9296 }
9297 /// Helper function to set range return attribute.
9298 static bool
9299 setRangeRetAttrIfisBetterRange(Attributor &A, const IRPosition &IRP,
9300 Instruction *I,
9301 const ConstantRange &AssumedConstantRange) {
9302 if (isBetterRange(Assumed: AssumedConstantRange, I: *I)) {
9303 A.manifestAttrs(IRP,
9304 DeducedAttrs: Attribute::get(Context&: I->getContext(), Kind: Attribute::Range,
9305 CR: AssumedConstantRange),
9306 /*ForceReplace*/ true);
9307 return true;
9308 }
9309 return false;
9310 }
9311
9312 /// See AbstractAttribute::manifest()
9313 ChangeStatus manifest(Attributor &A) override {
9314 ChangeStatus Changed = ChangeStatus::UNCHANGED;
9315 ConstantRange AssumedConstantRange = getAssumedConstantRange(A);
9316 assert(!AssumedConstantRange.isFullSet() && "Invalid state");
9317
9318 auto &V = getAssociatedValue();
9319 if (!AssumedConstantRange.isEmptySet() &&
9320 !AssumedConstantRange.isSingleElement()) {
9321 if (Instruction *I = dyn_cast<Instruction>(Val: &V)) {
9322 assert(I == getCtxI() && "Should not annotate an instruction which is "
9323 "not the context instruction");
9324 if (isa<LoadInst>(Val: I))
9325 if (setRangeMetadataIfisBetterRange(I, AssumedConstantRange))
9326 Changed = ChangeStatus::CHANGED;
9327 if (isa<CallInst>(Val: I))
9328 if (setRangeRetAttrIfisBetterRange(A, IRP: getIRPosition(), I,
9329 AssumedConstantRange))
9330 Changed = ChangeStatus::CHANGED;
9331 }
9332 }
9333
9334 return Changed;
9335 }
9336};
9337
9338struct AAValueConstantRangeArgument final
9339 : AAArgumentFromCallSiteArguments<
9340 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9341 true /* BridgeCallBaseContext */> {
9342 using Base = AAArgumentFromCallSiteArguments<
9343 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9344 true /* BridgeCallBaseContext */>;
9345 AAValueConstantRangeArgument(const IRPosition &IRP, Attributor &A)
9346 : Base(IRP, A) {}
9347
9348 /// See AbstractAttribute::trackStatistics()
9349 void trackStatistics() const override {
9350 STATS_DECLTRACK_ARG_ATTR(value_range)
9351 }
9352};
9353
9354struct AAValueConstantRangeReturned
9355 : AAReturnedFromReturnedValues<AAValueConstantRange,
9356 AAValueConstantRangeImpl,
9357 AAValueConstantRangeImpl::StateType,
9358 /* PropagateCallBaseContext */ true> {
9359 using Base =
9360 AAReturnedFromReturnedValues<AAValueConstantRange,
9361 AAValueConstantRangeImpl,
9362 AAValueConstantRangeImpl::StateType,
9363 /* PropagateCallBaseContext */ true>;
9364 AAValueConstantRangeReturned(const IRPosition &IRP, Attributor &A)
9365 : Base(IRP, A) {}
9366
9367 /// See AbstractAttribute::initialize(...).
9368 void initialize(Attributor &A) override {
9369 if (!A.isFunctionIPOAmendable(F: *getAssociatedFunction()))
9370 indicatePessimisticFixpoint();
9371 }
9372
9373 /// See AbstractAttribute::trackStatistics()
9374 void trackStatistics() const override {
9375 STATS_DECLTRACK_FNRET_ATTR(value_range)
9376 }
9377};
9378
9379struct AAValueConstantRangeFloating : AAValueConstantRangeImpl {
9380 AAValueConstantRangeFloating(const IRPosition &IRP, Attributor &A)
9381 : AAValueConstantRangeImpl(IRP, A) {}
9382
9383 /// See AbstractAttribute::initialize(...).
9384 void initialize(Attributor &A) override {
9385 AAValueConstantRangeImpl::initialize(A);
9386 if (isAtFixpoint())
9387 return;
9388
9389 Value &V = getAssociatedValue();
9390
9391 if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
9392 unionAssumed(R: ConstantRange(C->getValue()));
9393 indicateOptimisticFixpoint();
9394 return;
9395 }
9396
9397 if (isa<UndefValue>(Val: &V)) {
9398 // Collapse the undef state to 0.
9399 unionAssumed(R: ConstantRange(APInt(getBitWidth(), 0)));
9400 indicateOptimisticFixpoint();
9401 return;
9402 }
9403
9404 if (isa<CallBase>(Val: &V))
9405 return;
9406
9407 if (isa<BinaryOperator>(Val: &V) || isa<CmpInst>(Val: &V) || isa<CastInst>(Val: &V))
9408 return;
9409
9410 // If it is a load instruction with range metadata, use it.
9411 if (LoadInst *LI = dyn_cast<LoadInst>(Val: &V))
9412 if (auto *RangeMD = LI->getMetadata(KindID: LLVMContext::MD_range)) {
9413 intersectKnown(R: getConstantRangeFromMetadata(RangeMD: *RangeMD));
9414 return;
9415 }
9416
9417 // We can work with PHI and select instruction as we traverse their operands
9418 // during update.
9419 if (isa<SelectInst>(Val: V) || isa<PHINode>(Val: V))
9420 return;
9421
9422 // Otherwise we give up.
9423 indicatePessimisticFixpoint();
9424
9425 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] We give up: "
9426 << getAssociatedValue() << "\n");
9427 }
9428
9429 bool calculateBinaryOperator(
9430 Attributor &A, BinaryOperator *BinOp, IntegerRangeState &T,
9431 const Instruction *CtxI,
9432 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9433 Value *LHS = BinOp->getOperand(i_nocapture: 0);
9434 Value *RHS = BinOp->getOperand(i_nocapture: 1);
9435
9436 // Simplify the operands first.
9437 bool UsedAssumedInformation = false;
9438 const auto &SimplifiedLHS = A.getAssumedSimplified(
9439 IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()), AA: *this,
9440 UsedAssumedInformation, S: AA::Interprocedural);
9441 if (!SimplifiedLHS.has_value())
9442 return true;
9443 if (!*SimplifiedLHS)
9444 return false;
9445 LHS = *SimplifiedLHS;
9446
9447 const auto &SimplifiedRHS = A.getAssumedSimplified(
9448 IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()), AA: *this,
9449 UsedAssumedInformation, S: AA::Interprocedural);
9450 if (!SimplifiedRHS.has_value())
9451 return true;
9452 if (!*SimplifiedRHS)
9453 return false;
9454 RHS = *SimplifiedRHS;
9455
9456 // TODO: Allow non integers as well.
9457 if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9458 return false;
9459
9460 auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9461 QueryingAA: *this, IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()),
9462 DepClass: DepClassTy::REQUIRED);
9463 if (!LHSAA)
9464 return false;
9465 QuerriedAAs.push_back(Elt: LHSAA);
9466 auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9467
9468 auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9469 QueryingAA: *this, IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()),
9470 DepClass: DepClassTy::REQUIRED);
9471 if (!RHSAA)
9472 return false;
9473 QuerriedAAs.push_back(Elt: RHSAA);
9474 auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9475
9476 auto AssumedRange = LHSAARange.binaryOp(BinOp: BinOp->getOpcode(), Other: RHSAARange);
9477
9478 T.unionAssumed(R: AssumedRange);
9479
9480 // TODO: Track a known state too.
9481
9482 return T.isValidState();
9483 }
9484
9485 bool calculateCastInst(
9486 Attributor &A, CastInst *CastI, IntegerRangeState &T,
9487 const Instruction *CtxI,
9488 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9489 assert(CastI->getNumOperands() == 1 && "Expected cast to be unary!");
9490 // TODO: Allow non integers as well.
9491 Value *OpV = CastI->getOperand(i_nocapture: 0);
9492
9493 // Simplify the operand first.
9494 bool UsedAssumedInformation = false;
9495 const auto &SimplifiedOpV = A.getAssumedSimplified(
9496 IRP: IRPosition::value(V: *OpV, CBContext: getCallBaseContext()), AA: *this,
9497 UsedAssumedInformation, S: AA::Interprocedural);
9498 if (!SimplifiedOpV.has_value())
9499 return true;
9500 if (!*SimplifiedOpV)
9501 return false;
9502 OpV = *SimplifiedOpV;
9503
9504 if (!OpV->getType()->isIntegerTy())
9505 return false;
9506
9507 auto *OpAA = A.getAAFor<AAValueConstantRange>(
9508 QueryingAA: *this, IRP: IRPosition::value(V: *OpV, CBContext: getCallBaseContext()),
9509 DepClass: DepClassTy::REQUIRED);
9510 if (!OpAA)
9511 return false;
9512 QuerriedAAs.push_back(Elt: OpAA);
9513 T.unionAssumed(R: OpAA->getAssumed().castOp(CastOp: CastI->getOpcode(),
9514 BitWidth: getState().getBitWidth()));
9515 return T.isValidState();
9516 }
9517
9518 bool
9519 calculateCmpInst(Attributor &A, CmpInst *CmpI, IntegerRangeState &T,
9520 const Instruction *CtxI,
9521 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9522 Value *LHS = CmpI->getOperand(i_nocapture: 0);
9523 Value *RHS = CmpI->getOperand(i_nocapture: 1);
9524
9525 // Simplify the operands first.
9526 bool UsedAssumedInformation = false;
9527 const auto &SimplifiedLHS = A.getAssumedSimplified(
9528 IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()), AA: *this,
9529 UsedAssumedInformation, S: AA::Interprocedural);
9530 if (!SimplifiedLHS.has_value())
9531 return true;
9532 if (!*SimplifiedLHS)
9533 return false;
9534 LHS = *SimplifiedLHS;
9535
9536 const auto &SimplifiedRHS = A.getAssumedSimplified(
9537 IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()), AA: *this,
9538 UsedAssumedInformation, S: AA::Interprocedural);
9539 if (!SimplifiedRHS.has_value())
9540 return true;
9541 if (!*SimplifiedRHS)
9542 return false;
9543 RHS = *SimplifiedRHS;
9544
9545 // TODO: Allow non integers as well.
9546 if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9547 return false;
9548
9549 auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9550 QueryingAA: *this, IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()),
9551 DepClass: DepClassTy::REQUIRED);
9552 if (!LHSAA)
9553 return false;
9554 QuerriedAAs.push_back(Elt: LHSAA);
9555 auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9556 QueryingAA: *this, IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()),
9557 DepClass: DepClassTy::REQUIRED);
9558 if (!RHSAA)
9559 return false;
9560 QuerriedAAs.push_back(Elt: RHSAA);
9561 auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9562 auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9563
9564 // If one of them is empty set, we can't decide.
9565 if (LHSAARange.isEmptySet() || RHSAARange.isEmptySet())
9566 return true;
9567
9568 bool MustTrue = false, MustFalse = false;
9569
9570 auto AllowedRegion =
9571 ConstantRange::makeAllowedICmpRegion(Pred: CmpI->getPredicate(), Other: RHSAARange);
9572
9573 if (AllowedRegion.intersectWith(CR: LHSAARange).isEmptySet())
9574 MustFalse = true;
9575
9576 if (LHSAARange.icmp(Pred: CmpI->getPredicate(), Other: RHSAARange))
9577 MustTrue = true;
9578
9579 assert((!MustTrue || !MustFalse) &&
9580 "Either MustTrue or MustFalse should be false!");
9581
9582 if (MustTrue)
9583 T.unionAssumed(R: ConstantRange(APInt(/* numBits */ 1, /* val */ 1)));
9584 else if (MustFalse)
9585 T.unionAssumed(R: ConstantRange(APInt(/* numBits */ 1, /* val */ 0)));
9586 else
9587 T.unionAssumed(R: ConstantRange(/* BitWidth */ 1, /* isFullSet */ true));
9588
9589 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] " << *CmpI << " after "
9590 << (MustTrue ? "true" : (MustFalse ? "false" : "unknown"))
9591 << ": " << T << "\n\t" << *LHSAA << "\t<op>\n\t"
9592 << *RHSAA);
9593
9594 // TODO: Track a known state too.
9595 return T.isValidState();
9596 }
9597
9598 /// See AbstractAttribute::updateImpl(...).
9599 ChangeStatus updateImpl(Attributor &A) override {
9600
9601 IntegerRangeState T(getBitWidth());
9602 auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
9603 Instruction *I = dyn_cast<Instruction>(Val: &V);
9604 if (!I || isa<CallBase>(Val: I)) {
9605
9606 // Simplify the operand first.
9607 bool UsedAssumedInformation = false;
9608 const auto &SimplifiedOpV = A.getAssumedSimplified(
9609 IRP: IRPosition::value(V, CBContext: getCallBaseContext()), AA: *this,
9610 UsedAssumedInformation, S: AA::Interprocedural);
9611 if (!SimplifiedOpV.has_value())
9612 return true;
9613 if (!*SimplifiedOpV)
9614 return false;
9615 Value *VPtr = *SimplifiedOpV;
9616
9617 // If the value is not instruction, we query AA to Attributor.
9618 const auto *AA = A.getAAFor<AAValueConstantRange>(
9619 QueryingAA: *this, IRP: IRPosition::value(V: *VPtr, CBContext: getCallBaseContext()),
9620 DepClass: DepClassTy::REQUIRED);
9621
9622 // Clamp operator is not used to utilize a program point CtxI.
9623 if (AA)
9624 T.unionAssumed(R: AA->getAssumedConstantRange(A, CtxI));
9625 else
9626 return false;
9627
9628 return T.isValidState();
9629 }
9630
9631 SmallVector<const AAValueConstantRange *, 4> QuerriedAAs;
9632 if (auto *BinOp = dyn_cast<BinaryOperator>(Val: I)) {
9633 if (!calculateBinaryOperator(A, BinOp, T, CtxI, QuerriedAAs))
9634 return false;
9635 } else if (auto *CmpI = dyn_cast<CmpInst>(Val: I)) {
9636 if (!calculateCmpInst(A, CmpI, T, CtxI, QuerriedAAs))
9637 return false;
9638 } else if (auto *CastI = dyn_cast<CastInst>(Val: I)) {
9639 if (!calculateCastInst(A, CastI, T, CtxI, QuerriedAAs))
9640 return false;
9641 } else {
9642 // Give up with other instructions.
9643 // TODO: Add other instructions
9644
9645 T.indicatePessimisticFixpoint();
9646 return false;
9647 }
9648
9649 // Catch circular reasoning in a pessimistic way for now.
9650 // TODO: Check how the range evolves and if we stripped anything, see also
9651 // AADereferenceable or AAAlign for similar situations.
9652 for (const AAValueConstantRange *QueriedAA : QuerriedAAs) {
9653 if (QueriedAA != this)
9654 continue;
9655 // If we are in a stady state we do not need to worry.
9656 if (T.getAssumed() == getState().getAssumed())
9657 continue;
9658 T.indicatePessimisticFixpoint();
9659 }
9660
9661 return T.isValidState();
9662 };
9663
9664 if (!VisitValueCB(getAssociatedValue(), getCtxI()))
9665 return indicatePessimisticFixpoint();
9666
9667 // Ensure that long def-use chains can't cause circular reasoning either by
9668 // introducing a cutoff below.
9669 if (clampStateAndIndicateChange(S&: getState(), R: T) == ChangeStatus::UNCHANGED)
9670 return ChangeStatus::UNCHANGED;
9671 if (++NumChanges > MaxNumChanges) {
9672 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] performed " << NumChanges
9673 << " but only " << MaxNumChanges
9674 << " are allowed to avoid cyclic reasoning.");
9675 return indicatePessimisticFixpoint();
9676 }
9677 return ChangeStatus::CHANGED;
9678 }
9679
9680 /// See AbstractAttribute::trackStatistics()
9681 void trackStatistics() const override {
9682 STATS_DECLTRACK_FLOATING_ATTR(value_range)
9683 }
9684
9685 /// Tracker to bail after too many widening steps of the constant range.
9686 int NumChanges = 0;
9687
9688 /// Upper bound for the number of allowed changes (=widening steps) for the
9689 /// constant range before we give up.
9690 static constexpr int MaxNumChanges = 5;
9691};
9692
9693struct AAValueConstantRangeFunction : AAValueConstantRangeImpl {
9694 AAValueConstantRangeFunction(const IRPosition &IRP, Attributor &A)
9695 : AAValueConstantRangeImpl(IRP, A) {}
9696
9697 /// See AbstractAttribute::initialize(...).
9698 ChangeStatus updateImpl(Attributor &A) override {
9699 llvm_unreachable("AAValueConstantRange(Function|CallSite)::updateImpl will "
9700 "not be called");
9701 }
9702
9703 /// See AbstractAttribute::trackStatistics()
9704 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(value_range) }
9705};
9706
9707struct AAValueConstantRangeCallSite : AAValueConstantRangeFunction {
9708 AAValueConstantRangeCallSite(const IRPosition &IRP, Attributor &A)
9709 : AAValueConstantRangeFunction(IRP, A) {}
9710
9711 /// See AbstractAttribute::trackStatistics()
9712 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(value_range) }
9713};
9714
9715struct AAValueConstantRangeCallSiteReturned
9716 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9717 AAValueConstantRangeImpl::StateType,
9718 /* IntroduceCallBaseContext */ true> {
9719 AAValueConstantRangeCallSiteReturned(const IRPosition &IRP, Attributor &A)
9720 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9721 AAValueConstantRangeImpl::StateType,
9722 /* IntroduceCallBaseContext */ true>(IRP, A) {}
9723
9724 /// See AbstractAttribute::initialize(...).
9725 void initialize(Attributor &A) override {
9726 // If it is a call instruction with range attribute, use the range.
9727 if (CallInst *CI = dyn_cast<CallInst>(Val: &getAssociatedValue())) {
9728 if (std::optional<ConstantRange> Range = CI->getRange())
9729 intersectKnown(R: *Range);
9730 }
9731
9732 AAValueConstantRangeImpl::initialize(A);
9733 }
9734
9735 /// See AbstractAttribute::trackStatistics()
9736 void trackStatistics() const override {
9737 STATS_DECLTRACK_CSRET_ATTR(value_range)
9738 }
9739};
9740struct AAValueConstantRangeCallSiteArgument : AAValueConstantRangeFloating {
9741 AAValueConstantRangeCallSiteArgument(const IRPosition &IRP, Attributor &A)
9742 : AAValueConstantRangeFloating(IRP, A) {}
9743
9744 /// See AbstractAttribute::manifest()
9745 ChangeStatus manifest(Attributor &A) override {
9746 return ChangeStatus::UNCHANGED;
9747 }
9748
9749 /// See AbstractAttribute::trackStatistics()
9750 void trackStatistics() const override {
9751 STATS_DECLTRACK_CSARG_ATTR(value_range)
9752 }
9753};
9754} // namespace
9755
9756/// ------------------ Potential Values Attribute -------------------------
9757
9758namespace {
9759struct AAPotentialConstantValuesImpl : AAPotentialConstantValues {
9760 using StateType = PotentialConstantIntValuesState;
9761
9762 AAPotentialConstantValuesImpl(const IRPosition &IRP, Attributor &A)
9763 : AAPotentialConstantValues(IRP, A) {}
9764
9765 /// See AbstractAttribute::initialize(..).
9766 void initialize(Attributor &A) override {
9767 if (A.hasSimplificationCallback(IRP: getIRPosition()))
9768 indicatePessimisticFixpoint();
9769 else
9770 AAPotentialConstantValues::initialize(A);
9771 }
9772
9773 bool fillSetWithConstantValues(Attributor &A, const IRPosition &IRP, SetTy &S,
9774 bool &ContainsUndef, bool ForSelf) {
9775 SmallVector<AA::ValueAndContext> Values;
9776 bool UsedAssumedInformation = false;
9777 if (!A.getAssumedSimplifiedValues(IRP, AA: *this, Values, S: AA::Interprocedural,
9778 UsedAssumedInformation)) {
9779 // Avoid recursion when the caller is computing constant values for this
9780 // IRP itself.
9781 if (ForSelf)
9782 return false;
9783 if (!IRP.getAssociatedType()->isIntegerTy())
9784 return false;
9785 auto *PotentialValuesAA = A.getAAFor<AAPotentialConstantValues>(
9786 QueryingAA: *this, IRP, DepClass: DepClassTy::REQUIRED);
9787 if (!PotentialValuesAA || !PotentialValuesAA->getState().isValidState())
9788 return false;
9789 ContainsUndef = PotentialValuesAA->getState().undefIsContained();
9790 S = PotentialValuesAA->getState().getAssumedSet();
9791 return true;
9792 }
9793
9794 // Copy all the constant values, except UndefValue. ContainsUndef is true
9795 // iff Values contains only UndefValue instances. If there are other known
9796 // constants, then UndefValue is dropped.
9797 ContainsUndef = false;
9798 for (auto &It : Values) {
9799 if (isa<UndefValue>(Val: It.getValue())) {
9800 ContainsUndef = true;
9801 continue;
9802 }
9803 auto *CI = dyn_cast<ConstantInt>(Val: It.getValue());
9804 if (!CI)
9805 return false;
9806 S.insert(X: CI->getValue());
9807 }
9808 ContainsUndef &= S.empty();
9809
9810 return true;
9811 }
9812
9813 /// See AbstractAttribute::getAsStr().
9814 const std::string getAsStr(Attributor *A) const override {
9815 std::string Str;
9816 llvm::raw_string_ostream OS(Str);
9817 OS << getState();
9818 return Str;
9819 }
9820
9821 /// See AbstractAttribute::updateImpl(...).
9822 ChangeStatus updateImpl(Attributor &A) override {
9823 return indicatePessimisticFixpoint();
9824 }
9825};
9826
9827struct AAPotentialConstantValuesArgument final
9828 : AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9829 AAPotentialConstantValuesImpl,
9830 PotentialConstantIntValuesState> {
9831 using Base = AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9832 AAPotentialConstantValuesImpl,
9833 PotentialConstantIntValuesState>;
9834 AAPotentialConstantValuesArgument(const IRPosition &IRP, Attributor &A)
9835 : Base(IRP, A) {}
9836
9837 /// See AbstractAttribute::trackStatistics()
9838 void trackStatistics() const override {
9839 STATS_DECLTRACK_ARG_ATTR(potential_values)
9840 }
9841};
9842
9843struct AAPotentialConstantValuesReturned
9844 : AAReturnedFromReturnedValues<AAPotentialConstantValues,
9845 AAPotentialConstantValuesImpl> {
9846 using Base = AAReturnedFromReturnedValues<AAPotentialConstantValues,
9847 AAPotentialConstantValuesImpl>;
9848 AAPotentialConstantValuesReturned(const IRPosition &IRP, Attributor &A)
9849 : Base(IRP, A) {}
9850
9851 void initialize(Attributor &A) override {
9852 if (!A.isFunctionIPOAmendable(F: *getAssociatedFunction()))
9853 indicatePessimisticFixpoint();
9854 Base::initialize(A);
9855 }
9856
9857 /// See AbstractAttribute::trackStatistics()
9858 void trackStatistics() const override {
9859 STATS_DECLTRACK_FNRET_ATTR(potential_values)
9860 }
9861};
9862
9863struct AAPotentialConstantValuesFloating : AAPotentialConstantValuesImpl {
9864 AAPotentialConstantValuesFloating(const IRPosition &IRP, Attributor &A)
9865 : AAPotentialConstantValuesImpl(IRP, A) {}
9866
9867 /// See AbstractAttribute::initialize(..).
9868 void initialize(Attributor &A) override {
9869 AAPotentialConstantValuesImpl::initialize(A);
9870 if (isAtFixpoint())
9871 return;
9872
9873 Value &V = getAssociatedValue();
9874
9875 if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
9876 unionAssumed(C: C->getValue());
9877 indicateOptimisticFixpoint();
9878 return;
9879 }
9880
9881 if (isa<UndefValue>(Val: &V)) {
9882 unionAssumedWithUndef();
9883 indicateOptimisticFixpoint();
9884 return;
9885 }
9886
9887 if (isa<BinaryOperator>(Val: &V) || isa<ICmpInst>(Val: &V) || isa<CastInst>(Val: &V))
9888 return;
9889
9890 if (isa<SelectInst>(Val: V) || isa<PHINode>(Val: V) || isa<LoadInst>(Val: V))
9891 return;
9892
9893 indicatePessimisticFixpoint();
9894
9895 LLVM_DEBUG(dbgs() << "[AAPotentialConstantValues] We give up: "
9896 << getAssociatedValue() << "\n");
9897 }
9898
9899 static bool calculateICmpInst(const ICmpInst *ICI, const APInt &LHS,
9900 const APInt &RHS) {
9901 return ICmpInst::compare(LHS, RHS, Pred: ICI->getPredicate());
9902 }
9903
9904 static APInt calculateCastInst(const CastInst *CI, const APInt &Src,
9905 uint32_t ResultBitWidth) {
9906 Instruction::CastOps CastOp = CI->getOpcode();
9907 switch (CastOp) {
9908 default:
9909 llvm_unreachable("unsupported or not integer cast");
9910 case Instruction::Trunc:
9911 return Src.trunc(width: ResultBitWidth);
9912 case Instruction::SExt:
9913 return Src.sext(width: ResultBitWidth);
9914 case Instruction::ZExt:
9915 return Src.zext(width: ResultBitWidth);
9916 case Instruction::BitCast:
9917 return Src;
9918 }
9919 }
9920
9921 static APInt calculateBinaryOperator(const BinaryOperator *BinOp,
9922 const APInt &LHS, const APInt &RHS,
9923 bool &SkipOperation, bool &Unsupported) {
9924 Instruction::BinaryOps BinOpcode = BinOp->getOpcode();
9925 // Unsupported is set to true when the binary operator is not supported.
9926 // SkipOperation is set to true when UB occur with the given operand pair
9927 // (LHS, RHS).
9928 // TODO: we should look at nsw and nuw keywords to handle operations
9929 // that create poison or undef value.
9930 switch (BinOpcode) {
9931 default:
9932 Unsupported = true;
9933 return LHS;
9934 case Instruction::Add:
9935 return LHS + RHS;
9936 case Instruction::Sub:
9937 return LHS - RHS;
9938 case Instruction::Mul:
9939 return LHS * RHS;
9940 case Instruction::UDiv:
9941 if (RHS.isZero()) {
9942 SkipOperation = true;
9943 return LHS;
9944 }
9945 return LHS.udiv(RHS);
9946 case Instruction::SDiv:
9947 if (RHS.isZero()) {
9948 SkipOperation = true;
9949 return LHS;
9950 }
9951 return LHS.sdiv(RHS);
9952 case Instruction::URem:
9953 if (RHS.isZero()) {
9954 SkipOperation = true;
9955 return LHS;
9956 }
9957 return LHS.urem(RHS);
9958 case Instruction::SRem:
9959 if (RHS.isZero()) {
9960 SkipOperation = true;
9961 return LHS;
9962 }
9963 return LHS.srem(RHS);
9964 case Instruction::Shl:
9965 return LHS.shl(ShiftAmt: RHS);
9966 case Instruction::LShr:
9967 return LHS.lshr(ShiftAmt: RHS);
9968 case Instruction::AShr:
9969 return LHS.ashr(ShiftAmt: RHS);
9970 case Instruction::And:
9971 return LHS & RHS;
9972 case Instruction::Or:
9973 return LHS | RHS;
9974 case Instruction::Xor:
9975 return LHS ^ RHS;
9976 }
9977 }
9978
9979 bool calculateBinaryOperatorAndTakeUnion(const BinaryOperator *BinOp,
9980 const APInt &LHS, const APInt &RHS) {
9981 bool SkipOperation = false;
9982 bool Unsupported = false;
9983 APInt Result =
9984 calculateBinaryOperator(BinOp, LHS, RHS, SkipOperation, Unsupported);
9985 if (Unsupported)
9986 return false;
9987 // If SkipOperation is true, we can ignore this operand pair (L, R).
9988 if (!SkipOperation)
9989 unionAssumed(C: Result);
9990 return isValidState();
9991 }
9992
9993 ChangeStatus updateWithICmpInst(Attributor &A, ICmpInst *ICI) {
9994 auto AssumedBefore = getAssumed();
9995 Value *LHS = ICI->getOperand(i_nocapture: 0);
9996 Value *RHS = ICI->getOperand(i_nocapture: 1);
9997
9998 bool LHSContainsUndef = false, RHSContainsUndef = false;
9999 SetTy LHSAAPVS, RHSAAPVS;
10000 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
10001 ContainsUndef&: LHSContainsUndef, /* ForSelf */ false) ||
10002 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
10003 ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
10004 return indicatePessimisticFixpoint();
10005
10006 // TODO: make use of undef flag to limit potential values aggressively.
10007 bool MaybeTrue = false, MaybeFalse = false;
10008 const APInt Zero(RHS->getType()->getIntegerBitWidth(), 0);
10009 if (LHSContainsUndef && RHSContainsUndef) {
10010 // The result of any comparison between undefs can be soundly replaced
10011 // with undef.
10012 unionAssumedWithUndef();
10013 } else if (LHSContainsUndef) {
10014 for (const APInt &R : RHSAAPVS) {
10015 bool CmpResult = calculateICmpInst(ICI, LHS: Zero, RHS: R);
10016 MaybeTrue |= CmpResult;
10017 MaybeFalse |= !CmpResult;
10018 if (MaybeTrue & MaybeFalse)
10019 return indicatePessimisticFixpoint();
10020 }
10021 } else if (RHSContainsUndef) {
10022 for (const APInt &L : LHSAAPVS) {
10023 bool CmpResult = calculateICmpInst(ICI, LHS: L, RHS: Zero);
10024 MaybeTrue |= CmpResult;
10025 MaybeFalse |= !CmpResult;
10026 if (MaybeTrue & MaybeFalse)
10027 return indicatePessimisticFixpoint();
10028 }
10029 } else {
10030 for (const APInt &L : LHSAAPVS) {
10031 for (const APInt &R : RHSAAPVS) {
10032 bool CmpResult = calculateICmpInst(ICI, LHS: L, RHS: R);
10033 MaybeTrue |= CmpResult;
10034 MaybeFalse |= !CmpResult;
10035 if (MaybeTrue & MaybeFalse)
10036 return indicatePessimisticFixpoint();
10037 }
10038 }
10039 }
10040 if (MaybeTrue)
10041 unionAssumed(C: APInt(/* numBits */ 1, /* val */ 1));
10042 if (MaybeFalse)
10043 unionAssumed(C: APInt(/* numBits */ 1, /* val */ 0));
10044 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10045 : ChangeStatus::CHANGED;
10046 }
10047
10048 ChangeStatus updateWithSelectInst(Attributor &A, SelectInst *SI) {
10049 auto AssumedBefore = getAssumed();
10050 Value *LHS = SI->getTrueValue();
10051 Value *RHS = SI->getFalseValue();
10052
10053 bool UsedAssumedInformation = false;
10054 std::optional<Constant *> C = A.getAssumedConstant(
10055 V: *SI->getCondition(), AA: *this, UsedAssumedInformation);
10056
10057 // Check if we only need one operand.
10058 bool OnlyLeft = false, OnlyRight = false;
10059 if (C && *C && (*C)->isOneValue())
10060 OnlyLeft = true;
10061 else if (C && *C && (*C)->isZeroValue())
10062 OnlyRight = true;
10063
10064 bool LHSContainsUndef = false, RHSContainsUndef = false;
10065 SetTy LHSAAPVS, RHSAAPVS;
10066 if (!OnlyRight &&
10067 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
10068 ContainsUndef&: LHSContainsUndef, /* ForSelf */ false))
10069 return indicatePessimisticFixpoint();
10070
10071 if (!OnlyLeft &&
10072 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
10073 ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
10074 return indicatePessimisticFixpoint();
10075
10076 if (OnlyLeft || OnlyRight) {
10077 // select (true/false), lhs, rhs
10078 auto *OpAA = OnlyLeft ? &LHSAAPVS : &RHSAAPVS;
10079 auto Undef = OnlyLeft ? LHSContainsUndef : RHSContainsUndef;
10080
10081 if (Undef)
10082 unionAssumedWithUndef();
10083 else {
10084 for (const auto &It : *OpAA)
10085 unionAssumed(C: It);
10086 }
10087
10088 } else if (LHSContainsUndef && RHSContainsUndef) {
10089 // select i1 *, undef , undef => undef
10090 unionAssumedWithUndef();
10091 } else {
10092 for (const auto &It : LHSAAPVS)
10093 unionAssumed(C: It);
10094 for (const auto &It : RHSAAPVS)
10095 unionAssumed(C: It);
10096 }
10097 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10098 : ChangeStatus::CHANGED;
10099 }
10100
10101 ChangeStatus updateWithCastInst(Attributor &A, CastInst *CI) {
10102 auto AssumedBefore = getAssumed();
10103 if (!CI->isIntegerCast())
10104 return indicatePessimisticFixpoint();
10105 assert(CI->getNumOperands() == 1 && "Expected cast to be unary!");
10106 uint32_t ResultBitWidth = CI->getDestTy()->getIntegerBitWidth();
10107 Value *Src = CI->getOperand(i_nocapture: 0);
10108
10109 bool SrcContainsUndef = false;
10110 SetTy SrcPVS;
10111 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *Src), S&: SrcPVS,
10112 ContainsUndef&: SrcContainsUndef, /* ForSelf */ false))
10113 return indicatePessimisticFixpoint();
10114
10115 if (SrcContainsUndef)
10116 unionAssumedWithUndef();
10117 else {
10118 for (const APInt &S : SrcPVS) {
10119 APInt T = calculateCastInst(CI, Src: S, ResultBitWidth);
10120 unionAssumed(C: T);
10121 }
10122 }
10123 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10124 : ChangeStatus::CHANGED;
10125 }
10126
10127 ChangeStatus updateWithBinaryOperator(Attributor &A, BinaryOperator *BinOp) {
10128 auto AssumedBefore = getAssumed();
10129 Value *LHS = BinOp->getOperand(i_nocapture: 0);
10130 Value *RHS = BinOp->getOperand(i_nocapture: 1);
10131
10132 bool LHSContainsUndef = false, RHSContainsUndef = false;
10133 SetTy LHSAAPVS, RHSAAPVS;
10134 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
10135 ContainsUndef&: LHSContainsUndef, /* ForSelf */ false) ||
10136 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
10137 ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
10138 return indicatePessimisticFixpoint();
10139
10140 const APInt Zero = APInt(LHS->getType()->getIntegerBitWidth(), 0);
10141
10142 // TODO: make use of undef flag to limit potential values aggressively.
10143 if (LHSContainsUndef && RHSContainsUndef) {
10144 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: Zero, RHS: Zero))
10145 return indicatePessimisticFixpoint();
10146 } else if (LHSContainsUndef) {
10147 for (const APInt &R : RHSAAPVS) {
10148 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: Zero, RHS: R))
10149 return indicatePessimisticFixpoint();
10150 }
10151 } else if (RHSContainsUndef) {
10152 for (const APInt &L : LHSAAPVS) {
10153 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: L, RHS: Zero))
10154 return indicatePessimisticFixpoint();
10155 }
10156 } else {
10157 for (const APInt &L : LHSAAPVS) {
10158 for (const APInt &R : RHSAAPVS) {
10159 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: L, RHS: R))
10160 return indicatePessimisticFixpoint();
10161 }
10162 }
10163 }
10164 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10165 : ChangeStatus::CHANGED;
10166 }
10167
10168 ChangeStatus updateWithInstruction(Attributor &A, Instruction *Inst) {
10169 auto AssumedBefore = getAssumed();
10170 SetTy Incoming;
10171 bool ContainsUndef;
10172 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *Inst), S&: Incoming,
10173 ContainsUndef, /* ForSelf */ true))
10174 return indicatePessimisticFixpoint();
10175 if (ContainsUndef) {
10176 unionAssumedWithUndef();
10177 } else {
10178 for (const auto &It : Incoming)
10179 unionAssumed(C: It);
10180 }
10181 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10182 : ChangeStatus::CHANGED;
10183 }
10184
10185 /// See AbstractAttribute::updateImpl(...).
10186 ChangeStatus updateImpl(Attributor &A) override {
10187 Value &V = getAssociatedValue();
10188 Instruction *I = dyn_cast<Instruction>(Val: &V);
10189
10190 if (auto *ICI = dyn_cast<ICmpInst>(Val: I))
10191 return updateWithICmpInst(A, ICI);
10192
10193 if (auto *SI = dyn_cast<SelectInst>(Val: I))
10194 return updateWithSelectInst(A, SI);
10195
10196 if (auto *CI = dyn_cast<CastInst>(Val: I))
10197 return updateWithCastInst(A, CI);
10198
10199 if (auto *BinOp = dyn_cast<BinaryOperator>(Val: I))
10200 return updateWithBinaryOperator(A, BinOp);
10201
10202 if (isa<PHINode>(Val: I) || isa<LoadInst>(Val: I))
10203 return updateWithInstruction(A, Inst: I);
10204
10205 return indicatePessimisticFixpoint();
10206 }
10207
10208 /// See AbstractAttribute::trackStatistics()
10209 void trackStatistics() const override {
10210 STATS_DECLTRACK_FLOATING_ATTR(potential_values)
10211 }
10212};
10213
10214struct AAPotentialConstantValuesFunction : AAPotentialConstantValuesImpl {
10215 AAPotentialConstantValuesFunction(const IRPosition &IRP, Attributor &A)
10216 : AAPotentialConstantValuesImpl(IRP, A) {}
10217
10218 /// See AbstractAttribute::initialize(...).
10219 ChangeStatus updateImpl(Attributor &A) override {
10220 llvm_unreachable(
10221 "AAPotentialConstantValues(Function|CallSite)::updateImpl will "
10222 "not be called");
10223 }
10224
10225 /// See AbstractAttribute::trackStatistics()
10226 void trackStatistics() const override {
10227 STATS_DECLTRACK_FN_ATTR(potential_values)
10228 }
10229};
10230
10231struct AAPotentialConstantValuesCallSite : AAPotentialConstantValuesFunction {
10232 AAPotentialConstantValuesCallSite(const IRPosition &IRP, Attributor &A)
10233 : AAPotentialConstantValuesFunction(IRP, A) {}
10234
10235 /// See AbstractAttribute::trackStatistics()
10236 void trackStatistics() const override {
10237 STATS_DECLTRACK_CS_ATTR(potential_values)
10238 }
10239};
10240
10241struct AAPotentialConstantValuesCallSiteReturned
10242 : AACalleeToCallSite<AAPotentialConstantValues,
10243 AAPotentialConstantValuesImpl> {
10244 AAPotentialConstantValuesCallSiteReturned(const IRPosition &IRP,
10245 Attributor &A)
10246 : AACalleeToCallSite<AAPotentialConstantValues,
10247 AAPotentialConstantValuesImpl>(IRP, A) {}
10248
10249 /// See AbstractAttribute::trackStatistics()
10250 void trackStatistics() const override {
10251 STATS_DECLTRACK_CSRET_ATTR(potential_values)
10252 }
10253};
10254
10255struct AAPotentialConstantValuesCallSiteArgument
10256 : AAPotentialConstantValuesFloating {
10257 AAPotentialConstantValuesCallSiteArgument(const IRPosition &IRP,
10258 Attributor &A)
10259 : AAPotentialConstantValuesFloating(IRP, A) {}
10260
10261 /// See AbstractAttribute::initialize(..).
10262 void initialize(Attributor &A) override {
10263 AAPotentialConstantValuesImpl::initialize(A);
10264 if (isAtFixpoint())
10265 return;
10266
10267 Value &V = getAssociatedValue();
10268
10269 if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
10270 unionAssumed(C: C->getValue());
10271 indicateOptimisticFixpoint();
10272 return;
10273 }
10274
10275 if (isa<UndefValue>(Val: &V)) {
10276 unionAssumedWithUndef();
10277 indicateOptimisticFixpoint();
10278 return;
10279 }
10280 }
10281
10282 /// See AbstractAttribute::updateImpl(...).
10283 ChangeStatus updateImpl(Attributor &A) override {
10284 Value &V = getAssociatedValue();
10285 auto AssumedBefore = getAssumed();
10286 auto *AA = A.getAAFor<AAPotentialConstantValues>(
10287 QueryingAA: *this, IRP: IRPosition::value(V), DepClass: DepClassTy::REQUIRED);
10288 if (!AA)
10289 return indicatePessimisticFixpoint();
10290 const auto &S = AA->getAssumed();
10291 unionAssumed(PVS: S);
10292 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10293 : ChangeStatus::CHANGED;
10294 }
10295
10296 /// See AbstractAttribute::trackStatistics()
10297 void trackStatistics() const override {
10298 STATS_DECLTRACK_CSARG_ATTR(potential_values)
10299 }
10300};
10301} // namespace
10302
10303/// ------------------------ NoUndef Attribute ---------------------------------
10304bool AANoUndef::isImpliedByIR(Attributor &A, const IRPosition &IRP,
10305 Attribute::AttrKind ImpliedAttributeKind,
10306 bool IgnoreSubsumingPositions) {
10307 assert(ImpliedAttributeKind == Attribute::NoUndef &&
10308 "Unexpected attribute kind");
10309 if (A.hasAttr(IRP, AKs: {Attribute::NoUndef}, IgnoreSubsumingPositions,
10310 ImpliedAttributeKind: Attribute::NoUndef))
10311 return true;
10312
10313 Value &Val = IRP.getAssociatedValue();
10314 if (IRP.getPositionKind() != IRPosition::IRP_RETURNED &&
10315 isGuaranteedNotToBeUndefOrPoison(V: &Val)) {
10316 LLVMContext &Ctx = Val.getContext();
10317 A.manifestAttrs(IRP, DeducedAttrs: Attribute::get(Context&: Ctx, Kind: Attribute::NoUndef));
10318 return true;
10319 }
10320
10321 return false;
10322}
10323
10324namespace {
10325struct AANoUndefImpl : AANoUndef {
10326 AANoUndefImpl(const IRPosition &IRP, Attributor &A) : AANoUndef(IRP, A) {}
10327
10328 /// See AbstractAttribute::initialize(...).
10329 void initialize(Attributor &A) override {
10330 Value &V = getAssociatedValue();
10331 if (isa<UndefValue>(Val: V))
10332 indicatePessimisticFixpoint();
10333 assert(!isImpliedByIR(A, getIRPosition(), Attribute::NoUndef));
10334 }
10335
10336 /// See followUsesInMBEC
10337 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10338 AANoUndef::StateType &State) {
10339 const Value *UseV = U->get();
10340 const DominatorTree *DT = nullptr;
10341 AssumptionCache *AC = nullptr;
10342 InformationCache &InfoCache = A.getInfoCache();
10343 if (Function *F = getAnchorScope()) {
10344 DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
10345 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
10346 }
10347 State.setKnown(isGuaranteedNotToBeUndefOrPoison(V: UseV, AC, CtxI: I, DT));
10348 bool TrackUse = false;
10349 // Track use for instructions which must produce undef or poison bits when
10350 // at least one operand contains such bits.
10351 if (isa<CastInst>(Val: *I) || isa<GetElementPtrInst>(Val: *I))
10352 TrackUse = true;
10353 return TrackUse;
10354 }
10355
10356 /// See AbstractAttribute::getAsStr().
10357 const std::string getAsStr(Attributor *A) const override {
10358 return getAssumed() ? "noundef" : "may-undef-or-poison";
10359 }
10360
10361 ChangeStatus manifest(Attributor &A) override {
10362 // We don't manifest noundef attribute for dead positions because the
10363 // associated values with dead positions would be replaced with undef
10364 // values.
10365 bool UsedAssumedInformation = false;
10366 if (A.isAssumedDead(IRP: getIRPosition(), QueryingAA: nullptr, FnLivenessAA: nullptr,
10367 UsedAssumedInformation))
10368 return ChangeStatus::UNCHANGED;
10369 // A position whose simplified value does not have any value is
10370 // considered to be dead. We don't manifest noundef in such positions for
10371 // the same reason above.
10372 if (!A.getAssumedSimplified(IRP: getIRPosition(), AA: *this, UsedAssumedInformation,
10373 S: AA::Interprocedural)
10374 .has_value())
10375 return ChangeStatus::UNCHANGED;
10376 return AANoUndef::manifest(A);
10377 }
10378};
10379
10380struct AANoUndefFloating : public AANoUndefImpl {
10381 AANoUndefFloating(const IRPosition &IRP, Attributor &A)
10382 : AANoUndefImpl(IRP, A) {}
10383
10384 /// See AbstractAttribute::initialize(...).
10385 void initialize(Attributor &A) override {
10386 AANoUndefImpl::initialize(A);
10387 if (!getState().isAtFixpoint() && getAnchorScope() &&
10388 !getAnchorScope()->isDeclaration())
10389 if (Instruction *CtxI = getCtxI())
10390 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
10391 }
10392
10393 /// See AbstractAttribute::updateImpl(...).
10394 ChangeStatus updateImpl(Attributor &A) override {
10395 auto VisitValueCB = [&](const IRPosition &IRP) -> bool {
10396 bool IsKnownNoUndef;
10397 return AA::hasAssumedIRAttr<Attribute::NoUndef>(
10398 A, QueryingAA: this, IRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoUndef);
10399 };
10400
10401 bool Stripped;
10402 bool UsedAssumedInformation = false;
10403 Value *AssociatedValue = &getAssociatedValue();
10404 SmallVector<AA::ValueAndContext> Values;
10405 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
10406 S: AA::AnyScope, UsedAssumedInformation))
10407 Stripped = false;
10408 else
10409 Stripped =
10410 Values.size() != 1 || Values.front().getValue() != AssociatedValue;
10411
10412 if (!Stripped) {
10413 // If we haven't stripped anything we might still be able to use a
10414 // different AA, but only if the IRP changes. Effectively when we
10415 // interpret this not as a call site value but as a floating/argument
10416 // value.
10417 const IRPosition AVIRP = IRPosition::value(V: *AssociatedValue);
10418 if (AVIRP == getIRPosition() || !VisitValueCB(AVIRP))
10419 return indicatePessimisticFixpoint();
10420 return ChangeStatus::UNCHANGED;
10421 }
10422
10423 for (const auto &VAC : Values)
10424 if (!VisitValueCB(IRPosition::value(V: *VAC.getValue())))
10425 return indicatePessimisticFixpoint();
10426
10427 return ChangeStatus::UNCHANGED;
10428 }
10429
10430 /// See AbstractAttribute::trackStatistics()
10431 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10432};
10433
10434struct AANoUndefReturned final
10435 : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl> {
10436 AANoUndefReturned(const IRPosition &IRP, Attributor &A)
10437 : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl>(IRP, A) {}
10438
10439 /// See AbstractAttribute::trackStatistics()
10440 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10441};
10442
10443struct AANoUndefArgument final
10444 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl> {
10445 AANoUndefArgument(const IRPosition &IRP, Attributor &A)
10446 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl>(IRP, A) {}
10447
10448 /// See AbstractAttribute::trackStatistics()
10449 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noundef) }
10450};
10451
10452struct AANoUndefCallSiteArgument final : AANoUndefFloating {
10453 AANoUndefCallSiteArgument(const IRPosition &IRP, Attributor &A)
10454 : AANoUndefFloating(IRP, A) {}
10455
10456 /// See AbstractAttribute::trackStatistics()
10457 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noundef) }
10458};
10459
10460struct AANoUndefCallSiteReturned final
10461 : AACalleeToCallSite<AANoUndef, AANoUndefImpl> {
10462 AANoUndefCallSiteReturned(const IRPosition &IRP, Attributor &A)
10463 : AACalleeToCallSite<AANoUndef, AANoUndefImpl>(IRP, A) {}
10464
10465 /// See AbstractAttribute::trackStatistics()
10466 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noundef) }
10467};
10468
10469/// ------------------------ NoFPClass Attribute -------------------------------
10470
10471struct AANoFPClassImpl : AANoFPClass {
10472 AANoFPClassImpl(const IRPosition &IRP, Attributor &A) : AANoFPClass(IRP, A) {}
10473
10474 void initialize(Attributor &A) override {
10475 const IRPosition &IRP = getIRPosition();
10476
10477 Value &V = IRP.getAssociatedValue();
10478 if (isa<UndefValue>(Val: V)) {
10479 indicateOptimisticFixpoint();
10480 return;
10481 }
10482
10483 SmallVector<Attribute> Attrs;
10484 A.getAttrs(IRP: getIRPosition(), AKs: {Attribute::NoFPClass}, Attrs, IgnoreSubsumingPositions: false);
10485 for (const auto &Attr : Attrs) {
10486 addKnownBits(Bits: Attr.getNoFPClass());
10487 }
10488
10489 Instruction *CtxI = getCtxI();
10490
10491 if (getPositionKind() != IRPosition::IRP_RETURNED) {
10492 const DataLayout &DL = A.getDataLayout();
10493 InformationCache &InfoCache = A.getInfoCache();
10494
10495 const DominatorTree *DT = nullptr;
10496 AssumptionCache *AC = nullptr;
10497 const TargetLibraryInfo *TLI = nullptr;
10498 Function *F = getAnchorScope();
10499 if (F) {
10500 TLI = InfoCache.getTargetLibraryInfoForFunction(F: *F);
10501 if (!F->isDeclaration()) {
10502 DT =
10503 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
10504 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
10505 }
10506 }
10507
10508 SimplifyQuery Q(DL, TLI, DT, AC, CtxI);
10509
10510 KnownFPClass KnownFPClass = computeKnownFPClass(V: &V, InterestedClasses: fcAllFlags, SQ: Q);
10511 addKnownBits(Bits: ~KnownFPClass.KnownFPClasses);
10512 }
10513
10514 if (CtxI)
10515 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
10516 }
10517
10518 /// See followUsesInMBEC
10519 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10520 AANoFPClass::StateType &State) {
10521 // TODO: Determine what instructions can be looked through.
10522 auto *CB = dyn_cast<CallBase>(Val: I);
10523 if (!CB)
10524 return false;
10525
10526 if (!CB->isArgOperand(U))
10527 return false;
10528
10529 unsigned ArgNo = CB->getArgOperandNo(U);
10530 IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
10531 if (auto *NoFPAA = A.getAAFor<AANoFPClass>(QueryingAA: *this, IRP, DepClass: DepClassTy::NONE))
10532 State.addKnownBits(Bits: NoFPAA->getState().getKnown());
10533 return false;
10534 }
10535
10536 const std::string getAsStr(Attributor *A) const override {
10537 std::string Result = "nofpclass";
10538 raw_string_ostream OS(Result);
10539 OS << getKnownNoFPClass() << '/' << getAssumedNoFPClass();
10540 return Result;
10541 }
10542
10543 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
10544 SmallVectorImpl<Attribute> &Attrs) const override {
10545 Attrs.emplace_back(Args: Attribute::getWithNoFPClass(Context&: Ctx, Mask: getAssumedNoFPClass()));
10546 }
10547};
10548
10549struct AANoFPClassFloating : public AANoFPClassImpl {
10550 AANoFPClassFloating(const IRPosition &IRP, Attributor &A)
10551 : AANoFPClassImpl(IRP, A) {}
10552
10553 /// See AbstractAttribute::updateImpl(...).
10554 ChangeStatus updateImpl(Attributor &A) override {
10555 SmallVector<AA::ValueAndContext> Values;
10556 bool UsedAssumedInformation = false;
10557 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
10558 S: AA::AnyScope, UsedAssumedInformation)) {
10559 Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
10560 }
10561
10562 StateType T;
10563 auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
10564 const auto *AA = A.getAAFor<AANoFPClass>(QueryingAA: *this, IRP: IRPosition::value(V),
10565 DepClass: DepClassTy::REQUIRED);
10566 if (!AA || this == AA) {
10567 T.indicatePessimisticFixpoint();
10568 } else {
10569 const AANoFPClass::StateType &S =
10570 static_cast<const AANoFPClass::StateType &>(AA->getState());
10571 T ^= S;
10572 }
10573 return T.isValidState();
10574 };
10575
10576 for (const auto &VAC : Values)
10577 if (!VisitValueCB(*VAC.getValue(), VAC.getCtxI()))
10578 return indicatePessimisticFixpoint();
10579
10580 return clampStateAndIndicateChange(S&: getState(), R: T);
10581 }
10582
10583 /// See AbstractAttribute::trackStatistics()
10584 void trackStatistics() const override {
10585 STATS_DECLTRACK_FNRET_ATTR(nofpclass)
10586 }
10587};
10588
10589struct AANoFPClassReturned final
10590 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10591 AANoFPClassImpl::StateType, false,
10592 Attribute::None, false> {
10593 AANoFPClassReturned(const IRPosition &IRP, Attributor &A)
10594 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10595 AANoFPClassImpl::StateType, false,
10596 Attribute::None, false>(IRP, A) {}
10597
10598 /// See AbstractAttribute::trackStatistics()
10599 void trackStatistics() const override {
10600 STATS_DECLTRACK_FNRET_ATTR(nofpclass)
10601 }
10602};
10603
10604struct AANoFPClassArgument final
10605 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl> {
10606 AANoFPClassArgument(const IRPosition &IRP, Attributor &A)
10607 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10608
10609 /// See AbstractAttribute::trackStatistics()
10610 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofpclass) }
10611};
10612
10613struct AANoFPClassCallSiteArgument final : AANoFPClassFloating {
10614 AANoFPClassCallSiteArgument(const IRPosition &IRP, Attributor &A)
10615 : AANoFPClassFloating(IRP, A) {}
10616
10617 /// See AbstractAttribute::trackStatistics()
10618 void trackStatistics() const override {
10619 STATS_DECLTRACK_CSARG_ATTR(nofpclass)
10620 }
10621};
10622
10623struct AANoFPClassCallSiteReturned final
10624 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl> {
10625 AANoFPClassCallSiteReturned(const IRPosition &IRP, Attributor &A)
10626 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10627
10628 /// See AbstractAttribute::trackStatistics()
10629 void trackStatistics() const override {
10630 STATS_DECLTRACK_CSRET_ATTR(nofpclass)
10631 }
10632};
10633
10634struct AACallEdgesImpl : public AACallEdges {
10635 AACallEdgesImpl(const IRPosition &IRP, Attributor &A) : AACallEdges(IRP, A) {}
10636
10637 const SetVector<Function *> &getOptimisticEdges() const override {
10638 return CalledFunctions;
10639 }
10640
10641 bool hasUnknownCallee() const override { return HasUnknownCallee; }
10642
10643 bool hasNonAsmUnknownCallee() const override {
10644 return HasUnknownCalleeNonAsm;
10645 }
10646
10647 const std::string getAsStr(Attributor *A) const override {
10648 return "CallEdges[" + std::to_string(val: HasUnknownCallee) + "," +
10649 std::to_string(val: CalledFunctions.size()) + "]";
10650 }
10651
10652 void trackStatistics() const override {}
10653
10654protected:
10655 void addCalledFunction(Function *Fn, ChangeStatus &Change) {
10656 if (CalledFunctions.insert(X: Fn)) {
10657 Change = ChangeStatus::CHANGED;
10658 LLVM_DEBUG(dbgs() << "[AACallEdges] New call edge: " << Fn->getName()
10659 << "\n");
10660 }
10661 }
10662
10663 void setHasUnknownCallee(bool NonAsm, ChangeStatus &Change) {
10664 if (!HasUnknownCallee)
10665 Change = ChangeStatus::CHANGED;
10666 if (NonAsm && !HasUnknownCalleeNonAsm)
10667 Change = ChangeStatus::CHANGED;
10668 HasUnknownCalleeNonAsm |= NonAsm;
10669 HasUnknownCallee = true;
10670 }
10671
10672private:
10673 /// Optimistic set of functions that might be called by this position.
10674 SetVector<Function *> CalledFunctions;
10675
10676 /// Is there any call with a unknown callee.
10677 bool HasUnknownCallee = false;
10678
10679 /// Is there any call with a unknown callee, excluding any inline asm.
10680 bool HasUnknownCalleeNonAsm = false;
10681};
10682
10683struct AACallEdgesCallSite : public AACallEdgesImpl {
10684 AACallEdgesCallSite(const IRPosition &IRP, Attributor &A)
10685 : AACallEdgesImpl(IRP, A) {}
10686 /// See AbstractAttribute::updateImpl(...).
10687 ChangeStatus updateImpl(Attributor &A) override {
10688 ChangeStatus Change = ChangeStatus::UNCHANGED;
10689
10690 auto VisitValue = [&](Value &V, const Instruction *CtxI) -> bool {
10691 if (Function *Fn = dyn_cast<Function>(Val: &V)) {
10692 addCalledFunction(Fn, Change);
10693 } else {
10694 LLVM_DEBUG(dbgs() << "[AACallEdges] Unrecognized value: " << V << "\n");
10695 setHasUnknownCallee(NonAsm: true, Change);
10696 }
10697
10698 // Explore all values.
10699 return true;
10700 };
10701
10702 SmallVector<AA::ValueAndContext> Values;
10703 // Process any value that we might call.
10704 auto ProcessCalledOperand = [&](Value *V, Instruction *CtxI) {
10705 if (isa<Constant>(Val: V)) {
10706 VisitValue(*V, CtxI);
10707 return;
10708 }
10709
10710 bool UsedAssumedInformation = false;
10711 Values.clear();
10712 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *V), AA: *this, Values,
10713 S: AA::AnyScope, UsedAssumedInformation)) {
10714 Values.push_back(Elt: {*V, CtxI});
10715 }
10716 for (auto &VAC : Values)
10717 VisitValue(*VAC.getValue(), VAC.getCtxI());
10718 };
10719
10720 CallBase *CB = cast<CallBase>(Val: getCtxI());
10721
10722 if (auto *IA = dyn_cast<InlineAsm>(Val: CB->getCalledOperand())) {
10723 if (IA->hasSideEffects() &&
10724 !hasAssumption(F: *CB->getCaller(), AssumptionStr: "ompx_no_call_asm") &&
10725 !hasAssumption(CB: *CB, AssumptionStr: "ompx_no_call_asm")) {
10726 setHasUnknownCallee(NonAsm: false, Change);
10727 }
10728 return Change;
10729 }
10730
10731 if (CB->isIndirectCall())
10732 if (auto *IndirectCallAA = A.getAAFor<AAIndirectCallInfo>(
10733 QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL))
10734 if (IndirectCallAA->foreachCallee(
10735 CB: [&](Function *Fn) { return VisitValue(*Fn, CB); }))
10736 return Change;
10737
10738 // The most simple case.
10739 ProcessCalledOperand(CB->getCalledOperand(), CB);
10740
10741 // Process callback functions.
10742 SmallVector<const Use *, 4u> CallbackUses;
10743 AbstractCallSite::getCallbackUses(CB: *CB, CallbackUses);
10744 for (const Use *U : CallbackUses)
10745 ProcessCalledOperand(U->get(), CB);
10746
10747 return Change;
10748 }
10749};
10750
10751struct AACallEdgesFunction : public AACallEdgesImpl {
10752 AACallEdgesFunction(const IRPosition &IRP, Attributor &A)
10753 : AACallEdgesImpl(IRP, A) {}
10754
10755 /// See AbstractAttribute::updateImpl(...).
10756 ChangeStatus updateImpl(Attributor &A) override {
10757 ChangeStatus Change = ChangeStatus::UNCHANGED;
10758
10759 auto ProcessCallInst = [&](Instruction &Inst) {
10760 CallBase &CB = cast<CallBase>(Val&: Inst);
10761
10762 auto *CBEdges = A.getAAFor<AACallEdges>(
10763 QueryingAA: *this, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::REQUIRED);
10764 if (!CBEdges)
10765 return false;
10766 if (CBEdges->hasNonAsmUnknownCallee())
10767 setHasUnknownCallee(NonAsm: true, Change);
10768 if (CBEdges->hasUnknownCallee())
10769 setHasUnknownCallee(NonAsm: false, Change);
10770
10771 for (Function *F : CBEdges->getOptimisticEdges())
10772 addCalledFunction(Fn: F, Change);
10773
10774 return true;
10775 };
10776
10777 // Visit all callable instructions.
10778 bool UsedAssumedInformation = false;
10779 if (!A.checkForAllCallLikeInstructions(Pred: ProcessCallInst, QueryingAA: *this,
10780 UsedAssumedInformation,
10781 /* CheckBBLivenessOnly */ true)) {
10782 // If we haven't looked at all call like instructions, assume that there
10783 // are unknown callees.
10784 setHasUnknownCallee(NonAsm: true, Change);
10785 }
10786
10787 return Change;
10788 }
10789};
10790
10791/// -------------------AAInterFnReachability Attribute--------------------------
10792
10793struct AAInterFnReachabilityFunction
10794 : public CachedReachabilityAA<AAInterFnReachability, Function> {
10795 using Base = CachedReachabilityAA<AAInterFnReachability, Function>;
10796 AAInterFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
10797 : Base(IRP, A) {}
10798
10799 bool instructionCanReach(
10800 Attributor &A, const Instruction &From, const Function &To,
10801 const AA::InstExclusionSetTy *ExclusionSet) const override {
10802 assert(From.getFunction() == getAnchorScope() && "Queried the wrong AA!");
10803 auto *NonConstThis = const_cast<AAInterFnReachabilityFunction *>(this);
10804
10805 RQITy StackRQI(A, From, To, ExclusionSet, false);
10806 RQITy::Reachable Result;
10807 if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
10808 return NonConstThis->isReachableImpl(A, RQI&: StackRQI,
10809 /*IsTemporaryRQI=*/true);
10810 return Result == RQITy::Reachable::Yes;
10811 }
10812
10813 bool isReachableImpl(Attributor &A, RQITy &RQI,
10814 bool IsTemporaryRQI) override {
10815 const Instruction *EntryI =
10816 &RQI.From->getFunction()->getEntryBlock().front();
10817 if (EntryI != RQI.From &&
10818 !instructionCanReach(A, From: *EntryI, To: *RQI.To, ExclusionSet: nullptr))
10819 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet: false,
10820 IsTemporaryRQI);
10821
10822 auto CheckReachableCallBase = [&](CallBase *CB) {
10823 auto *CBEdges = A.getAAFor<AACallEdges>(
10824 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL);
10825 if (!CBEdges || !CBEdges->getState().isValidState())
10826 return false;
10827 // TODO Check To backwards in this case.
10828 if (CBEdges->hasUnknownCallee())
10829 return false;
10830
10831 for (Function *Fn : CBEdges->getOptimisticEdges()) {
10832 if (Fn == RQI.To)
10833 return false;
10834
10835 if (Fn->isDeclaration()) {
10836 if (Fn->hasFnAttribute(Kind: Attribute::NoCallback))
10837 continue;
10838 // TODO Check To backwards in this case.
10839 return false;
10840 }
10841
10842 if (Fn == getAnchorScope()) {
10843 if (EntryI == RQI.From)
10844 continue;
10845 return false;
10846 }
10847
10848 const AAInterFnReachability *InterFnReachability =
10849 A.getAAFor<AAInterFnReachability>(QueryingAA: *this, IRP: IRPosition::function(F: *Fn),
10850 DepClass: DepClassTy::OPTIONAL);
10851
10852 const Instruction &FnFirstInst = Fn->getEntryBlock().front();
10853 if (!InterFnReachability ||
10854 InterFnReachability->instructionCanReach(A, Inst: FnFirstInst, Fn: *RQI.To,
10855 ExclusionSet: RQI.ExclusionSet))
10856 return false;
10857 }
10858 return true;
10859 };
10860
10861 const auto *IntraFnReachability = A.getAAFor<AAIntraFnReachability>(
10862 QueryingAA: *this, IRP: IRPosition::function(F: *RQI.From->getFunction()),
10863 DepClass: DepClassTy::OPTIONAL);
10864
10865 // Determine call like instructions that we can reach from the inst.
10866 auto CheckCallBase = [&](Instruction &CBInst) {
10867 // There are usually less nodes in the call graph, check inter function
10868 // reachability first.
10869 if (CheckReachableCallBase(cast<CallBase>(Val: &CBInst)))
10870 return true;
10871 return IntraFnReachability && !IntraFnReachability->isAssumedReachable(
10872 A, From: *RQI.From, To: CBInst, ExclusionSet: RQI.ExclusionSet);
10873 };
10874
10875 bool UsedExclusionSet = /* conservative */ true;
10876 bool UsedAssumedInformation = false;
10877 if (!A.checkForAllCallLikeInstructions(Pred: CheckCallBase, QueryingAA: *this,
10878 UsedAssumedInformation,
10879 /* CheckBBLivenessOnly */ true))
10880 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
10881 IsTemporaryRQI);
10882
10883 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
10884 IsTemporaryRQI);
10885 }
10886
10887 void trackStatistics() const override {}
10888};
10889} // namespace
10890
10891template <typename AAType>
10892static std::optional<Constant *>
10893askForAssumedConstant(Attributor &A, const AbstractAttribute &QueryingAA,
10894 const IRPosition &IRP, Type &Ty) {
10895 if (!Ty.isIntegerTy())
10896 return nullptr;
10897
10898 // This will also pass the call base context.
10899 const auto *AA = A.getAAFor<AAType>(QueryingAA, IRP, DepClassTy::NONE);
10900 if (!AA)
10901 return nullptr;
10902
10903 std::optional<Constant *> COpt = AA->getAssumedConstant(A);
10904
10905 if (!COpt.has_value()) {
10906 A.recordDependence(FromAA: *AA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
10907 return std::nullopt;
10908 }
10909 if (auto *C = *COpt) {
10910 A.recordDependence(FromAA: *AA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
10911 return C;
10912 }
10913 return nullptr;
10914}
10915
10916Value *AAPotentialValues::getSingleValue(
10917 Attributor &A, const AbstractAttribute &AA, const IRPosition &IRP,
10918 SmallVectorImpl<AA::ValueAndContext> &Values) {
10919 Type &Ty = *IRP.getAssociatedType();
10920 std::optional<Value *> V;
10921 for (auto &It : Values) {
10922 V = AA::combineOptionalValuesInAAValueLatice(A: V, B: It.getValue(), Ty: &Ty);
10923 if (V.has_value() && !*V)
10924 break;
10925 }
10926 if (!V.has_value())
10927 return UndefValue::get(T: &Ty);
10928 return *V;
10929}
10930
10931namespace {
10932struct AAPotentialValuesImpl : AAPotentialValues {
10933 using StateType = PotentialLLVMValuesState;
10934
10935 AAPotentialValuesImpl(const IRPosition &IRP, Attributor &A)
10936 : AAPotentialValues(IRP, A) {}
10937
10938 /// See AbstractAttribute::initialize(..).
10939 void initialize(Attributor &A) override {
10940 if (A.hasSimplificationCallback(IRP: getIRPosition())) {
10941 indicatePessimisticFixpoint();
10942 return;
10943 }
10944 Value *Stripped = getAssociatedValue().stripPointerCasts();
10945 if (isa<Constant>(Val: Stripped) && !isa<ConstantExpr>(Val: Stripped)) {
10946 addValue(A, State&: getState(), V&: *Stripped, CtxI: getCtxI(), S: AA::AnyScope,
10947 AnchorScope: getAnchorScope());
10948 indicateOptimisticFixpoint();
10949 return;
10950 }
10951 AAPotentialValues::initialize(A);
10952 }
10953
10954 /// See AbstractAttribute::getAsStr().
10955 const std::string getAsStr(Attributor *A) const override {
10956 std::string Str;
10957 llvm::raw_string_ostream OS(Str);
10958 OS << getState();
10959 return Str;
10960 }
10961
10962 template <typename AAType>
10963 static std::optional<Value *> askOtherAA(Attributor &A,
10964 const AbstractAttribute &AA,
10965 const IRPosition &IRP, Type &Ty) {
10966 if (isa<Constant>(Val: IRP.getAssociatedValue()))
10967 return &IRP.getAssociatedValue();
10968 std::optional<Constant *> C = askForAssumedConstant<AAType>(A, AA, IRP, Ty);
10969 if (!C)
10970 return std::nullopt;
10971 if (*C)
10972 if (auto *CC = AA::getWithType(V&: **C, Ty))
10973 return CC;
10974 return nullptr;
10975 }
10976
10977 virtual void addValue(Attributor &A, StateType &State, Value &V,
10978 const Instruction *CtxI, AA::ValueScope S,
10979 Function *AnchorScope) const {
10980
10981 IRPosition ValIRP = IRPosition::value(V);
10982 if (auto *CB = dyn_cast_or_null<CallBase>(Val: CtxI)) {
10983 for (const auto &U : CB->args()) {
10984 if (U.get() != &V)
10985 continue;
10986 ValIRP = IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U));
10987 break;
10988 }
10989 }
10990
10991 Value *VPtr = &V;
10992 if (ValIRP.getAssociatedType()->isIntegerTy()) {
10993 Type &Ty = *getAssociatedType();
10994 std::optional<Value *> SimpleV =
10995 askOtherAA<AAValueConstantRange>(A, AA: *this, IRP: ValIRP, Ty);
10996 if (SimpleV.has_value() && !*SimpleV) {
10997 auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
10998 QueryingAA: *this, IRP: ValIRP, DepClass: DepClassTy::OPTIONAL);
10999 if (PotentialConstantsAA && PotentialConstantsAA->isValidState()) {
11000 for (const auto &It : PotentialConstantsAA->getAssumedSet())
11001 State.unionAssumed(C: {{*ConstantInt::get(Ty: &Ty, V: It), nullptr}, S});
11002 if (PotentialConstantsAA->undefIsContained())
11003 State.unionAssumed(C: {{*UndefValue::get(T: &Ty), nullptr}, S});
11004 return;
11005 }
11006 }
11007 if (!SimpleV.has_value())
11008 return;
11009
11010 if (*SimpleV)
11011 VPtr = *SimpleV;
11012 }
11013
11014 if (isa<ConstantInt>(Val: VPtr))
11015 CtxI = nullptr;
11016 if (!AA::isValidInScope(V: *VPtr, Scope: AnchorScope))
11017 S = AA::ValueScope(S | AA::Interprocedural);
11018
11019 State.unionAssumed(C: {{*VPtr, CtxI}, S});
11020 }
11021
11022 /// Helper struct to tie a value+context pair together with the scope for
11023 /// which this is the simplified version.
11024 struct ItemInfo {
11025 AA::ValueAndContext I;
11026 AA::ValueScope S;
11027
11028 bool operator==(const ItemInfo &II) const {
11029 return II.I == I && II.S == S;
11030 };
11031 bool operator<(const ItemInfo &II) const {
11032 return std::tie(args: I, args: S) < std::tie(args: II.I, args: II.S);
11033 };
11034 };
11035
11036 bool recurseForValue(Attributor &A, const IRPosition &IRP, AA::ValueScope S) {
11037 SmallMapVector<AA::ValueAndContext, int, 8> ValueScopeMap;
11038 for (auto CS : {AA::Intraprocedural, AA::Interprocedural}) {
11039 if (!(CS & S))
11040 continue;
11041
11042 bool UsedAssumedInformation = false;
11043 SmallVector<AA::ValueAndContext> Values;
11044 if (!A.getAssumedSimplifiedValues(IRP, AA: this, Values, S: CS,
11045 UsedAssumedInformation))
11046 return false;
11047
11048 for (auto &It : Values)
11049 ValueScopeMap[It] += CS;
11050 }
11051 for (auto &It : ValueScopeMap)
11052 addValue(A, State&: getState(), V&: *It.first.getValue(), CtxI: It.first.getCtxI(),
11053 S: AA::ValueScope(It.second), AnchorScope: getAnchorScope());
11054
11055 return true;
11056 }
11057
11058 void giveUpOnIntraprocedural(Attributor &A) {
11059 auto NewS = StateType::getBestState(PVS: getState());
11060 for (const auto &It : getAssumedSet()) {
11061 if (It.second == AA::Intraprocedural)
11062 continue;
11063 addValue(A, State&: NewS, V&: *It.first.getValue(), CtxI: It.first.getCtxI(),
11064 S: AA::Interprocedural, AnchorScope: getAnchorScope());
11065 }
11066 assert(!undefIsContained() && "Undef should be an explicit value!");
11067 addValue(A, State&: NewS, V&: getAssociatedValue(), CtxI: getCtxI(), S: AA::Intraprocedural,
11068 AnchorScope: getAnchorScope());
11069 getState() = NewS;
11070 }
11071
11072 /// See AbstractState::indicatePessimisticFixpoint(...).
11073 ChangeStatus indicatePessimisticFixpoint() override {
11074 getState() = StateType::getBestState(PVS: getState());
11075 getState().unionAssumed(C: {{getAssociatedValue(), getCtxI()}, AA::AnyScope});
11076 AAPotentialValues::indicateOptimisticFixpoint();
11077 return ChangeStatus::CHANGED;
11078 }
11079
11080 /// See AbstractAttribute::updateImpl(...).
11081 ChangeStatus updateImpl(Attributor &A) override {
11082 return indicatePessimisticFixpoint();
11083 }
11084
11085 /// See AbstractAttribute::manifest(...).
11086 ChangeStatus manifest(Attributor &A) override {
11087 SmallVector<AA::ValueAndContext> Values;
11088 for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {
11089 Values.clear();
11090 if (!getAssumedSimplifiedValues(A, Values, S))
11091 continue;
11092 Value &OldV = getAssociatedValue();
11093 if (isa<UndefValue>(Val: OldV))
11094 continue;
11095 Value *NewV = getSingleValue(A, AA: *this, IRP: getIRPosition(), Values);
11096 if (!NewV || NewV == &OldV)
11097 continue;
11098 if (getCtxI() &&
11099 !AA::isValidAtPosition(VAC: {*NewV, *getCtxI()}, InfoCache&: A.getInfoCache()))
11100 continue;
11101 if (A.changeAfterManifest(IRP: getIRPosition(), NV&: *NewV))
11102 return ChangeStatus::CHANGED;
11103 }
11104 return ChangeStatus::UNCHANGED;
11105 }
11106
11107 bool getAssumedSimplifiedValues(
11108 Attributor &A, SmallVectorImpl<AA::ValueAndContext> &Values,
11109 AA::ValueScope S, bool RecurseForSelectAndPHI = false) const override {
11110 if (!isValidState())
11111 return false;
11112 bool UsedAssumedInformation = false;
11113 for (const auto &It : getAssumedSet())
11114 if (It.second & S) {
11115 if (RecurseForSelectAndPHI && (isa<PHINode>(Val: It.first.getValue()) ||
11116 isa<SelectInst>(Val: It.first.getValue()))) {
11117 if (A.getAssumedSimplifiedValues(
11118 IRP: IRPosition::inst(I: *cast<Instruction>(Val: It.first.getValue())),
11119 AA: this, Values, S, UsedAssumedInformation))
11120 continue;
11121 }
11122 Values.push_back(Elt: It.first);
11123 }
11124 assert(!undefIsContained() && "Undef should be an explicit value!");
11125 return true;
11126 }
11127};
11128
11129struct AAPotentialValuesFloating : AAPotentialValuesImpl {
11130 AAPotentialValuesFloating(const IRPosition &IRP, Attributor &A)
11131 : AAPotentialValuesImpl(IRP, A) {}
11132
11133 /// See AbstractAttribute::updateImpl(...).
11134 ChangeStatus updateImpl(Attributor &A) override {
11135 auto AssumedBefore = getAssumed();
11136
11137 genericValueTraversal(A, InitialV: &getAssociatedValue());
11138
11139 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11140 : ChangeStatus::CHANGED;
11141 }
11142
11143 /// Helper struct to remember which AAIsDead instances we actually used.
11144 struct LivenessInfo {
11145 const AAIsDead *LivenessAA = nullptr;
11146 bool AnyDead = false;
11147 };
11148
11149 /// Check if \p Cmp is a comparison we can simplify.
11150 ///
11151 /// We handle multiple cases, one in which at least one operand is an
11152 /// (assumed) nullptr. If so, try to simplify it using AANonNull on the other
11153 /// operand. Return true if successful, in that case Worklist will be updated.
11154 bool handleCmp(Attributor &A, Value &Cmp, Value *LHS, Value *RHS,
11155 CmpInst::Predicate Pred, ItemInfo II,
11156 SmallVectorImpl<ItemInfo> &Worklist) {
11157
11158 // Simplify the operands first.
11159 bool UsedAssumedInformation = false;
11160 SmallVector<AA::ValueAndContext> LHSValues, RHSValues;
11161 auto GetSimplifiedValues = [&](Value &V,
11162 SmallVector<AA::ValueAndContext> &Values) {
11163 if (!A.getAssumedSimplifiedValues(
11164 IRP: IRPosition::value(V, CBContext: getCallBaseContext()), AA: this, Values,
11165 S: AA::Intraprocedural, UsedAssumedInformation)) {
11166 Values.clear();
11167 Values.push_back(Elt: AA::ValueAndContext{V, II.I.getCtxI()});
11168 }
11169 return Values.empty();
11170 };
11171 if (GetSimplifiedValues(*LHS, LHSValues))
11172 return true;
11173 if (GetSimplifiedValues(*RHS, RHSValues))
11174 return true;
11175
11176 LLVMContext &Ctx = LHS->getContext();
11177
11178 InformationCache &InfoCache = A.getInfoCache();
11179 Instruction *CmpI = dyn_cast<Instruction>(Val: &Cmp);
11180 Function *F = CmpI ? CmpI->getFunction() : nullptr;
11181 const auto *DT =
11182 F ? InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F)
11183 : nullptr;
11184 const auto *TLI =
11185 F ? A.getInfoCache().getTargetLibraryInfoForFunction(F: *F) : nullptr;
11186 auto *AC =
11187 F ? InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F)
11188 : nullptr;
11189
11190 const DataLayout &DL = A.getDataLayout();
11191 SimplifyQuery Q(DL, TLI, DT, AC, CmpI);
11192
11193 auto CheckPair = [&](Value &LHSV, Value &RHSV) {
11194 if (isa<UndefValue>(Val: LHSV) || isa<UndefValue>(Val: RHSV)) {
11195 addValue(A, State&: getState(), V&: *UndefValue::get(T: Cmp.getType()),
11196 /* CtxI */ nullptr, S: II.S, AnchorScope: getAnchorScope());
11197 return true;
11198 }
11199
11200 // Handle the trivial case first in which we don't even need to think
11201 // about null or non-null.
11202 if (&LHSV == &RHSV &&
11203 (CmpInst::isTrueWhenEqual(predicate: Pred) || CmpInst::isFalseWhenEqual(predicate: Pred))) {
11204 Constant *NewV = ConstantInt::get(Ty: Type::getInt1Ty(C&: Ctx),
11205 V: CmpInst::isTrueWhenEqual(predicate: Pred));
11206 addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11207 AnchorScope: getAnchorScope());
11208 return true;
11209 }
11210
11211 auto *TypedLHS = AA::getWithType(V&: LHSV, Ty&: *LHS->getType());
11212 auto *TypedRHS = AA::getWithType(V&: RHSV, Ty&: *RHS->getType());
11213 if (TypedLHS && TypedRHS) {
11214 Value *NewV = simplifyCmpInst(Predicate: Pred, LHS: TypedLHS, RHS: TypedRHS, Q);
11215 if (NewV && NewV != &Cmp) {
11216 addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11217 AnchorScope: getAnchorScope());
11218 return true;
11219 }
11220 }
11221
11222 // From now on we only handle equalities (==, !=).
11223 if (!CmpInst::isEquality(pred: Pred))
11224 return false;
11225
11226 bool LHSIsNull = isa<ConstantPointerNull>(Val: LHSV);
11227 bool RHSIsNull = isa<ConstantPointerNull>(Val: RHSV);
11228 if (!LHSIsNull && !RHSIsNull)
11229 return false;
11230
11231 // Left is the nullptr ==/!= non-nullptr case. We'll use AANonNull on the
11232 // non-nullptr operand and if we assume it's non-null we can conclude the
11233 // result of the comparison.
11234 assert((LHSIsNull || RHSIsNull) &&
11235 "Expected nullptr versus non-nullptr comparison at this point");
11236
11237 // The index is the operand that we assume is not null.
11238 unsigned PtrIdx = LHSIsNull;
11239 bool IsKnownNonNull;
11240 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
11241 A, QueryingAA: this, IRP: IRPosition::value(V: *(PtrIdx ? &RHSV : &LHSV)),
11242 DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNonNull);
11243 if (!IsAssumedNonNull)
11244 return false;
11245
11246 // The new value depends on the predicate, true for != and false for ==.
11247 Constant *NewV =
11248 ConstantInt::get(Ty: Type::getInt1Ty(C&: Ctx), V: Pred == CmpInst::ICMP_NE);
11249 addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11250 AnchorScope: getAnchorScope());
11251 return true;
11252 };
11253
11254 for (auto &LHSValue : LHSValues)
11255 for (auto &RHSValue : RHSValues)
11256 if (!CheckPair(*LHSValue.getValue(), *RHSValue.getValue()))
11257 return false;
11258 return true;
11259 }
11260
11261 bool handleSelectInst(Attributor &A, SelectInst &SI, ItemInfo II,
11262 SmallVectorImpl<ItemInfo> &Worklist) {
11263 const Instruction *CtxI = II.I.getCtxI();
11264 bool UsedAssumedInformation = false;
11265
11266 std::optional<Constant *> C =
11267 A.getAssumedConstant(V: *SI.getCondition(), AA: *this, UsedAssumedInformation);
11268 bool NoValueYet = !C.has_value();
11269 if (NoValueYet || isa_and_nonnull<UndefValue>(Val: *C))
11270 return true;
11271 if (auto *CI = dyn_cast_or_null<ConstantInt>(Val: *C)) {
11272 if (CI->isZero())
11273 Worklist.push_back(Elt: {.I: {*SI.getFalseValue(), CtxI}, .S: II.S});
11274 else
11275 Worklist.push_back(Elt: {.I: {*SI.getTrueValue(), CtxI}, .S: II.S});
11276 } else if (&SI == &getAssociatedValue()) {
11277 // We could not simplify the condition, assume both values.
11278 Worklist.push_back(Elt: {.I: {*SI.getTrueValue(), CtxI}, .S: II.S});
11279 Worklist.push_back(Elt: {.I: {*SI.getFalseValue(), CtxI}, .S: II.S});
11280 } else {
11281 std::optional<Value *> SimpleV = A.getAssumedSimplified(
11282 IRP: IRPosition::inst(I: SI), AA: *this, UsedAssumedInformation, S: II.S);
11283 if (!SimpleV.has_value())
11284 return true;
11285 if (*SimpleV) {
11286 addValue(A, State&: getState(), V&: **SimpleV, CtxI, S: II.S, AnchorScope: getAnchorScope());
11287 return true;
11288 }
11289 return false;
11290 }
11291 return true;
11292 }
11293
11294 bool handleLoadInst(Attributor &A, LoadInst &LI, ItemInfo II,
11295 SmallVectorImpl<ItemInfo> &Worklist) {
11296 SmallSetVector<Value *, 4> PotentialCopies;
11297 SmallSetVector<Instruction *, 4> PotentialValueOrigins;
11298 bool UsedAssumedInformation = false;
11299 if (!AA::getPotentiallyLoadedValues(A, LI, PotentialValues&: PotentialCopies,
11300 PotentialValueOrigins, QueryingAA: *this,
11301 UsedAssumedInformation,
11302 /* OnlyExact */ true)) {
11303 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Failed to get potentially "
11304 "loaded values for load instruction "
11305 << LI << "\n");
11306 return false;
11307 }
11308
11309 // Do not simplify loads that are only used in llvm.assume if we cannot also
11310 // remove all stores that may feed into the load. The reason is that the
11311 // assume is probably worth something as long as the stores are around.
11312 InformationCache &InfoCache = A.getInfoCache();
11313 if (InfoCache.isOnlyUsedByAssume(I: LI)) {
11314 if (!llvm::all_of(Range&: PotentialValueOrigins, P: [&](Instruction *I) {
11315 if (!I || isa<AssumeInst>(Val: I))
11316 return true;
11317 if (auto *SI = dyn_cast<StoreInst>(Val: I))
11318 return A.isAssumedDead(U: SI->getOperandUse(i: 0), QueryingAA: this,
11319 /* LivenessAA */ FnLivenessAA: nullptr,
11320 UsedAssumedInformation,
11321 /* CheckBBLivenessOnly */ false);
11322 return A.isAssumedDead(I: *I, QueryingAA: this, /* LivenessAA */ nullptr,
11323 UsedAssumedInformation,
11324 /* CheckBBLivenessOnly */ false);
11325 })) {
11326 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Load is onl used by assumes "
11327 "and we cannot delete all the stores: "
11328 << LI << "\n");
11329 return false;
11330 }
11331 }
11332
11333 // Values have to be dynamically unique or we loose the fact that a
11334 // single llvm::Value might represent two runtime values (e.g.,
11335 // stack locations in different recursive calls).
11336 const Instruction *CtxI = II.I.getCtxI();
11337 bool ScopeIsLocal = (II.S & AA::Intraprocedural);
11338 bool AllLocal = ScopeIsLocal;
11339 bool DynamicallyUnique = llvm::all_of(Range&: PotentialCopies, P: [&](Value *PC) {
11340 AllLocal &= AA::isValidInScope(V: *PC, Scope: getAnchorScope());
11341 return AA::isDynamicallyUnique(A, QueryingAA: *this, V: *PC);
11342 });
11343 if (!DynamicallyUnique) {
11344 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Not all potentially loaded "
11345 "values are dynamically unique: "
11346 << LI << "\n");
11347 return false;
11348 }
11349
11350 for (auto *PotentialCopy : PotentialCopies) {
11351 if (AllLocal) {
11352 Worklist.push_back(Elt: {.I: {*PotentialCopy, CtxI}, .S: II.S});
11353 } else {
11354 Worklist.push_back(Elt: {.I: {*PotentialCopy, CtxI}, .S: AA::Interprocedural});
11355 }
11356 }
11357 if (!AllLocal && ScopeIsLocal)
11358 addValue(A, State&: getState(), V&: LI, CtxI, S: AA::Intraprocedural, AnchorScope: getAnchorScope());
11359 return true;
11360 }
11361
11362 bool handlePHINode(
11363 Attributor &A, PHINode &PHI, ItemInfo II,
11364 SmallVectorImpl<ItemInfo> &Worklist,
11365 SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
11366 auto GetLivenessInfo = [&](const Function &F) -> LivenessInfo & {
11367 LivenessInfo &LI = LivenessAAs[&F];
11368 if (!LI.LivenessAA)
11369 LI.LivenessAA = A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: IRPosition::function(F),
11370 DepClass: DepClassTy::NONE);
11371 return LI;
11372 };
11373
11374 if (&PHI == &getAssociatedValue()) {
11375 LivenessInfo &LI = GetLivenessInfo(*PHI.getFunction());
11376 const auto *CI =
11377 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
11378 F: *PHI.getFunction());
11379
11380 Cycle *C = nullptr;
11381 bool CyclePHI = mayBeInCycle(CI, I: &PHI, /* HeaderOnly */ true, CPtr: &C);
11382 for (unsigned u = 0, e = PHI.getNumIncomingValues(); u < e; u++) {
11383 BasicBlock *IncomingBB = PHI.getIncomingBlock(i: u);
11384 if (LI.LivenessAA &&
11385 LI.LivenessAA->isEdgeDead(From: IncomingBB, To: PHI.getParent())) {
11386 LI.AnyDead = true;
11387 continue;
11388 }
11389 Value *V = PHI.getIncomingValue(i: u);
11390 if (V == &PHI)
11391 continue;
11392
11393 // If the incoming value is not the PHI but an instruction in the same
11394 // cycle we might have multiple versions of it flying around.
11395 if (CyclePHI && isa<Instruction>(Val: V) &&
11396 (!C || C->contains(Block: cast<Instruction>(Val: V)->getParent())))
11397 return false;
11398
11399 Worklist.push_back(Elt: {.I: {*V, IncomingBB->getTerminator()}, .S: II.S});
11400 }
11401 return true;
11402 }
11403
11404 bool UsedAssumedInformation = false;
11405 std::optional<Value *> SimpleV = A.getAssumedSimplified(
11406 IRP: IRPosition::inst(I: PHI), AA: *this, UsedAssumedInformation, S: II.S);
11407 if (!SimpleV.has_value())
11408 return true;
11409 if (!(*SimpleV))
11410 return false;
11411 addValue(A, State&: getState(), V&: **SimpleV, CtxI: &PHI, S: II.S, AnchorScope: getAnchorScope());
11412 return true;
11413 }
11414
11415 /// Use the generic, non-optimistic InstSimplfy functionality if we managed to
11416 /// simplify any operand of the instruction \p I. Return true if successful,
11417 /// in that case Worklist will be updated.
11418 bool handleGenericInst(Attributor &A, Instruction &I, ItemInfo II,
11419 SmallVectorImpl<ItemInfo> &Worklist) {
11420 bool SomeSimplified = false;
11421 bool UsedAssumedInformation = false;
11422
11423 SmallVector<Value *, 8> NewOps(I.getNumOperands());
11424 int Idx = 0;
11425 for (Value *Op : I.operands()) {
11426 const auto &SimplifiedOp = A.getAssumedSimplified(
11427 IRP: IRPosition::value(V: *Op, CBContext: getCallBaseContext()), AA: *this,
11428 UsedAssumedInformation, S: AA::Intraprocedural);
11429 // If we are not sure about any operand we are not sure about the entire
11430 // instruction, we'll wait.
11431 if (!SimplifiedOp.has_value())
11432 return true;
11433
11434 if (*SimplifiedOp)
11435 NewOps[Idx] = *SimplifiedOp;
11436 else
11437 NewOps[Idx] = Op;
11438
11439 SomeSimplified |= (NewOps[Idx] != Op);
11440 ++Idx;
11441 }
11442
11443 // We won't bother with the InstSimplify interface if we didn't simplify any
11444 // operand ourselves.
11445 if (!SomeSimplified)
11446 return false;
11447
11448 InformationCache &InfoCache = A.getInfoCache();
11449 Function *F = I.getFunction();
11450 const auto *DT =
11451 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
11452 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
11453 auto *AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
11454
11455 const DataLayout &DL = I.getDataLayout();
11456 SimplifyQuery Q(DL, TLI, DT, AC, &I);
11457 Value *NewV = simplifyInstructionWithOperands(I: &I, NewOps, Q);
11458 if (!NewV || NewV == &I)
11459 return false;
11460
11461 LLVM_DEBUG(dbgs() << "Generic inst " << I << " assumed simplified to "
11462 << *NewV << "\n");
11463 Worklist.push_back(Elt: {.I: {*NewV, II.I.getCtxI()}, .S: II.S});
11464 return true;
11465 }
11466
11467 bool simplifyInstruction(
11468 Attributor &A, Instruction &I, ItemInfo II,
11469 SmallVectorImpl<ItemInfo> &Worklist,
11470 SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
11471 if (auto *CI = dyn_cast<CmpInst>(Val: &I))
11472 return handleCmp(A, Cmp&: *CI, LHS: CI->getOperand(i_nocapture: 0), RHS: CI->getOperand(i_nocapture: 1),
11473 Pred: CI->getPredicate(), II, Worklist);
11474
11475 switch (I.getOpcode()) {
11476 case Instruction::Select:
11477 return handleSelectInst(A, SI&: cast<SelectInst>(Val&: I), II, Worklist);
11478 case Instruction::PHI:
11479 return handlePHINode(A, PHI&: cast<PHINode>(Val&: I), II, Worklist, LivenessAAs);
11480 case Instruction::Load:
11481 return handleLoadInst(A, LI&: cast<LoadInst>(Val&: I), II, Worklist);
11482 default:
11483 return handleGenericInst(A, I, II, Worklist);
11484 };
11485 return false;
11486 }
11487
11488 void genericValueTraversal(Attributor &A, Value *InitialV) {
11489 SmallMapVector<const Function *, LivenessInfo, 4> LivenessAAs;
11490
11491 SmallSet<ItemInfo, 16> Visited;
11492 SmallVector<ItemInfo, 16> Worklist;
11493 Worklist.push_back(Elt: {.I: {*InitialV, getCtxI()}, .S: AA::AnyScope});
11494
11495 int Iteration = 0;
11496 do {
11497 ItemInfo II = Worklist.pop_back_val();
11498 Value *V = II.I.getValue();
11499 assert(V);
11500 const Instruction *CtxI = II.I.getCtxI();
11501 AA::ValueScope S = II.S;
11502
11503 // Check if we should process the current value. To prevent endless
11504 // recursion keep a record of the values we followed!
11505 if (!Visited.insert(V: II).second)
11506 continue;
11507
11508 // Make sure we limit the compile time for complex expressions.
11509 if (Iteration++ >= MaxPotentialValuesIterations) {
11510 LLVM_DEBUG(dbgs() << "Generic value traversal reached iteration limit: "
11511 << Iteration << "!\n");
11512 addValue(A, State&: getState(), V&: *V, CtxI, S, AnchorScope: getAnchorScope());
11513 continue;
11514 }
11515
11516 // Explicitly look through calls with a "returned" attribute if we do
11517 // not have a pointer as stripPointerCasts only works on them.
11518 Value *NewV = nullptr;
11519 if (V->getType()->isPointerTy()) {
11520 NewV = AA::getWithType(V&: *V->stripPointerCasts(), Ty&: *V->getType());
11521 } else {
11522 if (auto *CB = dyn_cast<CallBase>(Val: V))
11523 if (auto *Callee =
11524 dyn_cast_if_present<Function>(Val: CB->getCalledOperand())) {
11525 for (Argument &Arg : Callee->args())
11526 if (Arg.hasReturnedAttr()) {
11527 NewV = CB->getArgOperand(i: Arg.getArgNo());
11528 break;
11529 }
11530 }
11531 }
11532 if (NewV && NewV != V) {
11533 Worklist.push_back(Elt: {.I: {*NewV, CtxI}, .S: S});
11534 continue;
11535 }
11536
11537 if (auto *I = dyn_cast<Instruction>(Val: V)) {
11538 if (simplifyInstruction(A, I&: *I, II, Worklist, LivenessAAs))
11539 continue;
11540 }
11541
11542 if (V != InitialV || isa<Argument>(Val: V))
11543 if (recurseForValue(A, IRP: IRPosition::value(V: *V), S: II.S))
11544 continue;
11545
11546 // If we haven't stripped anything we give up.
11547 if (V == InitialV && CtxI == getCtxI()) {
11548 indicatePessimisticFixpoint();
11549 return;
11550 }
11551
11552 addValue(A, State&: getState(), V&: *V, CtxI, S, AnchorScope: getAnchorScope());
11553 } while (!Worklist.empty());
11554
11555 // If we actually used liveness information so we have to record a
11556 // dependence.
11557 for (auto &It : LivenessAAs)
11558 if (It.second.AnyDead)
11559 A.recordDependence(FromAA: *It.second.LivenessAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
11560 }
11561
11562 /// See AbstractAttribute::trackStatistics()
11563 void trackStatistics() const override {
11564 STATS_DECLTRACK_FLOATING_ATTR(potential_values)
11565 }
11566};
11567
11568struct AAPotentialValuesArgument final : AAPotentialValuesImpl {
11569 using Base = AAPotentialValuesImpl;
11570 AAPotentialValuesArgument(const IRPosition &IRP, Attributor &A)
11571 : Base(IRP, A) {}
11572
11573 /// See AbstractAttribute::initialize(..).
11574 void initialize(Attributor &A) override {
11575 auto &Arg = cast<Argument>(Val&: getAssociatedValue());
11576 if (Arg.hasPointeeInMemoryValueAttr())
11577 indicatePessimisticFixpoint();
11578 }
11579
11580 /// See AbstractAttribute::updateImpl(...).
11581 ChangeStatus updateImpl(Attributor &A) override {
11582 auto AssumedBefore = getAssumed();
11583
11584 unsigned ArgNo = getCalleeArgNo();
11585
11586 bool UsedAssumedInformation = false;
11587 SmallVector<AA::ValueAndContext> Values;
11588 auto CallSitePred = [&](AbstractCallSite ACS) {
11589 const auto CSArgIRP = IRPosition::callsite_argument(ACS, ArgNo);
11590 if (CSArgIRP.getPositionKind() == IRP_INVALID)
11591 return false;
11592
11593 if (!A.getAssumedSimplifiedValues(IRP: CSArgIRP, AA: this, Values,
11594 S: AA::Interprocedural,
11595 UsedAssumedInformation))
11596 return false;
11597
11598 return isValidState();
11599 };
11600
11601 if (!A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this,
11602 /* RequireAllCallSites */ true,
11603 UsedAssumedInformation))
11604 return indicatePessimisticFixpoint();
11605
11606 Function *Fn = getAssociatedFunction();
11607 bool AnyNonLocal = false;
11608 for (auto &It : Values) {
11609 if (isa<Constant>(Val: It.getValue())) {
11610 addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::AnyScope,
11611 AnchorScope: getAnchorScope());
11612 continue;
11613 }
11614 if (!AA::isDynamicallyUnique(A, QueryingAA: *this, V: *It.getValue()))
11615 return indicatePessimisticFixpoint();
11616
11617 if (auto *Arg = dyn_cast<Argument>(Val: It.getValue()))
11618 if (Arg->getParent() == Fn) {
11619 addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::AnyScope,
11620 AnchorScope: getAnchorScope());
11621 continue;
11622 }
11623 addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::Interprocedural,
11624 AnchorScope: getAnchorScope());
11625 AnyNonLocal = true;
11626 }
11627 assert(!undefIsContained() && "Undef should be an explicit value!");
11628 if (AnyNonLocal)
11629 giveUpOnIntraprocedural(A);
11630
11631 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11632 : ChangeStatus::CHANGED;
11633 }
11634
11635 /// See AbstractAttribute::trackStatistics()
11636 void trackStatistics() const override {
11637 STATS_DECLTRACK_ARG_ATTR(potential_values)
11638 }
11639};
11640
11641struct AAPotentialValuesReturned : public AAPotentialValuesFloating {
11642 using Base = AAPotentialValuesFloating;
11643 AAPotentialValuesReturned(const IRPosition &IRP, Attributor &A)
11644 : Base(IRP, A) {}
11645
11646 /// See AbstractAttribute::initialize(..).
11647 void initialize(Attributor &A) override {
11648 Function *F = getAssociatedFunction();
11649 if (!F || F->isDeclaration() || F->getReturnType()->isVoidTy()) {
11650 indicatePessimisticFixpoint();
11651 return;
11652 }
11653
11654 for (Argument &Arg : F->args())
11655 if (Arg.hasReturnedAttr()) {
11656 addValue(A, State&: getState(), V&: Arg, CtxI: nullptr, S: AA::AnyScope, AnchorScope: F);
11657 ReturnedArg = &Arg;
11658 break;
11659 }
11660 if (!A.isFunctionIPOAmendable(F: *F) ||
11661 A.hasSimplificationCallback(IRP: getIRPosition())) {
11662 if (!ReturnedArg)
11663 indicatePessimisticFixpoint();
11664 else
11665 indicateOptimisticFixpoint();
11666 }
11667 }
11668
11669 /// See AbstractAttribute::updateImpl(...).
11670 ChangeStatus updateImpl(Attributor &A) override {
11671 auto AssumedBefore = getAssumed();
11672 bool UsedAssumedInformation = false;
11673
11674 SmallVector<AA::ValueAndContext> Values;
11675 Function *AnchorScope = getAnchorScope();
11676 auto HandleReturnedValue = [&](Value &V, Instruction *CtxI,
11677 bool AddValues) {
11678 for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {
11679 Values.clear();
11680 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V), AA: this, Values, S,
11681 UsedAssumedInformation,
11682 /* RecurseForSelectAndPHI */ true))
11683 return false;
11684 if (!AddValues)
11685 continue;
11686
11687 bool AllInterAreIntra = false;
11688 if (S == AA::Interprocedural)
11689 AllInterAreIntra =
11690 llvm::all_of(Range&: Values, P: [&](const AA::ValueAndContext &VAC) {
11691 return AA::isValidInScope(V: *VAC.getValue(), Scope: AnchorScope);
11692 });
11693
11694 for (const AA::ValueAndContext &VAC : Values) {
11695 addValue(A, State&: getState(), V&: *VAC.getValue(),
11696 CtxI: VAC.getCtxI() ? VAC.getCtxI() : CtxI,
11697 S: AllInterAreIntra ? AA::AnyScope : S, AnchorScope);
11698 }
11699 if (AllInterAreIntra)
11700 break;
11701 }
11702 return true;
11703 };
11704
11705 if (ReturnedArg) {
11706 HandleReturnedValue(*ReturnedArg, nullptr, true);
11707 } else {
11708 auto RetInstPred = [&](Instruction &RetI) {
11709 bool AddValues = true;
11710 if (isa<PHINode>(Val: RetI.getOperand(i: 0)) ||
11711 isa<SelectInst>(Val: RetI.getOperand(i: 0))) {
11712 addValue(A, State&: getState(), V&: *RetI.getOperand(i: 0), CtxI: &RetI, S: AA::AnyScope,
11713 AnchorScope);
11714 AddValues = false;
11715 }
11716 return HandleReturnedValue(*RetI.getOperand(i: 0), &RetI, AddValues);
11717 };
11718
11719 if (!A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
11720 UsedAssumedInformation,
11721 /* CheckBBLivenessOnly */ true))
11722 return indicatePessimisticFixpoint();
11723 }
11724
11725 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11726 : ChangeStatus::CHANGED;
11727 }
11728
11729 ChangeStatus manifest(Attributor &A) override {
11730 if (ReturnedArg)
11731 return ChangeStatus::UNCHANGED;
11732 SmallVector<AA::ValueAndContext> Values;
11733 if (!getAssumedSimplifiedValues(A, Values, S: AA::ValueScope::Intraprocedural,
11734 /* RecurseForSelectAndPHI */ true))
11735 return ChangeStatus::UNCHANGED;
11736 Value *NewVal = getSingleValue(A, AA: *this, IRP: getIRPosition(), Values);
11737 if (!NewVal)
11738 return ChangeStatus::UNCHANGED;
11739
11740 ChangeStatus Changed = ChangeStatus::UNCHANGED;
11741 if (auto *Arg = dyn_cast<Argument>(Val: NewVal)) {
11742 STATS_DECLTRACK(UniqueReturnValue, FunctionReturn,
11743 "Number of function with unique return");
11744 Changed |= A.manifestAttrs(
11745 IRP: IRPosition::argument(Arg: *Arg),
11746 DeducedAttrs: {Attribute::get(Context&: Arg->getContext(), Kind: Attribute::Returned)});
11747 STATS_DECLTRACK_ARG_ATTR(returned);
11748 }
11749
11750 auto RetInstPred = [&](Instruction &RetI) {
11751 Value *RetOp = RetI.getOperand(i: 0);
11752 if (isa<UndefValue>(Val: RetOp) || RetOp == NewVal)
11753 return true;
11754 if (AA::isValidAtPosition(VAC: {*NewVal, RetI}, InfoCache&: A.getInfoCache()))
11755 if (A.changeUseAfterManifest(U&: RetI.getOperandUse(i: 0), NV&: *NewVal))
11756 Changed = ChangeStatus::CHANGED;
11757 return true;
11758 };
11759 bool UsedAssumedInformation = false;
11760 (void)A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
11761 UsedAssumedInformation,
11762 /* CheckBBLivenessOnly */ true);
11763 return Changed;
11764 }
11765
11766 ChangeStatus indicatePessimisticFixpoint() override {
11767 return AAPotentialValues::indicatePessimisticFixpoint();
11768 }
11769
11770 /// See AbstractAttribute::trackStatistics()
11771 void trackStatistics() const override{
11772 STATS_DECLTRACK_FNRET_ATTR(potential_values)}
11773
11774 /// The argumented with an existing `returned` attribute.
11775 Argument *ReturnedArg = nullptr;
11776};
11777
11778struct AAPotentialValuesFunction : AAPotentialValuesImpl {
11779 AAPotentialValuesFunction(const IRPosition &IRP, Attributor &A)
11780 : AAPotentialValuesImpl(IRP, A) {}
11781
11782 /// See AbstractAttribute::updateImpl(...).
11783 ChangeStatus updateImpl(Attributor &A) override {
11784 llvm_unreachable("AAPotentialValues(Function|CallSite)::updateImpl will "
11785 "not be called");
11786 }
11787
11788 /// See AbstractAttribute::trackStatistics()
11789 void trackStatistics() const override {
11790 STATS_DECLTRACK_FN_ATTR(potential_values)
11791 }
11792};
11793
11794struct AAPotentialValuesCallSite : AAPotentialValuesFunction {
11795 AAPotentialValuesCallSite(const IRPosition &IRP, Attributor &A)
11796 : AAPotentialValuesFunction(IRP, A) {}
11797
11798 /// See AbstractAttribute::trackStatistics()
11799 void trackStatistics() const override {
11800 STATS_DECLTRACK_CS_ATTR(potential_values)
11801 }
11802};
11803
11804struct AAPotentialValuesCallSiteReturned : AAPotentialValuesImpl {
11805 AAPotentialValuesCallSiteReturned(const IRPosition &IRP, Attributor &A)
11806 : AAPotentialValuesImpl(IRP, A) {}
11807
11808 /// See AbstractAttribute::updateImpl(...).
11809 ChangeStatus updateImpl(Attributor &A) override {
11810 auto AssumedBefore = getAssumed();
11811
11812 Function *Callee = getAssociatedFunction();
11813 if (!Callee)
11814 return indicatePessimisticFixpoint();
11815
11816 bool UsedAssumedInformation = false;
11817 auto *CB = cast<CallBase>(Val: getCtxI());
11818 if (CB->isMustTailCall() &&
11819 !A.isAssumedDead(IRP: IRPosition::inst(I: *CB), QueryingAA: this, FnLivenessAA: nullptr,
11820 UsedAssumedInformation))
11821 return indicatePessimisticFixpoint();
11822
11823 Function *Caller = CB->getCaller();
11824
11825 auto AddScope = [&](AA::ValueScope S) {
11826 SmallVector<AA::ValueAndContext> Values;
11827 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::returned(F: *Callee), AA: this,
11828 Values, S, UsedAssumedInformation))
11829 return false;
11830
11831 for (auto &It : Values) {
11832 Value *V = It.getValue();
11833 std::optional<Value *> CallerV = A.translateArgumentToCallSiteContent(
11834 V, CB&: *CB, AA: *this, UsedAssumedInformation);
11835 if (!CallerV.has_value()) {
11836 // Nothing to do as long as no value was determined.
11837 continue;
11838 }
11839 V = *CallerV ? *CallerV : V;
11840 if (*CallerV && AA::isDynamicallyUnique(A, QueryingAA: *this, V: *V)) {
11841 if (recurseForValue(A, IRP: IRPosition::value(V: *V), S))
11842 continue;
11843 }
11844 if (S == AA::Intraprocedural && !AA::isValidInScope(V: *V, Scope: Caller)) {
11845 giveUpOnIntraprocedural(A);
11846 return true;
11847 }
11848 addValue(A, State&: getState(), V&: *V, CtxI: CB, S, AnchorScope: getAnchorScope());
11849 }
11850 return true;
11851 };
11852 if (!AddScope(AA::Intraprocedural))
11853 return indicatePessimisticFixpoint();
11854 if (!AddScope(AA::Interprocedural))
11855 return indicatePessimisticFixpoint();
11856 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11857 : ChangeStatus::CHANGED;
11858 }
11859
11860 ChangeStatus indicatePessimisticFixpoint() override {
11861 return AAPotentialValues::indicatePessimisticFixpoint();
11862 }
11863
11864 /// See AbstractAttribute::trackStatistics()
11865 void trackStatistics() const override {
11866 STATS_DECLTRACK_CSRET_ATTR(potential_values)
11867 }
11868};
11869
11870struct AAPotentialValuesCallSiteArgument : AAPotentialValuesFloating {
11871 AAPotentialValuesCallSiteArgument(const IRPosition &IRP, Attributor &A)
11872 : AAPotentialValuesFloating(IRP, A) {}
11873
11874 /// See AbstractAttribute::trackStatistics()
11875 void trackStatistics() const override {
11876 STATS_DECLTRACK_CSARG_ATTR(potential_values)
11877 }
11878};
11879} // namespace
11880
11881/// ---------------------- Assumption Propagation ------------------------------
11882namespace {
11883struct AAAssumptionInfoImpl : public AAAssumptionInfo {
11884 AAAssumptionInfoImpl(const IRPosition &IRP, Attributor &A,
11885 const DenseSet<StringRef> &Known)
11886 : AAAssumptionInfo(IRP, A, Known) {}
11887
11888 /// See AbstractAttribute::manifest(...).
11889 ChangeStatus manifest(Attributor &A) override {
11890 // Don't manifest a universal set if it somehow made it here.
11891 if (getKnown().isUniversal())
11892 return ChangeStatus::UNCHANGED;
11893
11894 const IRPosition &IRP = getIRPosition();
11895 SmallVector<StringRef, 0> Set(getAssumed().getSet().begin(),
11896 getAssumed().getSet().end());
11897 llvm::sort(C&: Set);
11898 return A.manifestAttrs(IRP,
11899 DeducedAttrs: Attribute::get(Context&: IRP.getAnchorValue().getContext(),
11900 Kind: AssumptionAttrKey,
11901 Val: llvm::join(R&: Set, Separator: ",")),
11902 /*ForceReplace=*/true);
11903 }
11904
11905 bool hasAssumption(const StringRef Assumption) const override {
11906 return isValidState() && setContains(Assumption);
11907 }
11908
11909 /// See AbstractAttribute::getAsStr()
11910 const std::string getAsStr(Attributor *A) const override {
11911 const SetContents &Known = getKnown();
11912 const SetContents &Assumed = getAssumed();
11913
11914 SmallVector<StringRef, 0> Set(Known.getSet().begin(), Known.getSet().end());
11915 llvm::sort(C&: Set);
11916 const std::string KnownStr = llvm::join(R&: Set, Separator: ",");
11917
11918 std::string AssumedStr = "Universal";
11919 if (!Assumed.isUniversal()) {
11920 Set.assign(in_start: Assumed.getSet().begin(), in_end: Assumed.getSet().end());
11921 AssumedStr = llvm::join(R&: Set, Separator: ",");
11922 }
11923 return "Known [" + KnownStr + "]," + " Assumed [" + AssumedStr + "]";
11924 }
11925};
11926
11927/// Propagates assumption information from parent functions to all of their
11928/// successors. An assumption can be propagated if the containing function
11929/// dominates the called function.
11930///
11931/// We start with a "known" set of assumptions already valid for the associated
11932/// function and an "assumed" set that initially contains all possible
11933/// assumptions. The assumed set is inter-procedurally updated by narrowing its
11934/// contents as concrete values are known. The concrete values are seeded by the
11935/// first nodes that are either entries into the call graph, or contains no
11936/// assumptions. Each node is updated as the intersection of the assumed state
11937/// with all of its predecessors.
11938struct AAAssumptionInfoFunction final : AAAssumptionInfoImpl {
11939 AAAssumptionInfoFunction(const IRPosition &IRP, Attributor &A)
11940 : AAAssumptionInfoImpl(IRP, A,
11941 getAssumptions(F: *IRP.getAssociatedFunction())) {}
11942
11943 /// See AbstractAttribute::updateImpl(...).
11944 ChangeStatus updateImpl(Attributor &A) override {
11945 bool Changed = false;
11946
11947 auto CallSitePred = [&](AbstractCallSite ACS) {
11948 const auto *AssumptionAA = A.getAAFor<AAAssumptionInfo>(
11949 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *ACS.getInstruction()),
11950 DepClass: DepClassTy::REQUIRED);
11951 if (!AssumptionAA)
11952 return false;
11953 // Get the set of assumptions shared by all of this function's callers.
11954 Changed |= getIntersection(RHS: AssumptionAA->getAssumed());
11955 return !getAssumed().empty() || !getKnown().empty();
11956 };
11957
11958 bool UsedAssumedInformation = false;
11959 // Get the intersection of all assumptions held by this node's predecessors.
11960 // If we don't know all the call sites then this is either an entry into the
11961 // call graph or an empty node. This node is known to only contain its own
11962 // assumptions and can be propagated to its successors.
11963 if (!A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this, RequireAllCallSites: true,
11964 UsedAssumedInformation))
11965 return indicatePessimisticFixpoint();
11966
11967 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
11968 }
11969
11970 void trackStatistics() const override {}
11971};
11972
11973/// Assumption Info defined for call sites.
11974struct AAAssumptionInfoCallSite final : AAAssumptionInfoImpl {
11975
11976 AAAssumptionInfoCallSite(const IRPosition &IRP, Attributor &A)
11977 : AAAssumptionInfoImpl(IRP, A, getInitialAssumptions(IRP)) {}
11978
11979 /// See AbstractAttribute::initialize(...).
11980 void initialize(Attributor &A) override {
11981 const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
11982 A.getAAFor<AAAssumptionInfo>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
11983 }
11984
11985 /// See AbstractAttribute::updateImpl(...).
11986 ChangeStatus updateImpl(Attributor &A) override {
11987 const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
11988 auto *AssumptionAA =
11989 A.getAAFor<AAAssumptionInfo>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
11990 if (!AssumptionAA)
11991 return indicatePessimisticFixpoint();
11992 bool Changed = getIntersection(RHS: AssumptionAA->getAssumed());
11993 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
11994 }
11995
11996 /// See AbstractAttribute::trackStatistics()
11997 void trackStatistics() const override {}
11998
11999private:
12000 /// Helper to initialized the known set as all the assumptions this call and
12001 /// the callee contain.
12002 DenseSet<StringRef> getInitialAssumptions(const IRPosition &IRP) {
12003 const CallBase &CB = cast<CallBase>(Val&: IRP.getAssociatedValue());
12004 auto Assumptions = getAssumptions(CB);
12005 if (const Function *F = CB.getCaller())
12006 set_union(S1&: Assumptions, S2: getAssumptions(F: *F));
12007 if (Function *F = IRP.getAssociatedFunction())
12008 set_union(S1&: Assumptions, S2: getAssumptions(F: *F));
12009 return Assumptions;
12010 }
12011};
12012} // namespace
12013
12014AACallGraphNode *AACallEdgeIterator::operator*() const {
12015 return static_cast<AACallGraphNode *>(const_cast<AACallEdges *>(
12016 A.getOrCreateAAFor<AACallEdges>(IRP: IRPosition::function(F: **I))));
12017}
12018
12019void AttributorCallGraph::print() { llvm::WriteGraph(O&: outs(), G: this); }
12020
12021/// ------------------------ UnderlyingObjects ---------------------------------
12022
12023namespace {
12024struct AAUnderlyingObjectsImpl
12025 : StateWrapper<BooleanState, AAUnderlyingObjects> {
12026 using BaseTy = StateWrapper<BooleanState, AAUnderlyingObjects>;
12027 AAUnderlyingObjectsImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
12028
12029 /// See AbstractAttribute::getAsStr().
12030 const std::string getAsStr(Attributor *A) const override {
12031 if (!isValidState())
12032 return "<invalid>";
12033 std::string Str;
12034 llvm::raw_string_ostream OS(Str);
12035 OS << "underlying objects: inter " << InterAssumedUnderlyingObjects.size()
12036 << " objects, intra " << IntraAssumedUnderlyingObjects.size()
12037 << " objects.\n";
12038 if (!InterAssumedUnderlyingObjects.empty()) {
12039 OS << "inter objects:\n";
12040 for (auto *Obj : InterAssumedUnderlyingObjects)
12041 OS << *Obj << '\n';
12042 }
12043 if (!IntraAssumedUnderlyingObjects.empty()) {
12044 OS << "intra objects:\n";
12045 for (auto *Obj : IntraAssumedUnderlyingObjects)
12046 OS << *Obj << '\n';
12047 }
12048 return Str;
12049 }
12050
12051 /// See AbstractAttribute::trackStatistics()
12052 void trackStatistics() const override {}
12053
12054 /// See AbstractAttribute::updateImpl(...).
12055 ChangeStatus updateImpl(Attributor &A) override {
12056 auto &Ptr = getAssociatedValue();
12057
12058 bool UsedAssumedInformation = false;
12059 auto DoUpdate = [&](SmallSetVector<Value *, 8> &UnderlyingObjects,
12060 AA::ValueScope Scope) {
12061 SmallPtrSet<Value *, 8> SeenObjects;
12062 SmallVector<AA::ValueAndContext> Values;
12063
12064 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: Ptr), AA: *this, Values,
12065 S: Scope, UsedAssumedInformation))
12066 return UnderlyingObjects.insert(X: &Ptr);
12067
12068 bool Changed = false;
12069
12070 for (unsigned I = 0; I < Values.size(); ++I) {
12071 auto &VAC = Values[I];
12072 auto *Obj = VAC.getValue();
12073 Value *UO = getUnderlyingObject(V: Obj);
12074 if (!SeenObjects.insert(Ptr: UO ? UO : Obj).second)
12075 continue;
12076 if (UO && UO != Obj) {
12077 if (isa<AllocaInst>(Val: UO) || isa<GlobalValue>(Val: UO)) {
12078 Changed |= UnderlyingObjects.insert(X: UO);
12079 continue;
12080 }
12081
12082 const auto *OtherAA = A.getAAFor<AAUnderlyingObjects>(
12083 QueryingAA: *this, IRP: IRPosition::value(V: *UO), DepClass: DepClassTy::OPTIONAL);
12084 auto Pred = [&](Value &V) {
12085 if (&V == UO)
12086 Changed |= UnderlyingObjects.insert(X: UO);
12087 else
12088 Values.emplace_back(Args&: V, Args: nullptr);
12089 return true;
12090 };
12091
12092 if (!OtherAA || !OtherAA->forallUnderlyingObjects(Pred, Scope))
12093 llvm_unreachable(
12094 "The forall call should not return false at this position");
12095 UsedAssumedInformation |= !OtherAA->getState().isAtFixpoint();
12096 continue;
12097 }
12098
12099 if (isa<SelectInst>(Val: Obj)) {
12100 Changed |= handleIndirect(A, V&: *Obj, UnderlyingObjects, Scope,
12101 UsedAssumedInformation);
12102 continue;
12103 }
12104 if (auto *PHI = dyn_cast<PHINode>(Val: Obj)) {
12105 // Explicitly look through PHIs as we do not care about dynamically
12106 // uniqueness.
12107 for (unsigned u = 0, e = PHI->getNumIncomingValues(); u < e; u++) {
12108 Changed |=
12109 handleIndirect(A, V&: *PHI->getIncomingValue(i: u), UnderlyingObjects,
12110 Scope, UsedAssumedInformation);
12111 }
12112 continue;
12113 }
12114
12115 Changed |= UnderlyingObjects.insert(X: Obj);
12116 }
12117
12118 return Changed;
12119 };
12120
12121 bool Changed = false;
12122 Changed |= DoUpdate(IntraAssumedUnderlyingObjects, AA::Intraprocedural);
12123 Changed |= DoUpdate(InterAssumedUnderlyingObjects, AA::Interprocedural);
12124 if (!UsedAssumedInformation)
12125 indicateOptimisticFixpoint();
12126 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
12127 }
12128
12129 bool forallUnderlyingObjects(
12130 function_ref<bool(Value &)> Pred,
12131 AA::ValueScope Scope = AA::Interprocedural) const override {
12132 if (!isValidState())
12133 return Pred(getAssociatedValue());
12134
12135 auto &AssumedUnderlyingObjects = Scope == AA::Intraprocedural
12136 ? IntraAssumedUnderlyingObjects
12137 : InterAssumedUnderlyingObjects;
12138 for (Value *Obj : AssumedUnderlyingObjects)
12139 if (!Pred(*Obj))
12140 return false;
12141
12142 return true;
12143 }
12144
12145private:
12146 /// Handle the case where the value is not the actual underlying value, such
12147 /// as a phi node or a select instruction.
12148 bool handleIndirect(Attributor &A, Value &V,
12149 SmallSetVector<Value *, 8> &UnderlyingObjects,
12150 AA::ValueScope Scope, bool &UsedAssumedInformation) {
12151 bool Changed = false;
12152 const auto *AA = A.getAAFor<AAUnderlyingObjects>(
12153 QueryingAA: *this, IRP: IRPosition::value(V), DepClass: DepClassTy::OPTIONAL);
12154 auto Pred = [&](Value &V) {
12155 Changed |= UnderlyingObjects.insert(X: &V);
12156 return true;
12157 };
12158 if (!AA || !AA->forallUnderlyingObjects(Pred, Scope))
12159 llvm_unreachable(
12160 "The forall call should not return false at this position");
12161 UsedAssumedInformation |= !AA->getState().isAtFixpoint();
12162 return Changed;
12163 }
12164
12165 /// All the underlying objects collected so far via intra procedural scope.
12166 SmallSetVector<Value *, 8> IntraAssumedUnderlyingObjects;
12167 /// All the underlying objects collected so far via inter procedural scope.
12168 SmallSetVector<Value *, 8> InterAssumedUnderlyingObjects;
12169};
12170
12171struct AAUnderlyingObjectsFloating final : AAUnderlyingObjectsImpl {
12172 AAUnderlyingObjectsFloating(const IRPosition &IRP, Attributor &A)
12173 : AAUnderlyingObjectsImpl(IRP, A) {}
12174};
12175
12176struct AAUnderlyingObjectsArgument final : AAUnderlyingObjectsImpl {
12177 AAUnderlyingObjectsArgument(const IRPosition &IRP, Attributor &A)
12178 : AAUnderlyingObjectsImpl(IRP, A) {}
12179};
12180
12181struct AAUnderlyingObjectsCallSite final : AAUnderlyingObjectsImpl {
12182 AAUnderlyingObjectsCallSite(const IRPosition &IRP, Attributor &A)
12183 : AAUnderlyingObjectsImpl(IRP, A) {}
12184};
12185
12186struct AAUnderlyingObjectsCallSiteArgument final : AAUnderlyingObjectsImpl {
12187 AAUnderlyingObjectsCallSiteArgument(const IRPosition &IRP, Attributor &A)
12188 : AAUnderlyingObjectsImpl(IRP, A) {}
12189};
12190
12191struct AAUnderlyingObjectsReturned final : AAUnderlyingObjectsImpl {
12192 AAUnderlyingObjectsReturned(const IRPosition &IRP, Attributor &A)
12193 : AAUnderlyingObjectsImpl(IRP, A) {}
12194};
12195
12196struct AAUnderlyingObjectsCallSiteReturned final : AAUnderlyingObjectsImpl {
12197 AAUnderlyingObjectsCallSiteReturned(const IRPosition &IRP, Attributor &A)
12198 : AAUnderlyingObjectsImpl(IRP, A) {}
12199};
12200
12201struct AAUnderlyingObjectsFunction final : AAUnderlyingObjectsImpl {
12202 AAUnderlyingObjectsFunction(const IRPosition &IRP, Attributor &A)
12203 : AAUnderlyingObjectsImpl(IRP, A) {}
12204};
12205} // namespace
12206
12207/// ------------------------ Global Value Info -------------------------------
12208namespace {
12209struct AAGlobalValueInfoFloating : public AAGlobalValueInfo {
12210 AAGlobalValueInfoFloating(const IRPosition &IRP, Attributor &A)
12211 : AAGlobalValueInfo(IRP, A) {}
12212
12213 /// See AbstractAttribute::initialize(...).
12214 void initialize(Attributor &A) override {}
12215
12216 bool checkUse(Attributor &A, const Use &U, bool &Follow,
12217 SmallVectorImpl<const Value *> &Worklist) {
12218 Instruction *UInst = dyn_cast<Instruction>(Val: U.getUser());
12219 if (!UInst) {
12220 Follow = true;
12221 return true;
12222 }
12223
12224 LLVM_DEBUG(dbgs() << "[AAGlobalValueInfo] Check use: " << *U.get() << " in "
12225 << *UInst << "\n");
12226
12227 if (auto *Cmp = dyn_cast<ICmpInst>(Val: U.getUser())) {
12228 int Idx = &Cmp->getOperandUse(i: 0) == &U;
12229 if (isa<Constant>(Val: Cmp->getOperand(i_nocapture: Idx)))
12230 return true;
12231 return U == &getAnchorValue();
12232 }
12233
12234 // Explicitly catch return instructions.
12235 if (isa<ReturnInst>(Val: UInst)) {
12236 auto CallSitePred = [&](AbstractCallSite ACS) {
12237 Worklist.push_back(Elt: ACS.getInstruction());
12238 return true;
12239 };
12240 bool UsedAssumedInformation = false;
12241 // TODO: We should traverse the uses or add a "non-call-site" CB.
12242 if (!A.checkForAllCallSites(Pred: CallSitePred, Fn: *UInst->getFunction(),
12243 /*RequireAllCallSites=*/true, QueryingAA: this,
12244 UsedAssumedInformation))
12245 return false;
12246 return true;
12247 }
12248
12249 // For now we only use special logic for call sites. However, the tracker
12250 // itself knows about a lot of other non-capturing cases already.
12251 auto *CB = dyn_cast<CallBase>(Val: UInst);
12252 if (!CB)
12253 return false;
12254 // Direct calls are OK uses.
12255 if (CB->isCallee(U: &U))
12256 return true;
12257 // Non-argument uses are scary.
12258 if (!CB->isArgOperand(U: &U))
12259 return false;
12260 // TODO: Iterate callees.
12261 auto *Fn = dyn_cast<Function>(Val: CB->getCalledOperand());
12262 if (!Fn || !A.isFunctionIPOAmendable(F: *Fn))
12263 return false;
12264
12265 unsigned ArgNo = CB->getArgOperandNo(U: &U);
12266 Worklist.push_back(Elt: Fn->getArg(i: ArgNo));
12267 return true;
12268 }
12269
12270 ChangeStatus updateImpl(Attributor &A) override {
12271 unsigned NumUsesBefore = Uses.size();
12272
12273 SmallPtrSet<const Value *, 8> Visited;
12274 SmallVector<const Value *> Worklist;
12275 Worklist.push_back(Elt: &getAnchorValue());
12276
12277 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
12278 Uses.insert(Ptr: &U);
12279 // TODO(captures): Make this more precise.
12280 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
12281 if (CI.isPassthrough()) {
12282 Follow = true;
12283 return true;
12284 }
12285 return checkUse(A, U, Follow, Worklist);
12286 };
12287 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
12288 Uses.insert(Ptr: &OldU);
12289 return true;
12290 };
12291
12292 while (!Worklist.empty()) {
12293 const Value *V = Worklist.pop_back_val();
12294 if (!Visited.insert(Ptr: V).second)
12295 continue;
12296 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: *V,
12297 /* CheckBBLivenessOnly */ true,
12298 LivenessDepClass: DepClassTy::OPTIONAL,
12299 /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
12300 return indicatePessimisticFixpoint();
12301 }
12302 }
12303
12304 return Uses.size() == NumUsesBefore ? ChangeStatus::UNCHANGED
12305 : ChangeStatus::CHANGED;
12306 }
12307
12308 bool isPotentialUse(const Use &U) const override {
12309 return !isValidState() || Uses.contains(Ptr: &U);
12310 }
12311
12312 /// See AbstractAttribute::manifest(...).
12313 ChangeStatus manifest(Attributor &A) override {
12314 return ChangeStatus::UNCHANGED;
12315 }
12316
12317 /// See AbstractAttribute::getAsStr().
12318 const std::string getAsStr(Attributor *A) const override {
12319 return "[" + std::to_string(val: Uses.size()) + " uses]";
12320 }
12321
12322 void trackStatistics() const override {
12323 STATS_DECLTRACK_FLOATING_ATTR(GlobalValuesTracked);
12324 }
12325
12326private:
12327 /// Set of (transitive) uses of this GlobalValue.
12328 SmallPtrSet<const Use *, 8> Uses;
12329};
12330} // namespace
12331
12332/// ------------------------ Indirect Call Info -------------------------------
12333namespace {
12334struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo {
12335 AAIndirectCallInfoCallSite(const IRPosition &IRP, Attributor &A)
12336 : AAIndirectCallInfo(IRP, A) {}
12337
12338 /// See AbstractAttribute::initialize(...).
12339 void initialize(Attributor &A) override {
12340 auto *MD = getCtxI()->getMetadata(KindID: LLVMContext::MD_callees);
12341 if (!MD && !A.isClosedWorldModule())
12342 return;
12343
12344 if (MD) {
12345 for (const auto &Op : MD->operands())
12346 if (Function *Callee = mdconst::dyn_extract_or_null<Function>(MD: Op))
12347 PotentialCallees.insert(X: Callee);
12348 } else if (A.isClosedWorldModule()) {
12349 ArrayRef<Function *> IndirectlyCallableFunctions =
12350 A.getInfoCache().getIndirectlyCallableFunctions(A);
12351 PotentialCallees.insert_range(R&: IndirectlyCallableFunctions);
12352 }
12353
12354 if (PotentialCallees.empty())
12355 indicateOptimisticFixpoint();
12356 }
12357
12358 ChangeStatus updateImpl(Attributor &A) override {
12359 CallBase *CB = cast<CallBase>(Val: getCtxI());
12360 const Use &CalleeUse = CB->getCalledOperandUse();
12361 Value *FP = CB->getCalledOperand();
12362
12363 SmallSetVector<Function *, 4> AssumedCalleesNow;
12364 bool AllCalleesKnownNow = AllCalleesKnown;
12365
12366 auto CheckPotentialCalleeUse = [&](Function &PotentialCallee,
12367 bool &UsedAssumedInformation) {
12368 const auto *GIAA = A.getAAFor<AAGlobalValueInfo>(
12369 QueryingAA: *this, IRP: IRPosition::value(V: PotentialCallee), DepClass: DepClassTy::OPTIONAL);
12370 if (!GIAA || GIAA->isPotentialUse(U: CalleeUse))
12371 return true;
12372 UsedAssumedInformation = !GIAA->isAtFixpoint();
12373 return false;
12374 };
12375
12376 auto AddPotentialCallees = [&]() {
12377 for (auto *PotentialCallee : PotentialCallees) {
12378 bool UsedAssumedInformation = false;
12379 if (CheckPotentialCalleeUse(*PotentialCallee, UsedAssumedInformation))
12380 AssumedCalleesNow.insert(X: PotentialCallee);
12381 }
12382 };
12383
12384 // Use simplification to find potential callees, if !callees was present,
12385 // fallback to that set if necessary.
12386 bool UsedAssumedInformation = false;
12387 SmallVector<AA::ValueAndContext> Values;
12388 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *FP), AA: this, Values,
12389 S: AA::ValueScope::AnyScope,
12390 UsedAssumedInformation)) {
12391 if (PotentialCallees.empty())
12392 return indicatePessimisticFixpoint();
12393 AddPotentialCallees();
12394 }
12395
12396 // Try to find a reason for \p Fn not to be a potential callee. If none was
12397 // found, add it to the assumed callees set.
12398 auto CheckPotentialCallee = [&](Function &Fn) {
12399 if (!PotentialCallees.empty() && !PotentialCallees.count(key: &Fn))
12400 return false;
12401
12402 auto &CachedResult = FilterResults[&Fn];
12403 if (CachedResult.has_value())
12404 return CachedResult.value();
12405
12406 bool UsedAssumedInformation = false;
12407 if (!CheckPotentialCalleeUse(Fn, UsedAssumedInformation)) {
12408 if (!UsedAssumedInformation)
12409 CachedResult = false;
12410 return false;
12411 }
12412
12413 int NumFnArgs = Fn.arg_size();
12414 int NumCBArgs = CB->arg_size();
12415
12416 // Check if any excess argument (which we fill up with poison) is known to
12417 // be UB on undef.
12418 for (int I = NumCBArgs; I < NumFnArgs; ++I) {
12419 bool IsKnown = false;
12420 if (AA::hasAssumedIRAttr<Attribute::NoUndef>(
12421 A, QueryingAA: this, IRP: IRPosition::argument(Arg: *Fn.getArg(i: I)),
12422 DepClass: DepClassTy::OPTIONAL, IsKnown)) {
12423 if (IsKnown)
12424 CachedResult = false;
12425 return false;
12426 }
12427 }
12428
12429 CachedResult = true;
12430 return true;
12431 };
12432
12433 // Check simplification result, prune known UB callees, also restrict it to
12434 // the !callees set, if present.
12435 for (auto &VAC : Values) {
12436 if (isa<UndefValue>(Val: VAC.getValue()))
12437 continue;
12438 if (isa<ConstantPointerNull>(Val: VAC.getValue()) &&
12439 VAC.getValue()->getType()->getPointerAddressSpace() == 0)
12440 continue;
12441 // TODO: Check for known UB, e.g., poison + noundef.
12442 if (auto *VACFn = dyn_cast<Function>(Val: VAC.getValue())) {
12443 if (CheckPotentialCallee(*VACFn))
12444 AssumedCalleesNow.insert(X: VACFn);
12445 continue;
12446 }
12447 if (!PotentialCallees.empty()) {
12448 AddPotentialCallees();
12449 break;
12450 }
12451 AllCalleesKnownNow = false;
12452 }
12453
12454 if (AssumedCalleesNow == AssumedCallees &&
12455 AllCalleesKnown == AllCalleesKnownNow)
12456 return ChangeStatus::UNCHANGED;
12457
12458 std::swap(LHS&: AssumedCallees, RHS&: AssumedCalleesNow);
12459 AllCalleesKnown = AllCalleesKnownNow;
12460 return ChangeStatus::CHANGED;
12461 }
12462
12463 /// See AbstractAttribute::manifest(...).
12464 ChangeStatus manifest(Attributor &A) override {
12465 // If we can't specialize at all, give up now.
12466 if (!AllCalleesKnown && AssumedCallees.empty())
12467 return ChangeStatus::UNCHANGED;
12468
12469 CallBase *CB = cast<CallBase>(Val: getCtxI());
12470 bool UsedAssumedInformation = false;
12471 if (A.isAssumedDead(I: *CB, QueryingAA: this, /*LivenessAA=*/nullptr,
12472 UsedAssumedInformation))
12473 return ChangeStatus::UNCHANGED;
12474
12475 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12476 Value *FP = CB->getCalledOperand();
12477 if (FP->getType()->getPointerAddressSpace())
12478 FP = new AddrSpaceCastInst(FP, PointerType::get(C&: FP->getContext(), AddressSpace: 0),
12479 FP->getName() + ".as0", CB->getIterator());
12480
12481 bool CBIsVoid = CB->getType()->isVoidTy();
12482 BasicBlock::iterator IP = CB->getIterator();
12483 FunctionType *CSFT = CB->getFunctionType();
12484 SmallVector<Value *> CSArgs(CB->args());
12485
12486 // If we know all callees and there are none, the call site is (effectively)
12487 // dead (or UB).
12488 if (AssumedCallees.empty()) {
12489 assert(AllCalleesKnown &&
12490 "Expected all callees to be known if there are none.");
12491 A.changeToUnreachableAfterManifest(I: CB);
12492 return ChangeStatus::CHANGED;
12493 }
12494
12495 // Special handling for the single callee case.
12496 if (AllCalleesKnown && AssumedCallees.size() == 1) {
12497 auto *NewCallee = AssumedCallees.front();
12498 if (isLegalToPromote(CB: *CB, Callee: NewCallee)) {
12499 promoteCall(CB&: *CB, Callee: NewCallee, RetBitCast: nullptr);
12500 NumIndirectCallsPromoted++;
12501 return ChangeStatus::CHANGED;
12502 }
12503 Instruction *NewCall =
12504 CallInst::Create(Func: FunctionCallee(CSFT, NewCallee), Args: CSArgs,
12505 NameStr: CB->getName(), InsertBefore: CB->getIterator());
12506 if (!CBIsVoid)
12507 A.changeAfterManifest(IRP: IRPosition::callsite_returned(CB: *CB), NV&: *NewCall);
12508 A.deleteAfterManifest(I&: *CB);
12509 return ChangeStatus::CHANGED;
12510 }
12511
12512 // For each potential value we create a conditional
12513 //
12514 // ```
12515 // if (ptr == value) value(args);
12516 // else ...
12517 // ```
12518 //
12519 bool SpecializedForAnyCallees = false;
12520 bool SpecializedForAllCallees = AllCalleesKnown;
12521 ICmpInst *LastCmp = nullptr;
12522 SmallVector<Function *, 8> SkippedAssumedCallees;
12523 SmallVector<std::pair<CallInst *, Instruction *>> NewCalls;
12524 for (Function *NewCallee : AssumedCallees) {
12525 if (!A.shouldSpecializeCallSiteForCallee(AA: *this, CB&: *CB, Callee&: *NewCallee,
12526 NumAssumedCallees: AssumedCallees.size())) {
12527 SkippedAssumedCallees.push_back(Elt: NewCallee);
12528 SpecializedForAllCallees = false;
12529 continue;
12530 }
12531 SpecializedForAnyCallees = true;
12532
12533 LastCmp = new ICmpInst(IP, llvm::CmpInst::ICMP_EQ, FP, NewCallee);
12534 Instruction *ThenTI =
12535 SplitBlockAndInsertIfThen(Cond: LastCmp, SplitBefore: IP, /* Unreachable */ false);
12536 BasicBlock *CBBB = CB->getParent();
12537 A.registerManifestAddedBasicBlock(BB&: *ThenTI->getParent());
12538 A.registerManifestAddedBasicBlock(BB&: *IP->getParent());
12539 auto *SplitTI = cast<BranchInst>(Val: LastCmp->getNextNode());
12540 BasicBlock *ElseBB;
12541 if (&*IP == CB) {
12542 ElseBB = BasicBlock::Create(Context&: ThenTI->getContext(), Name: "",
12543 Parent: ThenTI->getFunction(), InsertBefore: CBBB);
12544 A.registerManifestAddedBasicBlock(BB&: *ElseBB);
12545 IP = BranchInst::Create(IfTrue: CBBB, InsertBefore: ElseBB)->getIterator();
12546 SplitTI->replaceUsesOfWith(From: CBBB, To: ElseBB);
12547 } else {
12548 ElseBB = IP->getParent();
12549 ThenTI->replaceUsesOfWith(From: ElseBB, To: CBBB);
12550 }
12551 CastInst *RetBC = nullptr;
12552 CallInst *NewCall = nullptr;
12553 if (isLegalToPromote(CB: *CB, Callee: NewCallee)) {
12554 auto *CBClone = cast<CallBase>(Val: CB->clone());
12555 CBClone->insertBefore(InsertPos: ThenTI->getIterator());
12556 NewCall = &cast<CallInst>(Val&: promoteCall(CB&: *CBClone, Callee: NewCallee, RetBitCast: &RetBC));
12557 NumIndirectCallsPromoted++;
12558 } else {
12559 NewCall = CallInst::Create(Func: FunctionCallee(CSFT, NewCallee), Args: CSArgs,
12560 NameStr: CB->getName(), InsertBefore: ThenTI->getIterator());
12561 }
12562 NewCalls.push_back(Elt: {NewCall, RetBC});
12563 }
12564
12565 auto AttachCalleeMetadata = [&](CallBase &IndirectCB) {
12566 if (!AllCalleesKnown)
12567 return ChangeStatus::UNCHANGED;
12568 MDBuilder MDB(IndirectCB.getContext());
12569 MDNode *Callees = MDB.createCallees(Callees: SkippedAssumedCallees);
12570 IndirectCB.setMetadata(KindID: LLVMContext::MD_callees, Node: Callees);
12571 return ChangeStatus::CHANGED;
12572 };
12573
12574 if (!SpecializedForAnyCallees)
12575 return AttachCalleeMetadata(*CB);
12576
12577 // Check if we need the fallback indirect call still.
12578 if (SpecializedForAllCallees) {
12579 LastCmp->replaceAllUsesWith(V: ConstantInt::getTrue(Context&: LastCmp->getContext()));
12580 LastCmp->eraseFromParent();
12581 new UnreachableInst(IP->getContext(), IP);
12582 IP->eraseFromParent();
12583 } else {
12584 auto *CBClone = cast<CallInst>(Val: CB->clone());
12585 CBClone->setName(CB->getName());
12586 CBClone->insertBefore(BB&: *IP->getParent(), InsertPos: IP);
12587 NewCalls.push_back(Elt: {CBClone, nullptr});
12588 AttachCalleeMetadata(*CBClone);
12589 }
12590
12591 // Check if we need a PHI to merge the results.
12592 if (!CBIsVoid) {
12593 auto *PHI = PHINode::Create(Ty: CB->getType(), NumReservedValues: NewCalls.size(),
12594 NameStr: CB->getName() + ".phi",
12595 InsertBefore: CB->getParent()->getFirstInsertionPt());
12596 for (auto &It : NewCalls) {
12597 CallBase *NewCall = It.first;
12598 Instruction *CallRet = It.second ? It.second : It.first;
12599 if (CallRet->getType() == CB->getType())
12600 PHI->addIncoming(V: CallRet, BB: CallRet->getParent());
12601 else if (NewCall->getType()->isVoidTy())
12602 PHI->addIncoming(V: PoisonValue::get(T: CB->getType()),
12603 BB: NewCall->getParent());
12604 else
12605 llvm_unreachable("Call return should match or be void!");
12606 }
12607 A.changeAfterManifest(IRP: IRPosition::callsite_returned(CB: *CB), NV&: *PHI);
12608 }
12609
12610 A.deleteAfterManifest(I&: *CB);
12611 Changed = ChangeStatus::CHANGED;
12612
12613 return Changed;
12614 }
12615
12616 /// See AbstractAttribute::getAsStr().
12617 const std::string getAsStr(Attributor *A) const override {
12618 return std::string(AllCalleesKnown ? "eliminate" : "specialize") +
12619 " indirect call site with " + std::to_string(val: AssumedCallees.size()) +
12620 " functions";
12621 }
12622
12623 void trackStatistics() const override {
12624 if (AllCalleesKnown) {
12625 STATS_DECLTRACK(
12626 Eliminated, CallSites,
12627 "Number of indirect call sites eliminated via specialization")
12628 } else {
12629 STATS_DECLTRACK(Specialized, CallSites,
12630 "Number of indirect call sites specialized")
12631 }
12632 }
12633
12634 bool foreachCallee(function_ref<bool(Function *)> CB) const override {
12635 return isValidState() && AllCalleesKnown && all_of(Range: AssumedCallees, P: CB);
12636 }
12637
12638private:
12639 /// Map to remember filter results.
12640 DenseMap<Function *, std::optional<bool>> FilterResults;
12641
12642 /// If the !callee metadata was present, this set will contain all potential
12643 /// callees (superset).
12644 SmallSetVector<Function *, 4> PotentialCallees;
12645
12646 /// This set contains all currently assumed calllees, which might grow over
12647 /// time.
12648 SmallSetVector<Function *, 4> AssumedCallees;
12649
12650 /// Flag to indicate if all possible callees are in the AssumedCallees set or
12651 /// if there could be others.
12652 bool AllCalleesKnown = true;
12653};
12654} // namespace
12655
12656/// --------------------- Invariant Load Pointer -------------------------------
12657namespace {
12658
12659struct AAInvariantLoadPointerImpl
12660 : public StateWrapper<BitIntegerState<uint8_t, 15>,
12661 AAInvariantLoadPointer> {
12662
12663 enum {
12664 // pointer does not alias within the bounds of the function
12665 IS_NOALIAS = 1 << 0,
12666 // pointer is not involved in any effectful instructions within the bounds
12667 // of the function
12668 IS_NOEFFECT = 1 << 1,
12669 // loads are invariant within the bounds of the function
12670 IS_LOCALLY_INVARIANT = 1 << 2,
12671 // memory lifetime is constrained within the bounds of the function
12672 IS_LOCALLY_CONSTRAINED = 1 << 3,
12673
12674 IS_BEST_STATE = IS_NOALIAS | IS_NOEFFECT | IS_LOCALLY_INVARIANT |
12675 IS_LOCALLY_CONSTRAINED,
12676 };
12677 static_assert(getBestState() == IS_BEST_STATE, "Unexpected best state");
12678
12679 using Base =
12680 StateWrapper<BitIntegerState<uint8_t, 15>, AAInvariantLoadPointer>;
12681
12682 // the BitIntegerState is optimistic about IS_NOALIAS and IS_NOEFFECT, but
12683 // pessimistic about IS_KNOWN_INVARIANT
12684 AAInvariantLoadPointerImpl(const IRPosition &IRP, Attributor &A)
12685 : Base(IRP) {}
12686
12687 bool isKnownInvariant() const final {
12688 return isKnownLocallyInvariant() && isKnown(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12689 }
12690
12691 bool isKnownLocallyInvariant() const final {
12692 if (isKnown(BitsEncoding: IS_LOCALLY_INVARIANT))
12693 return true;
12694 return isKnown(BitsEncoding: IS_NOALIAS | IS_NOEFFECT);
12695 }
12696
12697 bool isAssumedInvariant() const final {
12698 return isAssumedLocallyInvariant() && isAssumed(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12699 }
12700
12701 bool isAssumedLocallyInvariant() const final {
12702 if (isAssumed(BitsEncoding: IS_LOCALLY_INVARIANT))
12703 return true;
12704 return isAssumed(BitsEncoding: IS_NOALIAS | IS_NOEFFECT);
12705 }
12706
12707 ChangeStatus updateImpl(Attributor &A) override {
12708 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12709
12710 Changed |= updateNoAlias(A);
12711 if (requiresNoAlias() && !isAssumed(BitsEncoding: IS_NOALIAS))
12712 return indicatePessimisticFixpoint();
12713
12714 Changed |= updateNoEffect(A);
12715
12716 Changed |= updateLocalInvariance(A);
12717
12718 return Changed;
12719 }
12720
12721 ChangeStatus manifest(Attributor &A) override {
12722 if (!isKnownInvariant())
12723 return ChangeStatus::UNCHANGED;
12724
12725 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12726 const Value *Ptr = &getAssociatedValue();
12727 const auto TagInvariantLoads = [&](const Use &U, bool &) {
12728 if (U.get() != Ptr)
12729 return true;
12730 auto *I = dyn_cast<Instruction>(Val: U.getUser());
12731 if (!I)
12732 return true;
12733
12734 // Ensure that we are only changing uses from the corresponding callgraph
12735 // SSC in the case that the AA isn't run on the entire module
12736 if (!A.isRunOn(Fn: I->getFunction()))
12737 return true;
12738
12739 if (I->hasMetadata(KindID: LLVMContext::MD_invariant_load))
12740 return true;
12741
12742 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
12743 LI->setMetadata(KindID: LLVMContext::MD_invariant_load,
12744 Node: MDNode::get(Context&: LI->getContext(), MDs: {}));
12745 Changed = ChangeStatus::CHANGED;
12746 }
12747 return true;
12748 };
12749
12750 (void)A.checkForAllUses(Pred: TagInvariantLoads, QueryingAA: *this, V: *Ptr);
12751 return Changed;
12752 }
12753
12754 /// See AbstractAttribute::getAsStr().
12755 const std::string getAsStr(Attributor *) const override {
12756 if (isKnownInvariant())
12757 return "load-invariant pointer";
12758 return "non-invariant pointer";
12759 }
12760
12761 /// See AbstractAttribute::trackStatistics().
12762 void trackStatistics() const override {}
12763
12764private:
12765 /// Indicate that noalias is required for the pointer to be invariant.
12766 bool requiresNoAlias() const {
12767 switch (getPositionKind()) {
12768 default:
12769 // Conservatively default to require noalias.
12770 return true;
12771 case IRP_FLOAT:
12772 case IRP_RETURNED:
12773 case IRP_CALL_SITE:
12774 return false;
12775 case IRP_CALL_SITE_RETURNED: {
12776 const auto &CB = cast<CallBase>(Val&: getAnchorValue());
12777 return !isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
12778 Call: &CB, /*MustPreserveNullness=*/false);
12779 }
12780 case IRP_ARGUMENT: {
12781 const Function *F = getAssociatedFunction();
12782 assert(F && "no associated function for argument");
12783 return !isCallableCC(CC: F->getCallingConv());
12784 }
12785 }
12786 }
12787
12788 bool isExternal() const {
12789 const Function *F = getAssociatedFunction();
12790 if (!F)
12791 return true;
12792 return isCallableCC(CC: F->getCallingConv()) &&
12793 getPositionKind() != IRP_CALL_SITE_RETURNED;
12794 }
12795
12796 ChangeStatus updateNoAlias(Attributor &A) {
12797 if (isKnown(BitsEncoding: IS_NOALIAS) || !isAssumed(BitsEncoding: IS_NOALIAS))
12798 return ChangeStatus::UNCHANGED;
12799
12800 // Try to use AANoAlias.
12801 if (const auto *ANoAlias = A.getOrCreateAAFor<AANoAlias>(
12802 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED)) {
12803 if (ANoAlias->isKnownNoAlias()) {
12804 addKnownBits(Bits: IS_NOALIAS);
12805 return ChangeStatus::CHANGED;
12806 }
12807
12808 if (!ANoAlias->isAssumedNoAlias()) {
12809 removeAssumedBits(BitsEncoding: IS_NOALIAS);
12810 return ChangeStatus::CHANGED;
12811 }
12812
12813 return ChangeStatus::UNCHANGED;
12814 }
12815
12816 // Try to infer noalias from argument attribute, since it is applicable for
12817 // the duration of the function.
12818 if (const Argument *Arg = getAssociatedArgument()) {
12819 if (Arg->hasNoAliasAttr()) {
12820 addKnownBits(Bits: IS_NOALIAS);
12821 return ChangeStatus::UNCHANGED;
12822 }
12823
12824 // Noalias information is not provided, and cannot be inferred,
12825 // so we conservatively assume the pointer aliases.
12826 removeAssumedBits(BitsEncoding: IS_NOALIAS);
12827 return ChangeStatus::CHANGED;
12828 }
12829
12830 return ChangeStatus::UNCHANGED;
12831 }
12832
12833 ChangeStatus updateNoEffect(Attributor &A) {
12834 if (isKnown(BitsEncoding: IS_NOEFFECT) || !isAssumed(BitsEncoding: IS_NOEFFECT))
12835 return ChangeStatus::UNCHANGED;
12836
12837 if (!getAssociatedFunction())
12838 return indicatePessimisticFixpoint();
12839
12840 if (isa<AllocaInst>(Val: &getAssociatedValue()))
12841 return indicatePessimisticFixpoint();
12842
12843 const auto HasNoEffectLoads = [&](const Use &U, bool &) {
12844 const auto *LI = dyn_cast<LoadInst>(Val: U.getUser());
12845 return !LI || !LI->mayHaveSideEffects();
12846 };
12847 if (!A.checkForAllUses(Pred: HasNoEffectLoads, QueryingAA: *this, V: getAssociatedValue()))
12848 return indicatePessimisticFixpoint();
12849
12850 if (const auto *AMemoryBehavior = A.getOrCreateAAFor<AAMemoryBehavior>(
12851 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED)) {
12852 // For non-instructions, try to use AAMemoryBehavior to infer the readonly
12853 // attribute
12854 if (!AMemoryBehavior->isAssumedReadOnly())
12855 return indicatePessimisticFixpoint();
12856
12857 if (AMemoryBehavior->isKnownReadOnly()) {
12858 addKnownBits(Bits: IS_NOEFFECT);
12859 return ChangeStatus::UNCHANGED;
12860 }
12861
12862 return ChangeStatus::UNCHANGED;
12863 }
12864
12865 if (const Argument *Arg = getAssociatedArgument()) {
12866 if (Arg->onlyReadsMemory()) {
12867 addKnownBits(Bits: IS_NOEFFECT);
12868 return ChangeStatus::UNCHANGED;
12869 }
12870
12871 // Readonly information is not provided, and cannot be inferred from
12872 // AAMemoryBehavior.
12873 return indicatePessimisticFixpoint();
12874 }
12875
12876 return ChangeStatus::UNCHANGED;
12877 }
12878
12879 ChangeStatus updateLocalInvariance(Attributor &A) {
12880 if (isKnown(BitsEncoding: IS_LOCALLY_INVARIANT) || !isAssumed(BitsEncoding: IS_LOCALLY_INVARIANT))
12881 return ChangeStatus::UNCHANGED;
12882
12883 // try to infer invariance from underlying objects
12884 const auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(
12885 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED);
12886 if (!AUO)
12887 return ChangeStatus::UNCHANGED;
12888
12889 bool UsedAssumedInformation = false;
12890 const auto IsLocallyInvariantLoadIfPointer = [&](const Value &V) {
12891 if (!V.getType()->isPointerTy())
12892 return true;
12893 const auto *IsInvariantLoadPointer =
12894 A.getOrCreateAAFor<AAInvariantLoadPointer>(IRP: IRPosition::value(V), QueryingAA: this,
12895 DepClass: DepClassTy::REQUIRED);
12896 // Conservatively fail if invariance cannot be inferred.
12897 if (!IsInvariantLoadPointer)
12898 return false;
12899
12900 if (IsInvariantLoadPointer->isKnownLocallyInvariant())
12901 return true;
12902 if (!IsInvariantLoadPointer->isAssumedLocallyInvariant())
12903 return false;
12904
12905 UsedAssumedInformation = true;
12906 return true;
12907 };
12908 if (!AUO->forallUnderlyingObjects(Pred: IsLocallyInvariantLoadIfPointer))
12909 return indicatePessimisticFixpoint();
12910
12911 if (const auto *CB = dyn_cast<CallBase>(Val: &getAnchorValue())) {
12912 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
12913 Call: CB, /*MustPreserveNullness=*/false)) {
12914 for (const Value *Arg : CB->args()) {
12915 if (!IsLocallyInvariantLoadIfPointer(*Arg))
12916 return indicatePessimisticFixpoint();
12917 }
12918 }
12919 }
12920
12921 if (!UsedAssumedInformation) {
12922 // Pointer is known and not just assumed to be locally invariant.
12923 addKnownBits(Bits: IS_LOCALLY_INVARIANT);
12924 return ChangeStatus::CHANGED;
12925 }
12926
12927 return ChangeStatus::UNCHANGED;
12928 }
12929};
12930
12931struct AAInvariantLoadPointerFloating final : AAInvariantLoadPointerImpl {
12932 AAInvariantLoadPointerFloating(const IRPosition &IRP, Attributor &A)
12933 : AAInvariantLoadPointerImpl(IRP, A) {}
12934};
12935
12936struct AAInvariantLoadPointerReturned final : AAInvariantLoadPointerImpl {
12937 AAInvariantLoadPointerReturned(const IRPosition &IRP, Attributor &A)
12938 : AAInvariantLoadPointerImpl(IRP, A) {}
12939
12940 void initialize(Attributor &) override {
12941 removeAssumedBits(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12942 }
12943};
12944
12945struct AAInvariantLoadPointerCallSiteReturned final
12946 : AAInvariantLoadPointerImpl {
12947 AAInvariantLoadPointerCallSiteReturned(const IRPosition &IRP, Attributor &A)
12948 : AAInvariantLoadPointerImpl(IRP, A) {}
12949
12950 void initialize(Attributor &A) override {
12951 const Function *F = getAssociatedFunction();
12952 assert(F && "no associated function for return from call");
12953
12954 if (!F->isDeclaration() && !F->isIntrinsic())
12955 return AAInvariantLoadPointerImpl::initialize(A);
12956
12957 const auto &CB = cast<CallBase>(Val&: getAnchorValue());
12958 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
12959 Call: &CB, /*MustPreserveNullness=*/false))
12960 return AAInvariantLoadPointerImpl::initialize(A);
12961
12962 if (F->onlyReadsMemory() && F->hasNoSync())
12963 return AAInvariantLoadPointerImpl::initialize(A);
12964
12965 // At this point, the function is opaque, so we conservatively assume
12966 // non-invariance.
12967 indicatePessimisticFixpoint();
12968 }
12969};
12970
12971struct AAInvariantLoadPointerArgument final : AAInvariantLoadPointerImpl {
12972 AAInvariantLoadPointerArgument(const IRPosition &IRP, Attributor &A)
12973 : AAInvariantLoadPointerImpl(IRP, A) {}
12974
12975 void initialize(Attributor &) override {
12976 const Function *F = getAssociatedFunction();
12977 assert(F && "no associated function for argument");
12978
12979 if (!isCallableCC(CC: F->getCallingConv())) {
12980 addKnownBits(Bits: IS_LOCALLY_CONSTRAINED);
12981 return;
12982 }
12983
12984 if (!F->hasLocalLinkage())
12985 removeAssumedBits(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12986 }
12987};
12988
12989struct AAInvariantLoadPointerCallSiteArgument final
12990 : AAInvariantLoadPointerImpl {
12991 AAInvariantLoadPointerCallSiteArgument(const IRPosition &IRP, Attributor &A)
12992 : AAInvariantLoadPointerImpl(IRP, A) {}
12993};
12994} // namespace
12995
12996/// ------------------------ Address Space ------------------------------------
12997namespace {
12998
12999template <typename InstType>
13000static bool makeChange(Attributor &A, InstType *MemInst, const Use &U,
13001 Value *OriginalValue, PointerType *NewPtrTy,
13002 bool UseOriginalValue) {
13003 if (U.getOperandNo() != InstType::getPointerOperandIndex())
13004 return false;
13005
13006 if (MemInst->isVolatile()) {
13007 auto *TTI = A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(
13008 *MemInst->getFunction());
13009 unsigned NewAS = NewPtrTy->getPointerAddressSpace();
13010 if (!TTI || !TTI->hasVolatileVariant(MemInst, NewAS))
13011 return false;
13012 }
13013
13014 if (UseOriginalValue) {
13015 A.changeUseAfterManifest(U&: const_cast<Use &>(U), NV&: *OriginalValue);
13016 return true;
13017 }
13018
13019 Instruction *CastInst = new AddrSpaceCastInst(OriginalValue, NewPtrTy);
13020 CastInst->insertBefore(MemInst->getIterator());
13021 A.changeUseAfterManifest(U&: const_cast<Use &>(U), NV&: *CastInst);
13022 return true;
13023}
13024
13025struct AAAddressSpaceImpl : public AAAddressSpace {
13026 AAAddressSpaceImpl(const IRPosition &IRP, Attributor &A)
13027 : AAAddressSpace(IRP, A) {}
13028
13029 uint32_t getAddressSpace() const override {
13030 assert(isValidState() && "the AA is invalid");
13031 return AssumedAddressSpace;
13032 }
13033
13034 /// See AbstractAttribute::initialize(...).
13035 void initialize(Attributor &A) override {
13036 assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
13037 "Associated value is not a pointer");
13038
13039 if (!A.getInfoCache().getFlatAddressSpace().has_value()) {
13040 indicatePessimisticFixpoint();
13041 return;
13042 }
13043
13044 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13045 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13046 if (AS != FlatAS) {
13047 [[maybe_unused]] bool R = takeAddressSpace(AS);
13048 assert(R && "The take should happen");
13049 indicateOptimisticFixpoint();
13050 }
13051 }
13052
13053 ChangeStatus updateImpl(Attributor &A) override {
13054 uint32_t OldAddressSpace = AssumedAddressSpace;
13055 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13056
13057 auto CheckAddressSpace = [&](Value &Obj) {
13058 // Ignore undef.
13059 if (isa<UndefValue>(Val: &Obj))
13060 return true;
13061
13062 // If the object already has a non-flat address space, we simply take it.
13063 unsigned ObjAS = Obj.getType()->getPointerAddressSpace();
13064 if (ObjAS != FlatAS)
13065 return takeAddressSpace(AS: ObjAS);
13066
13067 // At this point, we know Obj is in the flat address space. For a final
13068 // attempt, we want to use getAssumedAddrSpace, but first we must get the
13069 // associated function, if possible.
13070 Function *F = nullptr;
13071 if (auto *Arg = dyn_cast<Argument>(Val: &Obj))
13072 F = Arg->getParent();
13073 else if (auto *I = dyn_cast<Instruction>(Val: &Obj))
13074 F = I->getFunction();
13075
13076 // Use getAssumedAddrSpace if the associated function exists.
13077 if (F) {
13078 auto *TTI =
13079 A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(F: *F);
13080 unsigned AssumedAS = TTI->getAssumedAddrSpace(V: &Obj);
13081 if (AssumedAS != ~0U)
13082 return takeAddressSpace(AS: AssumedAS);
13083 }
13084
13085 // Now we can't do anything else but to take the flat AS.
13086 return takeAddressSpace(AS: FlatAS);
13087 };
13088
13089 auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(IRP: getIRPosition(), QueryingAA: this,
13090 DepClass: DepClassTy::REQUIRED);
13091 if (!AUO->forallUnderlyingObjects(Pred: CheckAddressSpace))
13092 return indicatePessimisticFixpoint();
13093
13094 return OldAddressSpace == AssumedAddressSpace ? ChangeStatus::UNCHANGED
13095 : ChangeStatus::CHANGED;
13096 }
13097
13098 /// See AbstractAttribute::manifest(...).
13099 ChangeStatus manifest(Attributor &A) override {
13100 unsigned NewAS = getAddressSpace();
13101
13102 if (NewAS == InvalidAddressSpace ||
13103 NewAS == getAssociatedType()->getPointerAddressSpace())
13104 return ChangeStatus::UNCHANGED;
13105
13106 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13107
13108 Value *AssociatedValue = &getAssociatedValue();
13109 Value *OriginalValue = peelAddrspacecast(V: AssociatedValue, FlatAS);
13110
13111 PointerType *NewPtrTy =
13112 PointerType::get(C&: getAssociatedType()->getContext(), AddressSpace: NewAS);
13113 bool UseOriginalValue =
13114 OriginalValue->getType()->getPointerAddressSpace() == NewAS;
13115
13116 bool Changed = false;
13117
13118 auto Pred = [&](const Use &U, bool &) {
13119 if (U.get() != AssociatedValue)
13120 return true;
13121 auto *Inst = dyn_cast<Instruction>(Val: U.getUser());
13122 if (!Inst)
13123 return true;
13124 // This is a WA to make sure we only change uses from the corresponding
13125 // CGSCC if the AA is run on CGSCC instead of the entire module.
13126 if (!A.isRunOn(Fn: Inst->getFunction()))
13127 return true;
13128 if (auto *LI = dyn_cast<LoadInst>(Val: Inst)) {
13129 Changed |=
13130 makeChange(A, MemInst: LI, U, OriginalValue, NewPtrTy, UseOriginalValue);
13131 } else if (auto *SI = dyn_cast<StoreInst>(Val: Inst)) {
13132 Changed |=
13133 makeChange(A, MemInst: SI, U, OriginalValue, NewPtrTy, UseOriginalValue);
13134 } else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: Inst)) {
13135 Changed |=
13136 makeChange(A, MemInst: RMW, U, OriginalValue, NewPtrTy, UseOriginalValue);
13137 } else if (auto *CmpX = dyn_cast<AtomicCmpXchgInst>(Val: Inst)) {
13138 Changed |=
13139 makeChange(A, MemInst: CmpX, U, OriginalValue, NewPtrTy, UseOriginalValue);
13140 }
13141 return true;
13142 };
13143
13144 // It doesn't matter if we can't check all uses as we can simply
13145 // conservatively ignore those that can not be visited.
13146 (void)A.checkForAllUses(Pred, QueryingAA: *this, V: getAssociatedValue(),
13147 /* CheckBBLivenessOnly */ true);
13148
13149 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
13150 }
13151
13152 /// See AbstractAttribute::getAsStr().
13153 const std::string getAsStr(Attributor *A) const override {
13154 if (!isValidState())
13155 return "addrspace(<invalid>)";
13156 return "addrspace(" +
13157 (AssumedAddressSpace == InvalidAddressSpace
13158 ? "none"
13159 : std::to_string(val: AssumedAddressSpace)) +
13160 ")";
13161 }
13162
13163private:
13164 uint32_t AssumedAddressSpace = InvalidAddressSpace;
13165
13166 bool takeAddressSpace(uint32_t AS) {
13167 if (AssumedAddressSpace == InvalidAddressSpace) {
13168 AssumedAddressSpace = AS;
13169 return true;
13170 }
13171 return AssumedAddressSpace == AS;
13172 }
13173
13174 static Value *peelAddrspacecast(Value *V, unsigned FlatAS) {
13175 if (auto *I = dyn_cast<AddrSpaceCastInst>(Val: V)) {
13176 assert(I->getSrcAddressSpace() != FlatAS &&
13177 "there should not be flat AS -> non-flat AS");
13178 return I->getPointerOperand();
13179 }
13180 if (auto *C = dyn_cast<ConstantExpr>(Val: V))
13181 if (C->getOpcode() == Instruction::AddrSpaceCast) {
13182 assert(C->getOperand(0)->getType()->getPointerAddressSpace() !=
13183 FlatAS &&
13184 "there should not be flat AS -> non-flat AS X");
13185 return C->getOperand(i_nocapture: 0);
13186 }
13187 return V;
13188 }
13189};
13190
13191struct AAAddressSpaceFloating final : AAAddressSpaceImpl {
13192 AAAddressSpaceFloating(const IRPosition &IRP, Attributor &A)
13193 : AAAddressSpaceImpl(IRP, A) {}
13194
13195 void trackStatistics() const override {
13196 STATS_DECLTRACK_FLOATING_ATTR(addrspace);
13197 }
13198};
13199
13200struct AAAddressSpaceReturned final : AAAddressSpaceImpl {
13201 AAAddressSpaceReturned(const IRPosition &IRP, Attributor &A)
13202 : AAAddressSpaceImpl(IRP, A) {}
13203
13204 /// See AbstractAttribute::initialize(...).
13205 void initialize(Attributor &A) override {
13206 // TODO: we don't rewrite function argument for now because it will need to
13207 // rewrite the function signature and all call sites.
13208 (void)indicatePessimisticFixpoint();
13209 }
13210
13211 void trackStatistics() const override {
13212 STATS_DECLTRACK_FNRET_ATTR(addrspace);
13213 }
13214};
13215
13216struct AAAddressSpaceCallSiteReturned final : AAAddressSpaceImpl {
13217 AAAddressSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
13218 : AAAddressSpaceImpl(IRP, A) {}
13219
13220 void trackStatistics() const override {
13221 STATS_DECLTRACK_CSRET_ATTR(addrspace);
13222 }
13223};
13224
13225struct AAAddressSpaceArgument final : AAAddressSpaceImpl {
13226 AAAddressSpaceArgument(const IRPosition &IRP, Attributor &A)
13227 : AAAddressSpaceImpl(IRP, A) {}
13228
13229 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(addrspace); }
13230};
13231
13232struct AAAddressSpaceCallSiteArgument final : AAAddressSpaceImpl {
13233 AAAddressSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
13234 : AAAddressSpaceImpl(IRP, A) {}
13235
13236 /// See AbstractAttribute::initialize(...).
13237 void initialize(Attributor &A) override {
13238 // TODO: we don't rewrite call site argument for now because it will need to
13239 // rewrite the function signature of the callee.
13240 (void)indicatePessimisticFixpoint();
13241 }
13242
13243 void trackStatistics() const override {
13244 STATS_DECLTRACK_CSARG_ATTR(addrspace);
13245 }
13246};
13247} // namespace
13248
13249/// ------------------------ No Alias Address Space ---------------------------
13250// This attribute assumes flat address space can alias all other address space
13251
13252// TODO: this is similar to AAAddressSpace, most of the code should be merged.
13253// But merging it created failing cased on gateway test that cannot be
13254// reproduced locally. So should open a separated PR to handle the merge of
13255// AANoAliasAddrSpace and AAAddressSpace attribute
13256
13257namespace {
13258struct AANoAliasAddrSpaceImpl : public AANoAliasAddrSpace {
13259 AANoAliasAddrSpaceImpl(const IRPosition &IRP, Attributor &A)
13260 : AANoAliasAddrSpace(IRP, A) {}
13261
13262 void initialize(Attributor &A) override {
13263 assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
13264 "Associated value is not a pointer");
13265
13266 resetASRanges(A);
13267
13268 std::optional<unsigned> FlatAS = A.getInfoCache().getFlatAddressSpace();
13269 if (!FlatAS.has_value()) {
13270 indicatePessimisticFixpoint();
13271 return;
13272 }
13273
13274 removeAS(AS: *FlatAS);
13275
13276 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13277 if (AS != *FlatAS) {
13278 removeAS(AS);
13279 indicateOptimisticFixpoint();
13280 }
13281 }
13282
13283 ChangeStatus updateImpl(Attributor &A) override {
13284 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13285 uint32_t OldAssumed = getAssumed();
13286
13287 auto CheckAddressSpace = [&](Value &Obj) {
13288 if (isa<PoisonValue>(Val: &Obj))
13289 return true;
13290
13291 unsigned AS = Obj.getType()->getPointerAddressSpace();
13292 if (AS == FlatAS)
13293 return false;
13294
13295 removeAS(AS: Obj.getType()->getPointerAddressSpace());
13296 return true;
13297 };
13298
13299 const AAUnderlyingObjects *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(
13300 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED);
13301 if (!AUO->forallUnderlyingObjects(Pred: CheckAddressSpace))
13302 return indicatePessimisticFixpoint();
13303
13304 return OldAssumed == getAssumed() ? ChangeStatus::UNCHANGED
13305 : ChangeStatus::CHANGED;
13306 }
13307
13308 /// See AbstractAttribute::manifest(...).
13309 ChangeStatus manifest(Attributor &A) override {
13310 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13311
13312 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13313 if (AS != FlatAS || Map.empty())
13314 return ChangeStatus::UNCHANGED;
13315
13316 LLVMContext &Ctx = getAssociatedValue().getContext();
13317 MDNode *NoAliasASNode = nullptr;
13318 MDBuilder MDB(Ctx);
13319 // Has to use iterator to get the range info.
13320 for (RangeMap::const_iterator I = Map.begin(); I != Map.end(); I++) {
13321 if (!I.value())
13322 continue;
13323 unsigned Upper = I.stop();
13324 unsigned Lower = I.start();
13325 if (!NoAliasASNode) {
13326 NoAliasASNode = MDB.createRange(Lo: APInt(32, Lower), Hi: APInt(32, Upper + 1));
13327 continue;
13328 }
13329 MDNode *ASRange = MDB.createRange(Lo: APInt(32, Lower), Hi: APInt(32, Upper + 1));
13330 NoAliasASNode = MDNode::getMostGenericRange(A: NoAliasASNode, B: ASRange);
13331 }
13332
13333 Value *AssociatedValue = &getAssociatedValue();
13334 bool Changed = false;
13335
13336 auto AddNoAliasAttr = [&](const Use &U, bool &) {
13337 if (U.get() != AssociatedValue)
13338 return true;
13339 Instruction *Inst = dyn_cast<Instruction>(Val: U.getUser());
13340 if (!Inst || Inst->hasMetadata(KindID: LLVMContext::MD_noalias_addrspace))
13341 return true;
13342 if (!isa<LoadInst>(Val: Inst) && !isa<StoreInst>(Val: Inst) &&
13343 !isa<AtomicCmpXchgInst>(Val: Inst) && !isa<AtomicRMWInst>(Val: Inst))
13344 return true;
13345 if (!A.isRunOn(Fn: Inst->getFunction()))
13346 return true;
13347 Inst->setMetadata(KindID: LLVMContext::MD_noalias_addrspace, Node: NoAliasASNode);
13348 Changed = true;
13349 return true;
13350 };
13351 (void)A.checkForAllUses(Pred: AddNoAliasAttr, QueryingAA: *this, V: *AssociatedValue,
13352 /*CheckBBLivenessOnly=*/true);
13353 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
13354 }
13355
13356 /// See AbstractAttribute::getAsStr().
13357 const std::string getAsStr(Attributor *A) const override {
13358 if (!isValidState())
13359 return "<invalid>";
13360 std::string Str;
13361 raw_string_ostream OS(Str);
13362 OS << "CanNotBeAddrSpace(";
13363 for (RangeMap::const_iterator I = Map.begin(); I != Map.end(); I++) {
13364 unsigned Upper = I.stop();
13365 unsigned Lower = I.start();
13366 OS << ' ' << '[' << Upper << ',' << Lower + 1 << ')';
13367 }
13368 OS << " )";
13369 return OS.str();
13370 }
13371
13372private:
13373 void removeAS(unsigned AS) {
13374 RangeMap::iterator I = Map.find(x: AS);
13375
13376 if (I != Map.end()) {
13377 unsigned Upper = I.stop();
13378 unsigned Lower = I.start();
13379 I.erase();
13380 if (Upper == Lower)
13381 return;
13382 if (AS != ~((unsigned)0) && AS + 1 <= Upper)
13383 Map.insert(a: AS + 1, b: Upper, /*what ever this variable name is=*/y: true);
13384 if (AS != 0 && Lower <= AS - 1)
13385 Map.insert(a: Lower, b: AS - 1, y: true);
13386 }
13387 }
13388
13389 void resetASRanges(Attributor &A) {
13390 Map.clear();
13391 Map.insert(a: 0, b: A.getInfoCache().getMaxAddrSpace(), y: true);
13392 }
13393};
13394
13395struct AANoAliasAddrSpaceFloating final : AANoAliasAddrSpaceImpl {
13396 AANoAliasAddrSpaceFloating(const IRPosition &IRP, Attributor &A)
13397 : AANoAliasAddrSpaceImpl(IRP, A) {}
13398
13399 void trackStatistics() const override {
13400 STATS_DECLTRACK_FLOATING_ATTR(noaliasaddrspace);
13401 }
13402};
13403
13404struct AANoAliasAddrSpaceReturned final : AANoAliasAddrSpaceImpl {
13405 AANoAliasAddrSpaceReturned(const IRPosition &IRP, Attributor &A)
13406 : AANoAliasAddrSpaceImpl(IRP, A) {}
13407
13408 void trackStatistics() const override {
13409 STATS_DECLTRACK_FNRET_ATTR(noaliasaddrspace);
13410 }
13411};
13412
13413struct AANoAliasAddrSpaceCallSiteReturned final : AANoAliasAddrSpaceImpl {
13414 AANoAliasAddrSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
13415 : AANoAliasAddrSpaceImpl(IRP, A) {}
13416
13417 void trackStatistics() const override {
13418 STATS_DECLTRACK_CSRET_ATTR(noaliasaddrspace);
13419 }
13420};
13421
13422struct AANoAliasAddrSpaceArgument final : AANoAliasAddrSpaceImpl {
13423 AANoAliasAddrSpaceArgument(const IRPosition &IRP, Attributor &A)
13424 : AANoAliasAddrSpaceImpl(IRP, A) {}
13425
13426 void trackStatistics() const override {
13427 STATS_DECLTRACK_ARG_ATTR(noaliasaddrspace);
13428 }
13429};
13430
13431struct AANoAliasAddrSpaceCallSiteArgument final : AANoAliasAddrSpaceImpl {
13432 AANoAliasAddrSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
13433 : AANoAliasAddrSpaceImpl(IRP, A) {}
13434
13435 void trackStatistics() const override {
13436 STATS_DECLTRACK_CSARG_ATTR(noaliasaddrspace);
13437 }
13438};
13439} // namespace
13440/// ----------- Allocation Info ----------
13441namespace {
13442struct AAAllocationInfoImpl : public AAAllocationInfo {
13443 AAAllocationInfoImpl(const IRPosition &IRP, Attributor &A)
13444 : AAAllocationInfo(IRP, A) {}
13445
13446 std::optional<TypeSize> getAllocatedSize() const override {
13447 assert(isValidState() && "the AA is invalid");
13448 return AssumedAllocatedSize;
13449 }
13450
13451 std::optional<TypeSize> findInitialAllocationSize(Instruction *I,
13452 const DataLayout &DL) {
13453
13454 // TODO: implement case for malloc like instructions
13455 switch (I->getOpcode()) {
13456 case Instruction::Alloca: {
13457 AllocaInst *AI = cast<AllocaInst>(Val: I);
13458 return AI->getAllocationSize(DL);
13459 }
13460 default:
13461 return std::nullopt;
13462 }
13463 }
13464
13465 ChangeStatus updateImpl(Attributor &A) override {
13466
13467 const IRPosition &IRP = getIRPosition();
13468 Instruction *I = IRP.getCtxI();
13469
13470 // TODO: update check for malloc like calls
13471 if (!isa<AllocaInst>(Val: I))
13472 return indicatePessimisticFixpoint();
13473
13474 bool IsKnownNoCapture;
13475 if (!AA::hasAssumedIRAttr<Attribute::Captures>(
13476 A, QueryingAA: this, IRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture))
13477 return indicatePessimisticFixpoint();
13478
13479 const AAPointerInfo *PI =
13480 A.getOrCreateAAFor<AAPointerInfo>(IRP, QueryingAA: *this, DepClass: DepClassTy::REQUIRED);
13481
13482 if (!PI)
13483 return indicatePessimisticFixpoint();
13484
13485 if (!PI->getState().isValidState() || PI->reachesReturn())
13486 return indicatePessimisticFixpoint();
13487
13488 const DataLayout &DL = A.getDataLayout();
13489 const auto AllocationSize = findInitialAllocationSize(I, DL);
13490
13491 // If allocation size is nullopt, we give up.
13492 if (!AllocationSize)
13493 return indicatePessimisticFixpoint();
13494
13495 // For zero sized allocations, we give up.
13496 // Since we can't reduce further
13497 if (*AllocationSize == 0)
13498 return indicatePessimisticFixpoint();
13499
13500 int64_t BinSize = PI->numOffsetBins();
13501
13502 // TODO: implement for multiple bins
13503 if (BinSize > 1)
13504 return indicatePessimisticFixpoint();
13505
13506 if (BinSize == 0) {
13507 auto NewAllocationSize = std::make_optional<TypeSize>(args: 0, args: false);
13508 if (!changeAllocationSize(Size: NewAllocationSize))
13509 return ChangeStatus::UNCHANGED;
13510 return ChangeStatus::CHANGED;
13511 }
13512
13513 // TODO: refactor this to be part of multiple bin case
13514 const auto &It = PI->begin();
13515
13516 // TODO: handle if Offset is not zero
13517 if (It->first.Offset != 0)
13518 return indicatePessimisticFixpoint();
13519
13520 uint64_t SizeOfBin = It->first.Offset + It->first.Size;
13521
13522 if (SizeOfBin >= *AllocationSize)
13523 return indicatePessimisticFixpoint();
13524
13525 auto NewAllocationSize = std::make_optional<TypeSize>(args: SizeOfBin * 8, args: false);
13526
13527 if (!changeAllocationSize(Size: NewAllocationSize))
13528 return ChangeStatus::UNCHANGED;
13529
13530 return ChangeStatus::CHANGED;
13531 }
13532
13533 /// See AbstractAttribute::manifest(...).
13534 ChangeStatus manifest(Attributor &A) override {
13535
13536 assert(isValidState() &&
13537 "Manifest should only be called if the state is valid.");
13538
13539 Instruction *I = getIRPosition().getCtxI();
13540
13541 auto FixedAllocatedSizeInBits = getAllocatedSize()->getFixedValue();
13542
13543 unsigned long NumBytesToAllocate = (FixedAllocatedSizeInBits + 7) / 8;
13544
13545 switch (I->getOpcode()) {
13546 // TODO: add case for malloc like calls
13547 case Instruction::Alloca: {
13548
13549 AllocaInst *AI = cast<AllocaInst>(Val: I);
13550
13551 Type *CharType = Type::getInt8Ty(C&: I->getContext());
13552
13553 auto *NumBytesToValue =
13554 ConstantInt::get(Context&: I->getContext(), V: APInt(32, NumBytesToAllocate));
13555
13556 BasicBlock::iterator insertPt = AI->getIterator();
13557 insertPt = std::next(x: insertPt);
13558 AllocaInst *NewAllocaInst =
13559 new AllocaInst(CharType, AI->getAddressSpace(), NumBytesToValue,
13560 AI->getAlign(), AI->getName(), insertPt);
13561
13562 if (A.changeAfterManifest(IRP: IRPosition::inst(I: *AI), NV&: *NewAllocaInst))
13563 return ChangeStatus::CHANGED;
13564
13565 break;
13566 }
13567 default:
13568 break;
13569 }
13570
13571 return ChangeStatus::UNCHANGED;
13572 }
13573
13574 /// See AbstractAttribute::getAsStr().
13575 const std::string getAsStr(Attributor *A) const override {
13576 if (!isValidState())
13577 return "allocationinfo(<invalid>)";
13578 return "allocationinfo(" +
13579 (AssumedAllocatedSize == HasNoAllocationSize
13580 ? "none"
13581 : std::to_string(val: AssumedAllocatedSize->getFixedValue())) +
13582 ")";
13583 }
13584
13585private:
13586 std::optional<TypeSize> AssumedAllocatedSize = HasNoAllocationSize;
13587
13588 // Maintain the computed allocation size of the object.
13589 // Returns (bool) weather the size of the allocation was modified or not.
13590 bool changeAllocationSize(std::optional<TypeSize> Size) {
13591 if (AssumedAllocatedSize == HasNoAllocationSize ||
13592 AssumedAllocatedSize != Size) {
13593 AssumedAllocatedSize = Size;
13594 return true;
13595 }
13596 return false;
13597 }
13598};
13599
13600struct AAAllocationInfoFloating : AAAllocationInfoImpl {
13601 AAAllocationInfoFloating(const IRPosition &IRP, Attributor &A)
13602 : AAAllocationInfoImpl(IRP, A) {}
13603
13604 void trackStatistics() const override {
13605 STATS_DECLTRACK_FLOATING_ATTR(allocationinfo);
13606 }
13607};
13608
13609struct AAAllocationInfoReturned : AAAllocationInfoImpl {
13610 AAAllocationInfoReturned(const IRPosition &IRP, Attributor &A)
13611 : AAAllocationInfoImpl(IRP, A) {}
13612
13613 /// See AbstractAttribute::initialize(...).
13614 void initialize(Attributor &A) override {
13615 // TODO: we don't rewrite function argument for now because it will need to
13616 // rewrite the function signature and all call sites
13617 (void)indicatePessimisticFixpoint();
13618 }
13619
13620 void trackStatistics() const override {
13621 STATS_DECLTRACK_FNRET_ATTR(allocationinfo);
13622 }
13623};
13624
13625struct AAAllocationInfoCallSiteReturned : AAAllocationInfoImpl {
13626 AAAllocationInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
13627 : AAAllocationInfoImpl(IRP, A) {}
13628
13629 void trackStatistics() const override {
13630 STATS_DECLTRACK_CSRET_ATTR(allocationinfo);
13631 }
13632};
13633
13634struct AAAllocationInfoArgument : AAAllocationInfoImpl {
13635 AAAllocationInfoArgument(const IRPosition &IRP, Attributor &A)
13636 : AAAllocationInfoImpl(IRP, A) {}
13637
13638 void trackStatistics() const override {
13639 STATS_DECLTRACK_ARG_ATTR(allocationinfo);
13640 }
13641};
13642
13643struct AAAllocationInfoCallSiteArgument : AAAllocationInfoImpl {
13644 AAAllocationInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
13645 : AAAllocationInfoImpl(IRP, A) {}
13646
13647 /// See AbstractAttribute::initialize(...).
13648 void initialize(Attributor &A) override {
13649
13650 (void)indicatePessimisticFixpoint();
13651 }
13652
13653 void trackStatistics() const override {
13654 STATS_DECLTRACK_CSARG_ATTR(allocationinfo);
13655 }
13656};
13657} // namespace
13658
13659const char AANoUnwind::ID = 0;
13660const char AANoSync::ID = 0;
13661const char AANoFree::ID = 0;
13662const char AANonNull::ID = 0;
13663const char AAMustProgress::ID = 0;
13664const char AANoRecurse::ID = 0;
13665const char AANonConvergent::ID = 0;
13666const char AAWillReturn::ID = 0;
13667const char AAUndefinedBehavior::ID = 0;
13668const char AANoAlias::ID = 0;
13669const char AAIntraFnReachability::ID = 0;
13670const char AANoReturn::ID = 0;
13671const char AAIsDead::ID = 0;
13672const char AADereferenceable::ID = 0;
13673const char AAAlign::ID = 0;
13674const char AAInstanceInfo::ID = 0;
13675const char AANoCapture::ID = 0;
13676const char AAValueSimplify::ID = 0;
13677const char AAHeapToStack::ID = 0;
13678const char AAPrivatizablePtr::ID = 0;
13679const char AAMemoryBehavior::ID = 0;
13680const char AAMemoryLocation::ID = 0;
13681const char AAValueConstantRange::ID = 0;
13682const char AAPotentialConstantValues::ID = 0;
13683const char AAPotentialValues::ID = 0;
13684const char AANoUndef::ID = 0;
13685const char AANoFPClass::ID = 0;
13686const char AACallEdges::ID = 0;
13687const char AAInterFnReachability::ID = 0;
13688const char AAPointerInfo::ID = 0;
13689const char AAAssumptionInfo::ID = 0;
13690const char AAUnderlyingObjects::ID = 0;
13691const char AAInvariantLoadPointer::ID = 0;
13692const char AAAddressSpace::ID = 0;
13693const char AANoAliasAddrSpace::ID = 0;
13694const char AAAllocationInfo::ID = 0;
13695const char AAIndirectCallInfo::ID = 0;
13696const char AAGlobalValueInfo::ID = 0;
13697const char AADenormalFPMath::ID = 0;
13698
13699// Macro magic to create the static generator function for attributes that
13700// follow the naming scheme.
13701
13702#define SWITCH_PK_INV(CLASS, PK, POS_NAME) \
13703 case IRPosition::PK: \
13704 llvm_unreachable("Cannot create " #CLASS " for a " POS_NAME " position!");
13705
13706#define SWITCH_PK_CREATE(CLASS, IRP, PK, SUFFIX) \
13707 case IRPosition::PK: \
13708 AA = new (A.Allocator) CLASS##SUFFIX(IRP, A); \
13709 ++NumAAs; \
13710 break;
13711
13712#define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13713 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13714 CLASS *AA = nullptr; \
13715 switch (IRP.getPositionKind()) { \
13716 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13717 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
13718 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
13719 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13720 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
13721 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
13722 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13723 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13724 } \
13725 return *AA; \
13726 }
13727
13728#define CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13729 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13730 CLASS *AA = nullptr; \
13731 switch (IRP.getPositionKind()) { \
13732 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13733 SWITCH_PK_INV(CLASS, IRP_FUNCTION, "function") \
13734 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
13735 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13736 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13737 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
13738 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13739 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13740 } \
13741 return *AA; \
13742 }
13743
13744#define CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(POS, SUFFIX, CLASS) \
13745 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13746 CLASS *AA = nullptr; \
13747 switch (IRP.getPositionKind()) { \
13748 SWITCH_PK_CREATE(CLASS, IRP, POS, SUFFIX) \
13749 default: \
13750 llvm_unreachable("Cannot create " #CLASS " for position otherthan " #POS \
13751 " position!"); \
13752 } \
13753 return *AA; \
13754 }
13755
13756#define CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13757 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13758 CLASS *AA = nullptr; \
13759 switch (IRP.getPositionKind()) { \
13760 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13761 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13762 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13763 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13764 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13765 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
13766 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13767 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13768 } \
13769 return *AA; \
13770 }
13771
13772#define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13773 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13774 CLASS *AA = nullptr; \
13775 switch (IRP.getPositionKind()) { \
13776 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13777 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
13778 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
13779 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13780 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
13781 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
13782 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
13783 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13784 } \
13785 return *AA; \
13786 }
13787
13788#define CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13789 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13790 CLASS *AA = nullptr; \
13791 switch (IRP.getPositionKind()) { \
13792 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13793 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13794 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13795 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13796 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13797 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13798 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13799 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13800 } \
13801 return *AA; \
13802 }
13803
13804CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUnwind)
13805CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoSync)
13806CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoRecurse)
13807CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAWillReturn)
13808CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoReturn)
13809CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryLocation)
13810CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AACallEdges)
13811CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAssumptionInfo)
13812CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMustProgress)
13813
13814CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonNull)
13815CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAlias)
13816CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPrivatizablePtr)
13817CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADereferenceable)
13818CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAlign)
13819CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInstanceInfo)
13820CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoCapture)
13821CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueConstantRange)
13822CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialConstantValues)
13823CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialValues)
13824CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUndef)
13825CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFPClass)
13826CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPointerInfo)
13827CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInvariantLoadPointer)
13828CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAddressSpace)
13829CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAliasAddrSpace)
13830CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAllocationInfo)
13831
13832CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueSimplify)
13833CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIsDead)
13834CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFree)
13835CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUnderlyingObjects)
13836
13837CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_CALL_SITE, CallSite,
13838 AAIndirectCallInfo)
13839CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_FLOAT, Floating,
13840 AAGlobalValueInfo)
13841
13842CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAHeapToStack)
13843CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUndefinedBehavior)
13844CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonConvergent)
13845CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIntraFnReachability)
13846CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInterFnReachability)
13847CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADenormalFPMath)
13848
13849CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryBehavior)
13850
13851#undef CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION
13852#undef CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION
13853#undef CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION
13854#undef CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION
13855#undef CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION
13856#undef CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION
13857#undef SWITCH_PK_CREATE
13858#undef SWITCH_PK_INV
13859