1//===- AttributorAttributes.cpp - Attributes for Attributor deduction -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// See the Attributor.h file comment and the class descriptions in that file for
10// more information.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/IPO/Attributor.h"
15
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/ADT/DenseMapInfo.h"
20#include "llvm/ADT/MapVector.h"
21#include "llvm/ADT/SCCIterator.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SetOperations.h"
24#include "llvm/ADT/SetVector.h"
25#include "llvm/ADT/SmallPtrSet.h"
26#include "llvm/ADT/SmallVector.h"
27#include "llvm/ADT/Statistic.h"
28#include "llvm/ADT/StringExtras.h"
29#include "llvm/Analysis/AliasAnalysis.h"
30#include "llvm/Analysis/AssumeBundleQueries.h"
31#include "llvm/Analysis/AssumptionCache.h"
32#include "llvm/Analysis/CaptureTracking.h"
33#include "llvm/Analysis/CycleAnalysis.h"
34#include "llvm/Analysis/InstructionSimplify.h"
35#include "llvm/Analysis/LazyValueInfo.h"
36#include "llvm/Analysis/MemoryBuiltins.h"
37#include "llvm/Analysis/ScalarEvolution.h"
38#include "llvm/Analysis/TargetTransformInfo.h"
39#include "llvm/Analysis/ValueTracking.h"
40#include "llvm/IR/Argument.h"
41#include "llvm/IR/Assumptions.h"
42#include "llvm/IR/Attributes.h"
43#include "llvm/IR/BasicBlock.h"
44#include "llvm/IR/Constant.h"
45#include "llvm/IR/Constants.h"
46#include "llvm/IR/DataLayout.h"
47#include "llvm/IR/DerivedTypes.h"
48#include "llvm/IR/GlobalValue.h"
49#include "llvm/IR/IRBuilder.h"
50#include "llvm/IR/InlineAsm.h"
51#include "llvm/IR/InstrTypes.h"
52#include "llvm/IR/Instruction.h"
53#include "llvm/IR/Instructions.h"
54#include "llvm/IR/IntrinsicInst.h"
55#include "llvm/IR/IntrinsicsAMDGPU.h"
56#include "llvm/IR/IntrinsicsNVPTX.h"
57#include "llvm/IR/LLVMContext.h"
58#include "llvm/IR/MDBuilder.h"
59#include "llvm/IR/NoFolder.h"
60#include "llvm/IR/Value.h"
61#include "llvm/IR/ValueHandle.h"
62#include "llvm/Support/Alignment.h"
63#include "llvm/Support/Casting.h"
64#include "llvm/Support/CommandLine.h"
65#include "llvm/Support/ErrorHandling.h"
66#include "llvm/Support/GraphWriter.h"
67#include "llvm/Support/InterleavedRange.h"
68#include "llvm/Support/KnownFPClass.h"
69#include "llvm/Support/MathExtras.h"
70#include "llvm/Support/TypeSize.h"
71#include "llvm/Support/raw_ostream.h"
72#include "llvm/Transforms/Utils/BasicBlockUtils.h"
73#include "llvm/Transforms/Utils/CallPromotionUtils.h"
74#include "llvm/Transforms/Utils/Local.h"
75#include "llvm/Transforms/Utils/ValueMapper.h"
76#include <cassert>
77#include <numeric>
78#include <optional>
79#include <string>
80
81using namespace llvm;
82
83#define DEBUG_TYPE "attributor"
84
85static cl::opt<bool> ManifestInternal(
86 "attributor-manifest-internal", cl::Hidden,
87 cl::desc("Manifest Attributor internal string attributes."),
88 cl::init(Val: false));
89
90static cl::opt<int> MaxHeapToStackSize("max-heap-to-stack-size", cl::init(Val: 128),
91 cl::Hidden);
92
93template <>
94unsigned llvm::PotentialConstantIntValuesState::MaxPotentialValues = 0;
95
96template <> unsigned llvm::PotentialLLVMValuesState::MaxPotentialValues = -1;
97
98static cl::opt<unsigned, true> MaxPotentialValues(
99 "attributor-max-potential-values", cl::Hidden,
100 cl::desc("Maximum number of potential values to be "
101 "tracked for each position."),
102 cl::location(L&: llvm::PotentialConstantIntValuesState::MaxPotentialValues),
103 cl::init(Val: 7));
104
105static cl::opt<int> MaxPotentialValuesIterations(
106 "attributor-max-potential-values-iterations", cl::Hidden,
107 cl::desc(
108 "Maximum number of iterations we keep dismantling potential values."),
109 cl::init(Val: 64));
110
111STATISTIC(NumAAs, "Number of abstract attributes created");
112STATISTIC(NumIndirectCallsPromoted, "Number of indirect calls promoted");
113
114// Some helper macros to deal with statistics tracking.
115//
116// Usage:
117// For simple IR attribute tracking overload trackStatistics in the abstract
118// attribute and choose the right STATS_DECLTRACK_********* macro,
119// e.g.,:
120// void trackStatistics() const override {
121// STATS_DECLTRACK_ARG_ATTR(returned)
122// }
123// If there is a single "increment" side one can use the macro
124// STATS_DECLTRACK with a custom message. If there are multiple increment
125// sides, STATS_DECL and STATS_TRACK can also be used separately.
126//
127#define BUILD_STAT_MSG_IR_ATTR(TYPE, NAME) \
128 ("Number of " #TYPE " marked '" #NAME "'")
129#define BUILD_STAT_NAME(NAME, TYPE) NumIR##TYPE##_##NAME
130#define STATS_DECL_(NAME, MSG) STATISTIC(NAME, MSG);
131#define STATS_DECL(NAME, TYPE, MSG) \
132 STATS_DECL_(BUILD_STAT_NAME(NAME, TYPE), MSG);
133#define STATS_TRACK(NAME, TYPE) ++(BUILD_STAT_NAME(NAME, TYPE));
134#define STATS_DECLTRACK(NAME, TYPE, MSG) \
135 {STATS_DECL(NAME, TYPE, MSG) STATS_TRACK(NAME, TYPE)}
136#define STATS_DECLTRACK_ARG_ATTR(NAME) \
137 STATS_DECLTRACK(NAME, Arguments, BUILD_STAT_MSG_IR_ATTR(arguments, NAME))
138#define STATS_DECLTRACK_CSARG_ATTR(NAME) \
139 STATS_DECLTRACK(NAME, CSArguments, \
140 BUILD_STAT_MSG_IR_ATTR(call site arguments, NAME))
141#define STATS_DECLTRACK_FN_ATTR(NAME) \
142 STATS_DECLTRACK(NAME, Function, BUILD_STAT_MSG_IR_ATTR(functions, NAME))
143#define STATS_DECLTRACK_CS_ATTR(NAME) \
144 STATS_DECLTRACK(NAME, CS, BUILD_STAT_MSG_IR_ATTR(call site, NAME))
145#define STATS_DECLTRACK_FNRET_ATTR(NAME) \
146 STATS_DECLTRACK(NAME, FunctionReturn, \
147 BUILD_STAT_MSG_IR_ATTR(function returns, NAME))
148#define STATS_DECLTRACK_CSRET_ATTR(NAME) \
149 STATS_DECLTRACK(NAME, CSReturn, \
150 BUILD_STAT_MSG_IR_ATTR(call site returns, NAME))
151#define STATS_DECLTRACK_FLOATING_ATTR(NAME) \
152 STATS_DECLTRACK(NAME, Floating, \
153 ("Number of floating values known to be '" #NAME "'"))
154
155// Specialization of the operator<< for abstract attributes subclasses. This
156// disambiguates situations where multiple operators are applicable.
157namespace llvm {
158#define PIPE_OPERATOR(CLASS) \
159 raw_ostream &operator<<(raw_ostream &OS, const CLASS &AA) { \
160 return OS << static_cast<const AbstractAttribute &>(AA); \
161 }
162
163PIPE_OPERATOR(AAIsDead)
164PIPE_OPERATOR(AANoUnwind)
165PIPE_OPERATOR(AANoSync)
166PIPE_OPERATOR(AANoRecurse)
167PIPE_OPERATOR(AANonConvergent)
168PIPE_OPERATOR(AAWillReturn)
169PIPE_OPERATOR(AANoReturn)
170PIPE_OPERATOR(AANonNull)
171PIPE_OPERATOR(AAMustProgress)
172PIPE_OPERATOR(AANoAlias)
173PIPE_OPERATOR(AADereferenceable)
174PIPE_OPERATOR(AAAlign)
175PIPE_OPERATOR(AAInstanceInfo)
176PIPE_OPERATOR(AANoCapture)
177PIPE_OPERATOR(AAValueSimplify)
178PIPE_OPERATOR(AANoFree)
179PIPE_OPERATOR(AAHeapToStack)
180PIPE_OPERATOR(AAIntraFnReachability)
181PIPE_OPERATOR(AAMemoryBehavior)
182PIPE_OPERATOR(AAMemoryLocation)
183PIPE_OPERATOR(AAValueConstantRange)
184PIPE_OPERATOR(AAPrivatizablePtr)
185PIPE_OPERATOR(AAUndefinedBehavior)
186PIPE_OPERATOR(AAPotentialConstantValues)
187PIPE_OPERATOR(AAPotentialValues)
188PIPE_OPERATOR(AANoUndef)
189PIPE_OPERATOR(AANoFPClass)
190PIPE_OPERATOR(AACallEdges)
191PIPE_OPERATOR(AAInterFnReachability)
192PIPE_OPERATOR(AAPointerInfo)
193PIPE_OPERATOR(AAAssumptionInfo)
194PIPE_OPERATOR(AAUnderlyingObjects)
195PIPE_OPERATOR(AAInvariantLoadPointer)
196PIPE_OPERATOR(AAAddressSpace)
197PIPE_OPERATOR(AANoAliasAddrSpace)
198PIPE_OPERATOR(AAAllocationInfo)
199PIPE_OPERATOR(AAIndirectCallInfo)
200PIPE_OPERATOR(AAGlobalValueInfo)
201PIPE_OPERATOR(AADenormalFPMath)
202
203#undef PIPE_OPERATOR
204
205template <>
206ChangeStatus clampStateAndIndicateChange<DerefState>(DerefState &S,
207 const DerefState &R) {
208 ChangeStatus CS0 =
209 clampStateAndIndicateChange(S&: S.DerefBytesState, R: R.DerefBytesState);
210 ChangeStatus CS1 = clampStateAndIndicateChange(S&: S.GlobalState, R: R.GlobalState);
211 return CS0 | CS1;
212}
213
214} // namespace llvm
215
216static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I,
217 bool HeaderOnly, CycleRef *CPtr = nullptr) {
218 if (!CI)
219 return true;
220 auto *BB = I->getParent();
221 CycleRef C = CI->getCycle(Block: BB);
222 if (!C)
223 return false;
224 if (CPtr)
225 *CPtr = C;
226 return !HeaderOnly || BB == CI->getHeader(C);
227}
228
229/// Checks if a type could have padding bytes.
230static bool isDenselyPacked(Type *Ty, const DataLayout &DL) {
231 // There is no size information, so be conservative.
232 if (!Ty->isSized())
233 return false;
234
235 // If the alloc size is not equal to the storage size, then there are padding
236 // bytes. For x86_fp80 on x86-64, size: 80 alloc size: 128.
237 if (DL.getTypeSizeInBits(Ty) != DL.getTypeAllocSizeInBits(Ty))
238 return false;
239
240 // FIXME: This isn't the right way to check for padding in vectors with
241 // non-byte-size elements.
242 if (VectorType *SeqTy = dyn_cast<VectorType>(Val: Ty))
243 return isDenselyPacked(Ty: SeqTy->getElementType(), DL);
244
245 // For array types, check for padding within members.
246 if (ArrayType *SeqTy = dyn_cast<ArrayType>(Val: Ty))
247 return isDenselyPacked(Ty: SeqTy->getElementType(), DL);
248
249 if (!isa<StructType>(Val: Ty))
250 return true;
251
252 // Check for padding within and between elements of a struct.
253 StructType *StructTy = cast<StructType>(Val: Ty);
254 const StructLayout *Layout = DL.getStructLayout(Ty: StructTy);
255 uint64_t StartPos = 0;
256 for (unsigned I = 0, E = StructTy->getNumElements(); I < E; ++I) {
257 Type *ElTy = StructTy->getElementType(N: I);
258 if (!isDenselyPacked(Ty: ElTy, DL))
259 return false;
260 if (StartPos != Layout->getElementOffsetInBits(Idx: I))
261 return false;
262 StartPos += DL.getTypeAllocSizeInBits(Ty: ElTy);
263 }
264
265 // Check for tail padding.
266 return StartPos == Layout->getSizeInBits();
267}
268
269/// Get pointer operand of memory accessing instruction. If \p I is
270/// not a memory accessing instruction, return nullptr. If \p AllowVolatile,
271/// is set to false and the instruction is volatile, return nullptr.
272static const Value *getPointerOperand(const Instruction *I,
273 bool AllowVolatile) {
274 if (!AllowVolatile && I->isVolatile())
275 return nullptr;
276
277 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
278 return LI->getPointerOperand();
279 }
280
281 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
282 return SI->getPointerOperand();
283 }
284
285 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
286 return CXI->getPointerOperand();
287 }
288
289 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: I)) {
290 return RMWI->getPointerOperand();
291 }
292
293 return nullptr;
294}
295
296/// Helper function to create a pointer based on \p Ptr, and advanced by \p
297/// Offset bytes.
298static Value *constructPointer(Value *Ptr, int64_t Offset,
299 IRBuilder<NoFolder> &IRB) {
300 LLVM_DEBUG(dbgs() << "Construct pointer: " << *Ptr << " + " << Offset
301 << "-bytes\n");
302
303 if (Offset)
304 Ptr = IRB.CreatePtrAdd(Ptr, Offset: IRB.getInt64(C: Offset),
305 Name: Ptr->getName() + ".b" + Twine(Offset));
306 return Ptr;
307}
308
309static const Value *
310stripAndAccumulateOffsets(Attributor &A, const AbstractAttribute &QueryingAA,
311 const Value *Val, const DataLayout &DL, APInt &Offset,
312 bool GetMinOffset, bool AllowNonInbounds,
313 bool UseAssumed = false) {
314
315 auto AttributorAnalysis = [&](Value &V, APInt &ROffset) -> bool {
316 const IRPosition &Pos = IRPosition::value(V);
317 // Only track dependence if we are going to use the assumed info.
318 const AAValueConstantRange *ValueConstantRangeAA =
319 A.getAAFor<AAValueConstantRange>(QueryingAA, IRP: Pos,
320 DepClass: UseAssumed ? DepClassTy::OPTIONAL
321 : DepClassTy::NONE);
322 if (!ValueConstantRangeAA)
323 return false;
324 ConstantRange Range = UseAssumed ? ValueConstantRangeAA->getAssumed()
325 : ValueConstantRangeAA->getKnown();
326 if (Range.isFullSet())
327 return false;
328
329 // We can only use the lower part of the range because the upper part can
330 // be higher than what the value can really be.
331 if (GetMinOffset)
332 ROffset = Range.getSignedMin();
333 else
334 ROffset = Range.getSignedMax();
335 return true;
336 };
337
338 return Val->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds,
339 /* AllowInvariant */ AllowInvariantGroup: true,
340 ExternalAnalysis: AttributorAnalysis);
341}
342
343static const Value *
344getMinimalBaseOfPointer(Attributor &A, const AbstractAttribute &QueryingAA,
345 const Value *Ptr, int64_t &BytesOffset,
346 const DataLayout &DL, bool AllowNonInbounds = false) {
347 APInt OffsetAPInt(DL.getIndexTypeSizeInBits(Ty: Ptr->getType()), 0);
348 const Value *Base =
349 stripAndAccumulateOffsets(A, QueryingAA, Val: Ptr, DL, Offset&: OffsetAPInt,
350 /* GetMinOffset */ true, AllowNonInbounds);
351
352 BytesOffset = OffsetAPInt.getSExtValue();
353 return Base;
354}
355
356/// Clamp the information known for all returned values of a function
357/// (identified by \p QueryingAA) into \p S.
358template <typename AAType, typename StateType = typename AAType::StateType,
359 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
360 bool RecurseForSelectAndPHI = true>
361static void clampReturnedValueStates(
362 Attributor &A, const AAType &QueryingAA, StateType &S,
363 const IRPosition::CallBaseContext *CBContext = nullptr) {
364 LLVM_DEBUG(dbgs() << "[Attributor] Clamp return value states for "
365 << QueryingAA << " into " << S << "\n");
366
367 assert((QueryingAA.getIRPosition().getPositionKind() ==
368 IRPosition::IRP_RETURNED ||
369 QueryingAA.getIRPosition().getPositionKind() ==
370 IRPosition::IRP_CALL_SITE_RETURNED) &&
371 "Can only clamp returned value states for a function returned or call "
372 "site returned position!");
373
374 // Use an optional state as there might not be any return values and we want
375 // to join (IntegerState::operator&) the state of all there are.
376 std::optional<StateType> T;
377
378 // Callback for each possibly returned value.
379 auto CheckReturnValue = [&](Value &RV) -> bool {
380 const IRPosition &RVPos = IRPosition::value(V: RV, CBContext);
381 // If possible, use the hasAssumedIRAttr interface.
382 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
383 bool IsKnown;
384 return AA::hasAssumedIRAttr<IRAttributeKind>(
385 A, &QueryingAA, RVPos, DepClassTy::REQUIRED, IsKnown);
386 }
387
388 const AAType *AA =
389 A.getAAFor<AAType>(QueryingAA, RVPos, DepClassTy::REQUIRED);
390 if (!AA)
391 return false;
392 LLVM_DEBUG(dbgs() << "[Attributor] RV: " << RV
393 << " AA: " << AA->getAsStr(&A) << " @ " << RVPos << "\n");
394 const StateType &AAS = AA->getState();
395 if (!T)
396 T = StateType::getBestState(AAS);
397 *T &= AAS;
398 LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " RV State: " << T
399 << "\n");
400 return T->isValidState();
401 };
402
403 if (!A.checkForAllReturnedValues(Pred: CheckReturnValue, QueryingAA,
404 S: AA::ValueScope::Intraprocedural,
405 RecurseForSelectAndPHI))
406 S.indicatePessimisticFixpoint();
407 else if (T)
408 S ^= *T;
409}
410
411namespace {
412/// Helper class for generic deduction: return value -> returned position.
413template <typename AAType, typename BaseType,
414 typename StateType = typename BaseType::StateType,
415 bool PropagateCallBaseContext = false,
416 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
417 bool RecurseForSelectAndPHI = true>
418struct AAReturnedFromReturnedValues : public BaseType {
419 AAReturnedFromReturnedValues(const IRPosition &IRP, Attributor &A)
420 : BaseType(IRP, A) {}
421
422 /// See AbstractAttribute::updateImpl(...).
423 ChangeStatus updateImpl(Attributor &A) override {
424 StateType S(StateType::getBestState(this->getState()));
425 clampReturnedValueStates<AAType, StateType, IRAttributeKind,
426 RecurseForSelectAndPHI>(
427 A, *this, S,
428 PropagateCallBaseContext ? this->getCallBaseContext() : nullptr);
429 // TODO: If we know we visited all returned values, thus no are assumed
430 // dead, we can take the known information from the state T.
431 return clampStateAndIndicateChange<StateType>(this->getState(), S);
432 }
433};
434
435/// Clamp the information known at all call sites for a given argument
436/// (identified by \p QueryingAA) into \p S.
437template <typename AAType, typename StateType = typename AAType::StateType,
438 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
439static void clampCallSiteArgumentStates(Attributor &A, const AAType &QueryingAA,
440 StateType &S) {
441 LLVM_DEBUG(dbgs() << "[Attributor] Clamp call site argument states for "
442 << QueryingAA << " into " << S << "\n");
443
444 assert(QueryingAA.getIRPosition().getPositionKind() ==
445 IRPosition::IRP_ARGUMENT &&
446 "Can only clamp call site argument states for an argument position!");
447
448 // Use an optional state as there might not be any return values and we want
449 // to join (IntegerState::operator&) the state of all there are.
450 std::optional<StateType> T;
451
452 // The argument number which is also the call site argument number.
453 unsigned ArgNo = QueryingAA.getIRPosition().getCallSiteArgNo();
454
455 auto CallSiteCheck = [&](AbstractCallSite ACS) {
456 const IRPosition &ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
457 // Check if a coresponding argument was found or if it is on not associated
458 // (which can happen for callback calls).
459 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
460 return false;
461
462 // If possible, use the hasAssumedIRAttr interface.
463 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
464 bool IsKnown;
465 return AA::hasAssumedIRAttr<IRAttributeKind>(
466 A, &QueryingAA, ACSArgPos, DepClassTy::REQUIRED, IsKnown);
467 }
468
469 const AAType *AA =
470 A.getAAFor<AAType>(QueryingAA, ACSArgPos, DepClassTy::REQUIRED);
471 if (!AA)
472 return false;
473 LLVM_DEBUG(dbgs() << "[Attributor] ACS: " << *ACS.getInstruction()
474 << " AA: " << AA->getAsStr(&A) << " @" << ACSArgPos
475 << "\n");
476 const StateType &AAS = AA->getState();
477 if (!T)
478 T = StateType::getBestState(AAS);
479 *T &= AAS;
480 LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " CSA State: " << T
481 << "\n");
482 return T->isValidState();
483 };
484
485 bool UsedAssumedInformation = false;
486 if (!A.checkForAllCallSites(CallSiteCheck, QueryingAA, true,
487 UsedAssumedInformation))
488 S.indicatePessimisticFixpoint();
489 else if (T)
490 S ^= *T;
491}
492
493/// This function is the bridge between argument position and the call base
494/// context.
495template <typename AAType, typename BaseType,
496 typename StateType = typename AAType::StateType,
497 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
498bool getArgumentStateFromCallBaseContext(Attributor &A,
499 BaseType &QueryingAttribute,
500 IRPosition &Pos, StateType &State) {
501 assert((Pos.getPositionKind() == IRPosition::IRP_ARGUMENT) &&
502 "Expected an 'argument' position !");
503 const CallBase *CBContext = Pos.getCallBaseContext();
504 if (!CBContext)
505 return false;
506
507 int ArgNo = Pos.getCallSiteArgNo();
508 assert(ArgNo >= 0 && "Invalid Arg No!");
509 const IRPosition CBArgPos = IRPosition::callsite_argument(CB: *CBContext, ArgNo);
510
511 // If possible, use the hasAssumedIRAttr interface.
512 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
513 bool IsKnown;
514 return AA::hasAssumedIRAttr<IRAttributeKind>(
515 A, &QueryingAttribute, CBArgPos, DepClassTy::REQUIRED, IsKnown);
516 }
517
518 const auto *AA =
519 A.getAAFor<AAType>(QueryingAttribute, CBArgPos, DepClassTy::REQUIRED);
520 if (!AA)
521 return false;
522 const StateType &CBArgumentState =
523 static_cast<const StateType &>(AA->getState());
524
525 LLVM_DEBUG(dbgs() << "[Attributor] Briding Call site context to argument"
526 << "Position:" << Pos << "CB Arg state:" << CBArgumentState
527 << "\n");
528
529 // NOTE: If we want to do call site grouping it should happen here.
530 State ^= CBArgumentState;
531 return true;
532}
533
534/// Helper class for generic deduction: call site argument -> argument position.
535template <typename AAType, typename BaseType,
536 typename StateType = typename AAType::StateType,
537 bool BridgeCallBaseContext = false,
538 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
539struct AAArgumentFromCallSiteArguments : public BaseType {
540 AAArgumentFromCallSiteArguments(const IRPosition &IRP, Attributor &A)
541 : BaseType(IRP, A) {}
542
543 /// See AbstractAttribute::updateImpl(...).
544 ChangeStatus updateImpl(Attributor &A) override {
545 StateType S = StateType::getBestState(this->getState());
546
547 if (BridgeCallBaseContext) {
548 bool Success =
549 getArgumentStateFromCallBaseContext<AAType, BaseType, StateType,
550 IRAttributeKind>(
551 A, *this, this->getIRPosition(), S);
552 if (Success)
553 return clampStateAndIndicateChange<StateType>(this->getState(), S);
554 }
555 clampCallSiteArgumentStates<AAType, StateType, IRAttributeKind>(A, *this,
556 S);
557
558 // TODO: If we know we visited all incoming values, thus no are assumed
559 // dead, we can take the known information from the state T.
560 return clampStateAndIndicateChange<StateType>(this->getState(), S);
561 }
562};
563
564/// Helper class for generic replication: function returned -> cs returned.
565template <typename AAType, typename BaseType,
566 typename StateType = typename BaseType::StateType,
567 bool IntroduceCallBaseContext = false,
568 Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
569struct AACalleeToCallSite : public BaseType {
570 AACalleeToCallSite(const IRPosition &IRP, Attributor &A) : BaseType(IRP, A) {}
571
572 /// See AbstractAttribute::updateImpl(...).
573 ChangeStatus updateImpl(Attributor &A) override {
574 auto IRPKind = this->getIRPosition().getPositionKind();
575 assert((IRPKind == IRPosition::IRP_CALL_SITE_RETURNED ||
576 IRPKind == IRPosition::IRP_CALL_SITE) &&
577 "Can only wrap function returned positions for call site "
578 "returned positions!");
579 auto &S = this->getState();
580
581 CallBase &CB = cast<CallBase>(this->getAnchorValue());
582 if (IntroduceCallBaseContext)
583 LLVM_DEBUG(dbgs() << "[Attributor] Introducing call base context:" << CB
584 << "\n");
585
586 ChangeStatus Changed = ChangeStatus::UNCHANGED;
587 auto CalleePred = [&](ArrayRef<const Function *> Callees) {
588 for (const Function *Callee : Callees) {
589 IRPosition FnPos =
590 IRPKind == llvm::IRPosition::IRP_CALL_SITE_RETURNED
591 ? IRPosition::returned(F: *Callee,
592 CBContext: IntroduceCallBaseContext ? &CB : nullptr)
593 : IRPosition::function(
594 F: *Callee, CBContext: IntroduceCallBaseContext ? &CB : nullptr);
595 // If possible, use the hasAssumedIRAttr interface.
596 if (Attribute::isEnumAttrKind(Kind: IRAttributeKind)) {
597 bool IsKnown;
598 if (!AA::hasAssumedIRAttr<IRAttributeKind>(
599 A, this, FnPos, DepClassTy::REQUIRED, IsKnown))
600 return false;
601 continue;
602 }
603
604 const AAType *AA =
605 A.getAAFor<AAType>(*this, FnPos, DepClassTy::REQUIRED);
606 if (!AA)
607 return false;
608 Changed |= clampStateAndIndicateChange(S, AA->getState());
609 if (S.isAtFixpoint())
610 return S.isValidState();
611 }
612 return true;
613 };
614 if (!A.checkForAllCallees(Pred: CalleePred, QueryingAA: *this, CB))
615 return S.indicatePessimisticFixpoint();
616 return Changed;
617 }
618};
619
620/// Helper function to accumulate uses.
621template <class AAType, typename StateType = typename AAType::StateType>
622static void followUsesInContext(AAType &AA, Attributor &A,
623 MustBeExecutedContextExplorer &Explorer,
624 const Instruction *CtxI,
625 SetVector<const Use *> &Uses,
626 StateType &State) {
627 auto EIt = Explorer.begin(PP: CtxI), EEnd = Explorer.end(CtxI);
628 for (unsigned u = 0; u < Uses.size(); ++u) {
629 const Use *U = Uses[u];
630 if (const Instruction *UserI = dyn_cast<Instruction>(Val: U->getUser())) {
631 bool Found = Explorer.findInContextOf(I: UserI, EIt, EEnd);
632 if (Found && AA.followUseInMBEC(A, U, UserI, State))
633 Uses.insert_range(R: llvm::make_pointer_range(Range: UserI->uses()));
634 }
635 }
636}
637
638/// Use the must-be-executed-context around \p I to add information into \p S.
639/// The AAType class is required to have `followUseInMBEC` method with the
640/// following signature and behaviour:
641///
642/// bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I)
643/// U - Underlying use.
644/// I - The user of the \p U.
645/// Returns true if the value should be tracked transitively.
646///
647template <class AAType, typename StateType = typename AAType::StateType>
648static void followUsesInMBEC(AAType &AA, Attributor &A, StateType &S,
649 Instruction &CtxI) {
650 const Value &Val = AA.getIRPosition().getAssociatedValue();
651 if (isa<ConstantData>(Val))
652 return;
653
654 MustBeExecutedContextExplorer *Explorer =
655 A.getInfoCache().getMustBeExecutedContextExplorer();
656 if (!Explorer)
657 return;
658
659 // Container for (transitive) uses of the associated value.
660 SetVector<const Use *> Uses;
661 for (const Use &U : Val.uses())
662 Uses.insert(X: &U);
663
664 followUsesInContext<AAType>(AA, A, *Explorer, &CtxI, Uses, S);
665
666 if (S.isAtFixpoint())
667 return;
668
669 SmallVector<const CondBrInst *, 4> BrInsts;
670 auto Pred = [&](const Instruction *I) {
671 if (const CondBrInst *Br = dyn_cast<CondBrInst>(Val: I))
672 BrInsts.push_back(Elt: Br);
673 return true;
674 };
675
676 // Here, accumulate conditional branch instructions in the context. We
677 // explore the child paths and collect the known states. The disjunction of
678 // those states can be merged to its own state. Let ParentState_i be a state
679 // to indicate the known information for an i-th branch instruction in the
680 // context. ChildStates are created for its successors respectively.
681 //
682 // ParentS_1 = ChildS_{1, 1} /\ ChildS_{1, 2} /\ ... /\ ChildS_{1, n_1}
683 // ParentS_2 = ChildS_{2, 1} /\ ChildS_{2, 2} /\ ... /\ ChildS_{2, n_2}
684 // ...
685 // ParentS_m = ChildS_{m, 1} /\ ChildS_{m, 2} /\ ... /\ ChildS_{m, n_m}
686 //
687 // Known State |= ParentS_1 \/ ParentS_2 \/... \/ ParentS_m
688 //
689 // FIXME: Currently, recursive branches are not handled. For example, we
690 // can't deduce that ptr must be dereferenced in below function.
691 //
692 // void f(int a, int c, int *ptr) {
693 // if(a)
694 // if (b) {
695 // *ptr = 0;
696 // } else {
697 // *ptr = 1;
698 // }
699 // else {
700 // if (b) {
701 // *ptr = 0;
702 // } else {
703 // *ptr = 1;
704 // }
705 // }
706 // }
707
708 Explorer->checkForAllContext(PP: &CtxI, Pred);
709 for (const CondBrInst *Br : BrInsts) {
710 StateType ParentState;
711
712 // The known state of the parent state is a conjunction of children's
713 // known states so it is initialized with a best state.
714 ParentState.indicateOptimisticFixpoint();
715
716 for (const BasicBlock *BB : Br->successors()) {
717 StateType ChildState;
718
719 size_t BeforeSize = Uses.size();
720 followUsesInContext(AA, A, *Explorer, &BB->front(), Uses, ChildState);
721
722 // Erase uses which only appear in the child.
723 for (auto It = Uses.begin() + BeforeSize; It != Uses.end();)
724 It = Uses.erase(I: It);
725
726 ParentState &= ChildState;
727 }
728
729 // Use only known state.
730 S += ParentState;
731 }
732}
733} // namespace
734
735/// ------------------------ PointerInfo ---------------------------------------
736
737namespace llvm {
738namespace AA {
739namespace PointerInfo {
740
741struct State;
742
743} // namespace PointerInfo
744} // namespace AA
745
746/// Helper for AA::PointerInfo::Access DenseMap/Set usage.
747template <>
748struct DenseMapInfo<AAPointerInfo::Access> : DenseMapInfo<Instruction *> {
749 using Access = AAPointerInfo::Access;
750 static unsigned getHashValue(const Access &A);
751 static bool isEqual(const Access &LHS, const Access &RHS);
752};
753
754/// Helper that allows RangeTy as a key in a DenseMap.
755template <> struct DenseMapInfo<AA::RangeTy> {
756 static unsigned getHashValue(const AA::RangeTy &Range) {
757 return detail::combineHashValue(
758 a: DenseMapInfo<int64_t>::getHashValue(Val: Range.Offset),
759 b: DenseMapInfo<int64_t>::getHashValue(Val: Range.Size));
760 }
761
762 static bool isEqual(const AA::RangeTy &A, const AA::RangeTy B) {
763 return A == B;
764 }
765};
766
767} // namespace llvm
768
769/// A type to track pointer/struct usage and accesses for AAPointerInfo.
770struct AA::PointerInfo::State : public AbstractState {
771 /// Return the best possible representable state.
772 static State getBestState(const State &SIS) { return State(); }
773
774 /// Return the worst possible representable state.
775 static State getWorstState(const State &SIS) {
776 State R;
777 R.indicatePessimisticFixpoint();
778 return R;
779 }
780
781 State() = default;
782 State(State &&SIS) = default;
783
784 const State &getAssumed() const { return *this; }
785
786 /// See AbstractState::isValidState().
787 bool isValidState() const override { return BS.isValidState(); }
788
789 /// See AbstractState::isAtFixpoint().
790 bool isAtFixpoint() const override { return BS.isAtFixpoint(); }
791
792 /// See AbstractState::indicateOptimisticFixpoint().
793 ChangeStatus indicateOptimisticFixpoint() override {
794 BS.indicateOptimisticFixpoint();
795 return ChangeStatus::UNCHANGED;
796 }
797
798 /// See AbstractState::indicatePessimisticFixpoint().
799 ChangeStatus indicatePessimisticFixpoint() override {
800 BS.indicatePessimisticFixpoint();
801 return ChangeStatus::CHANGED;
802 }
803
804 State &operator=(const State &R) {
805 if (this == &R)
806 return *this;
807 BS = R.BS;
808 AccessList = R.AccessList;
809 OffsetBins = R.OffsetBins;
810 RemoteIMap = R.RemoteIMap;
811 ReturnedOffsets = R.ReturnedOffsets;
812 return *this;
813 }
814
815 State &operator=(State &&R) {
816 if (this == &R)
817 return *this;
818 std::swap(a&: BS, b&: R.BS);
819 std::swap(LHS&: AccessList, RHS&: R.AccessList);
820 std::swap(a&: OffsetBins, b&: R.OffsetBins);
821 std::swap(a&: RemoteIMap, b&: R.RemoteIMap);
822 std::swap(a&: ReturnedOffsets, b&: R.ReturnedOffsets);
823 return *this;
824 }
825
826 /// Add a new Access to the state at offset \p Offset and with size \p Size.
827 /// The access is associated with \p I, writes \p Content (if anything), and
828 /// is of kind \p Kind. If an Access already exists for the same \p I and same
829 /// \p RemoteI, the two are combined, potentially losing information about
830 /// offset and size. The resulting access must now be moved from its original
831 /// OffsetBin to the bin for its new offset.
832 ///
833 /// \Returns CHANGED, if the state changed, UNCHANGED otherwise.
834 ChangeStatus addAccess(Attributor &A, const AAPointerInfo::RangeList &Ranges,
835 Instruction &I, std::optional<Value *> Content,
836 AAPointerInfo::AccessKind Kind, Type *Ty,
837 Instruction *RemoteI = nullptr);
838
839 AAPointerInfo::const_bin_iterator begin() const { return OffsetBins.begin(); }
840 AAPointerInfo::const_bin_iterator end() const { return OffsetBins.end(); }
841 int64_t numOffsetBins() const { return OffsetBins.size(); }
842
843 const AAPointerInfo::Access &getAccess(unsigned Index) const {
844 return AccessList[Index];
845 }
846
847protected:
848 // Every memory instruction results in an Access object. We maintain a list of
849 // all Access objects that we own, along with the following maps:
850 //
851 // - OffsetBins: RangeTy -> { Access }
852 // - RemoteIMap: RemoteI x LocalI -> Access
853 //
854 // A RemoteI is any instruction that accesses memory. RemoteI is different
855 // from LocalI if and only if LocalI is a call; then RemoteI is some
856 // instruction in the callgraph starting from LocalI. Multiple paths in the
857 // callgraph from LocalI to RemoteI may produce multiple accesses, but these
858 // are all combined into a single Access object. This may result in loss of
859 // information in RangeTy in the Access object.
860 SmallVector<AAPointerInfo::Access> AccessList;
861 AAPointerInfo::OffsetBinsTy OffsetBins;
862 DenseMap<const Instruction *, SmallVector<unsigned>> RemoteIMap;
863
864 /// Flag to determine if the underlying pointer is reaching a return statement
865 /// in the associated function or not. Returns in other functions cause
866 /// invalidation.
867 AAPointerInfo::OffsetInfo ReturnedOffsets;
868
869 /// See AAPointerInfo::forallInterferingAccesses.
870 template <typename F>
871 bool forallInterferingAccesses(AA::RangeTy Range, F CB) const {
872 if (!isValidState() || !ReturnedOffsets.isUnassigned())
873 return false;
874
875 for (const auto &It : OffsetBins) {
876 AA::RangeTy ItRange = It.getFirst();
877 if (!Range.mayOverlap(Range: ItRange))
878 continue;
879 bool IsExact = Range == ItRange && !Range.offsetOrSizeAreUnknown();
880 for (auto Index : It.getSecond()) {
881 auto &Access = AccessList[Index];
882 if (!CB(Access, IsExact))
883 return false;
884 }
885 }
886 return true;
887 }
888
889 /// See AAPointerInfo::forallInterferingAccesses.
890 template <typename F>
891 bool forallInterferingAccesses(Instruction &I, F CB,
892 AA::RangeTy &Range) const {
893 if (!isValidState() || !ReturnedOffsets.isUnassigned())
894 return false;
895
896 auto LocalList = RemoteIMap.find(Val: &I);
897 if (LocalList == RemoteIMap.end()) {
898 return true;
899 }
900
901 for (unsigned Index : LocalList->getSecond()) {
902 for (auto &R : AccessList[Index]) {
903 Range &= R;
904 if (Range.offsetAndSizeAreUnknown())
905 break;
906 }
907 }
908 return forallInterferingAccesses(Range, CB);
909 }
910
911private:
912 /// State to track fixpoint and validity.
913 BooleanState BS;
914};
915
916ChangeStatus AA::PointerInfo::State::addAccess(
917 Attributor &A, const AAPointerInfo::RangeList &Ranges, Instruction &I,
918 std::optional<Value *> Content, AAPointerInfo::AccessKind Kind, Type *Ty,
919 Instruction *RemoteI) {
920 RemoteI = RemoteI ? RemoteI : &I;
921
922 // Check if we have an access for this instruction, if not, simply add it.
923 auto &LocalList = RemoteIMap[RemoteI];
924 bool AccExists = false;
925 unsigned AccIndex = AccessList.size();
926 for (auto Index : LocalList) {
927 auto &A = AccessList[Index];
928 if (A.getLocalInst() == &I) {
929 AccExists = true;
930 AccIndex = Index;
931 break;
932 }
933 }
934
935 auto AddToBins = [&](const AAPointerInfo::RangeList &ToAdd) {
936 LLVM_DEBUG(if (ToAdd.size()) dbgs()
937 << "[AAPointerInfo] Inserting access in new offset bins\n";);
938
939 for (auto Key : ToAdd) {
940 LLVM_DEBUG(dbgs() << " key " << Key << "\n");
941 OffsetBins[Key].insert(V: AccIndex);
942 }
943 };
944
945 if (!AccExists) {
946 AccessList.emplace_back(Args: &I, Args&: RemoteI, Args: Ranges, Args&: Content, Args&: Kind, Args&: Ty);
947 assert((AccessList.size() == AccIndex + 1) &&
948 "New Access should have been at AccIndex");
949 LocalList.push_back(Elt: AccIndex);
950 AddToBins(AccessList[AccIndex].getRanges());
951 return ChangeStatus::CHANGED;
952 }
953
954 // Combine the new Access with the existing Access, and then update the
955 // mapping in the offset bins.
956 AAPointerInfo::Access Acc(&I, RemoteI, Ranges, Content, Kind, Ty);
957 auto &Current = AccessList[AccIndex];
958 auto Before = Current;
959 Current &= Acc;
960 if (Current == Before)
961 return ChangeStatus::UNCHANGED;
962
963 auto &ExistingRanges = Before.getRanges();
964 auto &NewRanges = Current.getRanges();
965
966 // Ranges that are in the old access but not the new access need to be removed
967 // from the offset bins.
968 AAPointerInfo::RangeList ToRemove;
969 AAPointerInfo::RangeList::set_difference(L: ExistingRanges, R: NewRanges, D&: ToRemove);
970 LLVM_DEBUG(if (ToRemove.size()) dbgs()
971 << "[AAPointerInfo] Removing access from old offset bins\n";);
972
973 for (auto Key : ToRemove) {
974 LLVM_DEBUG(dbgs() << " key " << Key << "\n");
975 assert(OffsetBins.count(Key) && "Existing Access must be in some bin.");
976 auto &Bin = OffsetBins[Key];
977 assert(Bin.count(AccIndex) &&
978 "Expected bin to actually contain the Access.");
979 Bin.erase(V: AccIndex);
980 }
981
982 // Ranges that are in the new access but not the old access need to be added
983 // to the offset bins.
984 AAPointerInfo::RangeList ToAdd;
985 AAPointerInfo::RangeList::set_difference(L: NewRanges, R: ExistingRanges, D&: ToAdd);
986 AddToBins(ToAdd);
987 return ChangeStatus::CHANGED;
988}
989
990namespace {
991
992#ifndef NDEBUG
993static raw_ostream &operator<<(raw_ostream &OS,
994 const AAPointerInfo::OffsetInfo &OI) {
995 OS << llvm::interleaved_array(OI);
996 return OS;
997}
998#endif // NDEBUG
999
1000struct AAPointerInfoImpl
1001 : public StateWrapper<AA::PointerInfo::State, AAPointerInfo> {
1002 using BaseTy = StateWrapper<AA::PointerInfo::State, AAPointerInfo>;
1003 AAPointerInfoImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
1004
1005 /// See AbstractAttribute::getAsStr().
1006 const std::string getAsStr(Attributor *A) const override {
1007 return std::string("PointerInfo ") +
1008 (isValidState() ? (std::string("#") +
1009 std::to_string(val: OffsetBins.size()) + " bins")
1010 : "<invalid>") +
1011 (reachesReturn()
1012 ? (" (returned:" +
1013 join(R: map_range(C: ReturnedOffsets,
1014 F: [](int64_t O) { return std::to_string(val: O); }),
1015 Separator: ", ") +
1016 ")")
1017 : "");
1018 }
1019
1020 /// See AbstractAttribute::manifest(...).
1021 ChangeStatus manifest(Attributor &A) override {
1022 return AAPointerInfo::manifest(A);
1023 }
1024
1025 const_bin_iterator begin() const override { return State::begin(); }
1026 const_bin_iterator end() const override { return State::end(); }
1027 int64_t numOffsetBins() const override { return State::numOffsetBins(); }
1028 bool reachesReturn() const override {
1029 return !ReturnedOffsets.isUnassigned();
1030 }
1031 void addReturnedOffsetsTo(OffsetInfo &OI) const override {
1032 if (ReturnedOffsets.isUnknown()) {
1033 OI.setUnknown();
1034 return;
1035 }
1036
1037 OffsetInfo MergedOI;
1038 for (auto Offset : ReturnedOffsets) {
1039 OffsetInfo TmpOI = OI;
1040 TmpOI.addToAll(Inc: Offset);
1041 MergedOI.merge(R: TmpOI);
1042 }
1043 OI = std::move(MergedOI);
1044 }
1045
1046 ChangeStatus setReachesReturn(const OffsetInfo &ReachedReturnedOffsets) {
1047 if (ReturnedOffsets.isUnknown())
1048 return ChangeStatus::UNCHANGED;
1049 if (ReachedReturnedOffsets.isUnknown()) {
1050 ReturnedOffsets.setUnknown();
1051 return ChangeStatus::CHANGED;
1052 }
1053 if (ReturnedOffsets.merge(R: ReachedReturnedOffsets))
1054 return ChangeStatus::CHANGED;
1055 return ChangeStatus::UNCHANGED;
1056 }
1057
1058 bool forallInterferingAccesses(
1059 AA::RangeTy Range,
1060 function_ref<bool(const AAPointerInfo::Access &, bool)> CB)
1061 const override {
1062 return State::forallInterferingAccesses(Range, CB);
1063 }
1064
1065 bool forallInterferingAccesses(
1066 Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I,
1067 bool FindInterferingWrites, bool FindInterferingReads,
1068 function_ref<bool(const Access &, bool)> UserCB, bool &HasBeenWrittenTo,
1069 AA::RangeTy &Range,
1070 function_ref<bool(const Access &)> SkipCB) const override {
1071 HasBeenWrittenTo = false;
1072
1073 SmallPtrSet<const Access *, 8> DominatingWrites;
1074 SmallVector<std::pair<const Access *, bool>, 8> InterferingAccesses;
1075
1076 Function &Scope = *I.getFunction();
1077 bool IsKnownNoSync;
1078 bool IsAssumedNoSync = AA::hasAssumedIRAttr<Attribute::NoSync>(
1079 A, QueryingAA: &QueryingAA, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL,
1080 IsKnown&: IsKnownNoSync);
1081 const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
1082 IRP: IRPosition::function(F: Scope), QueryingAA: &QueryingAA, DepClass: DepClassTy::NONE);
1083 bool AllInSameNoSyncFn = IsAssumedNoSync;
1084 bool InstIsExecutedByInitialThreadOnly =
1085 ExecDomainAA && ExecDomainAA->isExecutedByInitialThreadOnly(I);
1086
1087 // If the function is not ending in aligned barriers, we need the stores to
1088 // be in aligned barriers. The load being in one is not sufficient since the
1089 // store might be executed by a thread that disappears after, causing the
1090 // aligned barrier guarding the load to unblock and the load to read a value
1091 // that has no CFG path to the load.
1092 bool InstIsExecutedInAlignedRegion =
1093 FindInterferingReads && ExecDomainAA &&
1094 ExecDomainAA->isExecutedInAlignedRegion(A, I);
1095
1096 if (InstIsExecutedInAlignedRegion || InstIsExecutedByInitialThreadOnly)
1097 A.recordDependence(FromAA: *ExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1098
1099 InformationCache &InfoCache = A.getInfoCache();
1100 bool IsThreadLocalObj =
1101 AA::isAssumedThreadLocalObject(A, Obj&: getAssociatedValue(), QueryingAA: *this);
1102
1103 // Helper to determine if we need to consider threading, which we cannot
1104 // right now. However, if the function is (assumed) nosync or the thread
1105 // executing all instructions is the main thread only we can ignore
1106 // threading. Also, thread-local objects do not require threading reasoning.
1107 // Finally, we can ignore threading if either access is executed in an
1108 // aligned region.
1109 auto CanIgnoreThreadingForInst = [&](const Instruction &I) -> bool {
1110 if (IsThreadLocalObj || AllInSameNoSyncFn)
1111 return true;
1112 const auto *FnExecDomainAA =
1113 I.getFunction() == &Scope
1114 ? ExecDomainAA
1115 : A.lookupAAFor<AAExecutionDomain>(
1116 IRP: IRPosition::function(F: *I.getFunction()), QueryingAA: &QueryingAA,
1117 DepClass: DepClassTy::NONE);
1118 if (!FnExecDomainAA)
1119 return false;
1120 if (InstIsExecutedInAlignedRegion ||
1121 (FindInterferingWrites &&
1122 FnExecDomainAA->isExecutedInAlignedRegion(A, I))) {
1123 A.recordDependence(FromAA: *FnExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1124 return true;
1125 }
1126 if (InstIsExecutedByInitialThreadOnly &&
1127 FnExecDomainAA->isExecutedByInitialThreadOnly(I)) {
1128 A.recordDependence(FromAA: *FnExecDomainAA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
1129 return true;
1130 }
1131 return false;
1132 };
1133
1134 // Helper to determine if the access is executed by the same thread as the
1135 // given instruction, for now it is sufficient to avoid any potential
1136 // threading effects as we cannot deal with them anyway.
1137 auto CanIgnoreThreading = [&](const Access &Acc) -> bool {
1138 return CanIgnoreThreadingForInst(*Acc.getRemoteInst()) ||
1139 (Acc.getRemoteInst() != Acc.getLocalInst() &&
1140 CanIgnoreThreadingForInst(*Acc.getLocalInst()));
1141 };
1142
1143 // TODO: Use inter-procedural reachability and dominance.
1144 bool IsKnownNoRecurse;
1145 AA::hasAssumedIRAttr<Attribute::NoRecurse>(
1146 A, QueryingAA: this, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL,
1147 IsKnown&: IsKnownNoRecurse);
1148
1149 // TODO: Use reaching kernels from AAKernelInfo (or move it to
1150 // AAExecutionDomain) such that we allow scopes other than kernels as long
1151 // as the reaching kernels are disjoint.
1152 bool InstInKernel = A.getInfoCache().isKernel(F: Scope);
1153 bool ObjHasKernelLifetime = false;
1154 const bool UseDominanceReasoning =
1155 FindInterferingWrites && IsKnownNoRecurse;
1156 const DominatorTree *DT =
1157 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: Scope);
1158
1159 // Helper to check if a value has "kernel lifetime", that is it will not
1160 // outlive a GPU kernel. This is true for shared, constant, and local
1161 // globals on AMD and NVIDIA GPUs.
1162 auto HasKernelLifetime = [&](Value *V, Module &M) {
1163 if (!AA::isGPU(M))
1164 return false;
1165 unsigned VAS = V->getType()->getPointerAddressSpace();
1166 return AA::isGPUSharedAddressSpace(M, AS: VAS) ||
1167 AA::isGPUConstantAddressSpace(M, AS: VAS) ||
1168 AA::isGPULocalAddressSpace(M, AS: VAS);
1169 };
1170
1171 // The IsLiveInCalleeCB will be used by the AA::isPotentiallyReachable query
1172 // to determine if we should look at reachability from the callee. For
1173 // certain pointers we know the lifetime and we do not have to step into the
1174 // callee to determine reachability as the pointer would be dead in the
1175 // callee. See the conditional initialization below.
1176 std::function<bool(const Function &)> IsLiveInCalleeCB;
1177
1178 if (auto *AI = dyn_cast<AllocaInst>(Val: &getAssociatedValue())) {
1179 // If the alloca containing function is not recursive the alloca
1180 // must be dead in the callee.
1181 const Function *AIFn = AI->getFunction();
1182 ObjHasKernelLifetime = A.getInfoCache().isKernel(F: *AIFn);
1183 bool IsKnownNoRecurse;
1184 if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
1185 A, QueryingAA: this, IRP: IRPosition::function(F: *AIFn), DepClass: DepClassTy::OPTIONAL,
1186 IsKnown&: IsKnownNoRecurse)) {
1187 IsLiveInCalleeCB = [AIFn](const Function &Fn) { return AIFn != &Fn; };
1188 }
1189 } else if (auto *GV = dyn_cast<GlobalValue>(Val: &getAssociatedValue())) {
1190 // If the global has kernel lifetime we can stop if we reach a kernel
1191 // as it is "dead" in the (unknown) callees.
1192 ObjHasKernelLifetime = HasKernelLifetime(GV, *GV->getParent());
1193 if (ObjHasKernelLifetime)
1194 IsLiveInCalleeCB = [&A](const Function &Fn) {
1195 return !A.getInfoCache().isKernel(F: Fn);
1196 };
1197 }
1198
1199 // Set of accesses/instructions that will overwrite the result and are
1200 // therefore blockers in the reachability traversal.
1201 AA::InstExclusionSetTy ExclusionSet;
1202
1203 auto AccessCB = [&](const Access &Acc, bool Exact) {
1204 Function *AccScope = Acc.getRemoteInst()->getFunction();
1205 bool AccInSameScope = AccScope == &Scope;
1206
1207 // If the object has kernel lifetime we can ignore accesses only reachable
1208 // by other kernels. For now we only skip accesses *in* other kernels.
1209 if (InstInKernel && ObjHasKernelLifetime && !AccInSameScope &&
1210 A.getInfoCache().isKernel(F: *AccScope))
1211 return true;
1212
1213 if (Exact && Acc.isMustAccess() && Acc.getRemoteInst() != &I) {
1214 if (Acc.isWrite() || (isa<LoadInst>(Val: I) && Acc.isWriteOrAssumption()))
1215 ExclusionSet.insert(Ptr: Acc.getRemoteInst());
1216 }
1217
1218 if ((!FindInterferingWrites || !Acc.isWriteOrAssumption()) &&
1219 (!FindInterferingReads || !Acc.isRead()))
1220 return true;
1221
1222 bool Dominates = FindInterferingWrites && DT && Exact &&
1223 Acc.isMustAccess() && AccInSameScope &&
1224 DT->dominates(Def: Acc.getRemoteInst(), User: &I);
1225 if (Dominates)
1226 DominatingWrites.insert(Ptr: &Acc);
1227
1228 // Track if all interesting accesses are in the same `nosync` function as
1229 // the given instruction.
1230 AllInSameNoSyncFn &= Acc.getRemoteInst()->getFunction() == &Scope;
1231
1232 InterferingAccesses.push_back(Elt: {&Acc, Exact});
1233 return true;
1234 };
1235 if (!State::forallInterferingAccesses(I, CB: AccessCB, Range))
1236 return false;
1237
1238 HasBeenWrittenTo = !DominatingWrites.empty();
1239
1240 // Dominating writes form a chain, find the least/lowest member.
1241 Instruction *LeastDominatingWriteInst = nullptr;
1242 for (const Access *Acc : DominatingWrites) {
1243 if (!LeastDominatingWriteInst) {
1244 LeastDominatingWriteInst = Acc->getRemoteInst();
1245 } else if (DT->dominates(Def: LeastDominatingWriteInst,
1246 User: Acc->getRemoteInst())) {
1247 LeastDominatingWriteInst = Acc->getRemoteInst();
1248 }
1249 }
1250
1251 // Helper to determine if we can skip a specific write access.
1252 auto CanSkipAccess = [&](const Access &Acc, bool Exact) {
1253 if (SkipCB && SkipCB(Acc))
1254 return true;
1255 if (!CanIgnoreThreading(Acc))
1256 return false;
1257
1258 // Check read (RAW) dependences and write (WAR) dependences as necessary.
1259 // If we successfully excluded all effects we are interested in, the
1260 // access can be skipped.
1261 bool ReadChecked = !FindInterferingReads;
1262 bool WriteChecked = !FindInterferingWrites;
1263
1264 // If the instruction cannot reach the access, the former does not
1265 // interfere with what the access reads.
1266 if (!ReadChecked) {
1267 if (!AA::isPotentiallyReachable(A, FromI: I, ToI: *Acc.getRemoteInst(), QueryingAA,
1268 ExclusionSet: &ExclusionSet, GoBackwardsCB: IsLiveInCalleeCB))
1269 ReadChecked = true;
1270 }
1271 // If the instruction cannot be reach from the access, the latter does not
1272 // interfere with what the instruction reads.
1273 if (!WriteChecked) {
1274 if (!AA::isPotentiallyReachable(A, FromI: *Acc.getRemoteInst(), ToI: I, QueryingAA,
1275 ExclusionSet: &ExclusionSet, GoBackwardsCB: IsLiveInCalleeCB))
1276 WriteChecked = true;
1277 }
1278
1279 // If we still might be affected by the write of the access but there are
1280 // dominating writes in the function of the instruction
1281 // (HasBeenWrittenTo), we can try to reason that the access is overwritten
1282 // by them. This would have happend above if they are all in the same
1283 // function, so we only check the inter-procedural case. Effectively, we
1284 // want to show that there is no call after the dominting write that might
1285 // reach the access, and when it returns reach the instruction with the
1286 // updated value. To this end, we iterate all call sites, check if they
1287 // might reach the instruction without going through another access
1288 // (ExclusionSet) and at the same time might reach the access. However,
1289 // that is all part of AAInterFnReachability.
1290 if (!WriteChecked && HasBeenWrittenTo &&
1291 Acc.getRemoteInst()->getFunction() != &Scope) {
1292
1293 const auto *FnReachabilityAA = A.getAAFor<AAInterFnReachability>(
1294 QueryingAA, IRP: IRPosition::function(F: Scope), DepClass: DepClassTy::OPTIONAL);
1295 if (FnReachabilityAA) {
1296 // Without going backwards in the call tree, can we reach the access
1297 // from the least dominating write. Do not allow to pass the
1298 // instruction itself either.
1299 bool Inserted = ExclusionSet.insert(Ptr: &I).second;
1300
1301 if (!FnReachabilityAA->instructionCanReach(
1302 A, Inst: *LeastDominatingWriteInst,
1303 Fn: *Acc.getRemoteInst()->getFunction(), ExclusionSet: &ExclusionSet))
1304 WriteChecked = true;
1305
1306 if (Inserted)
1307 ExclusionSet.erase(Ptr: &I);
1308 }
1309 }
1310
1311 if (ReadChecked && WriteChecked)
1312 return true;
1313
1314 if (!DT || !UseDominanceReasoning)
1315 return false;
1316 if (!DominatingWrites.count(Ptr: &Acc))
1317 return false;
1318 return LeastDominatingWriteInst != Acc.getRemoteInst();
1319 };
1320
1321 // Run the user callback on all accesses we cannot skip and return if
1322 // that succeeded for all or not.
1323 for (auto &It : InterferingAccesses) {
1324 if ((!AllInSameNoSyncFn && !IsThreadLocalObj && !ExecDomainAA) ||
1325 !CanSkipAccess(*It.first, It.second)) {
1326 if (!UserCB(*It.first, It.second))
1327 return false;
1328 }
1329 }
1330 return true;
1331 }
1332
1333 ChangeStatus translateAndAddStateFromCallee(Attributor &A,
1334 const AAPointerInfo &OtherAA,
1335 CallBase &CB) {
1336 using namespace AA::PointerInfo;
1337 if (!OtherAA.getState().isValidState() || !isValidState())
1338 return indicatePessimisticFixpoint();
1339
1340 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1341 const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1342 bool IsByval = OtherAAImpl.getAssociatedArgument()->hasByValAttr();
1343 Changed |= setReachesReturn(OtherAAImpl.ReturnedOffsets);
1344
1345 // Combine the accesses bin by bin.
1346 const auto &State = OtherAAImpl.getState();
1347 for (const auto &It : State) {
1348 for (auto Index : It.getSecond()) {
1349 const auto &RAcc = State.getAccess(Index);
1350 if (IsByval && !RAcc.isRead())
1351 continue;
1352 bool UsedAssumedInformation = false;
1353 AccessKind AK = RAcc.getKind();
1354 auto Content = A.translateArgumentToCallSiteContent(
1355 V: RAcc.getContent(), CB, AA: *this, UsedAssumedInformation);
1356 AK = AccessKind(AK & (IsByval ? AccessKind::AK_R : AccessKind::AK_RW));
1357 AK = AccessKind(AK | (RAcc.isMayAccess() ? AK_MAY : AK_MUST));
1358
1359 Changed |= addAccess(A, Ranges: RAcc.getRanges(), I&: CB, Content, Kind: AK,
1360 Ty: RAcc.getType(), RemoteI: RAcc.getRemoteInst());
1361 }
1362 }
1363 return Changed;
1364 }
1365
1366 ChangeStatus translateAndAddState(Attributor &A, const AAPointerInfo &OtherAA,
1367 const OffsetInfo &Offsets, CallBase &CB,
1368 bool IsMustAcc) {
1369 using namespace AA::PointerInfo;
1370 if (!OtherAA.getState().isValidState() || !isValidState())
1371 return indicatePessimisticFixpoint();
1372
1373 const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
1374
1375 // Combine the accesses bin by bin.
1376 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1377 const auto &State = OtherAAImpl.getState();
1378 for (const auto &It : State) {
1379 for (auto Index : It.getSecond()) {
1380 const auto &RAcc = State.getAccess(Index);
1381 if (!IsMustAcc && RAcc.isAssumption())
1382 continue;
1383 for (auto Offset : Offsets) {
1384 auto NewRanges = Offset == AA::RangeTy::Unknown
1385 ? AA::RangeTy::getUnknown()
1386 : RAcc.getRanges();
1387 if (!NewRanges.isUnknown()) {
1388 NewRanges.addToAllOffsets(Inc: Offset);
1389 }
1390 AccessKind AK = RAcc.getKind();
1391 if (!IsMustAcc)
1392 AK = AccessKind((AK & ~AK_MUST) | AK_MAY);
1393 Changed |= addAccess(A, Ranges: NewRanges, I&: CB, Content: RAcc.getContent(), Kind: AK,
1394 Ty: RAcc.getType(), RemoteI: RAcc.getRemoteInst());
1395 }
1396 }
1397 }
1398 return Changed;
1399 }
1400
1401 /// Statistic tracking for all AAPointerInfo implementations.
1402 /// See AbstractAttribute::trackStatistics().
1403 void trackPointerInfoStatistics(const IRPosition &IRP) const {}
1404
1405 /// Dump the state into \p O.
1406 void dumpState(raw_ostream &O) {
1407 for (auto &It : OffsetBins) {
1408 O << "[" << It.first.Offset << "-" << It.first.Offset + It.first.Size
1409 << "] : " << It.getSecond().size() << "\n";
1410 for (auto AccIndex : It.getSecond()) {
1411 auto &Acc = AccessList[AccIndex];
1412 O << " - " << Acc.getKind() << " - " << *Acc.getLocalInst() << "\n";
1413 if (Acc.getLocalInst() != Acc.getRemoteInst())
1414 O << " --> " << *Acc.getRemoteInst()
1415 << "\n";
1416 if (!Acc.isWrittenValueYetUndetermined()) {
1417 if (isa_and_nonnull<Function>(Val: Acc.getWrittenValue()))
1418 O << " - c: func " << Acc.getWrittenValue()->getName()
1419 << "\n";
1420 else if (Acc.getWrittenValue())
1421 O << " - c: " << *Acc.getWrittenValue() << "\n";
1422 else
1423 O << " - c: <unknown>\n";
1424 }
1425 }
1426 }
1427 }
1428};
1429
1430struct AAPointerInfoFloating : public AAPointerInfoImpl {
1431 using AccessKind = AAPointerInfo::AccessKind;
1432 AAPointerInfoFloating(const IRPosition &IRP, Attributor &A)
1433 : AAPointerInfoImpl(IRP, A) {}
1434
1435 /// Deal with an access and signal if it was handled successfully.
1436 bool handleAccess(Attributor &A, Instruction &I,
1437 std::optional<Value *> Content, AccessKind Kind,
1438 OffsetInfo::VecTy &Offsets, ChangeStatus &Changed,
1439 Type &Ty) {
1440 using namespace AA::PointerInfo;
1441 auto Size = AA::RangeTy::Unknown;
1442 const DataLayout &DL = A.getDataLayout();
1443 TypeSize AccessSize = DL.getTypeStoreSize(Ty: &Ty);
1444 if (!AccessSize.isScalable())
1445 Size = AccessSize.getFixedValue();
1446
1447 // Make a strictly ascending list of offsets as required by addAccess()
1448 SmallVector<int64_t> OffsetsSorted(Offsets.begin(), Offsets.end());
1449 llvm::sort(C&: OffsetsSorted);
1450
1451 VectorType *VT = dyn_cast<VectorType>(Val: &Ty);
1452 if (!VT || VT->getElementCount().isScalable() ||
1453 !Content.value_or(u: nullptr) || !isa<Constant>(Val: *Content) ||
1454 (*Content)->getType() != VT ||
1455 DL.getTypeStoreSize(Ty: VT->getElementType()).isScalable()) {
1456 Changed =
1457 Changed | addAccess(A, Ranges: {OffsetsSorted, Size}, I, Content, Kind, Ty: &Ty);
1458 } else {
1459 // Handle vector stores with constant content element-wise.
1460 // TODO: We could look for the elements or create instructions
1461 // representing them.
1462 // TODO: We need to push the Content into the range abstraction
1463 // (AA::RangeTy) to allow different content values for different
1464 // ranges. ranges. Hence, support vectors storing different values.
1465 Type *ElementType = VT->getElementType();
1466 int64_t ElementSize = DL.getTypeStoreSize(Ty: ElementType).getFixedValue();
1467 auto *ConstContent = cast<Constant>(Val: *Content);
1468 Type *Int32Ty = Type::getInt32Ty(C&: ElementType->getContext());
1469 SmallVector<int64_t> ElementOffsets(Offsets.begin(), Offsets.end());
1470
1471 for (int i = 0, e = VT->getElementCount().getFixedValue(); i != e; ++i) {
1472 Value *ElementContent = ConstantExpr::getExtractElement(
1473 Vec: ConstContent, Idx: ConstantInt::get(Ty: Int32Ty, V: i));
1474
1475 // Add the element access.
1476 Changed = Changed | addAccess(A, Ranges: {ElementOffsets, ElementSize}, I,
1477 Content: ElementContent, Kind, Ty: ElementType);
1478
1479 // Advance the offsets for the next element.
1480 for (auto &ElementOffset : ElementOffsets)
1481 ElementOffset += ElementSize;
1482 }
1483 }
1484 return true;
1485 };
1486
1487 /// See AbstractAttribute::updateImpl(...).
1488 ChangeStatus updateImpl(Attributor &A) override;
1489
1490 /// If the indices to \p GEP can be traced to constants, incorporate all
1491 /// of these into \p UsrOI.
1492 ///
1493 /// \return true iff \p UsrOI is updated.
1494 bool collectConstantsForGEP(Attributor &A, const DataLayout &DL,
1495 OffsetInfo &UsrOI, const OffsetInfo &PtrOI,
1496 const GEPOperator *GEP);
1497
1498 /// See AbstractAttribute::trackStatistics()
1499 void trackStatistics() const override {
1500 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1501 }
1502};
1503
1504bool AAPointerInfoFloating::collectConstantsForGEP(Attributor &A,
1505 const DataLayout &DL,
1506 OffsetInfo &UsrOI,
1507 const OffsetInfo &PtrOI,
1508 const GEPOperator *GEP) {
1509 unsigned BitWidth = DL.getIndexTypeSizeInBits(Ty: GEP->getType());
1510 SmallMapVector<Value *, APInt, 4> VariableOffsets;
1511 APInt ConstantOffset(BitWidth, 0);
1512
1513 assert(!UsrOI.isUnknown() && !PtrOI.isUnknown() &&
1514 "Don't look for constant values if the offset has already been "
1515 "determined to be unknown.");
1516
1517 if (!GEP->collectOffset(DL, BitWidth, VariableOffsets, ConstantOffset)) {
1518 UsrOI.setUnknown();
1519 return true;
1520 }
1521
1522 LLVM_DEBUG(dbgs() << "[AAPointerInfo] GEP offset is "
1523 << (VariableOffsets.empty() ? "" : "not") << " constant "
1524 << *GEP << "\n");
1525
1526 auto Union = PtrOI;
1527 Union.addToAll(Inc: ConstantOffset.getSExtValue());
1528
1529 // Each VI in VariableOffsets has a set of potential constant values. Every
1530 // combination of elements, picked one each from these sets, is separately
1531 // added to the original set of offsets, thus resulting in more offsets.
1532 for (const auto &VI : VariableOffsets) {
1533 auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
1534 QueryingAA: *this, IRP: IRPosition::value(V: *VI.first), DepClass: DepClassTy::OPTIONAL);
1535 if (!PotentialConstantsAA || !PotentialConstantsAA->isValidState()) {
1536 UsrOI.setUnknown();
1537 return true;
1538 }
1539
1540 // UndefValue is treated as a zero, which leaves Union as is.
1541 if (PotentialConstantsAA->undefIsContained())
1542 continue;
1543
1544 // We need at least one constant in every set to compute an actual offset.
1545 // Otherwise, we end up pessimizing AAPointerInfo by respecting offsets that
1546 // don't actually exist. In other words, the absence of constant values
1547 // implies that the operation can be assumed dead for now.
1548 auto &AssumedSet = PotentialConstantsAA->getAssumedSet();
1549 if (AssumedSet.empty())
1550 return false;
1551
1552 OffsetInfo Product;
1553 for (const auto &ConstOffset : AssumedSet) {
1554 auto CopyPerOffset = Union;
1555 CopyPerOffset.addToAll(Inc: ConstOffset.getSExtValue() *
1556 VI.second.getZExtValue());
1557 Product.merge(R: CopyPerOffset);
1558 }
1559 Union = Product;
1560 }
1561
1562 UsrOI = std::move(Union);
1563 return true;
1564}
1565
1566ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
1567 using namespace AA::PointerInfo;
1568 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1569 const DataLayout &DL = A.getDataLayout();
1570 Value &AssociatedValue = getAssociatedValue();
1571
1572 DenseMap<Value *, OffsetInfo> OffsetInfoMap;
1573 OffsetInfoMap[&AssociatedValue].insert(Offset: 0);
1574
1575 auto HandlePassthroughUser = [&](Value *Usr, Value *CurPtr, bool &Follow) {
1576 // One does not simply walk into a map and assign a reference to a possibly
1577 // new location. That can cause an invalidation before the assignment
1578 // happens, like so:
1579 //
1580 // OffsetInfoMap[Usr] = OffsetInfoMap[CurPtr]; /* bad idea! */
1581 //
1582 // The RHS is a reference that may be invalidated by an insertion caused by
1583 // the LHS. So we ensure that the side-effect of the LHS happens first.
1584
1585 assert(OffsetInfoMap.contains(CurPtr) &&
1586 "CurPtr does not exist in the map!");
1587
1588 auto &UsrOI = OffsetInfoMap[Usr];
1589 auto &PtrOI = OffsetInfoMap[CurPtr];
1590 assert(!PtrOI.isUnassigned() &&
1591 "Cannot pass through if the input Ptr was not visited!");
1592 UsrOI.merge(R: PtrOI);
1593 Follow = true;
1594 return true;
1595 };
1596
1597 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
1598 Value *CurPtr = U.get();
1599 User *Usr = U.getUser();
1600 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Analyze " << *CurPtr << " in " << *Usr
1601 << "\n");
1602 assert(OffsetInfoMap.count(CurPtr) &&
1603 "The current pointer offset should have been seeded!");
1604 assert(!OffsetInfoMap[CurPtr].isUnassigned() &&
1605 "Current pointer should be assigned");
1606
1607 // Only address space casts are followed; other casts of a pointer (e.g.,
1608 // ptrtoint, ptrtoaddr, or a bitcast to a byte type) do not produce a
1609 // pointer that can be tracked.
1610 if (isa<AddrSpaceCastOperator>(Val: Usr))
1611 return HandlePassthroughUser(Usr, CurPtr, Follow);
1612
1613 if (auto *GEP = dyn_cast<GEPOperator>(Val: Usr)) {
1614 // Note the order here, the Usr access might change the map, CurPtr is
1615 // already in it though.
1616 auto &UsrOI = OffsetInfoMap[Usr];
1617 auto &PtrOI = OffsetInfoMap[CurPtr];
1618
1619 if (UsrOI.isUnknown())
1620 return true;
1621
1622 if (PtrOI.isUnknown()) {
1623 Follow = true;
1624 UsrOI.setUnknown();
1625 return true;
1626 }
1627
1628 Follow = collectConstantsForGEP(A, DL, UsrOI, PtrOI, GEP);
1629 return true;
1630 }
1631 if (isa<SelectInst>(Val: Usr))
1632 return HandlePassthroughUser(Usr, CurPtr, Follow);
1633 // Returns are allowed if they are in the associated functions. Users can
1634 // then check the call site return. Returns from other functions can't be
1635 // tracked and are cause for invalidation.
1636 if (auto *RI = dyn_cast<ReturnInst>(Val: Usr)) {
1637 if (RI->getFunction() == getAssociatedFunction()) {
1638 auto &PtrOI = OffsetInfoMap[CurPtr];
1639 Changed |= setReachesReturn(PtrOI);
1640 return true;
1641 }
1642 return false;
1643 }
1644
1645 // For PHIs we need to take care of the recurrence explicitly as the value
1646 // might change while we iterate through a loop. For now, we give up if
1647 // the PHI is not invariant.
1648 if (auto *PHI = dyn_cast<PHINode>(Val: Usr)) {
1649 // Note the order here, the Usr access might change the map, CurPtr is
1650 // already in it though.
1651 auto [PhiIt, IsFirstPHIUser] = OffsetInfoMap.try_emplace(Key: PHI);
1652 auto &UsrOI = PhiIt->second;
1653 auto &PtrOI = OffsetInfoMap[CurPtr];
1654
1655 // Check if the PHI operand has already an unknown offset as we can't
1656 // improve on that anymore.
1657 if (PtrOI.isUnknown()) {
1658 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand offset unknown "
1659 << *CurPtr << " in " << *PHI << "\n");
1660 Follow = !UsrOI.isUnknown();
1661 UsrOI.setUnknown();
1662 return true;
1663 }
1664
1665 // Check if the PHI is invariant (so far).
1666 if (UsrOI == PtrOI) {
1667 assert(!PtrOI.isUnassigned() &&
1668 "Cannot assign if the current Ptr was not visited!");
1669 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant (so far)");
1670 return true;
1671 }
1672
1673 // Check if the PHI operand can be traced back to AssociatedValue.
1674 APInt Offset(
1675 DL.getIndexSizeInBits(AS: CurPtr->getType()->getPointerAddressSpace()),
1676 0);
1677 Value *CurPtrBase = CurPtr->stripAndAccumulateConstantOffsets(
1678 DL, Offset, /* AllowNonInbounds */ true);
1679 auto It = OffsetInfoMap.find(Val: CurPtrBase);
1680 if (It == OffsetInfoMap.end()) {
1681 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand is too complex "
1682 << *CurPtr << " in " << *PHI
1683 << " (base: " << *CurPtrBase << ")\n");
1684 UsrOI.setUnknown();
1685 Follow = true;
1686 return true;
1687 }
1688
1689 // Check if the PHI operand is not dependent on the PHI itself. Every
1690 // recurrence is a cyclic net of PHIs in the data flow, and has an
1691 // equivalent Cycle in the control flow. One of those PHIs must be in the
1692 // header of that control flow Cycle. This is independent of the choice of
1693 // Cycles reported by CycleInfo. It is sufficient to check the PHIs in
1694 // every Cycle header; if such a node is marked unknown, this will
1695 // eventually propagate through the whole net of PHIs in the recurrence.
1696 const auto *CI =
1697 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
1698 F: *PHI->getFunction());
1699 if (mayBeInCycle(CI, I: cast<Instruction>(Val: Usr), /* HeaderOnly */ true)) {
1700 auto BaseOI = It->getSecond();
1701 BaseOI.addToAll(Inc: Offset.getZExtValue());
1702 if (IsFirstPHIUser || BaseOI == UsrOI) {
1703 LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant " << *CurPtr
1704 << " in " << *Usr << "\n");
1705 return HandlePassthroughUser(Usr, CurPtr, Follow);
1706 }
1707
1708 LLVM_DEBUG(
1709 dbgs() << "[AAPointerInfo] PHI operand pointer offset mismatch "
1710 << *CurPtr << " in " << *PHI << "\n");
1711 UsrOI.setUnknown();
1712 Follow = true;
1713 return true;
1714 }
1715
1716 UsrOI.merge(R: PtrOI);
1717 Follow = true;
1718 return true;
1719 }
1720
1721 if (auto *LoadI = dyn_cast<LoadInst>(Val: Usr)) {
1722 // If the access is to a pointer that may or may not be the associated
1723 // value, e.g. due to a PHI, we cannot assume it will be read.
1724 AccessKind AK = AccessKind::AK_R;
1725 if (getUnderlyingObject(V: CurPtr) == &AssociatedValue)
1726 AK = AccessKind(AK | AccessKind::AK_MUST);
1727 else
1728 AK = AccessKind(AK | AccessKind::AK_MAY);
1729 if (!handleAccess(A, I&: *LoadI, /* Content */ nullptr, Kind: AK,
1730 Offsets&: OffsetInfoMap[CurPtr].Offsets, Changed,
1731 Ty&: *LoadI->getType()))
1732 return false;
1733
1734 auto IsAssumption = [](Instruction &I) {
1735 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
1736 return II->isAssumeLikeIntrinsic();
1737 return false;
1738 };
1739
1740 auto IsImpactedInRange = [&](Instruction *FromI, Instruction *ToI) {
1741 // Check if the assumption and the load are executed together without
1742 // memory modification.
1743 do {
1744 if (FromI->mayWriteToMemory() && !IsAssumption(*FromI))
1745 return true;
1746 FromI = FromI->getNextNode();
1747 } while (FromI && FromI != ToI);
1748 return false;
1749 };
1750
1751 BasicBlock *BB = LoadI->getParent();
1752 auto IsValidAssume = [&](IntrinsicInst &IntrI) {
1753 if (IntrI.getIntrinsicID() != Intrinsic::assume)
1754 return false;
1755 BasicBlock *IntrBB = IntrI.getParent();
1756 if (IntrI.getParent() == BB) {
1757 if (IsImpactedInRange(LoadI->getNextNode(), &IntrI))
1758 return false;
1759 } else {
1760 auto PredIt = pred_begin(BB: IntrBB);
1761 if (PredIt == pred_end(BB: IntrBB))
1762 return false;
1763 if ((*PredIt) != BB)
1764 return false;
1765 if (++PredIt != pred_end(BB: IntrBB))
1766 return false;
1767 for (auto *SuccBB : successors(BB)) {
1768 if (SuccBB == IntrBB)
1769 continue;
1770 if (isa<UnreachableInst>(Val: SuccBB->getTerminator()))
1771 continue;
1772 return false;
1773 }
1774 if (IsImpactedInRange(LoadI->getNextNode(), BB->getTerminator()))
1775 return false;
1776 if (IsImpactedInRange(&IntrBB->front(), &IntrI))
1777 return false;
1778 }
1779 return true;
1780 };
1781
1782 std::pair<Value *, IntrinsicInst *> Assumption;
1783 for (const Use &LoadU : LoadI->uses()) {
1784 if (auto *CmpI = dyn_cast<CmpInst>(Val: LoadU.getUser())) {
1785 if (!CmpI->isEquality() || !CmpI->isTrueWhenEqual())
1786 continue;
1787 for (const Use &CmpU : CmpI->uses()) {
1788 if (auto *IntrI = dyn_cast<IntrinsicInst>(Val: CmpU.getUser())) {
1789 if (!IsValidAssume(*IntrI))
1790 continue;
1791 int Idx = CmpI->getOperandUse(i: 0) == LoadU;
1792 Assumption = {CmpI->getOperand(i_nocapture: Idx), IntrI};
1793 break;
1794 }
1795 }
1796 }
1797 if (Assumption.first)
1798 break;
1799 }
1800
1801 // Check if we found an assumption associated with this load.
1802 if (!Assumption.first || !Assumption.second)
1803 return true;
1804
1805 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Assumption found "
1806 << *Assumption.second << ": " << *LoadI
1807 << " == " << *Assumption.first << "\n");
1808 bool UsedAssumedInformation = false;
1809 std::optional<Value *> Content = nullptr;
1810 if (Assumption.first)
1811 Content =
1812 A.getAssumedSimplified(V: *Assumption.first, AA: *this,
1813 UsedAssumedInformation, S: AA::Interprocedural);
1814 return handleAccess(
1815 A, I&: *Assumption.second, Content, Kind: AccessKind::AK_ASSUMPTION,
1816 Offsets&: OffsetInfoMap[CurPtr].Offsets, Changed, Ty&: *LoadI->getType());
1817 }
1818
1819 auto HandleStoreLike = [&](Instruction &I, Value *ValueOp, Type &ValueTy,
1820 ArrayRef<Value *> OtherOps, AccessKind AK) {
1821 for (auto *OtherOp : OtherOps) {
1822 if (OtherOp == CurPtr) {
1823 LLVM_DEBUG(
1824 dbgs()
1825 << "[AAPointerInfo] Escaping use in store like instruction " << I
1826 << "\n");
1827 return false;
1828 }
1829 }
1830
1831 // If the access is to a pointer that may or may not be the associated
1832 // value, e.g. due to a PHI, we cannot assume it will be written.
1833 if (getUnderlyingObject(V: CurPtr) == &AssociatedValue)
1834 AK = AccessKind(AK | AccessKind::AK_MUST);
1835 else
1836 AK = AccessKind(AK | AccessKind::AK_MAY);
1837 bool UsedAssumedInformation = false;
1838 std::optional<Value *> Content = nullptr;
1839 if (ValueOp)
1840 Content = A.getAssumedSimplified(
1841 V: *ValueOp, AA: *this, UsedAssumedInformation, S: AA::Interprocedural);
1842 return handleAccess(A, I, Content, Kind: AK, Offsets&: OffsetInfoMap[CurPtr].Offsets,
1843 Changed, Ty&: ValueTy);
1844 };
1845
1846 if (auto *StoreI = dyn_cast<StoreInst>(Val: Usr))
1847 return HandleStoreLike(*StoreI, StoreI->getValueOperand(),
1848 *StoreI->getValueOperand()->getType(),
1849 {StoreI->getValueOperand()}, AccessKind::AK_W);
1850 if (auto *RMWI = dyn_cast<AtomicRMWInst>(Val: Usr))
1851 return HandleStoreLike(*RMWI, nullptr, *RMWI->getValOperand()->getType(),
1852 {RMWI->getValOperand()}, AccessKind::AK_RW);
1853 if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Val: Usr))
1854 return HandleStoreLike(
1855 *CXI, nullptr, *CXI->getNewValOperand()->getType(),
1856 {CXI->getCompareOperand(), CXI->getNewValOperand()},
1857 AccessKind::AK_RW);
1858
1859 if (auto *CB = dyn_cast<CallBase>(Val: Usr)) {
1860 if (CB->isLifetimeStartOrEnd())
1861 return true;
1862 const auto *TLI =
1863 A.getInfoCache().getTargetLibraryInfoForFunction(F: *CB->getFunction());
1864 if (getFreedOperand(CB, TLI) == U)
1865 return true;
1866 if (CB->isArgOperand(U: &U)) {
1867 unsigned ArgNo = CB->getArgOperandNo(U: &U);
1868 const auto *CSArgPI = A.getAAFor<AAPointerInfo>(
1869 QueryingAA: *this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
1870 DepClass: DepClassTy::REQUIRED);
1871 if (!CSArgPI)
1872 return false;
1873 bool IsArgMustAcc = (getUnderlyingObject(V: CurPtr) == &AssociatedValue);
1874 Changed = translateAndAddState(A, OtherAA: *CSArgPI, Offsets: OffsetInfoMap[CurPtr], CB&: *CB,
1875 IsMustAcc: IsArgMustAcc) |
1876 Changed;
1877 if (!CSArgPI->reachesReturn())
1878 return isValidState();
1879
1880 Function *Callee = CB->getCalledFunction();
1881 if (!Callee || Callee->arg_size() <= ArgNo)
1882 return false;
1883 bool UsedAssumedInformation = false;
1884 auto ReturnedValue = A.getAssumedSimplified(
1885 IRP: IRPosition::returned(F: *Callee), AA: *this, UsedAssumedInformation,
1886 S: AA::ValueScope::Intraprocedural);
1887 auto *ReturnedArg =
1888 dyn_cast_or_null<Argument>(Val: ReturnedValue.value_or(u: nullptr));
1889 auto *Arg = Callee->getArg(i: ArgNo);
1890 if (ReturnedArg && Arg != ReturnedArg)
1891 return true;
1892 bool IsRetMustAcc = IsArgMustAcc && (ReturnedArg == Arg);
1893 const auto *CSRetPI = A.getAAFor<AAPointerInfo>(
1894 QueryingAA: *this, IRP: IRPosition::callsite_returned(CB: *CB), DepClass: DepClassTy::REQUIRED);
1895 if (!CSRetPI)
1896 return false;
1897 OffsetInfo OI = OffsetInfoMap[CurPtr];
1898 CSArgPI->addReturnedOffsetsTo(OI);
1899 Changed =
1900 translateAndAddState(A, OtherAA: *CSRetPI, Offsets: OI, CB&: *CB, IsMustAcc: IsRetMustAcc) | Changed;
1901 return isValidState();
1902 }
1903 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Call user not handled " << *CB
1904 << "\n");
1905 return false;
1906 }
1907
1908 LLVM_DEBUG(dbgs() << "[AAPointerInfo] User not handled " << *Usr << "\n");
1909 return false;
1910 };
1911 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
1912 assert(OffsetInfoMap.count(OldU) && "Old use should be known already!");
1913 assert(!OffsetInfoMap[OldU].isUnassigned() && "Old use should be assinged");
1914 if (OffsetInfoMap.count(Val: NewU)) {
1915 LLVM_DEBUG({
1916 if (!(OffsetInfoMap[NewU] == OffsetInfoMap[OldU])) {
1917 dbgs() << "[AAPointerInfo] Equivalent use callback failed: "
1918 << OffsetInfoMap[NewU] << " vs " << OffsetInfoMap[OldU]
1919 << "\n";
1920 }
1921 });
1922 return OffsetInfoMap[NewU] == OffsetInfoMap[OldU];
1923 }
1924 bool Unused;
1925 return HandlePassthroughUser(NewU.get(), OldU.get(), Unused);
1926 };
1927 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: AssociatedValue,
1928 /* CheckBBLivenessOnly */ true, LivenessDepClass: DepClassTy::OPTIONAL,
1929 /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
1930 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Check for all uses failed, abort!\n");
1931 return indicatePessimisticFixpoint();
1932 }
1933
1934 LLVM_DEBUG({
1935 dbgs() << "Accesses by bin after update:\n";
1936 dumpState(dbgs());
1937 });
1938
1939 return Changed;
1940}
1941
1942struct AAPointerInfoReturned final : AAPointerInfoImpl {
1943 AAPointerInfoReturned(const IRPosition &IRP, Attributor &A)
1944 : AAPointerInfoImpl(IRP, A) {}
1945
1946 /// See AbstractAttribute::updateImpl(...).
1947 ChangeStatus updateImpl(Attributor &A) override {
1948 return indicatePessimisticFixpoint();
1949 }
1950
1951 /// See AbstractAttribute::trackStatistics()
1952 void trackStatistics() const override {
1953 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1954 }
1955};
1956
1957struct AAPointerInfoArgument final : AAPointerInfoFloating {
1958 AAPointerInfoArgument(const IRPosition &IRP, Attributor &A)
1959 : AAPointerInfoFloating(IRP, A) {}
1960
1961 /// See AbstractAttribute::trackStatistics()
1962 void trackStatistics() const override {
1963 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
1964 }
1965};
1966
1967struct AAPointerInfoCallSiteArgument final : AAPointerInfoFloating {
1968 AAPointerInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
1969 : AAPointerInfoFloating(IRP, A) {}
1970
1971 /// See AbstractAttribute::updateImpl(...).
1972 ChangeStatus updateImpl(Attributor &A) override {
1973 using namespace AA::PointerInfo;
1974 // We handle memory intrinsics explicitly, at least the first (=
1975 // destination) and second (=source) arguments as we know how they are
1976 // accessed.
1977 if (auto *MI = dyn_cast_or_null<MemIntrinsic>(Val: getCtxI())) {
1978 int64_t LengthVal = AA::RangeTy::Unknown;
1979 if (auto Length = MI->getLengthInBytes())
1980 LengthVal = Length->getSExtValue();
1981 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
1982 ChangeStatus Changed = ChangeStatus::UNCHANGED;
1983 if (ArgNo > 1) {
1984 LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled memory intrinsic "
1985 << *MI << "\n");
1986 return indicatePessimisticFixpoint();
1987 } else {
1988 auto Kind =
1989 ArgNo == 0 ? AccessKind::AK_MUST_WRITE : AccessKind::AK_MUST_READ;
1990 Changed =
1991 Changed | addAccess(A, Ranges: {0, LengthVal}, I&: *MI, Content: nullptr, Kind, Ty: nullptr);
1992 }
1993 LLVM_DEBUG({
1994 dbgs() << "Accesses by bin after update:\n";
1995 dumpState(dbgs());
1996 });
1997
1998 return Changed;
1999 }
2000
2001 // TODO: Once we have call site specific value information we can provide
2002 // call site specific liveness information and then it makes
2003 // sense to specialize attributes for call sites arguments instead of
2004 // redirecting requests to the callee argument.
2005 Argument *Arg = getAssociatedArgument();
2006 if (Arg) {
2007 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
2008 auto *ArgAA =
2009 A.getAAFor<AAPointerInfo>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
2010 if (ArgAA && ArgAA->getState().isValidState())
2011 return translateAndAddStateFromCallee(A, OtherAA: *ArgAA,
2012 CB&: *cast<CallBase>(Val: getCtxI()));
2013 if (!Arg->getParent()->isDeclaration())
2014 return indicatePessimisticFixpoint();
2015 }
2016
2017 bool IsKnownNoCapture;
2018 if (!AA::hasAssumedIRAttr<Attribute::Captures>(
2019 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture))
2020 return indicatePessimisticFixpoint();
2021
2022 bool IsKnown = false;
2023 if (AA::isAssumedReadNone(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
2024 return ChangeStatus::UNCHANGED;
2025 bool ReadOnly = AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown);
2026 auto Kind =
2027 ReadOnly ? AccessKind::AK_MAY_READ : AccessKind::AK_MAY_READ_WRITE;
2028 return addAccess(A, Ranges: AA::RangeTy::getUnknown(), I&: *getCtxI(), Content: nullptr, Kind,
2029 Ty: nullptr);
2030 }
2031
2032 /// See AbstractAttribute::trackStatistics()
2033 void trackStatistics() const override {
2034 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
2035 }
2036};
2037
2038struct AAPointerInfoCallSiteReturned final : AAPointerInfoFloating {
2039 AAPointerInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
2040 : AAPointerInfoFloating(IRP, A) {}
2041
2042 /// See AbstractAttribute::trackStatistics()
2043 void trackStatistics() const override {
2044 AAPointerInfoImpl::trackPointerInfoStatistics(IRP: getIRPosition());
2045 }
2046};
2047} // namespace
2048
2049/// -----------------------NoUnwind Function Attribute--------------------------
2050
2051namespace {
2052struct AANoUnwindImpl : AANoUnwind {
2053 AANoUnwindImpl(const IRPosition &IRP, Attributor &A) : AANoUnwind(IRP, A) {}
2054
2055 /// See AbstractAttribute::initialize(...).
2056 void initialize(Attributor &A) override {
2057 bool IsKnown;
2058 assert(!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
2059 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2060 (void)IsKnown;
2061 }
2062
2063 const std::string getAsStr(Attributor *A) const override {
2064 return getAssumed() ? "nounwind" : "may-unwind";
2065 }
2066
2067 /// See AbstractAttribute::updateImpl(...).
2068 ChangeStatus updateImpl(Attributor &A) override {
2069 auto Opcodes = {
2070 (unsigned)Instruction::Invoke, (unsigned)Instruction::CallBr,
2071 (unsigned)Instruction::Call, (unsigned)Instruction::CleanupRet,
2072 (unsigned)Instruction::CatchSwitch, (unsigned)Instruction::Resume};
2073
2074 auto CheckForNoUnwind = [&](Instruction &I) {
2075 if (!I.mayThrow(/* IncludePhaseOneUnwind */ true))
2076 return true;
2077
2078 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
2079 bool IsKnownNoUnwind;
2080 return AA::hasAssumedIRAttr<Attribute::NoUnwind>(
2081 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::REQUIRED,
2082 IsKnown&: IsKnownNoUnwind);
2083 }
2084 return false;
2085 };
2086
2087 bool UsedAssumedInformation = false;
2088 if (!A.checkForAllInstructions(Pred: CheckForNoUnwind, QueryingAA: *this, Opcodes,
2089 UsedAssumedInformation))
2090 return indicatePessimisticFixpoint();
2091
2092 return ChangeStatus::UNCHANGED;
2093 }
2094};
2095
2096struct AANoUnwindFunction final : public AANoUnwindImpl {
2097 AANoUnwindFunction(const IRPosition &IRP, Attributor &A)
2098 : AANoUnwindImpl(IRP, A) {}
2099
2100 /// See AbstractAttribute::trackStatistics()
2101 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nounwind) }
2102};
2103
2104/// NoUnwind attribute deduction for a call sites.
2105struct AANoUnwindCallSite final
2106 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl> {
2107 AANoUnwindCallSite(const IRPosition &IRP, Attributor &A)
2108 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl>(IRP, A) {}
2109
2110 /// See AbstractAttribute::trackStatistics()
2111 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nounwind); }
2112};
2113} // namespace
2114
2115/// ------------------------ NoSync Function Attribute -------------------------
2116
2117bool AANoSync::isAlignedBarrier(const CallBase &CB, bool ExecutedAligned) {
2118 switch (CB.getIntrinsicID()) {
2119 case Intrinsic::nvvm_barrier_cta_sync_aligned_all:
2120 case Intrinsic::nvvm_barrier_cta_sync_aligned_count:
2121 case Intrinsic::nvvm_barrier_cta_red_and_aligned_all:
2122 case Intrinsic::nvvm_barrier_cta_red_and_aligned_count:
2123 case Intrinsic::nvvm_barrier_cta_red_or_aligned_all:
2124 case Intrinsic::nvvm_barrier_cta_red_or_aligned_count:
2125 case Intrinsic::nvvm_barrier_cta_red_popc_aligned_all:
2126 case Intrinsic::nvvm_barrier_cta_red_popc_aligned_count:
2127 return true;
2128 case Intrinsic::amdgcn_s_barrier:
2129 if (ExecutedAligned)
2130 return true;
2131 break;
2132 default:
2133 break;
2134 }
2135 return hasAssumption(CB, AssumptionStr: KnownAssumptionString("ompx_aligned_barrier"));
2136}
2137
2138bool AANoSync::isNonRelaxedAtomic(const Instruction *I) {
2139 if (!I->isAtomic())
2140 return false;
2141
2142 if (auto *FI = dyn_cast<FenceInst>(Val: I))
2143 // All legal orderings for fence are stronger than monotonic.
2144 return FI->getSyncScopeID() != SyncScope::SingleThread;
2145 if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
2146 // Unordered is not a legal ordering for cmpxchg.
2147 return (AI->getSuccessOrdering() != AtomicOrdering::Monotonic ||
2148 AI->getFailureOrdering() != AtomicOrdering::Monotonic);
2149 }
2150
2151 AtomicOrdering Ordering;
2152 switch (I->getOpcode()) {
2153 case Instruction::AtomicRMW:
2154 Ordering = cast<AtomicRMWInst>(Val: I)->getOrdering();
2155 break;
2156 case Instruction::Store:
2157 Ordering = cast<StoreInst>(Val: I)->getOrdering();
2158 break;
2159 case Instruction::Load:
2160 Ordering = cast<LoadInst>(Val: I)->getOrdering();
2161 break;
2162 default:
2163 llvm_unreachable(
2164 "New atomic operations need to be known in the attributor.");
2165 }
2166
2167 return (Ordering != AtomicOrdering::Unordered &&
2168 Ordering != AtomicOrdering::Monotonic);
2169}
2170
2171namespace {
2172struct AANoSyncImpl : AANoSync {
2173 AANoSyncImpl(const IRPosition &IRP, Attributor &A) : AANoSync(IRP, A) {}
2174
2175 /// See AbstractAttribute::initialize(...).
2176 void initialize(Attributor &A) override {
2177 bool IsKnown;
2178 assert(!AA::hasAssumedIRAttr<Attribute::NoSync>(A, nullptr, getIRPosition(),
2179 DepClassTy::NONE, IsKnown));
2180 (void)IsKnown;
2181 }
2182
2183 const std::string getAsStr(Attributor *A) const override {
2184 return getAssumed() ? "nosync" : "may-sync";
2185 }
2186
2187 /// See AbstractAttribute::updateImpl(...).
2188 ChangeStatus updateImpl(Attributor &A) override;
2189};
2190
2191ChangeStatus AANoSyncImpl::updateImpl(Attributor &A) {
2192
2193 auto CheckRWInstForNoSync = [&](Instruction &I) {
2194 return AA::isNoSyncInst(A, I, QueryingAA: *this);
2195 };
2196
2197 auto CheckForNoSync = [&](Instruction &I) {
2198 // At this point we handled all read/write effects and they are all
2199 // nosync, so they can be skipped.
2200 if (I.mayReadOrWriteMemory())
2201 return true;
2202
2203 bool IsKnown;
2204 CallBase &CB = cast<CallBase>(Val&: I);
2205 if (AA::hasAssumedIRAttr<Attribute::NoSync>(
2206 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::OPTIONAL,
2207 IsKnown))
2208 return true;
2209
2210 // non-convergent and readnone imply nosync.
2211 return !CB.isConvergent();
2212 };
2213
2214 bool UsedAssumedInformation = false;
2215 if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInstForNoSync, QueryingAA&: *this,
2216 UsedAssumedInformation) ||
2217 !A.checkForAllCallLikeInstructions(Pred: CheckForNoSync, QueryingAA: *this,
2218 UsedAssumedInformation))
2219 return indicatePessimisticFixpoint();
2220
2221 return ChangeStatus::UNCHANGED;
2222}
2223
2224struct AANoSyncFunction final : public AANoSyncImpl {
2225 AANoSyncFunction(const IRPosition &IRP, Attributor &A)
2226 : AANoSyncImpl(IRP, A) {}
2227
2228 /// See AbstractAttribute::trackStatistics()
2229 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nosync) }
2230};
2231
2232/// NoSync attribute deduction for a call sites.
2233struct AANoSyncCallSite final : AACalleeToCallSite<AANoSync, AANoSyncImpl> {
2234 AANoSyncCallSite(const IRPosition &IRP, Attributor &A)
2235 : AACalleeToCallSite<AANoSync, AANoSyncImpl>(IRP, A) {}
2236
2237 /// See AbstractAttribute::trackStatistics()
2238 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nosync); }
2239};
2240} // namespace
2241
2242/// ------------------------ No-Free Attributes ----------------------------
2243
2244namespace {
2245struct AANoFreeImpl : public AANoFree {
2246 AANoFreeImpl(const IRPosition &IRP, Attributor &A) : AANoFree(IRP, A) {}
2247
2248 /// See AbstractAttribute::initialize(...).
2249 void initialize(Attributor &A) override {
2250 bool IsKnown;
2251 assert(!AA::hasAssumedIRAttr<Attribute::NoFree>(A, nullptr, getIRPosition(),
2252 DepClassTy::NONE, IsKnown));
2253 (void)IsKnown;
2254 }
2255
2256 /// See AbstractAttribute::updateImpl(...).
2257 ChangeStatus updateImpl(Attributor &A) override {
2258 auto CheckForNoFree = [&](Instruction &I) {
2259 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
2260 bool IsKnown;
2261 return AA::hasAssumedIRAttr<Attribute::NoFree>(
2262 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::REQUIRED,
2263 IsKnown);
2264 }
2265 // Make sure that synchronization cannot establish happens-before with a
2266 // free on another thread.
2267 return AA::isNoSyncInst(A, I, QueryingAA: *this);
2268 };
2269
2270 bool UsedAssumedInformation = false;
2271 if (!A.checkForAllReadWriteInstructions(Pred: CheckForNoFree, QueryingAA&: *this,
2272 UsedAssumedInformation) ||
2273 !A.checkForAllCallLikeInstructions(Pred: CheckForNoFree, QueryingAA: *this,
2274 UsedAssumedInformation))
2275 return indicatePessimisticFixpoint();
2276
2277 return ChangeStatus::UNCHANGED;
2278 }
2279
2280 /// See AbstractAttribute::getAsStr().
2281 const std::string getAsStr(Attributor *A) const override {
2282 return getAssumed() ? "nofree" : "may-free";
2283 }
2284};
2285
2286struct AANoFreeFunction final : public AANoFreeImpl {
2287 AANoFreeFunction(const IRPosition &IRP, Attributor &A)
2288 : AANoFreeImpl(IRP, A) {}
2289
2290 /// See AbstractAttribute::trackStatistics()
2291 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nofree) }
2292};
2293
2294/// NoFree attribute deduction for a call sites.
2295struct AANoFreeCallSite final : AACalleeToCallSite<AANoFree, AANoFreeImpl> {
2296 AANoFreeCallSite(const IRPosition &IRP, Attributor &A)
2297 : AACalleeToCallSite<AANoFree, AANoFreeImpl>(IRP, A) {}
2298
2299 /// See AbstractAttribute::trackStatistics()
2300 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nofree); }
2301};
2302
2303/// NoFree attribute for floating values.
2304struct AANoFreeFloating : AANoFreeImpl {
2305 AANoFreeFloating(const IRPosition &IRP, Attributor &A)
2306 : AANoFreeImpl(IRP, A) {}
2307
2308 /// See AbstractAttribute::trackStatistics()
2309 void trackStatistics() const override{STATS_DECLTRACK_FLOATING_ATTR(nofree)}
2310
2311 /// See Abstract Attribute::updateImpl(...).
2312 ChangeStatus updateImpl(Attributor &A) override {
2313 const IRPosition &IRP = getIRPosition();
2314
2315 bool IsKnown;
2316 if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, QueryingAA: this,
2317 IRP: IRPosition::function_scope(IRP),
2318 DepClass: DepClassTy::OPTIONAL, IsKnown))
2319 return ChangeStatus::UNCHANGED;
2320
2321 Value &AssociatedValue = getIRPosition().getAssociatedValue();
2322 auto Pred = [&](const Use &U, bool &Follow) -> bool {
2323 Instruction *UserI = cast<Instruction>(Val: U.getUser());
2324 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
2325 if (CB->isBundleOperand(U: &U))
2326 return false;
2327 if (!CB->isArgOperand(U: &U))
2328 return true;
2329 unsigned ArgNo = CB->getArgOperandNo(U: &U);
2330
2331 // Even if the argument is nofree, we still need to check for nocapture,
2332 // as the call may capture the argument without freeing it, and the
2333 // captured argument is freed later.
2334 bool IsKnown;
2335 if (!AA::hasAssumedIRAttr<Attribute::NoFree>(
2336 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
2337 DepClass: DepClassTy::REQUIRED, IsKnown))
2338 return false;
2339
2340 const AANoCapture *NoCaptureAA = nullptr;
2341 if (!AA::hasAssumedIRAttr<Attribute::Captures>(
2342 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
2343 DepClass: DepClassTy::REQUIRED, IsKnown,
2344 /*IgnoreSubsumingPositions=*/false, AAPtr: &NoCaptureAA)) {
2345 if (NoCaptureAA && NoCaptureAA->isAssumedNoCaptureMaybeReturned()) {
2346 Follow = true;
2347 return true;
2348 }
2349 return false;
2350 }
2351
2352 return true;
2353 }
2354
2355 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
2356 if (!capturesAnyProvenance(CC: CI))
2357 return true;
2358 if (capturesAnyProvenance(CC: CI.ResultCC)) {
2359 Follow = true;
2360 return true;
2361 }
2362
2363 if (isa<ReturnInst>(Val: UserI) && getIRPosition().isArgumentPosition())
2364 return true;
2365
2366 // Capturing user.
2367 return false;
2368 };
2369 if (!A.checkForAllUses(Pred, QueryingAA: *this, V: AssociatedValue))
2370 return indicatePessimisticFixpoint();
2371
2372 return ChangeStatus::UNCHANGED;
2373 }
2374};
2375
2376/// NoFree attribute for a call site argument.
2377struct AANoFreeArgument final : AANoFreeFloating {
2378 AANoFreeArgument(const IRPosition &IRP, Attributor &A)
2379 : AANoFreeFloating(IRP, A) {}
2380
2381 /// See AbstractAttribute::trackStatistics()
2382 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofree) }
2383};
2384
2385/// NoFree attribute for call site arguments.
2386struct AANoFreeCallSiteArgument final : AANoFreeFloating {
2387 AANoFreeCallSiteArgument(const IRPosition &IRP, Attributor &A)
2388 : AANoFreeFloating(IRP, A) {}
2389
2390 /// See AbstractAttribute::updateImpl(...).
2391 ChangeStatus updateImpl(Attributor &A) override {
2392 // TODO: Once we have call site specific value information we can provide
2393 // call site specific liveness information and then it makes
2394 // sense to specialize attributes for call sites arguments instead of
2395 // redirecting requests to the callee argument.
2396 Argument *Arg = getAssociatedArgument();
2397 if (!Arg)
2398 return indicatePessimisticFixpoint();
2399 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
2400 bool IsKnown;
2401 if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, QueryingAA: this, IRP: ArgPos,
2402 DepClass: DepClassTy::REQUIRED, IsKnown))
2403 return ChangeStatus::UNCHANGED;
2404 return indicatePessimisticFixpoint();
2405 }
2406
2407 /// See AbstractAttribute::trackStatistics()
2408 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nofree) };
2409};
2410
2411/// NoFree attribute for function return value.
2412struct AANoFreeReturned final : AANoFreeFloating {
2413 AANoFreeReturned(const IRPosition &IRP, Attributor &A)
2414 : AANoFreeFloating(IRP, A) {
2415 llvm_unreachable("NoFree is not applicable to function returns!");
2416 }
2417
2418 /// See AbstractAttribute::initialize(...).
2419 void initialize(Attributor &A) override {
2420 llvm_unreachable("NoFree is not applicable to function returns!");
2421 }
2422
2423 /// See AbstractAttribute::updateImpl(...).
2424 ChangeStatus updateImpl(Attributor &A) override {
2425 llvm_unreachable("NoFree is not applicable to function returns!");
2426 }
2427
2428 /// See AbstractAttribute::trackStatistics()
2429 void trackStatistics() const override {}
2430};
2431
2432/// NoFree attribute deduction for a call site return value.
2433struct AANoFreeCallSiteReturned final : AANoFreeFloating {
2434 AANoFreeCallSiteReturned(const IRPosition &IRP, Attributor &A)
2435 : AANoFreeFloating(IRP, A) {}
2436
2437 ChangeStatus manifest(Attributor &A) override {
2438 return ChangeStatus::UNCHANGED;
2439 }
2440 /// See AbstractAttribute::trackStatistics()
2441 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nofree) }
2442};
2443} // namespace
2444
2445/// ------------------------ NonNull Argument Attribute ------------------------
2446
2447bool AANonNull::isImpliedByIR(Attributor &A, const IRPosition &IRP,
2448 Attribute::AttrKind ImpliedAttributeKind,
2449 bool IgnoreSubsumingPositions) {
2450 SmallVector<Attribute::AttrKind, 2> AttrKinds;
2451 AttrKinds.push_back(Elt: Attribute::NonNull);
2452 if (!NullPointerIsDefined(F: IRP.getAnchorScope(),
2453 AS: IRP.getAssociatedType()->getPointerAddressSpace()))
2454 AttrKinds.push_back(Elt: Attribute::Dereferenceable);
2455 if (A.hasAttr(IRP, AKs: AttrKinds, IgnoreSubsumingPositions, ImpliedAttributeKind: Attribute::NonNull))
2456 return true;
2457
2458 DominatorTree *DT = nullptr;
2459 AssumptionCache *AC = nullptr;
2460 InformationCache &InfoCache = A.getInfoCache();
2461 if (const Function *Fn = IRP.getAnchorScope()) {
2462 if (!Fn->isDeclaration()) {
2463 DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *Fn);
2464 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *Fn);
2465 }
2466 }
2467
2468 SmallVector<AA::ValueAndContext> Worklist;
2469 if (IRP.getPositionKind() != IRP_RETURNED) {
2470 Worklist.push_back(Elt: {IRP.getAssociatedValue(), IRP.getCtxI()});
2471 } else {
2472 bool UsedAssumedInformation = false;
2473 if (!A.checkForAllInstructions(
2474 Pred: [&](Instruction &I) {
2475 Worklist.push_back(Elt: {*cast<ReturnInst>(Val&: I).getReturnValue(), &I});
2476 return true;
2477 },
2478 Fn: IRP.getAssociatedFunction(), QueryingAA: nullptr, Opcodes: {Instruction::Ret},
2479 UsedAssumedInformation, CheckBBLivenessOnly: false, /*CheckPotentiallyDead=*/true))
2480 return false;
2481 }
2482
2483 if (llvm::any_of(Range&: Worklist, P: [&](AA::ValueAndContext VAC) {
2484 return !isKnownNonZero(
2485 V: VAC.getValue(),
2486 Q: SimplifyQuery(A.getDataLayout(), DT, AC, VAC.getCtxI()));
2487 }))
2488 return false;
2489
2490 A.manifestAttrs(IRP, DeducedAttrs: {Attribute::get(Context&: IRP.getAnchorValue().getContext(),
2491 Kind: Attribute::NonNull)});
2492 return true;
2493}
2494
2495namespace {
2496static int64_t getKnownNonNullAndDerefBytesForUse(
2497 Attributor &A, const AbstractAttribute &QueryingAA, Value &AssociatedValue,
2498 const Use *U, const Instruction *I, bool &IsNonNull, bool &TrackUse) {
2499 TrackUse = false;
2500
2501 const Value *UseV = U->get();
2502 if (!UseV->getType()->isPointerTy())
2503 return 0;
2504
2505 // We need to follow common pointer manipulation uses to the accesses they
2506 // feed into. We can try to be smart to avoid looking through things we do not
2507 // like for now, e.g., non-inbounds GEPs.
2508 if (isa<CastInst>(Val: I)) {
2509 TrackUse = true;
2510 return 0;
2511 }
2512
2513 if (isa<GetElementPtrInst>(Val: I)) {
2514 TrackUse = true;
2515 return 0;
2516 }
2517
2518 Type *PtrTy = UseV->getType();
2519 const Function *F = I->getFunction();
2520 bool NullPointerIsDefined =
2521 F ? llvm::NullPointerIsDefined(F, AS: PtrTy->getPointerAddressSpace()) : true;
2522 const DataLayout &DL = A.getInfoCache().getDL();
2523 if (const auto *CB = dyn_cast<CallBase>(Val: I)) {
2524 if (CB->isBundleOperand(U)) {
2525 if (RetainedKnowledge RK = getKnowledgeFromUse(
2526 U, AttrKinds: {Attribute::NonNull, Attribute::Dereferenceable})) {
2527 IsNonNull |=
2528 (RK.AttrKind == Attribute::NonNull || !NullPointerIsDefined);
2529 return RK.ArgValue;
2530 }
2531 return 0;
2532 }
2533
2534 if (CB->isCallee(U)) {
2535 IsNonNull |= !NullPointerIsDefined;
2536 return 0;
2537 }
2538
2539 unsigned ArgNo = CB->getArgOperandNo(U);
2540 IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
2541 // As long as we only use known information there is no need to track
2542 // dependences here.
2543 bool IsKnownNonNull;
2544 AA::hasAssumedIRAttr<Attribute::NonNull>(A, QueryingAA: &QueryingAA, IRP,
2545 DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
2546 IsNonNull |= IsKnownNonNull;
2547 auto *DerefAA =
2548 A.getAAFor<AADereferenceable>(QueryingAA, IRP, DepClass: DepClassTy::NONE);
2549 return DerefAA ? DerefAA->getKnownDereferenceableBytes() : 0;
2550 }
2551
2552 std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(Inst: I);
2553 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() ||
2554 Loc->Size.isScalable() || I->isVolatile())
2555 return 0;
2556
2557 int64_t Offset;
2558 const Value *Base =
2559 getMinimalBaseOfPointer(A, QueryingAA, Ptr: Loc->Ptr, BytesOffset&: Offset, DL);
2560 if (Base && Base == &AssociatedValue) {
2561 int64_t DerefBytes = Loc->Size.getValue() + Offset;
2562 IsNonNull |= !NullPointerIsDefined;
2563 return std::max(a: int64_t(0), b: DerefBytes);
2564 }
2565
2566 /// Corner case when an offset is 0.
2567 Base = GetPointerBaseWithConstantOffset(Ptr: Loc->Ptr, Offset, DL,
2568 /*AllowNonInbounds*/ true);
2569 if (Base && Base == &AssociatedValue && Offset == 0) {
2570 int64_t DerefBytes = Loc->Size.getValue();
2571 IsNonNull |= !NullPointerIsDefined;
2572 return std::max(a: int64_t(0), b: DerefBytes);
2573 }
2574
2575 return 0;
2576}
2577
2578struct AANonNullImpl : AANonNull {
2579 AANonNullImpl(const IRPosition &IRP, Attributor &A) : AANonNull(IRP, A) {}
2580
2581 /// See AbstractAttribute::initialize(...).
2582 void initialize(Attributor &A) override {
2583 Value &V = *getAssociatedValue().stripPointerCasts();
2584 if (isa<ConstantPointerNull>(Val: V)) {
2585 indicatePessimisticFixpoint();
2586 return;
2587 }
2588
2589 if (Instruction *CtxI = getCtxI())
2590 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
2591 }
2592
2593 /// See followUsesInMBEC
2594 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
2595 AANonNull::StateType &State) {
2596 bool IsNonNull = false;
2597 bool TrackUse = false;
2598 getKnownNonNullAndDerefBytesForUse(A, QueryingAA: *this, AssociatedValue&: getAssociatedValue(), U, I,
2599 IsNonNull, TrackUse);
2600 State.setKnown(IsNonNull);
2601 return TrackUse;
2602 }
2603
2604 /// See AbstractAttribute::getAsStr().
2605 const std::string getAsStr(Attributor *A) const override {
2606 return getAssumed() ? "nonnull" : "may-null";
2607 }
2608};
2609
2610/// NonNull attribute for a floating value.
2611struct AANonNullFloating : public AANonNullImpl {
2612 AANonNullFloating(const IRPosition &IRP, Attributor &A)
2613 : AANonNullImpl(IRP, A) {}
2614
2615 /// See AbstractAttribute::updateImpl(...).
2616 ChangeStatus updateImpl(Attributor &A) override {
2617 auto CheckIRP = [&](const IRPosition &IRP) {
2618 bool IsKnownNonNull;
2619 return AA::hasAssumedIRAttr<Attribute::NonNull>(
2620 A, QueryingAA: *this, IRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNonNull);
2621 };
2622
2623 bool Stripped;
2624 bool UsedAssumedInformation = false;
2625 Value *AssociatedValue = &getAssociatedValue();
2626 SmallVector<AA::ValueAndContext> Values;
2627 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
2628 S: AA::AnyScope, UsedAssumedInformation))
2629 Stripped = false;
2630 else
2631 Stripped =
2632 Values.size() != 1 || Values.front().getValue() != AssociatedValue;
2633
2634 if (!Stripped) {
2635 bool IsKnown;
2636 if (auto *PHI = dyn_cast<PHINode>(Val: AssociatedValue))
2637 if (llvm::all_of(Range: PHI->incoming_values(), P: [&](Value *Op) {
2638 return AA::hasAssumedIRAttr<Attribute::NonNull>(
2639 A, QueryingAA: this, IRP: IRPosition::value(V: *Op), DepClass: DepClassTy::OPTIONAL,
2640 IsKnown);
2641 }))
2642 return ChangeStatus::UNCHANGED;
2643 if (auto *Select = dyn_cast<SelectInst>(Val: AssociatedValue))
2644 if (AA::hasAssumedIRAttr<Attribute::NonNull>(
2645 A, QueryingAA: this, IRP: IRPosition::value(V: *Select->getFalseValue()),
2646 DepClass: DepClassTy::OPTIONAL, IsKnown) &&
2647 AA::hasAssumedIRAttr<Attribute::NonNull>(
2648 A, QueryingAA: this, IRP: IRPosition::value(V: *Select->getTrueValue()),
2649 DepClass: DepClassTy::OPTIONAL, IsKnown))
2650 return ChangeStatus::UNCHANGED;
2651
2652 // If we haven't stripped anything we might still be able to use a
2653 // different AA, but only if the IRP changes. Effectively when we
2654 // interpret this not as a call site value but as a floating/argument
2655 // value.
2656 const IRPosition AVIRP = IRPosition::value(V: *AssociatedValue);
2657 if (AVIRP == getIRPosition() || !CheckIRP(AVIRP))
2658 return indicatePessimisticFixpoint();
2659 return ChangeStatus::UNCHANGED;
2660 }
2661
2662 for (const auto &VAC : Values)
2663 if (!CheckIRP(IRPosition::value(V: *VAC.getValue())))
2664 return indicatePessimisticFixpoint();
2665
2666 return ChangeStatus::UNCHANGED;
2667 }
2668
2669 /// See AbstractAttribute::trackStatistics()
2670 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2671};
2672
2673/// NonNull attribute for function return value.
2674struct AANonNullReturned final
2675 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2676 false, AANonNull::IRAttributeKind, false> {
2677 AANonNullReturned(const IRPosition &IRP, Attributor &A)
2678 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2679 false, Attribute::NonNull, false>(IRP, A) {
2680 }
2681
2682 /// See AbstractAttribute::getAsStr().
2683 const std::string getAsStr(Attributor *A) const override {
2684 return getAssumed() ? "nonnull" : "may-null";
2685 }
2686
2687 /// See AbstractAttribute::trackStatistics()
2688 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
2689};
2690
2691/// NonNull attribute for function argument.
2692struct AANonNullArgument final
2693 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl> {
2694 AANonNullArgument(const IRPosition &IRP, Attributor &A)
2695 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl>(IRP, A) {}
2696
2697 /// See AbstractAttribute::trackStatistics()
2698 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nonnull) }
2699};
2700
2701struct AANonNullCallSiteArgument final : AANonNullFloating {
2702 AANonNullCallSiteArgument(const IRPosition &IRP, Attributor &A)
2703 : AANonNullFloating(IRP, A) {}
2704
2705 /// See AbstractAttribute::trackStatistics()
2706 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nonnull) }
2707};
2708
2709/// NonNull attribute for a call site return position.
2710struct AANonNullCallSiteReturned final
2711 : AACalleeToCallSite<AANonNull, AANonNullImpl> {
2712 AANonNullCallSiteReturned(const IRPosition &IRP, Attributor &A)
2713 : AACalleeToCallSite<AANonNull, AANonNullImpl>(IRP, A) {}
2714
2715 /// See AbstractAttribute::trackStatistics()
2716 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nonnull) }
2717};
2718} // namespace
2719
2720/// ------------------------ Must-Progress Attributes --------------------------
2721namespace {
2722struct AAMustProgressImpl : public AAMustProgress {
2723 AAMustProgressImpl(const IRPosition &IRP, Attributor &A)
2724 : AAMustProgress(IRP, A) {}
2725
2726 /// See AbstractAttribute::initialize(...).
2727 void initialize(Attributor &A) override {
2728 bool IsKnown;
2729 assert(!AA::hasAssumedIRAttr<Attribute::MustProgress>(
2730 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2731 (void)IsKnown;
2732 }
2733
2734 /// See AbstractAttribute::getAsStr()
2735 const std::string getAsStr(Attributor *A) const override {
2736 return getAssumed() ? "mustprogress" : "may-not-progress";
2737 }
2738};
2739
2740struct AAMustProgressFunction final : AAMustProgressImpl {
2741 AAMustProgressFunction(const IRPosition &IRP, Attributor &A)
2742 : AAMustProgressImpl(IRP, A) {}
2743
2744 /// See AbstractAttribute::updateImpl(...).
2745 ChangeStatus updateImpl(Attributor &A) override {
2746 bool IsKnown;
2747 if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
2748 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown)) {
2749 if (IsKnown)
2750 return indicateOptimisticFixpoint();
2751 return ChangeStatus::UNCHANGED;
2752 }
2753
2754 auto CheckForMustProgress = [&](AbstractCallSite ACS) {
2755 IRPosition IPos = IRPosition::callsite_function(CB: *ACS.getInstruction());
2756 bool IsKnownMustProgress;
2757 return AA::hasAssumedIRAttr<Attribute::MustProgress>(
2758 A, QueryingAA: this, IRP: IPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownMustProgress,
2759 /* IgnoreSubsumingPositions */ true);
2760 };
2761
2762 bool AllCallSitesKnown = true;
2763 if (!A.checkForAllCallSites(Pred: CheckForMustProgress, QueryingAA: *this,
2764 /* RequireAllCallSites */ true,
2765 UsedAssumedInformation&: AllCallSitesKnown))
2766 return indicatePessimisticFixpoint();
2767
2768 return ChangeStatus::UNCHANGED;
2769 }
2770
2771 /// See AbstractAttribute::trackStatistics()
2772 void trackStatistics() const override {
2773 STATS_DECLTRACK_FN_ATTR(mustprogress)
2774 }
2775};
2776
2777/// MustProgress attribute deduction for a call sites.
2778struct AAMustProgressCallSite final : AAMustProgressImpl {
2779 AAMustProgressCallSite(const IRPosition &IRP, Attributor &A)
2780 : AAMustProgressImpl(IRP, A) {}
2781
2782 /// See AbstractAttribute::updateImpl(...).
2783 ChangeStatus updateImpl(Attributor &A) override {
2784 // TODO: Once we have call site specific value information we can provide
2785 // call site specific liveness information and then it makes
2786 // sense to specialize attributes for call sites arguments instead of
2787 // redirecting requests to the callee argument.
2788 const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
2789 bool IsKnownMustProgress;
2790 if (!AA::hasAssumedIRAttr<Attribute::MustProgress>(
2791 A, QueryingAA: this, IRP: FnPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownMustProgress))
2792 return indicatePessimisticFixpoint();
2793 return ChangeStatus::UNCHANGED;
2794 }
2795
2796 /// See AbstractAttribute::trackStatistics()
2797 void trackStatistics() const override {
2798 STATS_DECLTRACK_CS_ATTR(mustprogress);
2799 }
2800};
2801} // namespace
2802
2803/// ------------------------ No-Recurse Attributes ----------------------------
2804
2805namespace {
2806struct AANoRecurseImpl : public AANoRecurse {
2807 AANoRecurseImpl(const IRPosition &IRP, Attributor &A) : AANoRecurse(IRP, A) {}
2808
2809 /// See AbstractAttribute::initialize(...).
2810 void initialize(Attributor &A) override {
2811 bool IsKnown;
2812 assert(!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
2813 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2814 (void)IsKnown;
2815 }
2816
2817 /// See AbstractAttribute::getAsStr()
2818 const std::string getAsStr(Attributor *A) const override {
2819 return getAssumed() ? "norecurse" : "may-recurse";
2820 }
2821};
2822
2823struct AANoRecurseFunction final : AANoRecurseImpl {
2824 AANoRecurseFunction(const IRPosition &IRP, Attributor &A)
2825 : AANoRecurseImpl(IRP, A) {}
2826
2827 /// See AbstractAttribute::updateImpl(...).
2828 ChangeStatus updateImpl(Attributor &A) override {
2829
2830 // If all live call sites are known to be no-recurse, we are as well.
2831 auto CallSitePred = [&](AbstractCallSite ACS) {
2832 bool IsKnownNoRecurse;
2833 if (!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
2834 A, QueryingAA: this,
2835 IRP: IRPosition::function(F: *ACS.getInstruction()->getFunction()),
2836 DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoRecurse))
2837 return false;
2838 return IsKnownNoRecurse;
2839 };
2840 bool UsedAssumedInformation = false;
2841 if (A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this, RequireAllCallSites: true,
2842 UsedAssumedInformation)) {
2843 // If we know all call sites and all are known no-recurse, we are done.
2844 // If all known call sites, which might not be all that exist, are known
2845 // to be no-recurse, we are not done but we can continue to assume
2846 // no-recurse. If one of the call sites we have not visited will become
2847 // live, another update is triggered.
2848 if (!UsedAssumedInformation)
2849 indicateOptimisticFixpoint();
2850 return ChangeStatus::UNCHANGED;
2851 }
2852
2853 const AAInterFnReachability *EdgeReachability =
2854 A.getAAFor<AAInterFnReachability>(QueryingAA: *this, IRP: getIRPosition(),
2855 DepClass: DepClassTy::REQUIRED);
2856 if (EdgeReachability && EdgeReachability->canReach(A, Fn: *getAnchorScope()))
2857 return indicatePessimisticFixpoint();
2858 return ChangeStatus::UNCHANGED;
2859 }
2860
2861 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(norecurse) }
2862};
2863
2864/// NoRecurse attribute deduction for a call sites.
2865struct AANoRecurseCallSite final
2866 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl> {
2867 AANoRecurseCallSite(const IRPosition &IRP, Attributor &A)
2868 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl>(IRP, A) {}
2869
2870 /// See AbstractAttribute::trackStatistics()
2871 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(norecurse); }
2872};
2873} // namespace
2874
2875/// ------------------------ No-Convergent Attribute --------------------------
2876
2877namespace {
2878struct AANonConvergentImpl : public AANonConvergent {
2879 AANonConvergentImpl(const IRPosition &IRP, Attributor &A)
2880 : AANonConvergent(IRP, A) {}
2881
2882 /// See AbstractAttribute::getAsStr()
2883 const std::string getAsStr(Attributor *A) const override {
2884 return getAssumed() ? "non-convergent" : "may-be-convergent";
2885 }
2886};
2887
2888struct AANonConvergentFunction final : AANonConvergentImpl {
2889 AANonConvergentFunction(const IRPosition &IRP, Attributor &A)
2890 : AANonConvergentImpl(IRP, A) {}
2891
2892 /// See AbstractAttribute::updateImpl(...).
2893 ChangeStatus updateImpl(Attributor &A) override {
2894 // If all function calls are known to not be convergent, we are not
2895 // convergent.
2896 auto CalleeIsNotConvergent = [&](Instruction &Inst) {
2897 CallBase &CB = cast<CallBase>(Val&: Inst);
2898 auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand());
2899 if (!Callee || Callee->isIntrinsic()) {
2900 return false;
2901 }
2902 if (Callee->isDeclaration()) {
2903 return !Callee->hasFnAttribute(Kind: Attribute::Convergent);
2904 }
2905 const auto *ConvergentAA = A.getAAFor<AANonConvergent>(
2906 QueryingAA: *this, IRP: IRPosition::function(F: *Callee), DepClass: DepClassTy::REQUIRED);
2907 return ConvergentAA && ConvergentAA->isAssumedNotConvergent();
2908 };
2909
2910 bool UsedAssumedInformation = false;
2911 if (!A.checkForAllCallLikeInstructions(Pred: CalleeIsNotConvergent, QueryingAA: *this,
2912 UsedAssumedInformation)) {
2913 return indicatePessimisticFixpoint();
2914 }
2915 return ChangeStatus::UNCHANGED;
2916 }
2917
2918 ChangeStatus manifest(Attributor &A) override {
2919 if (isKnownNotConvergent() &&
2920 A.hasAttr(IRP: getIRPosition(), AKs: Attribute::Convergent)) {
2921 A.removeAttrs(IRP: getIRPosition(), AttrKinds: {Attribute::Convergent});
2922 return ChangeStatus::CHANGED;
2923 }
2924 return ChangeStatus::UNCHANGED;
2925 }
2926
2927 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(convergent) }
2928};
2929} // namespace
2930
2931/// -------------------- Undefined-Behavior Attributes ------------------------
2932
2933namespace {
2934struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior {
2935 AAUndefinedBehaviorImpl(const IRPosition &IRP, Attributor &A)
2936 : AAUndefinedBehavior(IRP, A) {}
2937
2938 struct UBInfo {
2939 enum Kind {
2940 NullPtrAccess,
2941 UndefPtrAccess,
2942 UndefBranchCondition,
2943 UndefReturnValue,
2944 NullReturnViolatesNonNull,
2945 UndefCallArgument,
2946 NullArgViolatesNonNull,
2947 };
2948
2949 Kind K;
2950 std::optional<unsigned> ArgNo;
2951
2952 UBInfo(Kind K) : K(K), ArgNo(std::nullopt) {}
2953
2954 UBInfo(Kind K, std::optional<unsigned> ArgNo) : K(K), ArgNo(ArgNo) {}
2955 };
2956
2957 /// See AbstractAttribute::updateImpl(...).
2958 // through a pointer (i.e. also branches etc.)
2959 ChangeStatus updateImpl(Attributor &A) override {
2960 const size_t UBPrevSize = KnownUBInsts.size();
2961 const size_t NoUBPrevSize = AssumedNoUBInsts.size();
2962
2963 auto InspectMemAccessInstForUB = [&](Instruction &I) {
2964 // Volatile accesses on null are not necessarily UB.
2965 if (I.isVolatile())
2966 return true;
2967
2968 // Skip instructions that are already saved.
2969 if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Key: &I))
2970 return true;
2971
2972 // If we reach here, we know we have an instruction
2973 // that accesses memory through a pointer operand,
2974 // for which getPointerOperand() should give it to us.
2975 Value *PtrOp =
2976 const_cast<Value *>(getPointerOperand(I: &I, /* AllowVolatile */ true));
2977 assert(PtrOp &&
2978 "Expected pointer operand of memory accessing instruction");
2979
2980 // Either we stopped and the appropriate action was taken,
2981 // or we got back a simplified value to continue.
2982 std::optional<Value *> SimplifiedPtrOp =
2983 stopOnUndefOrAssumed(A, V: PtrOp, I: &I, K: UBInfo::UndefPtrAccess);
2984 if (!SimplifiedPtrOp || !*SimplifiedPtrOp)
2985 return true;
2986 const Value *PtrOpVal = *SimplifiedPtrOp;
2987
2988 // A memory access through a pointer is considered UB
2989 // only if the pointer has constant null value.
2990 // TODO: Expand it to not only check constant values.
2991 if (!isa<ConstantPointerNull>(Val: PtrOpVal)) {
2992 AssumedNoUBInsts.insert(Ptr: &I);
2993 return true;
2994 }
2995 const Type *PtrTy = PtrOpVal->getType();
2996
2997 // Because we only consider instructions inside functions,
2998 // assume that a parent function exists.
2999 const Function *F = I.getFunction();
3000
3001 // A memory access using constant null pointer is only considered UB
3002 // if null pointer is _not_ defined for the target platform.
3003 if (llvm::NullPointerIsDefined(F, AS: PtrTy->getPointerAddressSpace()))
3004 AssumedNoUBInsts.insert(Ptr: &I);
3005 else
3006 KnownUBInsts.try_emplace(Key: &I, Args: UBInfo::NullPtrAccess);
3007 return true;
3008 };
3009
3010 auto InspectBrInstForUB = [&](Instruction &I) {
3011 // A conditional branch instruction is considered UB if it has `undef`
3012 // condition.
3013
3014 // Skip instructions that are already saved.
3015 if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Key: &I))
3016 return true;
3017
3018 // We know we have a branch instruction.
3019 auto *BrInst = cast<CondBrInst>(Val: &I);
3020
3021 // Either we stopped and the appropriate action was taken,
3022 // or we got back a simplified value to continue.
3023 std::optional<Value *> SimplifiedCond = stopOnUndefOrAssumed(
3024 A, V: BrInst->getCondition(), I: BrInst, K: UBInfo::UndefBranchCondition);
3025 if (!SimplifiedCond || !*SimplifiedCond)
3026 return true;
3027 AssumedNoUBInsts.insert(Ptr: &I);
3028 return true;
3029 };
3030
3031 auto InspectCallSiteForUB = [&](Instruction &I) {
3032 // Check whether a callsite always cause UB or not
3033
3034 // Skip instructions that are already saved.
3035 if (AssumedNoUBInsts.count(Ptr: &I) || KnownUBInsts.count(Key: &I))
3036 return true;
3037
3038 // Check nonnull and noundef argument attribute violation for each
3039 // callsite.
3040 CallBase &CB = cast<CallBase>(Val&: I);
3041 auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand());
3042 if (!Callee)
3043 return true;
3044 for (unsigned idx = 0; idx < CB.arg_size(); idx++) {
3045 // If current argument is known to be simplified to null pointer and the
3046 // corresponding argument position is known to have nonnull attribute,
3047 // the argument is poison. Furthermore, if the argument is poison and
3048 // the position is known to have noundef attriubte, this callsite is
3049 // considered UB.
3050 if (idx >= Callee->arg_size())
3051 break;
3052 Value *ArgVal = CB.getArgOperand(i: idx);
3053 if (!ArgVal)
3054 continue;
3055 // Here, we handle three cases.
3056 // (1) Not having a value means it is dead. (we can replace the value
3057 // with undef)
3058 // (2) Simplified to undef. The argument violate noundef attriubte.
3059 // (3) Simplified to null pointer where known to be nonnull.
3060 // The argument is a poison value and violate noundef attribute.
3061 IRPosition CalleeArgumentIRP = IRPosition::callsite_argument(CB, ArgNo: idx);
3062 bool IsKnownNoUndef;
3063 AA::hasAssumedIRAttr<Attribute::NoUndef>(
3064 A, QueryingAA: this, IRP: CalleeArgumentIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoUndef);
3065 if (!IsKnownNoUndef)
3066 continue;
3067 bool UsedAssumedInformation = false;
3068 std::optional<Value *> SimplifiedVal =
3069 A.getAssumedSimplified(IRP: IRPosition::value(V: *ArgVal), AA: *this,
3070 UsedAssumedInformation, S: AA::Interprocedural);
3071 if (UsedAssumedInformation)
3072 continue;
3073 if (SimplifiedVal && !*SimplifiedVal)
3074 return true;
3075 if (!SimplifiedVal || isa<UndefValue>(Val: **SimplifiedVal)) {
3076 KnownUBInsts.try_emplace(Key: &I, Args: UBInfo(UBInfo::UndefCallArgument, idx));
3077 continue;
3078 }
3079 if (!ArgVal->getType()->isPointerTy() ||
3080 !isa<ConstantPointerNull>(Val: **SimplifiedVal))
3081 continue;
3082 bool IsKnownNonNull;
3083 AA::hasAssumedIRAttr<Attribute::NonNull>(
3084 A, QueryingAA: this, IRP: CalleeArgumentIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
3085 if (IsKnownNonNull)
3086 KnownUBInsts.try_emplace(Key: &I,
3087 Args: UBInfo(UBInfo::NullArgViolatesNonNull, idx));
3088 }
3089 return true;
3090 };
3091
3092 auto InspectReturnInstForUB = [&](Instruction &I) {
3093 auto &RI = cast<ReturnInst>(Val&: I);
3094 // Either we stopped and the appropriate action was taken,
3095 // or we got back a simplified return value to continue.
3096 std::optional<Value *> SimplifiedRetValue = stopOnUndefOrAssumed(
3097 A, V: RI.getReturnValue(), I: &I, K: UBInfo::UndefReturnValue);
3098 if (!SimplifiedRetValue || !*SimplifiedRetValue)
3099 return true;
3100
3101 // Check if a return instruction always cause UB or not
3102 // Note: It is guaranteed that the returned position of the anchor
3103 // scope has noundef attribute when this is called.
3104 // We also ensure the return position is not "assumed dead"
3105 // because the returned value was then potentially simplified to
3106 // `undef` in AAReturnedValues without removing the `noundef`
3107 // attribute yet.
3108
3109 // When the returned position has noundef attriubte, UB occurs in the
3110 // following cases.
3111 // (1) Returned value is known to be undef.
3112 // (2) The value is known to be a null pointer and the returned
3113 // position has nonnull attribute (because the returned value is
3114 // poison).
3115 if (isa<ConstantPointerNull>(Val: *SimplifiedRetValue)) {
3116 bool IsKnownNonNull;
3117 AA::hasAssumedIRAttr<Attribute::NonNull>(
3118 A, QueryingAA: this, IRP: IRPosition::returned(F: *getAnchorScope()), DepClass: DepClassTy::NONE,
3119 IsKnown&: IsKnownNonNull);
3120 if (IsKnownNonNull)
3121 KnownUBInsts.try_emplace(Key: &I, Args: UBInfo::NullReturnViolatesNonNull);
3122 }
3123
3124 return true;
3125 };
3126
3127 bool UsedAssumedInformation = false;
3128 A.checkForAllInstructions(Pred: InspectMemAccessInstForUB, QueryingAA: *this,
3129 Opcodes: {Instruction::Load, Instruction::Store,
3130 Instruction::AtomicCmpXchg,
3131 Instruction::AtomicRMW},
3132 UsedAssumedInformation,
3133 /* CheckBBLivenessOnly */ true);
3134 A.checkForAllInstructions(Pred: InspectBrInstForUB, QueryingAA: *this, Opcodes: {Instruction::CondBr},
3135 UsedAssumedInformation,
3136 /* CheckBBLivenessOnly */ true);
3137 A.checkForAllCallLikeInstructions(Pred: InspectCallSiteForUB, QueryingAA: *this,
3138 UsedAssumedInformation);
3139
3140 // If the returned position of the anchor scope has noundef attriubte, check
3141 // all returned instructions.
3142 if (!getAnchorScope()->getReturnType()->isVoidTy()) {
3143 const IRPosition &ReturnIRP = IRPosition::returned(F: *getAnchorScope());
3144 if (!A.isAssumedDead(IRP: ReturnIRP, QueryingAA: this, FnLivenessAA: nullptr, UsedAssumedInformation)) {
3145 bool IsKnownNoUndef;
3146 AA::hasAssumedIRAttr<Attribute::NoUndef>(
3147 A, QueryingAA: this, IRP: ReturnIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoUndef);
3148 if (IsKnownNoUndef)
3149 A.checkForAllInstructions(Pred: InspectReturnInstForUB, QueryingAA: *this,
3150 Opcodes: {Instruction::Ret}, UsedAssumedInformation,
3151 /* CheckBBLivenessOnly */ true);
3152 }
3153 }
3154
3155 if (NoUBPrevSize != AssumedNoUBInsts.size() ||
3156 UBPrevSize != KnownUBInsts.size())
3157 return ChangeStatus::CHANGED;
3158 return ChangeStatus::UNCHANGED;
3159 }
3160
3161 bool isKnownToCauseUB(Instruction *I) const override {
3162 return KnownUBInsts.count(Key: I);
3163 }
3164
3165 bool isAssumedToCauseUB(Instruction *I) const override {
3166 // In simple words, if an instruction is not in the assumed to _not_
3167 // cause UB, then it is assumed UB (that includes those
3168 // in the KnownUBInsts set). The rest is boilerplate
3169 // is to ensure that it is one of the instructions we test
3170 // for UB.
3171
3172 switch (I->getOpcode()) {
3173 case Instruction::Load:
3174 case Instruction::Store:
3175 case Instruction::AtomicCmpXchg:
3176 case Instruction::AtomicRMW:
3177 case Instruction::CondBr:
3178 return !AssumedNoUBInsts.count(Ptr: I);
3179 default:
3180 return false;
3181 }
3182 return false;
3183 }
3184
3185 /// Emit an optimization remark explaining why \p I is known to cause UB,
3186 /// per \p Info, right before it is replaced with 'unreachable'.
3187 static void emitUBRemark(Attributor &A, Instruction *I, const UBInfo &Info) {
3188 auto Remark = [&](OptimizationRemark OR) {
3189 switch (Info.K) {
3190 case UBInfo::NullPtrAccess:
3191 case UBInfo::UndefPtrAccess: {
3192 return OR << "Memory access through a pointer known to be "
3193 << ore::NV("Pointer",
3194 getPointerOperand(I, /*AllowVolatile*/ true))
3195 << " is undefined behavior; replacing with 'unreachable'.";
3196 }
3197 case UBInfo::UndefBranchCondition:
3198 return OR << "Branch condition known to be "
3199 << ore::NV("Condition", cast<CondBrInst>(Val: I)->getCondition())
3200 << " is undefined behavior; replacing with 'unreachable'.";
3201 case UBInfo::UndefReturnValue:
3202 case UBInfo::NullReturnViolatesNonNull:
3203 return OR << "Value returned known to be "
3204 << ore::NV("ReturnValue",
3205 cast<ReturnInst>(Val: I)->getReturnValue())
3206 << " is undefined behavior; replacing with 'unreachable'.";
3207 case UBInfo::UndefCallArgument:
3208 case UBInfo::NullArgViolatesNonNull: {
3209 bool IsUndef = Info.K == UBInfo::UndefCallArgument;
3210 CallBase &CB = *cast<CallBase>(Val: I);
3211 OR << "Argument " << ore::NV("ArgNo", *Info.ArgNo)
3212 << " passed to parameter of ";
3213 if (auto *Callee = dyn_cast_if_present<Function>(Val: CB.getCalledOperand()))
3214 OR << ore::NV("Callee", Callee);
3215 else
3216 OR << "the callee";
3217 return OR << " known to be "
3218 << ore::NV("Argument", IsUndef ? "undef" : "null")
3219 << " is undefined behavior; replacing with 'unreachable'.";
3220 }
3221 }
3222 llvm_unreachable("Unknown UBInfo::Kind");
3223 };
3224 A.emitRemark<OptimizationRemark>(I, RemarkName: "UndefinedBehavior", RemarkCB&: Remark);
3225 }
3226
3227 ChangeStatus manifest(Attributor &A) override {
3228 if (KnownUBInsts.empty())
3229 return ChangeStatus::UNCHANGED;
3230 for (const auto &[I, Info] : KnownUBInsts) {
3231 emitUBRemark(A, I, Info);
3232 A.changeToUnreachableAfterManifest(I);
3233 }
3234 return ChangeStatus::CHANGED;
3235 }
3236
3237 /// See AbstractAttribute::getAsStr()
3238 const std::string getAsStr(Attributor *A) const override {
3239 return getAssumed() ? "undefined-behavior" : "no-ub";
3240 }
3241
3242 /// Note: The correctness of this analysis depends on the fact that the
3243 /// following 2 sets will stop changing after some point.
3244 /// "Change" here means that their size changes.
3245 /// The size of each set is monotonically increasing
3246 /// (we only add items to them) and it is upper bounded by the number of
3247 /// instructions in the processed function (we can never save more
3248 /// elements in either set than this number). Hence, at some point,
3249 /// they will stop increasing.
3250 /// Consequently, at some point, both sets will have stopped
3251 /// changing, effectively making the analysis reach a fixpoint.
3252
3253 /// Note: These 2 sets are disjoint and an instruction can be considered
3254 /// one of 3 things:
3255 /// 1) Known to cause UB (AAUndefinedBehavior could prove it) and put it in
3256 /// the KnownUBInsts set.
3257 /// 2) Assumed to cause UB (in every updateImpl, AAUndefinedBehavior
3258 /// has a reason to assume it).
3259 /// 3) Assumed to not cause UB. very other instruction - AAUndefinedBehavior
3260 /// could not find a reason to assume or prove that it can cause UB,
3261 /// hence it assumes it doesn't. We have a set for these instructions
3262 /// so that we don't reprocess them in every update.
3263 /// Note however that instructions in this set may cause UB.
3264
3265protected:
3266 /// A map from all live instructions _known_ to cause UB to the reason why,
3267 /// used to build actionable optimization remarks in manifest().
3268 MapVector<Instruction *, UBInfo> KnownUBInsts;
3269
3270private:
3271 /// A set of all the (live) instructions that are assumed to _not_ cause UB.
3272 SmallPtrSet<Instruction *, 8> AssumedNoUBInsts;
3273
3274 // Should be called on updates in which if we're processing an instruction
3275 // \p I that depends on a value \p V, one of the following has to happen:
3276 // - If the value is assumed, then stop.
3277 // - If the value is known but undef, then consider it UB for \p K.
3278 // - Otherwise, do specific processing with the simplified value.
3279 // We return std::nullopt in the first 2 cases to signify that an appropriate
3280 // action was taken and the caller should stop.
3281 // Otherwise, we return the simplified value that the caller should
3282 // use for specific processing.
3283 std::optional<Value *> stopOnUndefOrAssumed(Attributor &A, Value *V,
3284 Instruction *I, UBInfo::Kind K) {
3285 bool UsedAssumedInformation = false;
3286 std::optional<Value *> SimplifiedV =
3287 A.getAssumedSimplified(IRP: IRPosition::value(V: *V), AA: *this,
3288 UsedAssumedInformation, S: AA::Interprocedural);
3289 if (!UsedAssumedInformation) {
3290 // Don't depend on assumed values.
3291 if (!SimplifiedV) {
3292 // If it is known (which we tested above) but it doesn't have a value,
3293 // then we can assume `undef` and hence the instruction is UB.
3294 KnownUBInsts.try_emplace(Key: I, Args&: K);
3295 return std::nullopt;
3296 }
3297 if (!*SimplifiedV)
3298 return nullptr;
3299 V = *SimplifiedV;
3300 }
3301 if (isa<UndefValue>(Val: V)) {
3302 KnownUBInsts.try_emplace(Key: I, Args&: K);
3303 return std::nullopt;
3304 }
3305 return V;
3306 }
3307};
3308
3309struct AAUndefinedBehaviorFunction final : AAUndefinedBehaviorImpl {
3310 AAUndefinedBehaviorFunction(const IRPosition &IRP, Attributor &A)
3311 : AAUndefinedBehaviorImpl(IRP, A) {}
3312
3313 /// See AbstractAttribute::trackStatistics()
3314 void trackStatistics() const override {
3315 STATS_DECL(UndefinedBehaviorInstruction, Instruction,
3316 "Number of instructions known to have UB");
3317 BUILD_STAT_NAME(UndefinedBehaviorInstruction, Instruction) +=
3318 KnownUBInsts.size();
3319 }
3320};
3321} // namespace
3322
3323/// ------------------------ Will-Return Attributes ----------------------------
3324
3325namespace {
3326// Helper function that checks whether a function has any cycle which we don't
3327// know if it is bounded or not.
3328// Loops with maximum trip count are considered bounded, any other cycle not.
3329static bool mayContainUnboundedCycle(Function &F, Attributor &A) {
3330 ScalarEvolution *SE =
3331 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(F);
3332 LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F);
3333 // If either SCEV or LoopInfo is not available for the function then we assume
3334 // any cycle to be unbounded cycle.
3335 // We use scc_iterator which uses Tarjan algorithm to find all the maximal
3336 // SCCs.To detect if there's a cycle, we only need to find the maximal ones.
3337 if (!SE || !LI) {
3338 for (scc_iterator<Function *> SCCI = scc_begin(G: &F); !SCCI.isAtEnd(); ++SCCI)
3339 if (SCCI.hasCycle())
3340 return true;
3341 return false;
3342 }
3343
3344 // If there's irreducible control, the function may contain non-loop cycles.
3345 if (mayContainIrreducibleControl(F, LI))
3346 return true;
3347
3348 // Any loop that does not have a max trip count is considered unbounded cycle.
3349 for (auto *L : LI->getLoopsInPreorder()) {
3350 if (!SE->getSmallConstantMaxTripCount(L))
3351 return true;
3352 }
3353 return false;
3354}
3355
3356struct AAWillReturnImpl : public AAWillReturn {
3357 AAWillReturnImpl(const IRPosition &IRP, Attributor &A)
3358 : AAWillReturn(IRP, A) {}
3359
3360 /// See AbstractAttribute::initialize(...).
3361 void initialize(Attributor &A) override {
3362 bool IsKnown;
3363 assert(!AA::hasAssumedIRAttr<Attribute::WillReturn>(
3364 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
3365 (void)IsKnown;
3366 }
3367
3368 /// Check for `mustprogress` and `readonly` as they imply `willreturn`.
3369 bool isImpliedByMustprogressAndReadonly(Attributor &A, bool KnownOnly) {
3370 if (!A.hasAttr(IRP: getIRPosition(), AKs: {Attribute::MustProgress}))
3371 return false;
3372
3373 bool IsKnown;
3374 if (AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
3375 return IsKnown || !KnownOnly;
3376 return false;
3377 }
3378
3379 /// See AbstractAttribute::updateImpl(...).
3380 ChangeStatus updateImpl(Attributor &A) override {
3381 if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3382 return ChangeStatus::UNCHANGED;
3383
3384 auto CheckForWillReturn = [&](Instruction &I) {
3385 IRPosition IPos = IRPosition::callsite_function(CB: cast<CallBase>(Val&: I));
3386 bool IsKnown;
3387 if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
3388 A, QueryingAA: this, IRP: IPos, DepClass: DepClassTy::REQUIRED, IsKnown)) {
3389 if (IsKnown)
3390 return true;
3391 } else {
3392 return false;
3393 }
3394 bool IsKnownNoRecurse;
3395 return AA::hasAssumedIRAttr<Attribute::NoRecurse>(
3396 A, QueryingAA: this, IRP: IPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoRecurse);
3397 };
3398
3399 bool UsedAssumedInformation = false;
3400 if (!A.checkForAllCallLikeInstructions(Pred: CheckForWillReturn, QueryingAA: *this,
3401 UsedAssumedInformation))
3402 return indicatePessimisticFixpoint();
3403
3404 auto CheckForVolatile = [&](Instruction &I) {
3405 // Volatile operations are not willreturn.
3406 return !I.isVolatile();
3407 };
3408 if (!A.checkForAllInstructions(Pred: CheckForVolatile, QueryingAA: *this,
3409 Opcodes: {Instruction::Load, Instruction::Store,
3410 Instruction::AtomicCmpXchg,
3411 Instruction::AtomicRMW},
3412 UsedAssumedInformation))
3413 return indicatePessimisticFixpoint();
3414
3415 return ChangeStatus::UNCHANGED;
3416 }
3417
3418 /// See AbstractAttribute::getAsStr()
3419 const std::string getAsStr(Attributor *A) const override {
3420 return getAssumed() ? "willreturn" : "may-noreturn";
3421 }
3422};
3423
3424struct AAWillReturnFunction final : AAWillReturnImpl {
3425 AAWillReturnFunction(const IRPosition &IRP, Attributor &A)
3426 : AAWillReturnImpl(IRP, A) {}
3427
3428 /// See AbstractAttribute::initialize(...).
3429 void initialize(Attributor &A) override {
3430 AAWillReturnImpl::initialize(A);
3431
3432 Function *F = getAnchorScope();
3433 assert(F && "Did expect an anchor function");
3434 if (F->isDeclaration() || mayContainUnboundedCycle(F&: *F, A))
3435 indicatePessimisticFixpoint();
3436 }
3437
3438 /// See AbstractAttribute::trackStatistics()
3439 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(willreturn) }
3440};
3441
3442/// WillReturn attribute deduction for a call sites.
3443struct AAWillReturnCallSite final
3444 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl> {
3445 AAWillReturnCallSite(const IRPosition &IRP, Attributor &A)
3446 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl>(IRP, A) {}
3447
3448 /// See AbstractAttribute::updateImpl(...).
3449 ChangeStatus updateImpl(Attributor &A) override {
3450 if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
3451 return ChangeStatus::UNCHANGED;
3452
3453 return AACalleeToCallSite::updateImpl(A);
3454 }
3455
3456 /// See AbstractAttribute::trackStatistics()
3457 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(willreturn); }
3458};
3459} // namespace
3460
3461/// -------------------AAIntraFnReachability Attribute--------------------------
3462
3463/// All information associated with a reachability query. This boilerplate code
3464/// is used by both AAIntraFnReachability and AAInterFnReachability, with
3465/// different \p ToTy values.
3466template <typename ToTy> struct ReachabilityQueryInfo {
3467 enum class Reachable {
3468 No,
3469 Yes,
3470 };
3471
3472 /// Start here,
3473 const Instruction *From = nullptr;
3474 /// reach this place,
3475 const ToTy *To = nullptr;
3476 /// without going through any of these instructions,
3477 const AA::InstExclusionSetTy *ExclusionSet = nullptr;
3478 /// and remember if it worked:
3479 Reachable Result = Reachable::No;
3480
3481 /// Precomputed hash for this RQI.
3482 unsigned Hash = 0;
3483
3484 unsigned computeHashValue() const {
3485 assert(Hash == 0 && "Computed hash twice!");
3486 using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;
3487 using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;
3488 return const_cast<ReachabilityQueryInfo<ToTy> *>(this)->Hash =
3489 detail::combineHashValue(a: PairDMI ::getHashValue({From, To}),
3490 b: InstSetDMI::getHashValue(BES: ExclusionSet));
3491 }
3492
3493 ReachabilityQueryInfo(const Instruction *From, const ToTy *To)
3494 : From(From), To(To) {}
3495
3496 /// Constructor replacement to ensure unique and stable sets are used for the
3497 /// cache.
3498 ReachabilityQueryInfo(Attributor &A, const Instruction &From, const ToTy &To,
3499 const AA::InstExclusionSetTy *ES, bool MakeUnique)
3500 : From(&From), To(&To), ExclusionSet(ES) {
3501
3502 if (!ES || ES->empty()) {
3503 ExclusionSet = nullptr;
3504 } else if (MakeUnique) {
3505 ExclusionSet = A.getInfoCache().getOrCreateUniqueBlockExecutionSet(BES: ES);
3506 }
3507 }
3508
3509 ReachabilityQueryInfo(const ReachabilityQueryInfo &RQI)
3510 : From(RQI.From), To(RQI.To), ExclusionSet(RQI.ExclusionSet) {}
3511};
3512
3513namespace llvm {
3514template <typename ToTy> struct DenseMapInfo<ReachabilityQueryInfo<ToTy> *> {
3515 using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;
3516 using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;
3517
3518 static unsigned getHashValue(const ReachabilityQueryInfo<ToTy> *RQI) {
3519 return RQI->Hash ? RQI->Hash : RQI->computeHashValue();
3520 }
3521 static bool isEqual(const ReachabilityQueryInfo<ToTy> *LHS,
3522 const ReachabilityQueryInfo<ToTy> *RHS) {
3523 if (!PairDMI::isEqual({LHS->From, LHS->To}, {RHS->From, RHS->To}))
3524 return false;
3525 return InstSetDMI::isEqual(LHS: LHS->ExclusionSet, RHS: RHS->ExclusionSet);
3526 }
3527};
3528
3529} // namespace llvm
3530
3531namespace {
3532
3533template <typename BaseTy, typename ToTy>
3534struct CachedReachabilityAA : public BaseTy {
3535 using RQITy = ReachabilityQueryInfo<ToTy>;
3536
3537 CachedReachabilityAA(const IRPosition &IRP, Attributor &A) : BaseTy(IRP, A) {}
3538
3539 /// See AbstractAttribute::isQueryAA.
3540 bool isQueryAA() const override { return true; }
3541
3542 /// See AbstractAttribute::updateImpl(...).
3543 ChangeStatus updateImpl(Attributor &A) override {
3544 ChangeStatus Changed = ChangeStatus::UNCHANGED;
3545 for (unsigned u = 0, e = QueryVector.size(); u < e; ++u) {
3546 RQITy *RQI = QueryVector[u];
3547 if (RQI->Result == RQITy::Reachable::No &&
3548 isReachableImpl(A, RQI&: *RQI, /*IsTemporaryRQI=*/false))
3549 Changed = ChangeStatus::CHANGED;
3550 }
3551 return Changed;
3552 }
3553
3554 virtual bool isReachableImpl(Attributor &A, RQITy &RQI,
3555 bool IsTemporaryRQI) = 0;
3556
3557 bool rememberResult(Attributor &A, typename RQITy::Reachable Result,
3558 RQITy &RQI, bool UsedExclusionSet, bool IsTemporaryRQI) {
3559 RQI.Result = Result;
3560
3561 // Remove the temporary RQI from the cache.
3562 if (IsTemporaryRQI)
3563 QueryCache.erase(&RQI);
3564
3565 // Insert a plain RQI (w/o exclusion set) if that makes sense. Two options:
3566 // 1) If it is reachable, it doesn't matter if we have an exclusion set for
3567 // this query. 2) We did not use the exclusion set, potentially because
3568 // there is none.
3569 if (Result == RQITy::Reachable::Yes || !UsedExclusionSet) {
3570 RQITy PlainRQI(RQI.From, RQI.To);
3571 if (!QueryCache.count(&PlainRQI)) {
3572 RQITy *RQIPtr = new (A.Allocator) RQITy(RQI.From, RQI.To);
3573 RQIPtr->Result = Result;
3574 QueryVector.push_back(RQIPtr);
3575 QueryCache.insert(RQIPtr);
3576 }
3577 }
3578
3579 // Check if we need to insert a new permanent RQI with the exclusion set.
3580 if (IsTemporaryRQI && Result != RQITy::Reachable::Yes && UsedExclusionSet) {
3581 assert((!RQI.ExclusionSet || !RQI.ExclusionSet->empty()) &&
3582 "Did not expect empty set!");
3583 RQITy *RQIPtr = new (A.Allocator)
3584 RQITy(A, *RQI.From, *RQI.To, RQI.ExclusionSet, true);
3585 assert(RQIPtr->Result == RQITy::Reachable::No && "Already reachable?");
3586 RQIPtr->Result = Result;
3587 assert(!QueryCache.count(RQIPtr));
3588 QueryVector.push_back(RQIPtr);
3589 QueryCache.insert(RQIPtr);
3590 }
3591
3592 if (Result == RQITy::Reachable::No && IsTemporaryRQI)
3593 A.registerForUpdate(AA&: *this);
3594 return Result == RQITy::Reachable::Yes;
3595 }
3596
3597 const std::string getAsStr(Attributor *A) const override {
3598 // TODO: Return the number of reachable queries.
3599 return "#queries(" + std::to_string(QueryVector.size()) + ")";
3600 }
3601
3602 bool checkQueryCache(Attributor &A, RQITy &StackRQI,
3603 typename RQITy::Reachable &Result) {
3604 if (!this->getState().isValidState()) {
3605 Result = RQITy::Reachable::Yes;
3606 return true;
3607 }
3608
3609 // If we have an exclusion set we might be able to find our answer by
3610 // ignoring it first.
3611 if (StackRQI.ExclusionSet) {
3612 RQITy PlainRQI(StackRQI.From, StackRQI.To);
3613 auto It = QueryCache.find(&PlainRQI);
3614 if (It != QueryCache.end() && (*It)->Result == RQITy::Reachable::No) {
3615 Result = RQITy::Reachable::No;
3616 return true;
3617 }
3618 }
3619
3620 auto It = QueryCache.find(&StackRQI);
3621 if (It != QueryCache.end()) {
3622 Result = (*It)->Result;
3623 return true;
3624 }
3625
3626 // Insert a temporary for recursive queries. We will replace it with a
3627 // permanent entry later.
3628 QueryCache.insert(&StackRQI);
3629 return false;
3630 }
3631
3632private:
3633 SmallVector<RQITy *> QueryVector;
3634 DenseSet<RQITy *> QueryCache;
3635};
3636
3637struct AAIntraFnReachabilityFunction final
3638 : public CachedReachabilityAA<AAIntraFnReachability, Instruction> {
3639 using Base = CachedReachabilityAA<AAIntraFnReachability, Instruction>;
3640 AAIntraFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
3641 : Base(IRP, A) {
3642 DT = A.getInfoCache().getAnalysisResultForFunction<DominatorTreeAnalysis>(
3643 F: *IRP.getAssociatedFunction());
3644 }
3645
3646 bool isAssumedReachable(
3647 Attributor &A, const Instruction &From, const Instruction &To,
3648 const AA::InstExclusionSetTy *ExclusionSet) const override {
3649 auto *NonConstThis = const_cast<AAIntraFnReachabilityFunction *>(this);
3650 if (&From == &To)
3651 return true;
3652
3653 RQITy StackRQI(A, From, To, ExclusionSet, false);
3654 RQITy::Reachable Result;
3655 if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
3656 return NonConstThis->isReachableImpl(A, RQI&: StackRQI,
3657 /*IsTemporaryRQI=*/true);
3658 return Result == RQITy::Reachable::Yes;
3659 }
3660
3661 ChangeStatus updateImpl(Attributor &A) override {
3662 // We only depend on liveness. DeadEdges is all we care about, check if any
3663 // of them changed.
3664 auto *LivenessAA =
3665 A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL);
3666 if (LivenessAA &&
3667 llvm::all_of(Range&: DeadEdges,
3668 P: [&](const auto &DeadEdge) {
3669 return LivenessAA->isEdgeDead(From: DeadEdge.first,
3670 To: DeadEdge.second);
3671 }) &&
3672 llvm::all_of(Range&: DeadBlocks, P: [&](const BasicBlock *BB) {
3673 return LivenessAA->isAssumedDead(BB);
3674 })) {
3675 return ChangeStatus::UNCHANGED;
3676 }
3677 DeadEdges.clear();
3678 DeadBlocks.clear();
3679 return Base::updateImpl(A);
3680 }
3681
3682 bool isReachableImpl(Attributor &A, RQITy &RQI,
3683 bool IsTemporaryRQI) override {
3684 const Instruction *Origin = RQI.From;
3685 bool UsedExclusionSet = false;
3686
3687 auto WillReachInBlock = [&](const Instruction &From, const Instruction &To,
3688 const AA::InstExclusionSetTy *ExclusionSet) {
3689 const Instruction *IP = &From;
3690 while (IP && IP != &To) {
3691 if (ExclusionSet && IP != Origin && ExclusionSet->count(Ptr: IP)) {
3692 UsedExclusionSet = true;
3693 break;
3694 }
3695 IP = IP->getNextNode();
3696 }
3697 return IP == &To;
3698 };
3699
3700 const BasicBlock *FromBB = RQI.From->getParent();
3701 const BasicBlock *ToBB = RQI.To->getParent();
3702 assert(FromBB->getParent() == ToBB->getParent() &&
3703 "Not an intra-procedural query!");
3704
3705 // Check intra-block reachability, however, other reaching paths are still
3706 // possible.
3707 if (FromBB == ToBB &&
3708 WillReachInBlock(*RQI.From, *RQI.To, RQI.ExclusionSet))
3709 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3710 IsTemporaryRQI);
3711
3712 // Check if reaching the ToBB block is sufficient or if even that would not
3713 // ensure reaching the target. In the latter case we are done.
3714 if (!WillReachInBlock(ToBB->front(), *RQI.To, RQI.ExclusionSet))
3715 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3716 IsTemporaryRQI);
3717
3718 const Function *Fn = FromBB->getParent();
3719 SmallPtrSet<const BasicBlock *, 16> ExclusionBlocks;
3720 if (RQI.ExclusionSet)
3721 for (auto *I : *RQI.ExclusionSet)
3722 if (I->getFunction() == Fn)
3723 ExclusionBlocks.insert(Ptr: I->getParent());
3724
3725 // Check if we make it out of the FromBB block at all.
3726 if (ExclusionBlocks.count(Ptr: FromBB) &&
3727 !WillReachInBlock(*RQI.From, *FromBB->getTerminator(),
3728 RQI.ExclusionSet))
3729 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet: true, IsTemporaryRQI);
3730
3731 auto *LivenessAA =
3732 A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL);
3733 if (LivenessAA && LivenessAA->isAssumedDead(BB: ToBB)) {
3734 DeadBlocks.insert(V: ToBB);
3735 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3736 IsTemporaryRQI);
3737 }
3738
3739 SmallPtrSet<const BasicBlock *, 16> Visited;
3740 SmallVector<const BasicBlock *, 16> Worklist;
3741 Worklist.push_back(Elt: FromBB);
3742
3743 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> LocalDeadEdges;
3744 while (!Worklist.empty()) {
3745 const BasicBlock *BB = Worklist.pop_back_val();
3746 if (!Visited.insert(Ptr: BB).second)
3747 continue;
3748 for (const BasicBlock *SuccBB : successors(BB)) {
3749 if (LivenessAA && LivenessAA->isEdgeDead(From: BB, To: SuccBB)) {
3750 LocalDeadEdges.insert(V: {BB, SuccBB});
3751 continue;
3752 }
3753 // We checked before if we just need to reach the ToBB block.
3754 if (SuccBB == ToBB)
3755 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3756 IsTemporaryRQI);
3757 if (DT && ExclusionBlocks.empty() && DT->dominates(A: BB, B: ToBB))
3758 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3759 IsTemporaryRQI);
3760
3761 if (ExclusionBlocks.count(Ptr: SuccBB)) {
3762 UsedExclusionSet = true;
3763 continue;
3764 }
3765 Worklist.push_back(Elt: SuccBB);
3766 }
3767 }
3768
3769 DeadEdges.insert_range(R&: LocalDeadEdges);
3770 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
3771 IsTemporaryRQI);
3772 }
3773
3774 /// See AbstractAttribute::trackStatistics()
3775 void trackStatistics() const override {}
3776
3777private:
3778 // Set of assumed dead blocks we used in the last query. If any changes we
3779 // update the state.
3780 DenseSet<const BasicBlock *> DeadBlocks;
3781
3782 // Set of assumed dead edges we used in the last query. If any changes we
3783 // update the state.
3784 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> DeadEdges;
3785
3786 /// The dominator tree of the function to short-circuit reasoning.
3787 const DominatorTree *DT = nullptr;
3788};
3789} // namespace
3790
3791/// ------------------------ NoAlias Argument Attribute ------------------------
3792
3793bool AANoAlias::isImpliedByIR(Attributor &A, const IRPosition &IRP,
3794 Attribute::AttrKind ImpliedAttributeKind,
3795 bool IgnoreSubsumingPositions) {
3796 assert(ImpliedAttributeKind == Attribute::NoAlias &&
3797 "Unexpected attribute kind");
3798 Value *Val = &IRP.getAssociatedValue();
3799 if (IRP.getPositionKind() != IRP_CALL_SITE_ARGUMENT) {
3800 if (isa<AllocaInst>(Val))
3801 return true;
3802 } else {
3803 IgnoreSubsumingPositions = true;
3804 }
3805
3806 if (isa<UndefValue>(Val))
3807 return true;
3808
3809 if (isa<ConstantPointerNull>(Val) &&
3810 !NullPointerIsDefined(F: IRP.getAnchorScope(),
3811 AS: Val->getType()->getPointerAddressSpace()))
3812 return true;
3813
3814 if (A.hasAttr(IRP, AKs: {Attribute::ByVal, Attribute::NoAlias},
3815 IgnoreSubsumingPositions, ImpliedAttributeKind: Attribute::NoAlias))
3816 return true;
3817
3818 return false;
3819}
3820
3821namespace {
3822struct AANoAliasImpl : AANoAlias {
3823 AANoAliasImpl(const IRPosition &IRP, Attributor &A) : AANoAlias(IRP, A) {
3824 assert(getAssociatedType()->isPointerTy() &&
3825 "Noalias is a pointer attribute");
3826 }
3827
3828 const std::string getAsStr(Attributor *A) const override {
3829 return getAssumed() ? "noalias" : "may-alias";
3830 }
3831};
3832
3833/// NoAlias attribute for a floating value.
3834struct AANoAliasFloating final : AANoAliasImpl {
3835 AANoAliasFloating(const IRPosition &IRP, Attributor &A)
3836 : AANoAliasImpl(IRP, A) {}
3837
3838 /// See AbstractAttribute::updateImpl(...).
3839 ChangeStatus updateImpl(Attributor &A) override {
3840 // TODO: Implement this.
3841 return indicatePessimisticFixpoint();
3842 }
3843
3844 /// See AbstractAttribute::trackStatistics()
3845 void trackStatistics() const override {
3846 STATS_DECLTRACK_FLOATING_ATTR(noalias)
3847 }
3848};
3849
3850/// NoAlias attribute for an argument.
3851struct AANoAliasArgument final
3852 : AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl> {
3853 using Base = AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl>;
3854 AANoAliasArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
3855
3856 /// See AbstractAttribute::update(...).
3857 ChangeStatus updateImpl(Attributor &A) override {
3858 // We have to make sure no-alias on the argument does not break
3859 // synchronization when this is a callback argument, see also [1] below.
3860 // If synchronization cannot be affected, we delegate to the base updateImpl
3861 // function, otherwise we give up for now.
3862
3863 // If the function is no-sync, no-alias cannot break synchronization.
3864 bool IsKnownNoSycn;
3865 if (AA::hasAssumedIRAttr<Attribute::NoSync>(
3866 A, QueryingAA: this, IRP: IRPosition::function_scope(IRP: getIRPosition()),
3867 DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoSycn))
3868 return Base::updateImpl(A);
3869
3870 // If the argument is read-only, no-alias cannot break synchronization.
3871 bool IsKnown;
3872 if (AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
3873 return Base::updateImpl(A);
3874
3875 // If the argument is never passed through callbacks, no-alias cannot break
3876 // synchronization.
3877 bool UsedAssumedInformation = false;
3878 if (A.checkForAllCallSites(
3879 Pred: [](AbstractCallSite ACS) { return !ACS.isCallbackCall(); }, QueryingAA: *this,
3880 RequireAllCallSites: true, UsedAssumedInformation))
3881 return Base::updateImpl(A);
3882
3883 // TODO: add no-alias but make sure it doesn't break synchronization by
3884 // introducing fake uses. See:
3885 // [1] Compiler Optimizations for OpenMP, J. Doerfert and H. Finkel,
3886 // International Workshop on OpenMP 2018,
3887 // http://compilers.cs.uni-saarland.de/people/doerfert/par_opt18.pdf
3888
3889 return indicatePessimisticFixpoint();
3890 }
3891
3892 /// See AbstractAttribute::trackStatistics()
3893 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noalias) }
3894};
3895
3896struct AANoAliasCallSiteArgument final : AANoAliasImpl {
3897 AANoAliasCallSiteArgument(const IRPosition &IRP, Attributor &A)
3898 : AANoAliasImpl(IRP, A) {}
3899
3900 /// Determine if the underlying value may alias with the call site argument
3901 /// \p OtherArgNo of \p ICS (= the underlying call site).
3902 bool mayAliasWithArgument(Attributor &A, AAResults *&AAR,
3903 const AAMemoryBehavior &MemBehaviorAA,
3904 const CallBase &CB, unsigned OtherArgNo) {
3905 // We do not need to worry about aliasing with the underlying IRP.
3906 if (this->getCallSiteArgNo() == (int)OtherArgNo)
3907 return false;
3908
3909 // If it is not a pointer or pointer vector we do not alias.
3910 const Value *ArgOp = CB.getArgOperand(i: OtherArgNo);
3911 if (!ArgOp->getType()->isPtrOrPtrVectorTy())
3912 return false;
3913
3914 auto *CBArgMemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
3915 QueryingAA: *this, IRP: IRPosition::callsite_argument(CB, ArgNo: OtherArgNo), DepClass: DepClassTy::NONE);
3916
3917 // If the argument is readnone, there is no read-write aliasing.
3918 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadNone()) {
3919 A.recordDependence(FromAA: *CBArgMemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3920 return false;
3921 }
3922
3923 // If the argument is readonly and the underlying value is readonly, there
3924 // is no read-write aliasing.
3925 bool IsReadOnly = MemBehaviorAA.isAssumedReadOnly();
3926 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadOnly() &&
3927 IsReadOnly) {
3928 A.recordDependence(FromAA: MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3929 A.recordDependence(FromAA: *CBArgMemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
3930 return false;
3931 }
3932
3933 // We have to utilize actual alias analysis queries so we need the object.
3934 if (!AAR)
3935 AAR = A.getInfoCache().getAnalysisResultForFunction<AAManager>(
3936 F: *getAnchorScope());
3937
3938 // Try to rule it out at the call site.
3939 bool IsAliasing = !AAR || !AAR->isNoAlias(V1: &getAssociatedValue(), V2: ArgOp);
3940 LLVM_DEBUG(dbgs() << "[NoAliasCSArg] Check alias between "
3941 "callsite arguments: "
3942 << getAssociatedValue() << " " << *ArgOp << " => "
3943 << (IsAliasing ? "" : "no-") << "alias \n");
3944
3945 return IsAliasing;
3946 }
3947
3948 bool isKnownNoAliasDueToNoAliasPreservation(
3949 Attributor &A, AAResults *&AAR, const AAMemoryBehavior &MemBehaviorAA) {
3950 // We can deduce "noalias" if the following conditions hold.
3951 // (i) Associated value is assumed to be noalias in the definition.
3952 // (ii) Associated value is assumed to be no-capture in all the uses
3953 // possibly executed before this callsite.
3954 // (iii) There is no other pointer argument which could alias with the
3955 // value.
3956
3957 const IRPosition &VIRP = IRPosition::value(V: getAssociatedValue());
3958 const Function *ScopeFn = VIRP.getAnchorScope();
3959 // Check whether the value is captured in the scope using AANoCapture.
3960 // Look at CFG and check only uses possibly executed before this
3961 // callsite.
3962 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
3963 Instruction *UserI = cast<Instruction>(Val: U.getUser());
3964
3965 // If UserI is the curr instruction and there is a single potential use of
3966 // the value in UserI we allow the use.
3967 // TODO: We should inspect the operands and allow those that cannot alias
3968 // with the value.
3969 if (UserI == getCtxI() && UserI->getNumOperands() == 1)
3970 return true;
3971
3972 if (ScopeFn) {
3973 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
3974 if (CB->isArgOperand(U: &U)) {
3975
3976 unsigned ArgNo = CB->getArgOperandNo(U: &U);
3977
3978 bool IsKnownNoCapture;
3979 if (AA::hasAssumedIRAttr<Attribute::Captures>(
3980 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
3981 DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture))
3982 return true;
3983 }
3984 }
3985
3986 if (!AA::isPotentiallyReachable(
3987 A, FromI: *UserI, ToI: *getCtxI(), QueryingAA: *this, /* ExclusionSet */ nullptr,
3988 GoBackwardsCB: [ScopeFn](const Function &Fn) { return &Fn != ScopeFn; }))
3989 return true;
3990 }
3991
3992 // TODO: We should track the capturing uses in AANoCapture but the problem
3993 // is CGSCC runs. For those we would need to "allow" AANoCapture for
3994 // a value in the module slice.
3995 // TODO(captures): Make this more precise.
3996 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
3997 if (capturesNothing(CC: CI))
3998 return true;
3999 if (CI.isPassthrough()) {
4000 Follow = true;
4001 return true;
4002 }
4003 LLVM_DEBUG(dbgs() << "[AANoAliasCSArg] Unknown user: " << *UserI << "\n");
4004 return false;
4005 };
4006
4007 bool IsKnownNoCapture;
4008 const AANoCapture *NoCaptureAA = nullptr;
4009 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
4010 A, QueryingAA: this, IRP: VIRP, DepClass: DepClassTy::NONE, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false, AAPtr: &NoCaptureAA);
4011 if (!IsAssumedNoCapture &&
4012 (!NoCaptureAA || !NoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
4013 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: getAssociatedValue())) {
4014 LLVM_DEBUG(
4015 dbgs() << "[AANoAliasCSArg] " << getAssociatedValue()
4016 << " cannot be noalias as it is potentially captured\n");
4017 return false;
4018 }
4019 }
4020 if (NoCaptureAA)
4021 A.recordDependence(FromAA: *NoCaptureAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
4022
4023 // Check there is no other pointer argument which could alias with the
4024 // value passed at this call site.
4025 // TODO: AbstractCallSite
4026 const auto &CB = cast<CallBase>(Val&: getAnchorValue());
4027 for (unsigned OtherArgNo = 0; OtherArgNo < CB.arg_size(); OtherArgNo++)
4028 if (mayAliasWithArgument(A, AAR, MemBehaviorAA, CB, OtherArgNo))
4029 return false;
4030
4031 return true;
4032 }
4033
4034 /// See AbstractAttribute::updateImpl(...).
4035 ChangeStatus updateImpl(Attributor &A) override {
4036 // If the argument is readnone we are done as there are no accesses via the
4037 // argument.
4038 auto *MemBehaviorAA =
4039 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::NONE);
4040 if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
4041 A.recordDependence(FromAA: *MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
4042 return ChangeStatus::UNCHANGED;
4043 }
4044
4045 bool IsKnownNoAlias;
4046 const IRPosition &VIRP = IRPosition::value(V: getAssociatedValue());
4047 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
4048 A, QueryingAA: this, IRP: VIRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoAlias)) {
4049 LLVM_DEBUG(dbgs() << "[AANoAlias] " << getAssociatedValue()
4050 << " is not no-alias at the definition\n");
4051 return indicatePessimisticFixpoint();
4052 }
4053
4054 AAResults *AAR = nullptr;
4055 if (MemBehaviorAA &&
4056 isKnownNoAliasDueToNoAliasPreservation(A, AAR, MemBehaviorAA: *MemBehaviorAA)) {
4057 LLVM_DEBUG(
4058 dbgs() << "[AANoAlias] No-Alias deduced via no-alias preservation\n");
4059 return ChangeStatus::UNCHANGED;
4060 }
4061
4062 return indicatePessimisticFixpoint();
4063 }
4064
4065 /// See AbstractAttribute::trackStatistics()
4066 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noalias) }
4067};
4068
4069/// NoAlias attribute for function return value.
4070struct AANoAliasReturned final : AANoAliasImpl {
4071 AANoAliasReturned(const IRPosition &IRP, Attributor &A)
4072 : AANoAliasImpl(IRP, A) {}
4073
4074 /// See AbstractAttribute::updateImpl(...).
4075 ChangeStatus updateImpl(Attributor &A) override {
4076
4077 auto CheckReturnValue = [&](Value &RV) -> bool {
4078 if (Constant *C = dyn_cast<Constant>(Val: &RV))
4079 if (C->isNullValue() || isa<UndefValue>(Val: C))
4080 return true;
4081
4082 /// For now, we can only deduce noalias if we have call sites.
4083 /// FIXME: add more support.
4084 if (!isa<CallBase>(Val: &RV))
4085 return false;
4086
4087 const IRPosition &RVPos = IRPosition::value(V: RV);
4088 bool IsKnownNoAlias;
4089 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
4090 A, QueryingAA: this, IRP: RVPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoAlias))
4091 return false;
4092
4093 bool IsKnownNoCapture;
4094 const AANoCapture *NoCaptureAA = nullptr;
4095 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
4096 A, QueryingAA: this, IRP: RVPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
4097 AAPtr: &NoCaptureAA);
4098 return IsAssumedNoCapture ||
4099 (NoCaptureAA && NoCaptureAA->isAssumedNoCaptureMaybeReturned());
4100 };
4101
4102 if (!A.checkForAllReturnedValues(Pred: CheckReturnValue, QueryingAA: *this))
4103 return indicatePessimisticFixpoint();
4104
4105 return ChangeStatus::UNCHANGED;
4106 }
4107
4108 /// See AbstractAttribute::trackStatistics()
4109 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noalias) }
4110};
4111
4112/// NoAlias attribute deduction for a call site return value.
4113struct AANoAliasCallSiteReturned final
4114 : AACalleeToCallSite<AANoAlias, AANoAliasImpl> {
4115 AANoAliasCallSiteReturned(const IRPosition &IRP, Attributor &A)
4116 : AACalleeToCallSite<AANoAlias, AANoAliasImpl>(IRP, A) {}
4117
4118 /// See AbstractAttribute::trackStatistics()
4119 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noalias); }
4120};
4121} // namespace
4122
4123/// -------------------AAIsDead Function Attribute-----------------------
4124
4125namespace {
4126struct AAIsDeadValueImpl : public AAIsDead {
4127 AAIsDeadValueImpl(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4128
4129 /// See AAIsDead::isAssumedDead().
4130 bool isAssumedDead() const override { return isAssumed(BitsEncoding: IS_DEAD); }
4131
4132 /// See AAIsDead::isKnownDead().
4133 bool isKnownDead() const override { return isKnown(BitsEncoding: IS_DEAD); }
4134
4135 /// See AAIsDead::isAssumedDead(BasicBlock *).
4136 bool isAssumedDead(const BasicBlock *BB) const override { return false; }
4137
4138 /// See AAIsDead::isKnownDead(BasicBlock *).
4139 bool isKnownDead(const BasicBlock *BB) const override { return false; }
4140
4141 /// See AAIsDead::isAssumedDead(Instruction *I).
4142 bool isAssumedDead(const Instruction *I) const override {
4143 return I == getCtxI() && isAssumedDead();
4144 }
4145
4146 /// See AAIsDead::isKnownDead(Instruction *I).
4147 bool isKnownDead(const Instruction *I) const override {
4148 return isAssumedDead(I) && isKnownDead();
4149 }
4150
4151 /// See AbstractAttribute::getAsStr().
4152 const std::string getAsStr(Attributor *A) const override {
4153 return isAssumedDead() ? "assumed-dead" : "assumed-live";
4154 }
4155
4156 /// Check if all uses are assumed dead.
4157 bool areAllUsesAssumedDead(Attributor &A, Value &V) {
4158 // Callers might not check the type, void has no uses.
4159 if (V.getType()->isVoidTy() || V.use_empty())
4160 return true;
4161
4162 // If we replace a value with a constant there are no uses left afterwards.
4163 if (!isa<Constant>(Val: V)) {
4164 if (auto *I = dyn_cast<Instruction>(Val: &V))
4165 if (!A.isRunOn(Fn&: *I->getFunction()))
4166 return false;
4167 bool UsedAssumedInformation = false;
4168 std::optional<Constant *> C =
4169 A.getAssumedConstant(V, AA: *this, UsedAssumedInformation);
4170 if (!C || *C)
4171 return true;
4172 }
4173
4174 auto UsePred = [&](const Use &U, bool &Follow) { return false; };
4175 // Explicitly set the dependence class to required because we want a long
4176 // chain of N dependent instructions to be considered live as soon as one is
4177 // without going through N update cycles. This is not required for
4178 // correctness.
4179 return A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V, /* CheckBBLivenessOnly */ false,
4180 LivenessDepClass: DepClassTy::REQUIRED,
4181 /* IgnoreDroppableUses */ false);
4182 }
4183
4184 /// Determine if \p I is assumed to be side-effect free.
4185 bool isAssumedSideEffectFree(Attributor &A, Instruction *I) {
4186 if (!I || wouldInstructionBeTriviallyDead(I))
4187 return true;
4188
4189 if (!I->isTerminator() && !I->mayHaveSideEffects())
4190 return true;
4191
4192 auto *CB = dyn_cast<CallBase>(Val: I);
4193 if (!CB || isa<IntrinsicInst>(Val: CB))
4194 return false;
4195
4196 const IRPosition &CallIRP = IRPosition::callsite_function(CB: *CB);
4197
4198 bool IsKnownNoUnwind;
4199 if (!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
4200 A, QueryingAA: this, IRP: CallIRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoUnwind))
4201 return false;
4202
4203 bool IsKnown;
4204 return AA::isAssumedReadOnly(A, IRP: CallIRP, QueryingAA: *this, IsKnown);
4205 }
4206};
4207
4208struct AAIsDeadFloating : public AAIsDeadValueImpl {
4209 AAIsDeadFloating(const IRPosition &IRP, Attributor &A)
4210 : AAIsDeadValueImpl(IRP, A) {}
4211
4212 /// See AbstractAttribute::initialize(...).
4213 void initialize(Attributor &A) override {
4214 AAIsDeadValueImpl::initialize(A);
4215
4216 if (isa<UndefValue>(Val: getAssociatedValue())) {
4217 indicatePessimisticFixpoint();
4218 return;
4219 }
4220
4221 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4222 if (!isAssumedSideEffectFree(A, I)) {
4223 if (!isa_and_nonnull<StoreInst>(Val: I) && !isa_and_nonnull<FenceInst>(Val: I))
4224 indicatePessimisticFixpoint();
4225 else
4226 removeAssumedBits(BitsEncoding: HAS_NO_EFFECT);
4227 }
4228 }
4229
4230 bool isDeadFence(Attributor &A, FenceInst &FI) {
4231 const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
4232 IRP: IRPosition::function(F: *FI.getFunction()), QueryingAA: *this, DepClass: DepClassTy::NONE);
4233 if (!ExecDomainAA || !ExecDomainAA->isNoOpFence(FI))
4234 return false;
4235 A.recordDependence(FromAA: *ExecDomainAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
4236 return true;
4237 }
4238
4239 bool isDeadStore(Attributor &A, StoreInst &SI,
4240 SmallSetVector<Instruction *, 8> *AssumeOnlyInst = nullptr) {
4241 // Lang ref now states volatile store is not UB/dead, let's skip them.
4242 if (SI.isVolatile())
4243 return false;
4244
4245 // If we are collecting assumes to be deleted we are in the manifest stage.
4246 // It's problematic to collect the potential copies again now so we use the
4247 // cached ones.
4248 bool UsedAssumedInformation = false;
4249 if (!AssumeOnlyInst) {
4250 PotentialCopies.clear();
4251 if (!AA::getPotentialCopiesOfStoredValue(A, SI, PotentialCopies, QueryingAA: *this,
4252 UsedAssumedInformation)) {
4253 LLVM_DEBUG(
4254 dbgs()
4255 << "[AAIsDead] Could not determine potential copies of store!\n");
4256 return false;
4257 }
4258 }
4259 LLVM_DEBUG(dbgs() << "[AAIsDead] Store has " << PotentialCopies.size()
4260 << " potential copies.\n");
4261
4262 InformationCache &InfoCache = A.getInfoCache();
4263 return llvm::all_of(Range&: PotentialCopies, P: [&](Value *V) {
4264 if (A.isAssumedDead(IRP: IRPosition::value(V: *V), QueryingAA: this, FnLivenessAA: nullptr,
4265 UsedAssumedInformation))
4266 return true;
4267 if (auto *LI = dyn_cast<LoadInst>(Val: V)) {
4268 if (llvm::all_of(Range: LI->uses(), P: [&](const Use &U) {
4269 auto &UserI = cast<Instruction>(Val&: *U.getUser());
4270 if (InfoCache.isOnlyUsedByAssume(I: UserI)) {
4271 if (AssumeOnlyInst)
4272 AssumeOnlyInst->insert(X: &UserI);
4273 return true;
4274 }
4275 return A.isAssumedDead(U, QueryingAA: this, FnLivenessAA: nullptr, UsedAssumedInformation);
4276 })) {
4277 return true;
4278 }
4279 }
4280 LLVM_DEBUG(dbgs() << "[AAIsDead] Potential copy " << *V
4281 << " is assumed live!\n");
4282 return false;
4283 });
4284 }
4285
4286 /// See AbstractAttribute::getAsStr().
4287 const std::string getAsStr(Attributor *A) const override {
4288 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4289 if (isa_and_nonnull<StoreInst>(Val: I))
4290 if (isValidState())
4291 return "assumed-dead-store";
4292 if (isa_and_nonnull<FenceInst>(Val: I))
4293 if (isValidState())
4294 return "assumed-dead-fence";
4295 return AAIsDeadValueImpl::getAsStr(A);
4296 }
4297
4298 /// See AbstractAttribute::updateImpl(...).
4299 ChangeStatus updateImpl(Attributor &A) override {
4300 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
4301 if (auto *SI = dyn_cast_or_null<StoreInst>(Val: I)) {
4302 if (!isDeadStore(A, SI&: *SI))
4303 return indicatePessimisticFixpoint();
4304 } else if (auto *FI = dyn_cast_or_null<FenceInst>(Val: I)) {
4305 if (!isDeadFence(A, FI&: *FI))
4306 return indicatePessimisticFixpoint();
4307 } else {
4308 if (!isAssumedSideEffectFree(A, I))
4309 return indicatePessimisticFixpoint();
4310 if (!areAllUsesAssumedDead(A, V&: getAssociatedValue()))
4311 return indicatePessimisticFixpoint();
4312 }
4313 return ChangeStatus::UNCHANGED;
4314 }
4315
4316 bool isRemovableStore() const override {
4317 return isAssumed(BitsEncoding: IS_REMOVABLE) && isa<StoreInst>(Val: &getAssociatedValue());
4318 }
4319
4320 /// See AbstractAttribute::manifest(...).
4321 ChangeStatus manifest(Attributor &A) override {
4322 Value &V = getAssociatedValue();
4323 if (auto *I = dyn_cast<Instruction>(Val: &V)) {
4324 // If we get here we basically know the users are all dead. We check if
4325 // isAssumedSideEffectFree returns true here again because it might not be
4326 // the case and only the users are dead but the instruction (=call) is
4327 // still needed.
4328 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
4329 SmallSetVector<Instruction *, 8> AssumeOnlyInst;
4330 bool IsDead = isDeadStore(A, SI&: *SI, AssumeOnlyInst: &AssumeOnlyInst);
4331 (void)IsDead;
4332 assert(IsDead && "Store was assumed to be dead!");
4333 A.deleteAfterManifest(I&: *I);
4334 for (size_t i = 0; i < AssumeOnlyInst.size(); ++i) {
4335 Instruction *AOI = AssumeOnlyInst[i];
4336 for (auto *Usr : AOI->users())
4337 AssumeOnlyInst.insert(X: cast<Instruction>(Val: Usr));
4338 A.deleteAfterManifest(I&: *AOI);
4339 }
4340 return ChangeStatus::CHANGED;
4341 }
4342 if (auto *FI = dyn_cast<FenceInst>(Val: I)) {
4343 assert(isDeadFence(A, *FI));
4344 A.deleteAfterManifest(I&: *FI);
4345 return ChangeStatus::CHANGED;
4346 }
4347 if (isAssumedSideEffectFree(A, I) && !I->isTerminator()) {
4348 A.deleteAfterManifest(I&: *I);
4349 return ChangeStatus::CHANGED;
4350 }
4351 }
4352 return ChangeStatus::UNCHANGED;
4353 }
4354
4355 /// See AbstractAttribute::trackStatistics()
4356 void trackStatistics() const override {
4357 STATS_DECLTRACK_FLOATING_ATTR(IsDead)
4358 }
4359
4360private:
4361 // The potential copies of a dead store, used for deletion during manifest.
4362 SmallSetVector<Value *, 4> PotentialCopies;
4363};
4364
4365struct AAIsDeadArgument : public AAIsDeadFloating {
4366 AAIsDeadArgument(const IRPosition &IRP, Attributor &A)
4367 : AAIsDeadFloating(IRP, A) {}
4368
4369 /// See AbstractAttribute::manifest(...).
4370 ChangeStatus manifest(Attributor &A) override {
4371 Argument &Arg = *getAssociatedArgument();
4372 if (A.isValidFunctionSignatureRewrite(Arg, /* ReplacementTypes */ {}))
4373 if (A.registerFunctionSignatureRewrite(
4374 Arg, /* ReplacementTypes */ {},
4375 CalleeRepairCB: Attributor::ArgumentReplacementInfo::CalleeRepairCBTy{},
4376 ACSRepairCB: Attributor::ArgumentReplacementInfo::ACSRepairCBTy{})) {
4377 return ChangeStatus::CHANGED;
4378 }
4379 return ChangeStatus::UNCHANGED;
4380 }
4381
4382 /// See AbstractAttribute::trackStatistics()
4383 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(IsDead) }
4384};
4385
4386struct AAIsDeadCallSiteArgument : public AAIsDeadValueImpl {
4387 AAIsDeadCallSiteArgument(const IRPosition &IRP, Attributor &A)
4388 : AAIsDeadValueImpl(IRP, A) {}
4389
4390 /// See AbstractAttribute::initialize(...).
4391 void initialize(Attributor &A) override {
4392 AAIsDeadValueImpl::initialize(A);
4393 if (isa<UndefValue>(Val: getAssociatedValue()))
4394 indicatePessimisticFixpoint();
4395 }
4396
4397 /// See AbstractAttribute::updateImpl(...).
4398 ChangeStatus updateImpl(Attributor &A) override {
4399 // TODO: Once we have call site specific value information we can provide
4400 // call site specific liveness information and then it makes
4401 // sense to specialize attributes for call sites arguments instead of
4402 // redirecting requests to the callee argument.
4403 Argument *Arg = getAssociatedArgument();
4404 if (!Arg)
4405 return indicatePessimisticFixpoint();
4406 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
4407 auto *ArgAA = A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
4408 if (!ArgAA)
4409 return indicatePessimisticFixpoint();
4410 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
4411 }
4412
4413 /// See AbstractAttribute::manifest(...).
4414 ChangeStatus manifest(Attributor &A) override {
4415 CallBase &CB = cast<CallBase>(Val&: getAnchorValue());
4416 Use &U = CB.getArgOperandUse(i: getCallSiteArgNo());
4417 assert(!isa<UndefValue>(U.get()) &&
4418 "Expected undef values to be filtered out!");
4419 UndefValue &UV = *UndefValue::get(T: U->getType());
4420 if (A.changeUseAfterManifest(U, NV&: UV))
4421 return ChangeStatus::CHANGED;
4422 return ChangeStatus::UNCHANGED;
4423 }
4424
4425 /// See AbstractAttribute::trackStatistics()
4426 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(IsDead) }
4427};
4428
4429struct AAIsDeadCallSiteReturned : public AAIsDeadFloating {
4430 AAIsDeadCallSiteReturned(const IRPosition &IRP, Attributor &A)
4431 : AAIsDeadFloating(IRP, A) {}
4432
4433 /// See AAIsDead::isAssumedDead().
4434 bool isAssumedDead() const override {
4435 return AAIsDeadFloating::isAssumedDead() && IsAssumedSideEffectFree;
4436 }
4437
4438 /// See AbstractAttribute::initialize(...).
4439 void initialize(Attributor &A) override {
4440 AAIsDeadFloating::initialize(A);
4441 if (isa<UndefValue>(Val: getAssociatedValue())) {
4442 indicatePessimisticFixpoint();
4443 return;
4444 }
4445
4446 // We track this separately as a secondary state.
4447 IsAssumedSideEffectFree = isAssumedSideEffectFree(A, I: getCtxI());
4448 }
4449
4450 /// See AbstractAttribute::updateImpl(...).
4451 ChangeStatus updateImpl(Attributor &A) override {
4452 ChangeStatus Changed = ChangeStatus::UNCHANGED;
4453 if (IsAssumedSideEffectFree && !isAssumedSideEffectFree(A, I: getCtxI())) {
4454 IsAssumedSideEffectFree = false;
4455 Changed = ChangeStatus::CHANGED;
4456 }
4457 if (!areAllUsesAssumedDead(A, V&: getAssociatedValue()))
4458 return indicatePessimisticFixpoint();
4459 return Changed;
4460 }
4461
4462 /// See AbstractAttribute::trackStatistics()
4463 void trackStatistics() const override {
4464 if (IsAssumedSideEffectFree)
4465 STATS_DECLTRACK_CSRET_ATTR(IsDead)
4466 else
4467 STATS_DECLTRACK_CSRET_ATTR(UnusedResult)
4468 }
4469
4470 /// See AbstractAttribute::getAsStr().
4471 const std::string getAsStr(Attributor *A) const override {
4472 return isAssumedDead()
4473 ? "assumed-dead"
4474 : (getAssumed() ? "assumed-dead-users" : "assumed-live");
4475 }
4476
4477private:
4478 bool IsAssumedSideEffectFree = true;
4479};
4480
4481struct AAIsDeadReturned : public AAIsDeadValueImpl {
4482 AAIsDeadReturned(const IRPosition &IRP, Attributor &A)
4483 : AAIsDeadValueImpl(IRP, A) {}
4484
4485 /// See AbstractAttribute::updateImpl(...).
4486 ChangeStatus updateImpl(Attributor &A) override {
4487
4488 bool UsedAssumedInformation = false;
4489 A.checkForAllInstructions(Pred: [](Instruction &) { return true; }, QueryingAA: *this,
4490 Opcodes: {Instruction::Ret}, UsedAssumedInformation);
4491
4492 auto PredForCallSite = [&](AbstractCallSite ACS) {
4493 if (ACS.isCallbackCall() || !ACS.getInstruction())
4494 return false;
4495 return areAllUsesAssumedDead(A, V&: *ACS.getInstruction());
4496 };
4497
4498 if (!A.checkForAllCallSites(Pred: PredForCallSite, QueryingAA: *this, RequireAllCallSites: true,
4499 UsedAssumedInformation))
4500 return indicatePessimisticFixpoint();
4501
4502 return ChangeStatus::UNCHANGED;
4503 }
4504
4505 /// See AbstractAttribute::manifest(...).
4506 ChangeStatus manifest(Attributor &A) override {
4507 // TODO: Rewrite the signature to return void?
4508 bool AnyChange = false;
4509 UndefValue &UV = *UndefValue::get(T: getAssociatedFunction()->getReturnType());
4510 auto RetInstPred = [&](Instruction &I) {
4511 ReturnInst &RI = cast<ReturnInst>(Val&: I);
4512 if (!isa<UndefValue>(Val: RI.getReturnValue()))
4513 AnyChange |= A.changeUseAfterManifest(U&: RI.getOperandUse(i: 0), NV&: UV);
4514 return true;
4515 };
4516 bool UsedAssumedInformation = false;
4517 A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
4518 UsedAssumedInformation);
4519 return AnyChange ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
4520 }
4521
4522 /// See AbstractAttribute::trackStatistics()
4523 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(IsDead) }
4524};
4525
4526struct AAIsDeadFunction : public AAIsDead {
4527 AAIsDeadFunction(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
4528
4529 /// See AbstractAttribute::initialize(...).
4530 void initialize(Attributor &A) override {
4531 Function *F = getAnchorScope();
4532 assert(F && "Did expect an anchor function");
4533 if (!isAssumedDeadInternalFunction(A)) {
4534 ToBeExploredFrom.insert(X: &F->getEntryBlock().front());
4535 assumeLive(A, BB: F->getEntryBlock());
4536 }
4537 }
4538
4539 bool isAssumedDeadInternalFunction(Attributor &A) {
4540 if (!getAnchorScope()->hasLocalLinkage())
4541 return false;
4542 // An AA that will not be updated is fixed pessimistically right after
4543 // initialization, so an assumed-dead result would be discarded. Checking
4544 // the call sites recursively initializes the liveness of every internal
4545 // caller, which in CGSCC cleanup repeats for each callee SCC.
4546 if (!A.shouldUpdateAA<AAIsDead>(IRP: getIRPosition()))
4547 return false;
4548 bool UsedAssumedInformation = false;
4549 return A.checkForAllCallSites(Pred: [](AbstractCallSite) { return false; }, QueryingAA: *this,
4550 RequireAllCallSites: true, UsedAssumedInformation);
4551 }
4552
4553 /// See AbstractAttribute::getAsStr().
4554 const std::string getAsStr(Attributor *A) const override {
4555 return "Live[#BB " + std::to_string(val: AssumedLiveBlocks.size()) + "/" +
4556 std::to_string(val: getAnchorScope()->size()) + "][#TBEP " +
4557 std::to_string(val: ToBeExploredFrom.size()) + "][#KDE " +
4558 std::to_string(val: KnownDeadEnds.size()) + "]";
4559 }
4560
4561 /// See AbstractAttribute::manifest(...).
4562 ChangeStatus manifest(Attributor &A) override {
4563 assert(getState().isValidState() &&
4564 "Attempted to manifest an invalid state!");
4565
4566 ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
4567 Function &F = *getAnchorScope();
4568
4569 if (AssumedLiveBlocks.empty()) {
4570 A.deleteAfterManifest(F);
4571 return ChangeStatus::CHANGED;
4572 }
4573
4574 // Flag to determine if we can change an invoke to a call assuming the
4575 // callee is nounwind. This is not possible if the personality of the
4576 // function allows to catch asynchronous exceptions.
4577 bool Invoke2CallAllowed = !mayCatchAsynchronousExceptions(F);
4578
4579 KnownDeadEnds.set_union(ToBeExploredFrom);
4580 for (const Instruction *DeadEndI : KnownDeadEnds) {
4581 auto *CB = dyn_cast<CallBase>(Val: DeadEndI);
4582 if (!CB)
4583 continue;
4584 bool IsKnownNoReturn;
4585 bool MayReturn = !AA::hasAssumedIRAttr<Attribute::NoReturn>(
4586 A, QueryingAA: this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL,
4587 IsKnown&: IsKnownNoReturn);
4588 if (MayReturn && (!Invoke2CallAllowed || !isa<InvokeInst>(Val: CB)))
4589 continue;
4590
4591 if (auto *II = dyn_cast<InvokeInst>(Val: DeadEndI))
4592 A.registerInvokeWithDeadSuccessor(II&: const_cast<InvokeInst &>(*II));
4593 else
4594 A.changeToUnreachableAfterManifest(
4595 I: const_cast<Instruction *>(DeadEndI->getNextNode()));
4596 HasChanged = ChangeStatus::CHANGED;
4597 }
4598
4599 STATS_DECL(AAIsDead, BasicBlock, "Number of dead basic blocks deleted.");
4600 for (BasicBlock &BB : F)
4601 if (!AssumedLiveBlocks.count(V: &BB)) {
4602 A.deleteAfterManifest(BB);
4603 ++BUILD_STAT_NAME(AAIsDead, BasicBlock);
4604 HasChanged = ChangeStatus::CHANGED;
4605 }
4606
4607 return HasChanged;
4608 }
4609
4610 /// See AbstractAttribute::updateImpl(...).
4611 ChangeStatus updateImpl(Attributor &A) override;
4612
4613 bool isEdgeDead(const BasicBlock *From, const BasicBlock *To) const override {
4614 assert(From->getParent() == getAnchorScope() &&
4615 To->getParent() == getAnchorScope() &&
4616 "Used AAIsDead of the wrong function");
4617 return isValidState() && !AssumedLiveEdges.count(V: std::make_pair(x&: From, y&: To));
4618 }
4619
4620 /// See AbstractAttribute::trackStatistics()
4621 void trackStatistics() const override {}
4622
4623 /// Returns true if the function is assumed dead.
4624 bool isAssumedDead() const override { return false; }
4625
4626 /// See AAIsDead::isKnownDead().
4627 bool isKnownDead() const override { return false; }
4628
4629 /// See AAIsDead::isAssumedDead(BasicBlock *).
4630 bool isAssumedDead(const BasicBlock *BB) const override {
4631 assert(BB->getParent() == getAnchorScope() &&
4632 "BB must be in the same anchor scope function.");
4633
4634 if (!getAssumed())
4635 return false;
4636 return !AssumedLiveBlocks.count(V: BB);
4637 }
4638
4639 /// See AAIsDead::isKnownDead(BasicBlock *).
4640 bool isKnownDead(const BasicBlock *BB) const override {
4641 return getKnown() && isAssumedDead(BB);
4642 }
4643
4644 /// See AAIsDead::isAssumed(Instruction *I).
4645 bool isAssumedDead(const Instruction *I) const override {
4646 assert(I->getParent()->getParent() == getAnchorScope() &&
4647 "Instruction must be in the same anchor scope function.");
4648
4649 if (!getAssumed())
4650 return false;
4651
4652 // If it is not in AssumedLiveBlocks then it for sure dead.
4653 // Otherwise, it can still be after noreturn call in a live block.
4654 if (!AssumedLiveBlocks.count(V: I->getParent()))
4655 return true;
4656
4657 // If it is not after a liveness barrier it is live.
4658 const Instruction *PrevI = I->getPrevNode();
4659 while (PrevI) {
4660 if (KnownDeadEnds.count(key: PrevI) || ToBeExploredFrom.count(key: PrevI))
4661 return true;
4662 PrevI = PrevI->getPrevNode();
4663 }
4664 return false;
4665 }
4666
4667 /// See AAIsDead::isKnownDead(Instruction *I).
4668 bool isKnownDead(const Instruction *I) const override {
4669 return getKnown() && isAssumedDead(I);
4670 }
4671
4672 /// Assume \p BB is (partially) live now and indicate to the Attributor \p A
4673 /// that internal function called from \p BB should now be looked at.
4674 bool assumeLive(Attributor &A, const BasicBlock &BB) {
4675 if (!AssumedLiveBlocks.insert(V: &BB).second)
4676 return false;
4677
4678 if (!A.isDuringDeduction())
4679 return true;
4680
4681 // We assume that all of BB is (probably) live now and if there are calls to
4682 // internal functions we will assume that those are now live as well. This
4683 // is a performance optimization for blocks with calls to a lot of internal
4684 // functions. It can however cause dead functions to be treated as live.
4685 for (const Instruction &I : BB)
4686 if (const auto *CB = dyn_cast<CallBase>(Val: &I))
4687 if (auto *F = dyn_cast_if_present<Function>(Val: CB->getCalledOperand()))
4688 if (F->hasLocalLinkage()) {
4689 LLVM_DEBUG({
4690 dbgs() << "[AAIsDead] Seeding live internal callee ";
4691 F->printAsOperand(dbgs(), /*PrintType=*/false);
4692 dbgs() << " from ";
4693 BB.getParent()->printAsOperand(dbgs(), /*PrintType=*/false);
4694 dbgs() << "\n";
4695 });
4696 A.markLiveInternalFunction(F: *F);
4697 }
4698 return true;
4699 }
4700
4701 /// Collection of instructions that need to be explored again, e.g., we
4702 /// did assume they do not transfer control to (one of their) successors.
4703 SmallSetVector<const Instruction *, 8> ToBeExploredFrom;
4704
4705 /// Collection of instructions that are known to not transfer control.
4706 SmallSetVector<const Instruction *, 8> KnownDeadEnds;
4707
4708 /// Collection of all assumed live edges
4709 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> AssumedLiveEdges;
4710
4711 /// Collection of all assumed live BasicBlocks.
4712 DenseSet<const BasicBlock *> AssumedLiveBlocks;
4713};
4714
4715static bool
4716identifyAliveSuccessors(Attributor &A, const CallBase &CB,
4717 AbstractAttribute &AA,
4718 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4719 const IRPosition &IPos = IRPosition::callsite_function(CB);
4720
4721 bool IsKnownNoReturn;
4722 if (AA::hasAssumedIRAttr<Attribute::NoReturn>(
4723 A, QueryingAA: &AA, IRP: IPos, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoReturn))
4724 return !IsKnownNoReturn;
4725 if (CB.isTerminator())
4726 AliveSuccessors.push_back(Elt: &CB.getSuccessor(Idx: 0)->front());
4727 else
4728 AliveSuccessors.push_back(Elt: CB.getNextNode());
4729 return false;
4730}
4731
4732static bool
4733identifyAliveSuccessors(Attributor &A, const InvokeInst &II,
4734 AbstractAttribute &AA,
4735 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4736 bool UsedAssumedInformation =
4737 identifyAliveSuccessors(A, CB: cast<CallBase>(Val: II), AA, AliveSuccessors);
4738
4739 // First, determine if we can change an invoke to a call assuming the
4740 // callee is nounwind. This is not possible if the personality of the
4741 // function allows to catch asynchronous exceptions.
4742 if (AAIsDeadFunction::mayCatchAsynchronousExceptions(F: *II.getFunction())) {
4743 AliveSuccessors.push_back(Elt: &II.getUnwindDest()->front());
4744 } else {
4745 const IRPosition &IPos = IRPosition::callsite_function(CB: II);
4746
4747 bool IsKnownNoUnwind;
4748 if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
4749 A, QueryingAA: &AA, IRP: IPos, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoUnwind)) {
4750 UsedAssumedInformation |= !IsKnownNoUnwind;
4751 } else {
4752 AliveSuccessors.push_back(Elt: &II.getUnwindDest()->front());
4753 }
4754 }
4755 return UsedAssumedInformation;
4756}
4757
4758static bool
4759identifyAliveSuccessors(Attributor &, const UncondBrInst &BI,
4760 AbstractAttribute &,
4761 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4762 AliveSuccessors.push_back(Elt: &BI.getSuccessor()->front());
4763 return false;
4764}
4765
4766static bool
4767identifyAliveSuccessors(Attributor &A, const CondBrInst &BI,
4768 AbstractAttribute &AA,
4769 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4770 bool UsedAssumedInformation = false;
4771 std::optional<Constant *> C =
4772 A.getAssumedConstant(V: *BI.getCondition(), AA, UsedAssumedInformation);
4773 if (!C || isa_and_nonnull<UndefValue>(Val: *C)) {
4774 // No value yet, assume both edges are dead.
4775 } else if (isa_and_nonnull<ConstantInt>(Val: *C)) {
4776 const BasicBlock *SuccBB =
4777 BI.getSuccessor(i: 1 - cast<ConstantInt>(Val: *C)->getValue().getZExtValue());
4778 AliveSuccessors.push_back(Elt: &SuccBB->front());
4779 } else {
4780 AliveSuccessors.push_back(Elt: &BI.getSuccessor(i: 0)->front());
4781 AliveSuccessors.push_back(Elt: &BI.getSuccessor(i: 1)->front());
4782 UsedAssumedInformation = false;
4783 }
4784 return UsedAssumedInformation;
4785}
4786
4787static bool
4788identifyAliveSuccessors(Attributor &A, const SwitchInst &SI,
4789 AbstractAttribute &AA,
4790 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4791 bool UsedAssumedInformation = false;
4792 SmallVector<AA::ValueAndContext> Values;
4793 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *SI.getCondition()), AA: &AA,
4794 Values, S: AA::AnyScope,
4795 UsedAssumedInformation)) {
4796 // Something went wrong, assume all successors are live.
4797 for (const BasicBlock *SuccBB : successors(BB: SI.getParent()))
4798 AliveSuccessors.push_back(Elt: &SuccBB->front());
4799 return false;
4800 }
4801
4802 if (Values.empty() ||
4803 (Values.size() == 1 &&
4804 isa_and_nonnull<UndefValue>(Val: Values.front().getValue()))) {
4805 // No valid value yet, assume all edges are dead.
4806 return UsedAssumedInformation;
4807 }
4808
4809 Type &Ty = *SI.getCondition()->getType();
4810 SmallPtrSet<ConstantInt *, 8> Constants;
4811 auto CheckForConstantInt = [&](Value *V) {
4812 if (auto *CI = dyn_cast_if_present<ConstantInt>(Val: AA::getWithType(V&: *V, Ty))) {
4813 Constants.insert(Ptr: CI);
4814 return true;
4815 }
4816 return false;
4817 };
4818
4819 if (!all_of(Range&: Values, P: [&](AA::ValueAndContext &VAC) {
4820 return CheckForConstantInt(VAC.getValue());
4821 })) {
4822 for (const BasicBlock *SuccBB : successors(BB: SI.getParent()))
4823 AliveSuccessors.push_back(Elt: &SuccBB->front());
4824 return UsedAssumedInformation;
4825 }
4826
4827 unsigned MatchedCases = 0;
4828 for (const auto &CaseIt : SI.cases()) {
4829 if (Constants.count(Ptr: CaseIt.getCaseValue())) {
4830 ++MatchedCases;
4831 AliveSuccessors.push_back(Elt: &CaseIt.getCaseSuccessor()->front());
4832 }
4833 }
4834
4835 // If all potential values have been matched, we will not visit the default
4836 // case.
4837 if (MatchedCases < Constants.size())
4838 AliveSuccessors.push_back(Elt: &SI.getDefaultDest()->front());
4839 return UsedAssumedInformation;
4840}
4841
4842ChangeStatus AAIsDeadFunction::updateImpl(Attributor &A) {
4843 ChangeStatus Change = ChangeStatus::UNCHANGED;
4844
4845 if (AssumedLiveBlocks.empty()) {
4846 if (isAssumedDeadInternalFunction(A))
4847 return ChangeStatus::UNCHANGED;
4848
4849 Function *F = getAnchorScope();
4850 ToBeExploredFrom.insert(X: &F->getEntryBlock().front());
4851 assumeLive(A, BB: F->getEntryBlock());
4852 Change = ChangeStatus::CHANGED;
4853 }
4854
4855 LLVM_DEBUG(dbgs() << "[AAIsDead] Live [" << AssumedLiveBlocks.size() << "/"
4856 << getAnchorScope()->size() << "] BBs and "
4857 << ToBeExploredFrom.size() << " exploration points and "
4858 << KnownDeadEnds.size() << " known dead ends\n");
4859
4860 // Copy and clear the list of instructions we need to explore from. It is
4861 // refilled with instructions the next update has to look at.
4862 SmallVector<const Instruction *, 8> Worklist(ToBeExploredFrom.begin(),
4863 ToBeExploredFrom.end());
4864 decltype(ToBeExploredFrom) NewToBeExploredFrom;
4865
4866 SmallVector<const Instruction *, 8> AliveSuccessors;
4867 while (!Worklist.empty()) {
4868 const Instruction *I = Worklist.pop_back_val();
4869 LLVM_DEBUG(dbgs() << "[AAIsDead] Exploration inst: " << *I << "\n");
4870
4871 // Fast forward for uninteresting instructions. We could look for UB here
4872 // though.
4873 while (!I->isTerminator() && !isa<CallBase>(Val: I))
4874 I = I->getNextNode();
4875
4876 AliveSuccessors.clear();
4877
4878 bool UsedAssumedInformation = false;
4879 switch (I->getOpcode()) {
4880 // TODO: look for (assumed) UB to backwards propagate "deadness".
4881 default:
4882 assert(I->isTerminator() &&
4883 "Expected non-terminators to be handled already!");
4884 for (const BasicBlock *SuccBB : successors(BB: I->getParent()))
4885 AliveSuccessors.push_back(Elt: &SuccBB->front());
4886 break;
4887 case Instruction::Call:
4888 UsedAssumedInformation = identifyAliveSuccessors(A, CB: cast<CallInst>(Val: *I),
4889 AA&: *this, AliveSuccessors);
4890 break;
4891 case Instruction::Invoke:
4892 UsedAssumedInformation = identifyAliveSuccessors(A, II: cast<InvokeInst>(Val: *I),
4893 AA&: *this, AliveSuccessors);
4894 break;
4895 case Instruction::UncondBr:
4896 UsedAssumedInformation = identifyAliveSuccessors(
4897 A, BI: cast<UncondBrInst>(Val: *I), *this, AliveSuccessors);
4898 break;
4899 case Instruction::CondBr:
4900 UsedAssumedInformation = identifyAliveSuccessors(A, BI: cast<CondBrInst>(Val: *I),
4901 AA&: *this, AliveSuccessors);
4902 break;
4903 case Instruction::Switch:
4904 UsedAssumedInformation = identifyAliveSuccessors(A, SI: cast<SwitchInst>(Val: *I),
4905 AA&: *this, AliveSuccessors);
4906 break;
4907 }
4908
4909 if (UsedAssumedInformation) {
4910 NewToBeExploredFrom.insert(X: I);
4911 } else if (AliveSuccessors.empty() ||
4912 (I->isTerminator() &&
4913 AliveSuccessors.size() < I->getNumSuccessors())) {
4914 if (KnownDeadEnds.insert(X: I))
4915 Change = ChangeStatus::CHANGED;
4916 }
4917
4918 LLVM_DEBUG(dbgs() << "[AAIsDead] #AliveSuccessors: "
4919 << AliveSuccessors.size() << " UsedAssumedInformation: "
4920 << UsedAssumedInformation << "\n");
4921
4922 for (const Instruction *AliveSuccessor : AliveSuccessors) {
4923 if (!I->isTerminator()) {
4924 assert(AliveSuccessors.size() == 1 &&
4925 "Non-terminator expected to have a single successor!");
4926 Worklist.push_back(Elt: AliveSuccessor);
4927 } else {
4928 // record the assumed live edge
4929 auto Edge = std::make_pair(x: I->getParent(), y: AliveSuccessor->getParent());
4930 if (AssumedLiveEdges.insert(V: Edge).second)
4931 Change = ChangeStatus::CHANGED;
4932 if (assumeLive(A, BB: *AliveSuccessor->getParent()))
4933 Worklist.push_back(Elt: AliveSuccessor);
4934 }
4935 }
4936 }
4937
4938 // Check if the content of ToBeExploredFrom changed, ignore the order.
4939 if (NewToBeExploredFrom.size() != ToBeExploredFrom.size() ||
4940 llvm::any_of(Range&: NewToBeExploredFrom, P: [&](const Instruction *I) {
4941 return !ToBeExploredFrom.count(key: I);
4942 })) {
4943 Change = ChangeStatus::CHANGED;
4944 ToBeExploredFrom = std::move(NewToBeExploredFrom);
4945 }
4946
4947 // If we know everything is live there is no need to query for liveness.
4948 // Instead, indicating a pessimistic fixpoint will cause the state to be
4949 // "invalid" and all queries to be answered conservatively without lookups.
4950 // To be in this state we have to (1) finished the exploration and (3) not
4951 // discovered any non-trivial dead end and (2) not ruled unreachable code
4952 // dead.
4953 if (ToBeExploredFrom.empty() &&
4954 getAnchorScope()->size() == AssumedLiveBlocks.size() &&
4955 llvm::all_of(Range&: KnownDeadEnds, P: [](const Instruction *DeadEndI) {
4956 return DeadEndI->isTerminator() && DeadEndI->getNumSuccessors() == 0;
4957 }))
4958 return indicatePessimisticFixpoint();
4959 return Change;
4960}
4961
4962/// Liveness information for a call sites.
4963struct AAIsDeadCallSite final : AAIsDeadFunction {
4964 AAIsDeadCallSite(const IRPosition &IRP, Attributor &A)
4965 : AAIsDeadFunction(IRP, A) {}
4966
4967 /// See AbstractAttribute::initialize(...).
4968 void initialize(Attributor &A) override {
4969 // TODO: Once we have call site specific value information we can provide
4970 // call site specific liveness information and then it makes
4971 // sense to specialize attributes for call sites instead of
4972 // redirecting requests to the callee.
4973 llvm_unreachable("Abstract attributes for liveness are not "
4974 "supported for call sites yet!");
4975 }
4976
4977 /// See AbstractAttribute::updateImpl(...).
4978 ChangeStatus updateImpl(Attributor &A) override {
4979 return indicatePessimisticFixpoint();
4980 }
4981
4982 /// See AbstractAttribute::trackStatistics()
4983 void trackStatistics() const override {}
4984};
4985} // namespace
4986
4987/// -------------------- Dereferenceable Argument Attribute --------------------
4988
4989namespace {
4990struct AADereferenceableImpl : AADereferenceable {
4991 AADereferenceableImpl(const IRPosition &IRP, Attributor &A)
4992 : AADereferenceable(IRP, A) {}
4993 using StateType = DerefState;
4994
4995 /// See AbstractAttribute::initialize(...).
4996 void initialize(Attributor &A) override {
4997 Value &V = *getAssociatedValue().stripPointerCasts();
4998 SmallVector<Attribute, 4> Attrs;
4999 A.getAttrs(IRP: getIRPosition(),
5000 AKs: {Attribute::Dereferenceable, Attribute::DereferenceableOrNull},
5001 Attrs, /* IgnoreSubsumingPositions */ false);
5002 for (const Attribute &Attr : Attrs)
5003 takeKnownDerefBytesMaximum(Bytes: Attr.getValueAsInt());
5004
5005 // Ensure we initialize the non-null AA (if necessary).
5006 bool IsKnownNonNull;
5007 AA::hasAssumedIRAttr<Attribute::NonNull>(
5008 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNonNull);
5009
5010 bool CanBeNull;
5011 takeKnownDerefBytesMaximum(Bytes: V.getPointerDereferenceableBytes(
5012 DL: A.getDataLayout(), CanBeNull, /*CanBeFreed=*/nullptr));
5013
5014 if (Instruction *CtxI = getCtxI())
5015 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
5016 }
5017
5018 /// See AbstractAttribute::getState()
5019 /// {
5020 StateType &getState() override { return *this; }
5021 const StateType &getState() const override { return *this; }
5022 /// }
5023
5024 /// Helper function for collecting accessed bytes in must-be-executed-context
5025 void addAccessedBytesForUse(Attributor &A, const Use *U, const Instruction *I,
5026 DerefState &State) {
5027 const Value *UseV = U->get();
5028 if (!UseV->getType()->isPointerTy())
5029 return;
5030
5031 std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(Inst: I);
5032 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() || I->isVolatile())
5033 return;
5034
5035 int64_t Offset;
5036 const Value *Base = GetPointerBaseWithConstantOffset(
5037 Ptr: Loc->Ptr, Offset, DL: A.getDataLayout(), /*AllowNonInbounds*/ true);
5038 if (Base && Base == &getAssociatedValue())
5039 State.addAccessedBytes(Offset, Size: Loc->Size.getValue());
5040 }
5041
5042 /// See followUsesInMBEC
5043 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
5044 AADereferenceable::StateType &State) {
5045 bool IsNonNull = false;
5046 bool TrackUse = false;
5047 int64_t DerefBytes = getKnownNonNullAndDerefBytesForUse(
5048 A, QueryingAA: *this, AssociatedValue&: getAssociatedValue(), U, I, IsNonNull, TrackUse);
5049 LLVM_DEBUG(dbgs() << "[AADereferenceable] Deref bytes: " << DerefBytes
5050 << " for instruction " << *I << "\n");
5051
5052 addAccessedBytesForUse(A, U, I, State);
5053 State.takeKnownDerefBytesMaximum(Bytes: DerefBytes);
5054 return TrackUse;
5055 }
5056
5057 /// See AbstractAttribute::manifest(...).
5058 ChangeStatus manifest(Attributor &A) override {
5059 ChangeStatus Change = AADereferenceable::manifest(A);
5060 bool IsKnownNonNull;
5061 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
5062 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
5063 if (IsAssumedNonNull &&
5064 A.hasAttr(IRP: getIRPosition(), AKs: Attribute::DereferenceableOrNull)) {
5065 A.removeAttrs(IRP: getIRPosition(), AttrKinds: {Attribute::DereferenceableOrNull});
5066 return ChangeStatus::CHANGED;
5067 }
5068 return Change;
5069 }
5070
5071 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5072 SmallVectorImpl<Attribute> &Attrs) const override {
5073 // TODO: Add *_globally support
5074 bool IsKnownNonNull;
5075 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
5076 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
5077 if (IsAssumedNonNull)
5078 Attrs.emplace_back(Args: Attribute::getWithDereferenceableBytes(
5079 Context&: Ctx, Bytes: getAssumedDereferenceableBytes()));
5080 else
5081 Attrs.emplace_back(Args: Attribute::getWithDereferenceableOrNullBytes(
5082 Context&: Ctx, Bytes: getAssumedDereferenceableBytes()));
5083 }
5084
5085 /// See AbstractAttribute::getAsStr().
5086 const std::string getAsStr(Attributor *A) const override {
5087 if (!getAssumedDereferenceableBytes())
5088 return "unknown-dereferenceable";
5089 bool IsKnownNonNull;
5090 bool IsAssumedNonNull = false;
5091 if (A)
5092 IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
5093 A&: *A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::NONE, IsKnown&: IsKnownNonNull);
5094 return std::string("dereferenceable") +
5095 (IsAssumedNonNull ? "" : "_or_null") +
5096 (isAssumedGlobal() ? "_globally" : "") + "<" +
5097 std::to_string(val: getKnownDereferenceableBytes()) + "-" +
5098 std::to_string(val: getAssumedDereferenceableBytes()) + ">" +
5099 (!A ? " [non-null is unknown]" : "");
5100 }
5101};
5102
5103/// Dereferenceable attribute for a floating value.
5104struct AADereferenceableFloating : AADereferenceableImpl {
5105 AADereferenceableFloating(const IRPosition &IRP, Attributor &A)
5106 : AADereferenceableImpl(IRP, A) {}
5107
5108 /// See AbstractAttribute::updateImpl(...).
5109 ChangeStatus updateImpl(Attributor &A) override {
5110 bool Stripped;
5111 bool UsedAssumedInformation = false;
5112 SmallVector<AA::ValueAndContext> Values;
5113 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
5114 S: AA::AnyScope, UsedAssumedInformation)) {
5115 Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
5116 Stripped = false;
5117 } else {
5118 Stripped = Values.size() != 1 ||
5119 Values.front().getValue() != &getAssociatedValue();
5120 }
5121
5122 const DataLayout &DL = A.getDataLayout();
5123 DerefState T;
5124
5125 auto VisitValueCB = [&](const Value &V) -> bool {
5126 unsigned IdxWidth =
5127 DL.getIndexSizeInBits(AS: V.getType()->getPointerAddressSpace());
5128 APInt Offset(IdxWidth, 0);
5129 const Value *Base = stripAndAccumulateOffsets(
5130 A, QueryingAA: *this, Val: &V, DL, Offset, /* GetMinOffset */ false,
5131 /* AllowNonInbounds */ true);
5132
5133 const auto *AA = A.getAAFor<AADereferenceable>(
5134 QueryingAA: *this, IRP: IRPosition::value(V: *Base), DepClass: DepClassTy::REQUIRED);
5135 int64_t DerefBytes = 0;
5136 if (!AA || (!Stripped && this == AA)) {
5137 // Use IR information if we did not strip anything.
5138 // TODO: track globally.
5139 bool CanBeNull;
5140 DerefBytes = Base->getPointerDereferenceableBytes(
5141 DL, CanBeNull, /*CanBeFreed=*/nullptr);
5142 T.GlobalState.indicatePessimisticFixpoint();
5143 } else {
5144 const DerefState &DS = AA->getState();
5145 DerefBytes = DS.DerefBytesState.getAssumed();
5146 T.GlobalState &= DS.GlobalState;
5147 }
5148
5149 // For now we do not try to "increase" dereferenceability due to negative
5150 // indices as we first have to come up with code to deal with loops and
5151 // for overflows of the dereferenceable bytes.
5152 int64_t OffsetSExt = Offset.getSExtValue();
5153 if (OffsetSExt < 0)
5154 OffsetSExt = 0;
5155
5156 T.takeAssumedDerefBytesMinimum(
5157 Bytes: std::max(a: int64_t(0), b: DerefBytes - OffsetSExt));
5158
5159 if (this == AA) {
5160 if (!Stripped) {
5161 // If nothing was stripped IR information is all we got.
5162 T.takeKnownDerefBytesMaximum(
5163 Bytes: std::max(a: int64_t(0), b: DerefBytes - OffsetSExt));
5164 T.indicatePessimisticFixpoint();
5165 } else if (OffsetSExt > 0) {
5166 // If something was stripped but there is circular reasoning we look
5167 // for the offset. If it is positive we basically decrease the
5168 // dereferenceable bytes in a circular loop now, which will simply
5169 // drive them down to the known value in a very slow way which we
5170 // can accelerate.
5171 T.indicatePessimisticFixpoint();
5172 }
5173 }
5174
5175 return T.isValidState();
5176 };
5177
5178 for (const auto &VAC : Values)
5179 if (!VisitValueCB(*VAC.getValue()))
5180 return indicatePessimisticFixpoint();
5181
5182 return clampStateAndIndicateChange(S&: getState(), R: T);
5183 }
5184
5185 /// See AbstractAttribute::trackStatistics()
5186 void trackStatistics() const override {
5187 STATS_DECLTRACK_FLOATING_ATTR(dereferenceable)
5188 }
5189};
5190
5191/// Dereferenceable attribute for a return value.
5192struct AADereferenceableReturned final
5193 : AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl> {
5194 using Base =
5195 AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl>;
5196 AADereferenceableReturned(const IRPosition &IRP, Attributor &A)
5197 : Base(IRP, A) {}
5198
5199 /// See AbstractAttribute::trackStatistics()
5200 void trackStatistics() const override {
5201 STATS_DECLTRACK_FNRET_ATTR(dereferenceable)
5202 }
5203};
5204
5205/// Dereferenceable attribute for an argument
5206struct AADereferenceableArgument final
5207 : AAArgumentFromCallSiteArguments<AADereferenceable,
5208 AADereferenceableImpl> {
5209 using Base =
5210 AAArgumentFromCallSiteArguments<AADereferenceable, AADereferenceableImpl>;
5211 AADereferenceableArgument(const IRPosition &IRP, Attributor &A)
5212 : Base(IRP, A) {}
5213
5214 /// See AbstractAttribute::trackStatistics()
5215 void trackStatistics() const override {
5216 STATS_DECLTRACK_ARG_ATTR(dereferenceable)
5217 }
5218};
5219
5220/// Dereferenceable attribute for a call site argument.
5221struct AADereferenceableCallSiteArgument final : AADereferenceableFloating {
5222 AADereferenceableCallSiteArgument(const IRPosition &IRP, Attributor &A)
5223 : AADereferenceableFloating(IRP, A) {}
5224
5225 /// See AbstractAttribute::trackStatistics()
5226 void trackStatistics() const override {
5227 STATS_DECLTRACK_CSARG_ATTR(dereferenceable)
5228 }
5229};
5230
5231/// Dereferenceable attribute deduction for a call site return value.
5232struct AADereferenceableCallSiteReturned final
5233 : AACalleeToCallSite<AADereferenceable, AADereferenceableImpl> {
5234 using Base = AACalleeToCallSite<AADereferenceable, AADereferenceableImpl>;
5235 AADereferenceableCallSiteReturned(const IRPosition &IRP, Attributor &A)
5236 : Base(IRP, A) {}
5237
5238 /// See AbstractAttribute::trackStatistics()
5239 void trackStatistics() const override {
5240 STATS_DECLTRACK_CS_ATTR(dereferenceable);
5241 }
5242};
5243} // namespace
5244
5245// ------------------------ Align Argument Attribute ------------------------
5246
5247namespace {
5248
5249static unsigned getKnownAlignForUse(Attributor &A, AAAlign &QueryingAA,
5250 Value &AssociatedValue, const Use *U,
5251 const Instruction *I, bool &TrackUse) {
5252 // We need to follow common pointer manipulation uses to the accesses they
5253 // feed into.
5254 if (isa<CastInst>(Val: I)) {
5255 // Follow all but ptr2int casts.
5256 TrackUse = !isa<PtrToIntInst>(Val: I);
5257 return 0;
5258 }
5259 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: I)) {
5260 if (GEP->hasAllConstantIndices())
5261 TrackUse = true;
5262 return 0;
5263 }
5264 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I))
5265 switch (II->getIntrinsicID()) {
5266 case Intrinsic::ptrmask: {
5267 // Is it appropriate to pull attribute in initialization?
5268 const auto *ConstVals = A.getAAFor<AAPotentialConstantValues>(
5269 QueryingAA, IRP: IRPosition::value(V: *II->getOperand(i_nocapture: 1)), DepClass: DepClassTy::NONE);
5270 const auto *AlignAA = A.getAAFor<AAAlign>(
5271 QueryingAA, IRP: IRPosition::value(V: *II), DepClass: DepClassTy::NONE);
5272 if (ConstVals && ConstVals->isValidState() && ConstVals->isAtFixpoint()) {
5273 unsigned ShiftValue = std::min(a: ConstVals->getAssumedMinTrailingZeros(),
5274 b: Value::MaxAlignmentExponent);
5275 Align ConstAlign = Align::fromLog2(Log2Value: ShiftValue);
5276 if (ConstAlign >= AlignAA->getKnownAlign())
5277 return Align(1).value();
5278 }
5279 if (AlignAA)
5280 return AlignAA->getKnownAlign().value();
5281 break;
5282 }
5283 case Intrinsic::amdgcn_make_buffer_rsrc: {
5284 const auto *AlignAA = A.getAAFor<AAAlign>(
5285 QueryingAA, IRP: IRPosition::value(V: *II), DepClass: DepClassTy::NONE);
5286 if (AlignAA)
5287 return AlignAA->getKnownAlign().value();
5288 break;
5289 }
5290 default:
5291 break;
5292 }
5293
5294 MaybeAlign MA;
5295 if (const auto *CB = dyn_cast<CallBase>(Val: I)) {
5296 if (CB->isBundleOperand(U) || CB->isCallee(U))
5297 return 0;
5298
5299 unsigned ArgNo = CB->getArgOperandNo(U);
5300 IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
5301 // As long as we only use known information there is no need to track
5302 // dependences here.
5303 auto *AlignAA = A.getAAFor<AAAlign>(QueryingAA, IRP, DepClass: DepClassTy::NONE);
5304 if (AlignAA)
5305 MA = MaybeAlign(AlignAA->getKnownAlign());
5306 }
5307
5308 const DataLayout &DL = A.getDataLayout();
5309 const Value *UseV = U->get();
5310 if (auto *SI = dyn_cast<StoreInst>(Val: I)) {
5311 if (SI->getPointerOperand() == UseV)
5312 MA = SI->getAlign();
5313 } else if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
5314 if (LI->getPointerOperand() == UseV)
5315 MA = LI->getAlign();
5316 } else if (auto *AI = dyn_cast<AtomicRMWInst>(Val: I)) {
5317 if (AI->getPointerOperand() == UseV)
5318 MA = AI->getAlign();
5319 } else if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: I)) {
5320 if (AI->getPointerOperand() == UseV)
5321 MA = AI->getAlign();
5322 }
5323
5324 if (!MA || *MA <= QueryingAA.getKnownAlign())
5325 return 0;
5326
5327 unsigned Alignment = MA->value();
5328 int64_t Offset;
5329
5330 if (const Value *Base = GetPointerBaseWithConstantOffset(Ptr: UseV, Offset, DL)) {
5331 if (Base == &AssociatedValue) {
5332 // BasePointerAddr + Offset = Alignment * Q for some integer Q.
5333 // So we can say that the maximum power of two which is a divisor of
5334 // gcd(Offset, Alignment) is an alignment.
5335
5336 uint32_t gcd = std::gcd(m: uint32_t(abs(x: (int32_t)Offset)), n: Alignment);
5337 Alignment = llvm::bit_floor(Value: gcd);
5338 }
5339 }
5340
5341 return Alignment;
5342}
5343
5344struct AAAlignImpl : AAAlign {
5345 AAAlignImpl(const IRPosition &IRP, Attributor &A) : AAAlign(IRP, A) {}
5346
5347 /// See AbstractAttribute::initialize(...).
5348 void initialize(Attributor &A) override {
5349 SmallVector<Attribute, 4> Attrs;
5350 A.getAttrs(IRP: getIRPosition(), AKs: {Attribute::Alignment}, Attrs);
5351 for (const Attribute &Attr : Attrs)
5352 takeKnownMaximum(Value: Attr.getValueAsInt());
5353
5354 Value &V = *getAssociatedValue().stripPointerCasts();
5355 takeKnownMaximum(Value: V.getPointerAlignment(DL: A.getDataLayout()).value());
5356
5357 if (Instruction *CtxI = getCtxI())
5358 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
5359 }
5360
5361 /// See AbstractAttribute::manifest(...).
5362 ChangeStatus manifest(Attributor &A) override {
5363 ChangeStatus InstrChanged = ChangeStatus::UNCHANGED;
5364
5365 // Check for users that allow alignment annotations.
5366 Value &AssociatedValue = getAssociatedValue();
5367 if (isa<ConstantData>(Val: AssociatedValue))
5368 return ChangeStatus::UNCHANGED;
5369
5370 for (const Use &U : AssociatedValue.uses()) {
5371 if (auto *SI = dyn_cast<StoreInst>(Val: U.getUser())) {
5372 if (SI->getPointerOperand() == &AssociatedValue)
5373 if (SI->getAlign() < getAssumedAlign()) {
5374 STATS_DECLTRACK(AAAlign, Store,
5375 "Number of times alignment added to a store");
5376 SI->setAlignment(getAssumedAlign());
5377 InstrChanged = ChangeStatus::CHANGED;
5378 }
5379 } else if (auto *LI = dyn_cast<LoadInst>(Val: U.getUser())) {
5380 if (LI->getPointerOperand() == &AssociatedValue)
5381 if (LI->getAlign() < getAssumedAlign()) {
5382 LI->setAlignment(getAssumedAlign());
5383 STATS_DECLTRACK(AAAlign, Load,
5384 "Number of times alignment added to a load");
5385 InstrChanged = ChangeStatus::CHANGED;
5386 }
5387 } else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: U.getUser())) {
5388 if (RMW->getPointerOperand() == &AssociatedValue) {
5389 if (RMW->getAlign() < getAssumedAlign()) {
5390 STATS_DECLTRACK(AAAlign, AtomicRMW,
5391 "Number of times alignment added to atomicrmw");
5392
5393 RMW->setAlignment(getAssumedAlign());
5394 InstrChanged = ChangeStatus::CHANGED;
5395 }
5396 }
5397 } else if (auto *CAS = dyn_cast<AtomicCmpXchgInst>(Val: U.getUser())) {
5398 if (CAS->getPointerOperand() == &AssociatedValue) {
5399 if (CAS->getAlign() < getAssumedAlign()) {
5400 STATS_DECLTRACK(AAAlign, AtomicCmpXchg,
5401 "Number of times alignment added to cmpxchg");
5402 CAS->setAlignment(getAssumedAlign());
5403 InstrChanged = ChangeStatus::CHANGED;
5404 }
5405 }
5406 }
5407 }
5408
5409 ChangeStatus Changed = AAAlign::manifest(A);
5410
5411 Align InheritAlign =
5412 getAssociatedValue().getPointerAlignment(DL: A.getDataLayout());
5413 if (InheritAlign >= getAssumedAlign())
5414 return InstrChanged;
5415 return Changed | InstrChanged;
5416 }
5417
5418 // TODO: Provide a helper to determine the implied ABI alignment and check in
5419 // the existing manifest method and a new one for AAAlignImpl that value
5420 // to avoid making the alignment explicit if it did not improve.
5421
5422 /// See AbstractAttribute::getDeducedAttributes
5423 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5424 SmallVectorImpl<Attribute> &Attrs) const override {
5425 if (getAssumedAlign() > 1)
5426 Attrs.emplace_back(
5427 Args: Attribute::getWithAlignment(Context&: Ctx, Alignment: Align(getAssumedAlign())));
5428 }
5429
5430 /// See followUsesInMBEC
5431 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
5432 AAAlign::StateType &State) {
5433 bool TrackUse = false;
5434
5435 unsigned int KnownAlign =
5436 getKnownAlignForUse(A, QueryingAA&: *this, AssociatedValue&: getAssociatedValue(), U, I, TrackUse);
5437 State.takeKnownMaximum(Value: KnownAlign);
5438
5439 return TrackUse;
5440 }
5441
5442 /// See AbstractAttribute::getAsStr().
5443 const std::string getAsStr(Attributor *A) const override {
5444 return "align<" + std::to_string(val: getKnownAlign().value()) + "-" +
5445 std::to_string(val: getAssumedAlign().value()) + ">";
5446 }
5447};
5448
5449/// Align attribute for a floating value.
5450struct AAAlignFloating : AAAlignImpl {
5451 AAAlignFloating(const IRPosition &IRP, Attributor &A) : AAAlignImpl(IRP, A) {}
5452
5453 /// See AbstractAttribute::updateImpl(...).
5454 ChangeStatus updateImpl(Attributor &A) override {
5455 const DataLayout &DL = A.getDataLayout();
5456
5457 bool Stripped;
5458 bool UsedAssumedInformation = false;
5459 SmallVector<AA::ValueAndContext> Values;
5460 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
5461 S: AA::AnyScope, UsedAssumedInformation)) {
5462 Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
5463 Stripped = false;
5464 } else {
5465 Stripped = Values.size() != 1 ||
5466 Values.front().getValue() != &getAssociatedValue();
5467 }
5468
5469 StateType T;
5470 auto VisitValueCB = [&](Value &V) -> bool {
5471 if (isa<UndefValue>(Val: V) || isa<ConstantPointerNull>(Val: V))
5472 return true;
5473 const auto *AA = A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V),
5474 DepClass: DepClassTy::REQUIRED);
5475 if (!AA || (!Stripped && this == AA)) {
5476 int64_t Offset;
5477 unsigned Alignment = 1;
5478 if (const Value *Base =
5479 GetPointerBaseWithConstantOffset(Ptr: &V, Offset, DL)) {
5480 // TODO: Use AAAlign for the base too.
5481 Align PA = Base->getPointerAlignment(DL);
5482 // BasePointerAddr + Offset = Alignment * Q for some integer Q.
5483 // So we can say that the maximum power of two which is a divisor of
5484 // gcd(Offset, Alignment) is an alignment.
5485
5486 uint32_t gcd =
5487 std::gcd(m: uint32_t(abs(x: (int32_t)Offset)), n: uint32_t(PA.value()));
5488 Alignment = llvm::bit_floor(Value: gcd);
5489 } else {
5490 Alignment = V.getPointerAlignment(DL).value();
5491 }
5492 // Use only IR information if we did not strip anything.
5493 T.takeKnownMaximum(Value: Alignment);
5494 T.indicatePessimisticFixpoint();
5495 } else {
5496 // Use abstract attribute information.
5497 const AAAlign::StateType &DS = AA->getState();
5498 T ^= DS;
5499 }
5500 return T.isValidState();
5501 };
5502
5503 for (const auto &VAC : Values) {
5504 if (!VisitValueCB(*VAC.getValue()))
5505 return indicatePessimisticFixpoint();
5506 }
5507
5508 // TODO: If we know we visited all incoming values, thus no are assumed
5509 // dead, we can take the known information from the state T.
5510 return clampStateAndIndicateChange(S&: getState(), R: T);
5511 }
5512
5513 /// See AbstractAttribute::trackStatistics()
5514 void trackStatistics() const override { STATS_DECLTRACK_FLOATING_ATTR(align) }
5515};
5516
5517/// Align attribute for function return value.
5518struct AAAlignReturned final
5519 : AAReturnedFromReturnedValues<AAAlign, AAAlignImpl> {
5520 using Base = AAReturnedFromReturnedValues<AAAlign, AAAlignImpl>;
5521 AAAlignReturned(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5522
5523 /// See AbstractAttribute::trackStatistics()
5524 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(aligned) }
5525};
5526
5527/// Align attribute for function argument.
5528struct AAAlignArgument final
5529 : AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl> {
5530 using Base = AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl>;
5531 AAAlignArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
5532
5533 /// See AbstractAttribute::manifest(...).
5534 ChangeStatus manifest(Attributor &A) override {
5535 // If the associated argument is involved in a must-tail call we give up
5536 // because we would need to keep the argument alignments of caller and
5537 // callee in-sync. Just does not seem worth the trouble right now.
5538 if (A.getInfoCache().isInvolvedInMustTailCall(Arg: *getAssociatedArgument()))
5539 return ChangeStatus::UNCHANGED;
5540 return Base::manifest(A);
5541 }
5542
5543 /// See AbstractAttribute::trackStatistics()
5544 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(aligned) }
5545};
5546
5547struct AAAlignCallSiteArgument final : AAAlignFloating {
5548 AAAlignCallSiteArgument(const IRPosition &IRP, Attributor &A)
5549 : AAAlignFloating(IRP, A) {}
5550
5551 /// See AbstractAttribute::manifest(...).
5552 ChangeStatus manifest(Attributor &A) override {
5553 // If the associated argument is involved in a must-tail call we give up
5554 // because we would need to keep the argument alignments of caller and
5555 // callee in-sync. Just does not seem worth the trouble right now.
5556 if (Argument *Arg = getAssociatedArgument())
5557 if (A.getInfoCache().isInvolvedInMustTailCall(Arg: *Arg))
5558 return ChangeStatus::UNCHANGED;
5559 ChangeStatus Changed = AAAlignImpl::manifest(A);
5560 Align InheritAlign =
5561 getAssociatedValue().getPointerAlignment(DL: A.getDataLayout());
5562 if (InheritAlign >= getAssumedAlign())
5563 Changed = ChangeStatus::UNCHANGED;
5564 return Changed;
5565 }
5566
5567 /// See AbstractAttribute::updateImpl(Attributor &A).
5568 ChangeStatus updateImpl(Attributor &A) override {
5569 ChangeStatus Changed = AAAlignFloating::updateImpl(A);
5570 if (Argument *Arg = getAssociatedArgument()) {
5571 // We only take known information from the argument
5572 // so we do not need to track a dependence.
5573 const auto *ArgAlignAA = A.getAAFor<AAAlign>(
5574 QueryingAA: *this, IRP: IRPosition::argument(Arg: *Arg), DepClass: DepClassTy::NONE);
5575 if (ArgAlignAA)
5576 takeKnownMaximum(Value: ArgAlignAA->getKnownAlign().value());
5577 }
5578 return Changed;
5579 }
5580
5581 /// See AbstractAttribute::trackStatistics()
5582 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(aligned) }
5583};
5584
5585/// Align attribute deduction for a call site return value.
5586struct AAAlignCallSiteReturned final
5587 : AACalleeToCallSite<AAAlign, AAAlignImpl> {
5588 using Base = AACalleeToCallSite<AAAlign, AAAlignImpl>;
5589 AAAlignCallSiteReturned(const IRPosition &IRP, Attributor &A)
5590 : Base(IRP, A) {}
5591
5592 ChangeStatus updateImpl(Attributor &A) override {
5593 Instruction *I = getIRPosition().getCtxI();
5594 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
5595 switch (II->getIntrinsicID()) {
5596 case Intrinsic::ptrmask: {
5597 Align Alignment;
5598 bool Valid = false;
5599
5600 const auto *ConstVals = A.getAAFor<AAPotentialConstantValues>(
5601 QueryingAA: *this, IRP: IRPosition::value(V: *II->getOperand(i_nocapture: 1)), DepClass: DepClassTy::REQUIRED);
5602 if (ConstVals && ConstVals->isValidState()) {
5603 unsigned ShiftValue =
5604 std::min(a: ConstVals->getAssumedMinTrailingZeros(),
5605 b: Value::MaxAlignmentExponent);
5606 Alignment = Align::fromLog2(Log2Value: ShiftValue);
5607 Valid = true;
5608 }
5609
5610 const auto *AlignAA =
5611 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *(II->getOperand(i_nocapture: 0))),
5612 DepClass: DepClassTy::REQUIRED);
5613 if (AlignAA) {
5614 Alignment = std::max(a: AlignAA->getAssumedAlign(), b: Alignment);
5615 Valid = true;
5616 }
5617
5618 if (Valid)
5619 return clampStateAndIndicateChange<StateType>(
5620 S&: this->getState(),
5621 R: std::min(a: this->getAssumedAlign(), b: Alignment).value());
5622 break;
5623 }
5624 // FIXME: Should introduce target specific sub-attributes and letting
5625 // getAAfor<AAAlign> lead to create sub-attribute to handle target
5626 // specific intrinsics.
5627 case Intrinsic::amdgcn_make_buffer_rsrc: {
5628 const auto *AlignAA =
5629 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *(II->getOperand(i_nocapture: 0))),
5630 DepClass: DepClassTy::REQUIRED);
5631 if (AlignAA)
5632 return clampStateAndIndicateChange<StateType>(
5633 S&: this->getState(), R: AlignAA->getAssumedAlign().value());
5634 break;
5635 }
5636 default:
5637 break;
5638 }
5639 }
5640 return Base::updateImpl(A);
5641 };
5642 /// See AbstractAttribute::trackStatistics()
5643 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(align); }
5644};
5645} // namespace
5646
5647/// ------------------ Function No-Return Attribute ----------------------------
5648namespace {
5649struct AANoReturnImpl : public AANoReturn {
5650 AANoReturnImpl(const IRPosition &IRP, Attributor &A) : AANoReturn(IRP, A) {}
5651
5652 /// See AbstractAttribute::initialize(...).
5653 void initialize(Attributor &A) override {
5654 bool IsKnown;
5655 assert(!AA::hasAssumedIRAttr<Attribute::NoReturn>(
5656 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5657 (void)IsKnown;
5658 }
5659
5660 /// See AbstractAttribute::getAsStr().
5661 const std::string getAsStr(Attributor *A) const override {
5662 return getAssumed() ? "noreturn" : "may-return";
5663 }
5664
5665 /// See AbstractAttribute::updateImpl(Attributor &A).
5666 ChangeStatus updateImpl(Attributor &A) override {
5667 auto CheckForNoReturn = [](Instruction &) { return false; };
5668 bool UsedAssumedInformation = false;
5669 if (!A.checkForAllInstructions(Pred: CheckForNoReturn, QueryingAA: *this,
5670 Opcodes: {(unsigned)Instruction::Ret},
5671 UsedAssumedInformation))
5672 return indicatePessimisticFixpoint();
5673 return ChangeStatus::UNCHANGED;
5674 }
5675};
5676
5677struct AANoReturnFunction final : AANoReturnImpl {
5678 AANoReturnFunction(const IRPosition &IRP, Attributor &A)
5679 : AANoReturnImpl(IRP, A) {}
5680
5681 /// See AbstractAttribute::trackStatistics()
5682 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(noreturn) }
5683};
5684
5685/// NoReturn attribute deduction for a call sites.
5686struct AANoReturnCallSite final
5687 : AACalleeToCallSite<AANoReturn, AANoReturnImpl> {
5688 AANoReturnCallSite(const IRPosition &IRP, Attributor &A)
5689 : AACalleeToCallSite<AANoReturn, AANoReturnImpl>(IRP, A) {}
5690
5691 /// See AbstractAttribute::trackStatistics()
5692 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(noreturn); }
5693};
5694} // namespace
5695
5696/// ----------------------- Instance Info ---------------------------------
5697
5698namespace {
5699/// A class to hold the state of for no-capture attributes.
5700struct AAInstanceInfoImpl : public AAInstanceInfo {
5701 AAInstanceInfoImpl(const IRPosition &IRP, Attributor &A)
5702 : AAInstanceInfo(IRP, A) {}
5703
5704 /// See AbstractAttribute::initialize(...).
5705 void initialize(Attributor &A) override {
5706 Value &V = getAssociatedValue();
5707 if (auto *C = dyn_cast<Constant>(Val: &V)) {
5708 if (C->isThreadDependent())
5709 indicatePessimisticFixpoint();
5710 else
5711 indicateOptimisticFixpoint();
5712 return;
5713 }
5714 if (auto *CB = dyn_cast<CallBase>(Val: &V))
5715 if (CB->arg_size() == 0 && !CB->mayHaveSideEffects() &&
5716 !CB->mayReadFromMemory()) {
5717 indicateOptimisticFixpoint();
5718 return;
5719 }
5720 if (auto *I = dyn_cast<Instruction>(Val: &V)) {
5721 const auto *CI =
5722 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
5723 F: *I->getFunction());
5724 if (mayBeInCycle(CI, I, /* HeaderOnly */ false)) {
5725 indicatePessimisticFixpoint();
5726 return;
5727 }
5728 }
5729 }
5730
5731 /// See AbstractAttribute::updateImpl(...).
5732 ChangeStatus updateImpl(Attributor &A) override {
5733 ChangeStatus Changed = ChangeStatus::UNCHANGED;
5734
5735 Value &V = getAssociatedValue();
5736 const Function *Scope = nullptr;
5737 if (auto *I = dyn_cast<Instruction>(Val: &V))
5738 Scope = I->getFunction();
5739 if (auto *A = dyn_cast<Argument>(Val: &V)) {
5740 Scope = A->getParent();
5741 if (!Scope->hasLocalLinkage())
5742 return Changed;
5743 }
5744 if (!Scope)
5745 return indicateOptimisticFixpoint();
5746
5747 bool IsKnownNoRecurse;
5748 if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
5749 A, QueryingAA: this, IRP: IRPosition::function(F: *Scope), DepClass: DepClassTy::OPTIONAL,
5750 IsKnown&: IsKnownNoRecurse))
5751 return Changed;
5752
5753 auto UsePred = [&](const Use &U, bool &Follow) {
5754 const Instruction *UserI = dyn_cast<Instruction>(Val: U.getUser());
5755 if (!UserI || isa<GetElementPtrInst>(Val: UserI) || isa<CastInst>(Val: UserI) ||
5756 isa<PHINode>(Val: UserI) || isa<SelectInst>(Val: UserI)) {
5757 Follow = true;
5758 return true;
5759 }
5760 if (isa<LoadInst>(Val: UserI) || isa<CmpInst>(Val: UserI) ||
5761 (isa<StoreInst>(Val: UserI) &&
5762 cast<StoreInst>(Val: UserI)->getValueOperand() != U.get()))
5763 return true;
5764 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
5765 // This check is not guaranteeing uniqueness but for now that we cannot
5766 // end up with two versions of \p U thinking it was one.
5767 auto *Callee = dyn_cast_if_present<Function>(Val: CB->getCalledOperand());
5768 if (!Callee || !Callee->hasLocalLinkage())
5769 return true;
5770 if (!CB->isArgOperand(U: &U))
5771 return false;
5772 const auto *ArgInstanceInfoAA = A.getAAFor<AAInstanceInfo>(
5773 QueryingAA: *this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U)),
5774 DepClass: DepClassTy::OPTIONAL);
5775 if (!ArgInstanceInfoAA ||
5776 !ArgInstanceInfoAA->isAssumedUniqueForAnalysis())
5777 return false;
5778 // If this call base might reach the scope again we might forward the
5779 // argument back here. This is very conservative.
5780 if (AA::isPotentiallyReachable(
5781 A, FromI: *CB, ToFn: *Scope, QueryingAA: *this, /* ExclusionSet */ nullptr,
5782 GoBackwardsCB: [Scope](const Function &Fn) { return &Fn != Scope; }))
5783 return false;
5784 return true;
5785 }
5786 return false;
5787 };
5788
5789 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
5790 if (auto *SI = dyn_cast<StoreInst>(Val: OldU.getUser())) {
5791 auto *Ptr = SI->getPointerOperand()->stripPointerCasts();
5792 if ((isa<AllocaInst>(Val: Ptr) || isNoAliasCall(V: Ptr)) &&
5793 AA::isDynamicallyUnique(A, QueryingAA: *this, V: *Ptr))
5794 return true;
5795 }
5796 return false;
5797 };
5798
5799 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V, /* CheckBBLivenessOnly */ true,
5800 LivenessDepClass: DepClassTy::OPTIONAL,
5801 /* IgnoreDroppableUses */ true, EquivalentUseCB))
5802 return indicatePessimisticFixpoint();
5803
5804 return Changed;
5805 }
5806
5807 /// See AbstractState::getAsStr().
5808 const std::string getAsStr(Attributor *A) const override {
5809 return isAssumedUniqueForAnalysis() ? "<unique [fAa]>" : "<unknown>";
5810 }
5811
5812 /// See AbstractAttribute::trackStatistics()
5813 void trackStatistics() const override {}
5814};
5815
5816/// InstanceInfo attribute for floating values.
5817struct AAInstanceInfoFloating : AAInstanceInfoImpl {
5818 AAInstanceInfoFloating(const IRPosition &IRP, Attributor &A)
5819 : AAInstanceInfoImpl(IRP, A) {}
5820};
5821
5822/// NoCapture attribute for function arguments.
5823struct AAInstanceInfoArgument final : AAInstanceInfoFloating {
5824 AAInstanceInfoArgument(const IRPosition &IRP, Attributor &A)
5825 : AAInstanceInfoFloating(IRP, A) {}
5826};
5827
5828/// InstanceInfo attribute for call site arguments.
5829struct AAInstanceInfoCallSiteArgument final : AAInstanceInfoImpl {
5830 AAInstanceInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
5831 : AAInstanceInfoImpl(IRP, A) {}
5832
5833 /// See AbstractAttribute::updateImpl(...).
5834 ChangeStatus updateImpl(Attributor &A) override {
5835 // TODO: Once we have call site specific value information we can provide
5836 // call site specific liveness information and then it makes
5837 // sense to specialize attributes for call sites arguments instead of
5838 // redirecting requests to the callee argument.
5839 Argument *Arg = getAssociatedArgument();
5840 if (!Arg)
5841 return indicatePessimisticFixpoint();
5842 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
5843 auto *ArgAA =
5844 A.getAAFor<AAInstanceInfo>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
5845 if (!ArgAA)
5846 return indicatePessimisticFixpoint();
5847 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
5848 }
5849};
5850
5851/// InstanceInfo attribute for function return value.
5852struct AAInstanceInfoReturned final : AAInstanceInfoImpl {
5853 AAInstanceInfoReturned(const IRPosition &IRP, Attributor &A)
5854 : AAInstanceInfoImpl(IRP, A) {
5855 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5856 }
5857
5858 /// See AbstractAttribute::initialize(...).
5859 void initialize(Attributor &A) override {
5860 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5861 }
5862
5863 /// See AbstractAttribute::updateImpl(...).
5864 ChangeStatus updateImpl(Attributor &A) override {
5865 llvm_unreachable("InstanceInfo is not applicable to function returns!");
5866 }
5867};
5868
5869/// InstanceInfo attribute deduction for a call site return value.
5870struct AAInstanceInfoCallSiteReturned final : AAInstanceInfoFloating {
5871 AAInstanceInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
5872 : AAInstanceInfoFloating(IRP, A) {}
5873};
5874} // namespace
5875
5876/// ----------------------- Variable Capturing ---------------------------------
5877bool AANoCapture::isImpliedByIR(Attributor &A, const IRPosition &IRP,
5878 Attribute::AttrKind ImpliedAttributeKind,
5879 bool IgnoreSubsumingPositions) {
5880 assert(ImpliedAttributeKind == Attribute::Captures &&
5881 "Unexpected attribute kind");
5882 Value &V = IRP.getAssociatedValue();
5883 if (!isa<Constant>(Val: V) && !IRP.isArgumentPosition())
5884 return V.use_empty();
5885
5886 // You cannot "capture" null if it is not a valid pointer.
5887 if (isa<UndefValue>(Val: V) ||
5888 (isa<ConstantPointerNull>(Val: V) &&
5889 !NullPointerIsDefined(F: IRP.getAnchorScope(),
5890 AS: V.getType()->getPointerAddressSpace()))) {
5891 return true;
5892 }
5893
5894 SmallVector<Attribute, 1> Attrs;
5895 A.getAttrs(IRP, AKs: {Attribute::Captures}, Attrs,
5896 /* IgnoreSubsumingPositions */ true);
5897 for (const Attribute &Attr : Attrs)
5898 if (capturesNothing(CC: Attr.getCaptureInfo()))
5899 return true;
5900
5901 if (IRP.getPositionKind() == IRP_CALL_SITE_ARGUMENT)
5902 if (Argument *Arg = IRP.getAssociatedArgument()) {
5903 SmallVector<Attribute, 1> Attrs;
5904 A.getAttrs(IRP: IRPosition::argument(Arg: *Arg),
5905 AKs: {Attribute::Captures, Attribute::ByVal}, Attrs,
5906 /* IgnoreSubsumingPositions */ true);
5907 bool ArgNoCapture = any_of(Range&: Attrs, P: [](Attribute Attr) {
5908 return Attr.getKindAsEnum() == Attribute::ByVal ||
5909 capturesNothing(CC: Attr.getCaptureInfo());
5910 });
5911 if (ArgNoCapture) {
5912 A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithCaptureInfo(
5913 Context&: V.getContext(), CI: CaptureInfo::none()));
5914 return true;
5915 }
5916 }
5917
5918 if (const Function *F = IRP.getAssociatedFunction()) {
5919 // Check what state the associated function can actually capture.
5920 AANoCapture::StateType State;
5921 determineFunctionCaptureCapabilities(IRP, F: *F, State);
5922 if (State.isKnown(BitsEncoding: NO_CAPTURE)) {
5923 A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithCaptureInfo(Context&: V.getContext(),
5924 CI: CaptureInfo::none()));
5925 return true;
5926 }
5927 }
5928
5929 return false;
5930}
5931
5932/// Set the NOT_CAPTURED_IN_MEM and NOT_CAPTURED_IN_RET bits in \p Known
5933/// depending on the ability of the function associated with \p IRP to capture
5934/// state in memory and through "returning/throwing", respectively.
5935void AANoCapture::determineFunctionCaptureCapabilities(const IRPosition &IRP,
5936 const Function &F,
5937 BitIntegerState &State) {
5938 // TODO: Once we have memory behavior attributes we should use them here.
5939
5940 // If we know we cannot communicate or write to memory, we do not care about
5941 // ptr2int anymore.
5942 bool ReadOnly = F.onlyReadsMemory();
5943 bool NoThrow = F.doesNotThrow();
5944 bool IsVoidReturn = F.getReturnType()->isVoidTy();
5945 if (ReadOnly && NoThrow && IsVoidReturn) {
5946 State.addKnownBits(Bits: NO_CAPTURE);
5947 return;
5948 }
5949
5950 // A function cannot capture state in memory if it only reads memory, it can
5951 // however return/throw state and the state might be influenced by the
5952 // pointer value, e.g., loading from a returned pointer might reveal a bit.
5953 if (ReadOnly)
5954 State.addKnownBits(Bits: NOT_CAPTURED_IN_MEM);
5955
5956 // A function cannot communicate state back if it does not through
5957 // exceptions and doesn not return values.
5958 if (NoThrow && IsVoidReturn)
5959 State.addKnownBits(Bits: NOT_CAPTURED_IN_RET);
5960
5961 // Check existing "returned" attributes.
5962 int ArgNo = IRP.getCalleeArgNo();
5963 if (!NoThrow || ArgNo < 0 ||
5964 !F.getAttributes().hasAttrSomewhere(Kind: Attribute::Returned))
5965 return;
5966
5967 for (unsigned U = 0, E = F.arg_size(); U < E; ++U)
5968 if (F.hasParamAttribute(ArgNo: U, Kind: Attribute::Returned)) {
5969 if (U == unsigned(ArgNo))
5970 State.removeAssumedBits(BitsEncoding: NOT_CAPTURED_IN_RET);
5971 else if (ReadOnly)
5972 State.addKnownBits(Bits: NO_CAPTURE);
5973 else
5974 State.addKnownBits(Bits: NOT_CAPTURED_IN_RET);
5975 break;
5976 }
5977}
5978
5979namespace {
5980/// A class to hold the state of for no-capture attributes.
5981struct AANoCaptureImpl : public AANoCapture {
5982 AANoCaptureImpl(const IRPosition &IRP, Attributor &A) : AANoCapture(IRP, A) {}
5983
5984 /// See AbstractAttribute::initialize(...).
5985 void initialize(Attributor &A) override {
5986 bool IsKnown;
5987 assert(!AA::hasAssumedIRAttr<Attribute::Captures>(
5988 A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5989 (void)IsKnown;
5990 }
5991
5992 /// See AbstractAttribute::updateImpl(...).
5993 ChangeStatus updateImpl(Attributor &A) override;
5994
5995 /// see AbstractAttribute::isAssumedNoCaptureMaybeReturned(...).
5996 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
5997 SmallVectorImpl<Attribute> &Attrs) const override {
5998 if (!isAssumedNoCaptureMaybeReturned())
5999 return;
6000
6001 if (isArgumentPosition()) {
6002 if (isAssumedNoCapture())
6003 Attrs.emplace_back(Args: Attribute::get(Context&: Ctx, Kind: Attribute::Captures));
6004 else if (ManifestInternal)
6005 Attrs.emplace_back(Args: Attribute::get(Context&: Ctx, Kind: "no-capture-maybe-returned"));
6006 }
6007 }
6008
6009 /// See AbstractState::getAsStr().
6010 const std::string getAsStr(Attributor *A) const override {
6011 if (isKnownNoCapture())
6012 return "known not-captured";
6013 if (isAssumedNoCapture())
6014 return "assumed not-captured";
6015 if (isKnownNoCaptureMaybeReturned())
6016 return "known not-captured-maybe-returned";
6017 if (isAssumedNoCaptureMaybeReturned())
6018 return "assumed not-captured-maybe-returned";
6019 return "assumed-captured";
6020 }
6021
6022 /// Check the use \p U and update \p State accordingly. Return true if we
6023 /// should continue to update the state.
6024 bool checkUse(Attributor &A, AANoCapture::StateType &State, const Use &U,
6025 bool &Follow) {
6026 Instruction *UInst = cast<Instruction>(Val: U.getUser());
6027 LLVM_DEBUG(dbgs() << "[AANoCapture] Check use: " << *U.get() << " in "
6028 << *UInst << "\n");
6029
6030 // Deal with ptr2int by following uses.
6031 if (isa<PtrToIntInst>(Val: UInst)) {
6032 LLVM_DEBUG(dbgs() << " - ptr2int assume the worst!\n");
6033 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
6034 /* Return */ CapturedInRet: true);
6035 }
6036
6037 // For stores we already checked if we can follow them, if they make it
6038 // here we give up.
6039 if (isa<StoreInst>(Val: UInst))
6040 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
6041 /* Return */ CapturedInRet: true);
6042
6043 // Explicitly catch return instructions.
6044 if (isa<ReturnInst>(Val: UInst)) {
6045 if (UInst->getFunction() == getAnchorScope())
6046 return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
6047 /* Return */ CapturedInRet: true);
6048 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
6049 /* Return */ CapturedInRet: true);
6050 }
6051
6052 // For now we only use special logic for call sites. However, the tracker
6053 // itself knows about a lot of other non-capturing cases already.
6054 auto *CB = dyn_cast<CallBase>(Val: UInst);
6055 if (!CB || !CB->isArgOperand(U: &U))
6056 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
6057 /* Return */ CapturedInRet: true);
6058
6059 unsigned ArgNo = CB->getArgOperandNo(U: &U);
6060 const IRPosition &CSArgPos = IRPosition::callsite_argument(CB: *CB, ArgNo);
6061 // If we have a abstract no-capture attribute for the argument we can use
6062 // it to justify a non-capture attribute here. This allows recursion!
6063 bool IsKnownNoCapture;
6064 const AANoCapture *ArgNoCaptureAA = nullptr;
6065 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
6066 A, QueryingAA: this, IRP: CSArgPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
6067 AAPtr: &ArgNoCaptureAA);
6068 if (IsAssumedNoCapture)
6069 return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
6070 /* Return */ CapturedInRet: false);
6071 if (ArgNoCaptureAA && ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned()) {
6072 Follow = true;
6073 return isCapturedIn(State, /* Memory */ CapturedInMem: false, /* Integer */ CapturedInInt: false,
6074 /* Return */ CapturedInRet: false);
6075 }
6076
6077 // Lastly, we could not find a reason no-capture can be assumed so we don't.
6078 return isCapturedIn(State, /* Memory */ CapturedInMem: true, /* Integer */ CapturedInInt: true,
6079 /* Return */ CapturedInRet: true);
6080 }
6081
6082 /// Update \p State according to \p CapturedInMem, \p CapturedInInt, and
6083 /// \p CapturedInRet, then return true if we should continue updating the
6084 /// state.
6085 static bool isCapturedIn(AANoCapture::StateType &State, bool CapturedInMem,
6086 bool CapturedInInt, bool CapturedInRet) {
6087 LLVM_DEBUG(dbgs() << " - captures [Mem " << CapturedInMem << "|Int "
6088 << CapturedInInt << "|Ret " << CapturedInRet << "]\n");
6089 if (CapturedInMem)
6090 State.removeAssumedBits(BitsEncoding: AANoCapture::NOT_CAPTURED_IN_MEM);
6091 if (CapturedInInt)
6092 State.removeAssumedBits(BitsEncoding: AANoCapture::NOT_CAPTURED_IN_INT);
6093 if (CapturedInRet)
6094 State.removeAssumedBits(BitsEncoding: AANoCapture::NOT_CAPTURED_IN_RET);
6095 return State.isAssumed(BitsEncoding: AANoCapture::NO_CAPTURE_MAYBE_RETURNED);
6096 }
6097};
6098
6099ChangeStatus AANoCaptureImpl::updateImpl(Attributor &A) {
6100 const IRPosition &IRP = getIRPosition();
6101 Value *V = isArgumentPosition() ? IRP.getAssociatedArgument()
6102 : &IRP.getAssociatedValue();
6103 if (!V)
6104 return indicatePessimisticFixpoint();
6105
6106 const Function *F =
6107 isArgumentPosition() ? IRP.getAssociatedFunction() : IRP.getAnchorScope();
6108
6109 // TODO: Is the checkForAllUses below useful for constants?
6110 if (!F)
6111 return indicatePessimisticFixpoint();
6112
6113 AANoCapture::StateType T;
6114 const IRPosition &FnPos = IRPosition::function(F: *F);
6115
6116 // Readonly means we cannot capture through memory.
6117 bool IsKnown;
6118 if (AA::isAssumedReadOnly(A, IRP: FnPos, QueryingAA: *this, IsKnown)) {
6119 T.addKnownBits(Bits: NOT_CAPTURED_IN_MEM);
6120 if (IsKnown)
6121 addKnownBits(Bits: NOT_CAPTURED_IN_MEM);
6122 }
6123
6124 // Make sure all returned values are different than the underlying value.
6125 // TODO: we could do this in a more sophisticated way inside
6126 // AAReturnedValues, e.g., track all values that escape through returns
6127 // directly somehow.
6128 auto CheckReturnedArgs = [&](bool &UsedAssumedInformation) {
6129 SmallVector<AA::ValueAndContext> Values;
6130 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::returned(F: *F), AA: this, Values,
6131 S: AA::ValueScope::Intraprocedural,
6132 UsedAssumedInformation))
6133 return false;
6134 bool SeenConstant = false;
6135 for (const AA::ValueAndContext &VAC : Values) {
6136 if (isa<Constant>(Val: VAC.getValue())) {
6137 if (SeenConstant)
6138 return false;
6139 SeenConstant = true;
6140 } else if (!isa<Argument>(Val: VAC.getValue()) ||
6141 VAC.getValue() == getAssociatedArgument())
6142 return false;
6143 }
6144 return true;
6145 };
6146
6147 bool IsKnownNoUnwind;
6148 if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
6149 A, QueryingAA: this, IRP: FnPos, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoUnwind)) {
6150 bool IsVoidTy = F->getReturnType()->isVoidTy();
6151 bool UsedAssumedInformation = false;
6152 if (IsVoidTy || CheckReturnedArgs(UsedAssumedInformation)) {
6153 T.addKnownBits(Bits: NOT_CAPTURED_IN_RET);
6154 if (T.isKnown(BitsEncoding: NOT_CAPTURED_IN_MEM))
6155 return ChangeStatus::UNCHANGED;
6156 if (IsKnownNoUnwind && (IsVoidTy || !UsedAssumedInformation)) {
6157 addKnownBits(Bits: NOT_CAPTURED_IN_RET);
6158 if (isKnown(BitsEncoding: NOT_CAPTURED_IN_MEM))
6159 return indicateOptimisticFixpoint();
6160 }
6161 }
6162 }
6163
6164 auto UseCheck = [&](const Use &U, bool &Follow) -> bool {
6165 // TODO(captures): Make this more precise.
6166 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
6167 if (capturesNothing(CC: CI))
6168 return true;
6169 if (CI.isPassthrough()) {
6170 Follow = true;
6171 return true;
6172 }
6173 return checkUse(A, State&: T, U, Follow);
6174 };
6175
6176 if (!A.checkForAllUses(Pred: UseCheck, QueryingAA: *this, V: *V))
6177 return indicatePessimisticFixpoint();
6178
6179 AANoCapture::StateType &S = getState();
6180 auto Assumed = S.getAssumed();
6181 S.intersectAssumedBits(BitsEncoding: T.getAssumed());
6182 if (!isAssumedNoCaptureMaybeReturned())
6183 return indicatePessimisticFixpoint();
6184 return Assumed == S.getAssumed() ? ChangeStatus::UNCHANGED
6185 : ChangeStatus::CHANGED;
6186}
6187
6188/// NoCapture attribute for function arguments.
6189struct AANoCaptureArgument final : AANoCaptureImpl {
6190 AANoCaptureArgument(const IRPosition &IRP, Attributor &A)
6191 : AANoCaptureImpl(IRP, A) {}
6192
6193 /// See AbstractAttribute::trackStatistics()
6194 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nocapture) }
6195};
6196
6197/// NoCapture attribute for call site arguments.
6198struct AANoCaptureCallSiteArgument final : AANoCaptureImpl {
6199 AANoCaptureCallSiteArgument(const IRPosition &IRP, Attributor &A)
6200 : AANoCaptureImpl(IRP, A) {}
6201
6202 /// See AbstractAttribute::updateImpl(...).
6203 ChangeStatus updateImpl(Attributor &A) override {
6204 // TODO: Once we have call site specific value information we can provide
6205 // call site specific liveness information and then it makes
6206 // sense to specialize attributes for call sites arguments instead of
6207 // redirecting requests to the callee argument.
6208 Argument *Arg = getAssociatedArgument();
6209 if (!Arg)
6210 return indicatePessimisticFixpoint();
6211 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
6212 bool IsKnownNoCapture;
6213 const AANoCapture *ArgAA = nullptr;
6214 if (AA::hasAssumedIRAttr<Attribute::Captures>(
6215 A, QueryingAA: this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
6216 AAPtr: &ArgAA))
6217 return ChangeStatus::UNCHANGED;
6218 if (!ArgAA || !ArgAA->isAssumedNoCaptureMaybeReturned())
6219 return indicatePessimisticFixpoint();
6220 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
6221 }
6222
6223 /// See AbstractAttribute::trackStatistics()
6224 void trackStatistics() const override {
6225 STATS_DECLTRACK_CSARG_ATTR(nocapture)
6226 };
6227};
6228
6229/// NoCapture attribute for floating values.
6230struct AANoCaptureFloating final : AANoCaptureImpl {
6231 AANoCaptureFloating(const IRPosition &IRP, Attributor &A)
6232 : AANoCaptureImpl(IRP, A) {}
6233
6234 /// See AbstractAttribute::trackStatistics()
6235 void trackStatistics() const override {
6236 STATS_DECLTRACK_FLOATING_ATTR(nocapture)
6237 }
6238};
6239
6240/// NoCapture attribute for function return value.
6241struct AANoCaptureReturned final : AANoCaptureImpl {
6242 AANoCaptureReturned(const IRPosition &IRP, Attributor &A)
6243 : AANoCaptureImpl(IRP, A) {
6244 llvm_unreachable("NoCapture is not applicable to function returns!");
6245 }
6246
6247 /// See AbstractAttribute::initialize(...).
6248 void initialize(Attributor &A) override {
6249 llvm_unreachable("NoCapture is not applicable to function returns!");
6250 }
6251
6252 /// See AbstractAttribute::updateImpl(...).
6253 ChangeStatus updateImpl(Attributor &A) override {
6254 llvm_unreachable("NoCapture is not applicable to function returns!");
6255 }
6256
6257 /// See AbstractAttribute::trackStatistics()
6258 void trackStatistics() const override {}
6259};
6260
6261/// NoCapture attribute deduction for a call site return value.
6262struct AANoCaptureCallSiteReturned final : AANoCaptureImpl {
6263 AANoCaptureCallSiteReturned(const IRPosition &IRP, Attributor &A)
6264 : AANoCaptureImpl(IRP, A) {}
6265
6266 /// See AbstractAttribute::initialize(...).
6267 void initialize(Attributor &A) override {
6268 const Function *F = getAnchorScope();
6269 // Check what state the associated function can actually capture.
6270 determineFunctionCaptureCapabilities(IRP: getIRPosition(), F: *F, State&: *this);
6271 }
6272
6273 /// See AbstractAttribute::trackStatistics()
6274 void trackStatistics() const override {
6275 STATS_DECLTRACK_CSRET_ATTR(nocapture)
6276 }
6277};
6278} // namespace
6279
6280/// ------------------ Value Simplify Attribute ----------------------------
6281
6282bool ValueSimplifyStateType::unionAssumed(std::optional<Value *> Other) {
6283 // FIXME: Add a typecast support.
6284 SimplifiedAssociatedValue = AA::combineOptionalValuesInAAValueLatice(
6285 A: SimplifiedAssociatedValue, B: Other, Ty);
6286 if (SimplifiedAssociatedValue == std::optional<Value *>(nullptr))
6287 return false;
6288
6289 LLVM_DEBUG({
6290 if (SimplifiedAssociatedValue)
6291 dbgs() << "[ValueSimplify] is assumed to be "
6292 << **SimplifiedAssociatedValue << "\n";
6293 else
6294 dbgs() << "[ValueSimplify] is assumed to be <none>\n";
6295 });
6296 return true;
6297}
6298
6299namespace {
6300struct AAValueSimplifyImpl : AAValueSimplify {
6301 AAValueSimplifyImpl(const IRPosition &IRP, Attributor &A)
6302 : AAValueSimplify(IRP, A) {}
6303
6304 /// See AbstractAttribute::initialize(...).
6305 void initialize(Attributor &A) override {
6306 if (getAssociatedValue().getType()->isVoidTy())
6307 indicatePessimisticFixpoint();
6308 if (A.hasSimplificationCallback(IRP: getIRPosition()))
6309 indicatePessimisticFixpoint();
6310 }
6311
6312 /// See AbstractAttribute::getAsStr().
6313 const std::string getAsStr(Attributor *A) const override {
6314 LLVM_DEBUG({
6315 dbgs() << "SAV: " << (bool)SimplifiedAssociatedValue << " ";
6316 if (SimplifiedAssociatedValue && *SimplifiedAssociatedValue)
6317 dbgs() << "SAV: " << **SimplifiedAssociatedValue << " ";
6318 });
6319 return isValidState() ? (isAtFixpoint() ? "simplified" : "maybe-simple")
6320 : "not-simple";
6321 }
6322
6323 /// See AbstractAttribute::trackStatistics()
6324 void trackStatistics() const override {}
6325
6326 /// See AAValueSimplify::getAssumedSimplifiedValue()
6327 std::optional<Value *>
6328 getAssumedSimplifiedValue(Attributor &A) const override {
6329 return SimplifiedAssociatedValue;
6330 }
6331
6332 /// Ensure the return value is \p V with type \p Ty, if not possible return
6333 /// nullptr. If \p Check is true we will only verify such an operation would
6334 /// suceed and return a non-nullptr value if that is the case. No IR is
6335 /// generated or modified.
6336 static Value *ensureType(Attributor &A, Value &V, Type &Ty, Instruction *CtxI,
6337 bool Check) {
6338 if (auto *TypedV = AA::getWithType(V, Ty))
6339 return TypedV;
6340 if (CtxI && V.getType()->canLosslesslyBitCastTo(Ty: &Ty))
6341 return Check ? &V
6342 : BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6343 S: &V, Ty: &Ty, Name: "", InsertBefore: CtxI->getIterator());
6344 return nullptr;
6345 }
6346
6347 /// Reproduce \p I with type \p Ty or return nullptr if that is not posisble.
6348 /// If \p Check is true we will only verify such an operation would suceed and
6349 /// return a non-nullptr value if that is the case. No IR is generated or
6350 /// modified.
6351 static Value *reproduceInst(Attributor &A,
6352 const AbstractAttribute &QueryingAA,
6353 Instruction &I, Type &Ty, Instruction *CtxI,
6354 bool Check, ValueToValueMapTy &VMap) {
6355 assert(CtxI && "Cannot reproduce an instruction without context!");
6356 if (Check && (I.mayReadFromMemory() ||
6357 !isSafeToSpeculativelyExecute(I: &I, CtxI, /* DT */ AC: nullptr,
6358 /* TLI */ DT: nullptr)))
6359 return nullptr;
6360 for (Value *Op : I.operands()) {
6361 Value *NewOp = reproduceValue(A, QueryingAA, V&: *Op, Ty, CtxI, Check, VMap);
6362 if (!NewOp) {
6363 assert(Check && "Manifest of new value unexpectedly failed!");
6364 return nullptr;
6365 }
6366 if (!Check)
6367 VMap[Op] = NewOp;
6368 }
6369 if (Check)
6370 return &I;
6371
6372 Instruction *CloneI = I.clone();
6373 // TODO: Try to salvage debug information here.
6374 CloneI->setDebugLoc(DebugLoc());
6375 VMap[&I] = CloneI;
6376 CloneI->insertBefore(InsertPos: CtxI->getIterator());
6377 RemapInstruction(I: CloneI, VM&: VMap);
6378 return CloneI;
6379 }
6380
6381 /// Reproduce \p V with type \p Ty or return nullptr if that is not posisble.
6382 /// If \p Check is true we will only verify such an operation would suceed and
6383 /// return a non-nullptr value if that is the case. No IR is generated or
6384 /// modified.
6385 static Value *reproduceValue(Attributor &A,
6386 const AbstractAttribute &QueryingAA, Value &V,
6387 Type &Ty, Instruction *CtxI, bool Check,
6388 ValueToValueMapTy &VMap) {
6389 if (const auto &NewV = VMap.lookup(Val: &V))
6390 return NewV;
6391 bool UsedAssumedInformation = false;
6392 std::optional<Value *> SimpleV = A.getAssumedSimplified(
6393 V, AA: QueryingAA, UsedAssumedInformation, S: AA::Interprocedural);
6394 if (!SimpleV.has_value())
6395 return PoisonValue::get(T: &Ty);
6396 Value *EffectiveV = &V;
6397 if (*SimpleV)
6398 EffectiveV = *SimpleV;
6399 if (auto *C = dyn_cast<Constant>(Val: EffectiveV))
6400 return C;
6401 if (CtxI && AA::isValidAtPosition(VAC: AA::ValueAndContext(*EffectiveV, *CtxI),
6402 InfoCache&: A.getInfoCache()))
6403 return ensureType(A, V&: *EffectiveV, Ty, CtxI, Check);
6404 if (auto *I = dyn_cast<Instruction>(Val: EffectiveV))
6405 if (Value *NewV = reproduceInst(A, QueryingAA, I&: *I, Ty, CtxI, Check, VMap))
6406 return ensureType(A, V&: *NewV, Ty, CtxI, Check);
6407 return nullptr;
6408 }
6409
6410 /// Return a value we can use as replacement for the associated one, or
6411 /// nullptr if we don't have one that makes sense.
6412 Value *manifestReplacementValue(Attributor &A, Instruction *CtxI) const {
6413 Value *NewV = SimplifiedAssociatedValue
6414 ? *SimplifiedAssociatedValue
6415 : UndefValue::get(T: getAssociatedType());
6416 if (NewV && NewV != &getAssociatedValue()) {
6417 ValueToValueMapTy VMap;
6418 // First verify we can reprduce the value with the required type at the
6419 // context location before we actually start modifying the IR.
6420 if (reproduceValue(A, QueryingAA: *this, V&: *NewV, Ty&: *getAssociatedType(), CtxI,
6421 /* CheckOnly */ Check: true, VMap))
6422 return reproduceValue(A, QueryingAA: *this, V&: *NewV, Ty&: *getAssociatedType(), CtxI,
6423 /* CheckOnly */ Check: false, VMap);
6424 }
6425 return nullptr;
6426 }
6427
6428 /// Helper function for querying AAValueSimplify and updating candidate.
6429 /// \param IRP The value position we are trying to unify with SimplifiedValue
6430 bool checkAndUpdate(Attributor &A, const AbstractAttribute &QueryingAA,
6431 const IRPosition &IRP, bool Simplify = true) {
6432 bool UsedAssumedInformation = false;
6433 std::optional<Value *> QueryingValueSimplified = &IRP.getAssociatedValue();
6434 if (Simplify)
6435 QueryingValueSimplified = A.getAssumedSimplified(
6436 IRP, AA: QueryingAA, UsedAssumedInformation, S: AA::Interprocedural);
6437 return unionAssumed(Other: QueryingValueSimplified);
6438 }
6439
6440 /// Returns a candidate is found or not
6441 template <typename AAType> bool askSimplifiedValueFor(Attributor &A) {
6442 if (!getAssociatedValue().getType()->isIntegerTy())
6443 return false;
6444
6445 // This will also pass the call base context.
6446 const auto *AA =
6447 A.getAAFor<AAType>(*this, getIRPosition(), DepClassTy::NONE);
6448 if (!AA)
6449 return false;
6450
6451 std::optional<Constant *> COpt = AA->getAssumedConstant(A);
6452
6453 if (!COpt) {
6454 SimplifiedAssociatedValue = std::nullopt;
6455 A.recordDependence(FromAA: *AA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
6456 return true;
6457 }
6458 if (auto *C = *COpt) {
6459 SimplifiedAssociatedValue = C;
6460 A.recordDependence(FromAA: *AA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
6461 return true;
6462 }
6463 return false;
6464 }
6465
6466 bool askSimplifiedValueForOtherAAs(Attributor &A) {
6467 if (askSimplifiedValueFor<AAValueConstantRange>(A))
6468 return true;
6469 if (askSimplifiedValueFor<AAPotentialConstantValues>(A))
6470 return true;
6471 return false;
6472 }
6473
6474 /// See AbstractAttribute::manifest(...).
6475 ChangeStatus manifest(Attributor &A) override {
6476 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6477 for (auto &U : getAssociatedValue().uses()) {
6478 // Check if we need to adjust the insertion point to make sure the IR is
6479 // valid.
6480 Instruction *IP = dyn_cast<Instruction>(Val: U.getUser());
6481 if (auto *PHI = dyn_cast_or_null<PHINode>(Val: IP))
6482 IP = PHI->getIncomingBlock(U)->getTerminator();
6483 if (auto *NewV = manifestReplacementValue(A, CtxI: IP)) {
6484 LLVM_DEBUG(dbgs() << "[ValueSimplify] " << getAssociatedValue()
6485 << " -> " << *NewV << " :: " << *this << "\n");
6486 if (A.changeUseAfterManifest(U, NV&: *NewV))
6487 Changed = ChangeStatus::CHANGED;
6488 }
6489 }
6490
6491 return Changed | AAValueSimplify::manifest(A);
6492 }
6493
6494 /// See AbstractState::indicatePessimisticFixpoint(...).
6495 ChangeStatus indicatePessimisticFixpoint() override {
6496 SimplifiedAssociatedValue = &getAssociatedValue();
6497 return AAValueSimplify::indicatePessimisticFixpoint();
6498 }
6499};
6500
6501struct AAValueSimplifyArgument final : AAValueSimplifyImpl {
6502 AAValueSimplifyArgument(const IRPosition &IRP, Attributor &A)
6503 : AAValueSimplifyImpl(IRP, A) {}
6504
6505 void initialize(Attributor &A) override {
6506 AAValueSimplifyImpl::initialize(A);
6507 if (A.hasAttr(IRP: getIRPosition(),
6508 AKs: {Attribute::InAlloca, Attribute::Preallocated,
6509 Attribute::StructRet, Attribute::Nest, Attribute::ByVal},
6510 /* IgnoreSubsumingPositions */ true))
6511 indicatePessimisticFixpoint();
6512 }
6513
6514 /// See AbstractAttribute::updateImpl(...).
6515 ChangeStatus updateImpl(Attributor &A) override {
6516 // Byval is only replacable if it is readonly otherwise we would write into
6517 // the replaced value and not the copy that byval creates implicitly.
6518 Argument *Arg = getAssociatedArgument();
6519 if (Arg->hasByValAttr()) {
6520 // TODO: We probably need to verify synchronization is not an issue, e.g.,
6521 // there is no race by not copying a constant byval.
6522 bool IsKnown;
6523 if (!AA::isAssumedReadOnly(A, IRP: getIRPosition(), QueryingAA: *this, IsKnown))
6524 return indicatePessimisticFixpoint();
6525 }
6526
6527 auto Before = SimplifiedAssociatedValue;
6528
6529 auto PredForCallSite = [&](AbstractCallSite ACS) {
6530 const IRPosition &ACSArgPos =
6531 IRPosition::callsite_argument(ACS, ArgNo: getCallSiteArgNo());
6532 // Check if a coresponding argument was found or if it is on not
6533 // associated (which can happen for callback calls).
6534 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
6535 return false;
6536
6537 // Simplify the argument operand explicitly and check if the result is
6538 // valid in the current scope. This avoids refering to simplified values
6539 // in other functions, e.g., we don't want to say a an argument in a
6540 // static function is actually an argument in a different function.
6541 bool UsedAssumedInformation = false;
6542 std::optional<Constant *> SimpleArgOp =
6543 A.getAssumedConstant(IRP: ACSArgPos, AA: *this, UsedAssumedInformation);
6544 if (!SimpleArgOp)
6545 return true;
6546 if (!*SimpleArgOp)
6547 return false;
6548 if (!AA::isDynamicallyUnique(A, QueryingAA: *this, V: **SimpleArgOp))
6549 return false;
6550 return unionAssumed(Other: *SimpleArgOp);
6551 };
6552
6553 // Generate a answer specific to a call site context.
6554 bool Success;
6555 bool UsedAssumedInformation = false;
6556 if (hasCallBaseContext() &&
6557 getCallBaseContext()->getCalledOperand() == Arg->getParent())
6558 Success = PredForCallSite(
6559 AbstractCallSite(&getCallBaseContext()->getCalledOperandUse()));
6560 else
6561 Success = A.checkForAllCallSites(Pred: PredForCallSite, QueryingAA: *this, RequireAllCallSites: true,
6562 UsedAssumedInformation);
6563
6564 if (!Success)
6565 if (!askSimplifiedValueForOtherAAs(A))
6566 return indicatePessimisticFixpoint();
6567
6568 // If a candidate was found in this update, return CHANGED.
6569 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6570 : ChangeStatus ::CHANGED;
6571 }
6572
6573 /// See AbstractAttribute::trackStatistics()
6574 void trackStatistics() const override {
6575 STATS_DECLTRACK_ARG_ATTR(value_simplify)
6576 }
6577};
6578
6579struct AAValueSimplifyReturned : AAValueSimplifyImpl {
6580 AAValueSimplifyReturned(const IRPosition &IRP, Attributor &A)
6581 : AAValueSimplifyImpl(IRP, A) {}
6582
6583 /// See AAValueSimplify::getAssumedSimplifiedValue()
6584 std::optional<Value *>
6585 getAssumedSimplifiedValue(Attributor &A) const override {
6586 if (!isValidState())
6587 return nullptr;
6588 return SimplifiedAssociatedValue;
6589 }
6590
6591 /// See AbstractAttribute::updateImpl(...).
6592 ChangeStatus updateImpl(Attributor &A) override {
6593 auto Before = SimplifiedAssociatedValue;
6594
6595 auto ReturnInstCB = [&](Instruction &I) {
6596 auto &RI = cast<ReturnInst>(Val&: I);
6597 return checkAndUpdate(
6598 A, QueryingAA: *this,
6599 IRP: IRPosition::value(V: *RI.getReturnValue(), CBContext: getCallBaseContext()));
6600 };
6601
6602 bool UsedAssumedInformation = false;
6603 if (!A.checkForAllInstructions(Pred: ReturnInstCB, QueryingAA: *this, Opcodes: {Instruction::Ret},
6604 UsedAssumedInformation))
6605 if (!askSimplifiedValueForOtherAAs(A))
6606 return indicatePessimisticFixpoint();
6607
6608 // If a candidate was found in this update, return CHANGED.
6609 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6610 : ChangeStatus ::CHANGED;
6611 }
6612
6613 ChangeStatus manifest(Attributor &A) override {
6614 // We queried AAValueSimplify for the returned values so they will be
6615 // replaced if a simplified form was found. Nothing to do here.
6616 return ChangeStatus::UNCHANGED;
6617 }
6618
6619 /// See AbstractAttribute::trackStatistics()
6620 void trackStatistics() const override {
6621 STATS_DECLTRACK_FNRET_ATTR(value_simplify)
6622 }
6623};
6624
6625struct AAValueSimplifyFloating : AAValueSimplifyImpl {
6626 AAValueSimplifyFloating(const IRPosition &IRP, Attributor &A)
6627 : AAValueSimplifyImpl(IRP, A) {}
6628
6629 /// See AbstractAttribute::initialize(...).
6630 void initialize(Attributor &A) override {
6631 AAValueSimplifyImpl::initialize(A);
6632 Value &V = getAnchorValue();
6633
6634 // TODO: add other stuffs
6635 if (isa<Constant>(Val: V))
6636 indicatePessimisticFixpoint();
6637 }
6638
6639 /// See AbstractAttribute::updateImpl(...).
6640 ChangeStatus updateImpl(Attributor &A) override {
6641 auto Before = SimplifiedAssociatedValue;
6642 if (!askSimplifiedValueForOtherAAs(A))
6643 return indicatePessimisticFixpoint();
6644
6645 // If a candidate was found in this update, return CHANGED.
6646 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6647 : ChangeStatus ::CHANGED;
6648 }
6649
6650 /// See AbstractAttribute::trackStatistics()
6651 void trackStatistics() const override {
6652 STATS_DECLTRACK_FLOATING_ATTR(value_simplify)
6653 }
6654};
6655
6656struct AAValueSimplifyFunction : AAValueSimplifyImpl {
6657 AAValueSimplifyFunction(const IRPosition &IRP, Attributor &A)
6658 : AAValueSimplifyImpl(IRP, A) {}
6659
6660 /// See AbstractAttribute::initialize(...).
6661 void initialize(Attributor &A) override {
6662 SimplifiedAssociatedValue = nullptr;
6663 indicateOptimisticFixpoint();
6664 }
6665 /// See AbstractAttribute::initialize(...).
6666 ChangeStatus updateImpl(Attributor &A) override {
6667 llvm_unreachable(
6668 "AAValueSimplify(Function|CallSite)::updateImpl will not be called");
6669 }
6670 /// See AbstractAttribute::trackStatistics()
6671 void trackStatistics() const override {
6672 STATS_DECLTRACK_FN_ATTR(value_simplify)
6673 }
6674};
6675
6676struct AAValueSimplifyCallSite : AAValueSimplifyFunction {
6677 AAValueSimplifyCallSite(const IRPosition &IRP, Attributor &A)
6678 : AAValueSimplifyFunction(IRP, A) {}
6679 /// See AbstractAttribute::trackStatistics()
6680 void trackStatistics() const override {
6681 STATS_DECLTRACK_CS_ATTR(value_simplify)
6682 }
6683};
6684
6685struct AAValueSimplifyCallSiteReturned : AAValueSimplifyImpl {
6686 AAValueSimplifyCallSiteReturned(const IRPosition &IRP, Attributor &A)
6687 : AAValueSimplifyImpl(IRP, A) {}
6688
6689 void initialize(Attributor &A) override {
6690 AAValueSimplifyImpl::initialize(A);
6691 Function *Fn = getAssociatedFunction();
6692 assert(Fn && "Did expect an associted function");
6693 for (Argument &Arg : Fn->args()) {
6694 if (Arg.hasReturnedAttr()) {
6695 auto IRP = IRPosition::callsite_argument(CB: *cast<CallBase>(Val: getCtxI()),
6696 ArgNo: Arg.getArgNo());
6697 if (IRP.getPositionKind() == IRPosition::IRP_CALL_SITE_ARGUMENT &&
6698 checkAndUpdate(A, QueryingAA: *this, IRP))
6699 indicateOptimisticFixpoint();
6700 else
6701 indicatePessimisticFixpoint();
6702 return;
6703 }
6704 }
6705 }
6706
6707 /// See AbstractAttribute::updateImpl(...).
6708 ChangeStatus updateImpl(Attributor &A) override {
6709 return indicatePessimisticFixpoint();
6710 }
6711
6712 void trackStatistics() const override {
6713 STATS_DECLTRACK_CSRET_ATTR(value_simplify)
6714 }
6715};
6716
6717struct AAValueSimplifyCallSiteArgument : AAValueSimplifyFloating {
6718 AAValueSimplifyCallSiteArgument(const IRPosition &IRP, Attributor &A)
6719 : AAValueSimplifyFloating(IRP, A) {}
6720
6721 /// See AbstractAttribute::manifest(...).
6722 ChangeStatus manifest(Attributor &A) override {
6723 ChangeStatus Changed = ChangeStatus::UNCHANGED;
6724 // TODO: We should avoid simplification duplication to begin with.
6725 auto *FloatAA = A.lookupAAFor<AAValueSimplify>(
6726 IRP: IRPosition::value(V: getAssociatedValue()), QueryingAA: this, DepClass: DepClassTy::NONE);
6727 if (FloatAA && FloatAA->getState().isValidState())
6728 return Changed;
6729
6730 if (auto *NewV = manifestReplacementValue(A, CtxI: getCtxI())) {
6731 Use &U = cast<CallBase>(Val: &getAnchorValue())
6732 ->getArgOperandUse(i: getCallSiteArgNo());
6733 if (A.changeUseAfterManifest(U, NV&: *NewV))
6734 Changed = ChangeStatus::CHANGED;
6735 }
6736
6737 return Changed | AAValueSimplify::manifest(A);
6738 }
6739
6740 void trackStatistics() const override {
6741 STATS_DECLTRACK_CSARG_ATTR(value_simplify)
6742 }
6743};
6744} // namespace
6745
6746/// ----------------------- Heap-To-Stack Conversion ---------------------------
6747namespace {
6748struct AAHeapToStackFunction final : public AAHeapToStack {
6749
6750 static bool isGlobalizedLocal(const CallBase &CB) {
6751 Attribute A = CB.getFnAttr(Kind: "alloc-family");
6752 return A.isValid() && A.getValueAsString() == "__kmpc_alloc_shared";
6753 }
6754
6755 struct AllocationInfo {
6756 /// The call that allocates the memory.
6757 CallBase *const CB;
6758
6759 /// Whether this allocation is an OpenMP globalized local variable.
6760 bool IsGlobalizedLocal = false;
6761
6762 /// The status wrt. a rewrite.
6763 enum {
6764 STACK_DUE_TO_USE,
6765 STACK_DUE_TO_FREE,
6766 INVALID,
6767 } Status = STACK_DUE_TO_USE;
6768
6769 /// Flag to indicate if we encountered a use that might free this allocation
6770 /// but which is not in the deallocation infos.
6771 bool HasPotentiallyFreeingUnknownUses = false;
6772
6773 /// Flag to indicate that we should place the new alloca in the function
6774 /// entry block rather than where the call site (CB) is.
6775 bool MoveAllocaIntoEntry = true;
6776
6777 /// The set of free calls that use this allocation.
6778 SmallSetVector<CallBase *, 1> PotentialFreeCalls{};
6779 };
6780
6781 struct DeallocationInfo {
6782 /// The call that deallocates the memory.
6783 CallBase *const CB;
6784 /// The value freed by the call.
6785 Value *FreedOp;
6786
6787 /// Flag to indicate if we don't know all objects this deallocation might
6788 /// free.
6789 bool MightFreeUnknownObjects = false;
6790
6791 /// The set of allocation calls that are potentially freed.
6792 SmallSetVector<CallBase *, 1> PotentialAllocationCalls{};
6793 };
6794
6795 AAHeapToStackFunction(const IRPosition &IRP, Attributor &A)
6796 : AAHeapToStack(IRP, A) {}
6797
6798 ~AAHeapToStackFunction() override {
6799 // Ensure we call the destructor so we release any memory allocated in the
6800 // sets.
6801 for (auto &It : AllocationInfos)
6802 It.second->~AllocationInfo();
6803 for (auto &It : DeallocationInfos)
6804 It.second->~DeallocationInfo();
6805 }
6806
6807 void initialize(Attributor &A) override {
6808 AAHeapToStack::initialize(A);
6809
6810 const Function *F = getAnchorScope();
6811 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6812
6813 auto AllocationIdentifierCB = [&](Instruction &I) {
6814 CallBase *CB = dyn_cast<CallBase>(Val: &I);
6815 if (!CB)
6816 return true;
6817 if (Value *FreedOp = getFreedOperand(CB, TLI)) {
6818 DeallocationInfos[CB] = new (A.Allocator) DeallocationInfo{.CB: CB, .FreedOp: FreedOp};
6819 return true;
6820 }
6821 // To do heap to stack, we need to know that the allocation itself is
6822 // removable once uses are rewritten, and that we can initialize the
6823 // alloca to the same pattern as the original allocation result.
6824 if (isRemovableAlloc(V: CB, TLI)) {
6825 auto *I8Ty = Type::getInt8Ty(C&: CB->getParent()->getContext());
6826 if (nullptr != getInitialValueOfAllocation(V: CB, TLI, Ty: I8Ty)) {
6827 AllocationInfo *AI = new (A.Allocator) AllocationInfo{.CB: CB};
6828 AllocationInfos[CB] = AI;
6829 AI->IsGlobalizedLocal = isGlobalizedLocal(CB: *CB);
6830 }
6831 }
6832 return true;
6833 };
6834
6835 bool UsedAssumedInformation = false;
6836 bool Success = A.checkForAllCallLikeInstructions(
6837 Pred: AllocationIdentifierCB, QueryingAA: *this, UsedAssumedInformation,
6838 /* CheckBBLivenessOnly */ false,
6839 /* CheckPotentiallyDead */ true);
6840 (void)Success;
6841 assert(Success && "Did not expect the call base visit callback to fail!");
6842
6843 Attributor::SimplifictionCallbackTy SCB =
6844 [](const IRPosition &, const AbstractAttribute *,
6845 bool &) -> std::optional<Value *> { return nullptr; };
6846 for (const auto &It : AllocationInfos)
6847 A.registerSimplificationCallback(IRP: IRPosition::callsite_returned(CB: *It.first),
6848 CB: SCB);
6849 for (const auto &It : DeallocationInfos)
6850 A.registerSimplificationCallback(IRP: IRPosition::callsite_returned(CB: *It.first),
6851 CB: SCB);
6852 }
6853
6854 const std::string getAsStr(Attributor *A) const override {
6855 unsigned NumH2SMallocs = 0, NumInvalidMallocs = 0;
6856 for (const auto &It : AllocationInfos) {
6857 if (It.second->Status == AllocationInfo::INVALID)
6858 ++NumInvalidMallocs;
6859 else
6860 ++NumH2SMallocs;
6861 }
6862 return "[H2S] Mallocs Good/Bad: " + std::to_string(val: NumH2SMallocs) + "/" +
6863 std::to_string(val: NumInvalidMallocs);
6864 }
6865
6866 /// See AbstractAttribute::trackStatistics().
6867 void trackStatistics() const override {
6868 STATS_DECL(
6869 MallocCalls, Function,
6870 "Number of malloc/calloc/aligned_alloc calls converted to allocas");
6871 for (const auto &It : AllocationInfos)
6872 if (It.second->Status != AllocationInfo::INVALID)
6873 ++BUILD_STAT_NAME(MallocCalls, Function);
6874 }
6875
6876 bool isAssumedHeapToStack(const CallBase &CB) const override {
6877 if (isValidState())
6878 if (AllocationInfo *AI =
6879 AllocationInfos.lookup(Key: const_cast<CallBase *>(&CB)))
6880 return AI->Status != AllocationInfo::INVALID;
6881 return false;
6882 }
6883
6884 bool isAssumedHeapToStackRemovedFree(CallBase &CB) const override {
6885 if (!isValidState())
6886 return false;
6887
6888 for (const auto &It : AllocationInfos) {
6889 AllocationInfo &AI = *It.second;
6890 if (AI.Status == AllocationInfo::INVALID)
6891 continue;
6892
6893 if (AI.PotentialFreeCalls.count(key: &CB))
6894 return true;
6895 }
6896
6897 return false;
6898 }
6899
6900 ChangeStatus manifest(Attributor &A) override {
6901 assert(getState().isValidState() &&
6902 "Attempted to manifest an invalid state!");
6903
6904 ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
6905 Function *F = getAnchorScope();
6906 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
6907
6908 for (auto &It : AllocationInfos) {
6909 AllocationInfo &AI = *It.second;
6910 if (AI.Status == AllocationInfo::INVALID)
6911 continue;
6912
6913 for (CallBase *FreeCall : AI.PotentialFreeCalls) {
6914 LLVM_DEBUG(dbgs() << "H2S: Removing free call: " << *FreeCall << "\n");
6915 A.deleteAfterManifest(I&: *FreeCall);
6916 HasChanged = ChangeStatus::CHANGED;
6917 }
6918
6919 LLVM_DEBUG(dbgs() << "H2S: Removing malloc-like call: " << *AI.CB
6920 << "\n");
6921
6922 auto Remark = [&](OptimizationRemark OR) {
6923 if (AI.IsGlobalizedLocal)
6924 return OR << "Moving globalized variable to the stack.";
6925 return OR << "Moving memory allocation from the heap to the stack.";
6926 };
6927 if (AI.IsGlobalizedLocal)
6928 A.emitRemark<OptimizationRemark>(I: AI.CB, RemarkName: "OMP110", RemarkCB&: Remark);
6929 else
6930 A.emitRemark<OptimizationRemark>(I: AI.CB, RemarkName: "HeapToStack", RemarkCB&: Remark);
6931
6932 const DataLayout &DL = A.getInfoCache().getDL();
6933 Value *Size;
6934 std::optional<APInt> SizeAPI = getSize(A, AA: *this, AI);
6935 if (SizeAPI) {
6936 Size = ConstantInt::get(Context&: AI.CB->getContext(), V: *SizeAPI);
6937 } else {
6938 ObjectSizeOpts Opts;
6939 ObjectSizeOffsetEvaluator Eval(*AI.CB->getModule(), TLI, Opts);
6940 SizeOffsetValue SizeOffsetPair = Eval.compute(V: AI.CB);
6941 assert(SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown() &&
6942 cast<ConstantInt>(SizeOffsetPair.Offset)->isZero());
6943 Size = SizeOffsetPair.Size;
6944 }
6945
6946 BasicBlock::iterator IP = AI.MoveAllocaIntoEntry
6947 ? F->getEntryBlock().begin()
6948 : AI.CB->getIterator();
6949
6950 Align Alignment(1);
6951 if (MaybeAlign RetAlign = AI.CB->getRetAlign())
6952 Alignment = std::max(a: Alignment, b: *RetAlign);
6953 if (Value *Align = getAllocAlignment(V: AI.CB, TLI)) {
6954 std::optional<APInt> AlignmentAPI = getAPInt(A, AA: *this, V&: *Align);
6955 assert(AlignmentAPI && AlignmentAPI->getZExtValue() > 0 &&
6956 "Expected an alignment during manifest!");
6957 Alignment =
6958 std::max(a: Alignment, b: assumeAligned(Value: AlignmentAPI->getZExtValue()));
6959 }
6960
6961 // TODO: Hoist the alloca towards the function entry.
6962 unsigned AS = DL.getAllocaAddrSpace();
6963 Instruction *Alloca =
6964 new AllocaInst(Type::getInt8Ty(C&: F->getContext()), AS, Size, Alignment,
6965 AI.CB->getName() + ".h2s", IP);
6966
6967 if (Alloca->getType() != AI.CB->getType())
6968 Alloca = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6969 S: Alloca, Ty: AI.CB->getType(), Name: "malloc_cast", InsertBefore: AI.CB->getIterator());
6970
6971 auto *I8Ty = Type::getInt8Ty(C&: F->getContext());
6972 auto *InitVal = getInitialValueOfAllocation(V: AI.CB, TLI, Ty: I8Ty);
6973 assert(InitVal &&
6974 "Must be able to materialize initial memory state of allocation");
6975
6976 A.changeAfterManifest(IRP: IRPosition::inst(I: *AI.CB), NV&: *Alloca);
6977
6978 if (auto *II = dyn_cast<InvokeInst>(Val: AI.CB)) {
6979 auto *NBB = II->getNormalDest();
6980 UncondBrInst::Create(Target: NBB, InsertBefore: AI.CB->getParent());
6981 A.deleteAfterManifest(I&: *AI.CB);
6982 } else {
6983 A.deleteAfterManifest(I&: *AI.CB);
6984 }
6985
6986 // Initialize the alloca with the same value as used by the allocation
6987 // function. We can skip undef as the initial value of an alloc is
6988 // undef, and the memset would simply end up being DSEd.
6989 if (!isa<UndefValue>(Val: InitVal)) {
6990 IRBuilder<> Builder(Alloca->getNextNode());
6991 // TODO: Use alignment above if align!=1
6992 Builder.CreateMemSet(Ptr: Alloca, Val: InitVal, Size, Align: std::nullopt);
6993 }
6994 HasChanged = ChangeStatus::CHANGED;
6995 }
6996
6997 return HasChanged;
6998 }
6999
7000 std::optional<APInt> getAPInt(Attributor &A, const AbstractAttribute &AA,
7001 Value &V) {
7002 bool UsedAssumedInformation = false;
7003 std::optional<Constant *> SimpleV =
7004 A.getAssumedConstant(V, AA, UsedAssumedInformation);
7005 if (!SimpleV)
7006 return APInt(64, 0);
7007 if (auto *CI = dyn_cast_or_null<ConstantInt>(Val: *SimpleV))
7008 return CI->getValue();
7009 return std::nullopt;
7010 }
7011
7012 std::optional<APInt> getSize(Attributor &A, const AbstractAttribute &AA,
7013 AllocationInfo &AI) {
7014 auto Mapper = [&](const Value *V) -> const Value * {
7015 bool UsedAssumedInformation = false;
7016 if (std::optional<Constant *> SimpleV =
7017 A.getAssumedConstant(V: *V, AA, UsedAssumedInformation))
7018 if (*SimpleV)
7019 return *SimpleV;
7020 return V;
7021 };
7022
7023 const Function *F = getAnchorScope();
7024 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
7025 return getAllocSize(CB: AI.CB, TLI, Mapper);
7026 }
7027
7028 /// Collection of all malloc-like calls in a function with associated
7029 /// information.
7030 MapVector<CallBase *, AllocationInfo *> AllocationInfos;
7031
7032 /// Collection of all free-like calls in a function with associated
7033 /// information.
7034 MapVector<CallBase *, DeallocationInfo *> DeallocationInfos;
7035
7036 ChangeStatus updateImpl(Attributor &A) override;
7037};
7038
7039ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) {
7040 ChangeStatus Changed = ChangeStatus::UNCHANGED;
7041 const Function *F = getAnchorScope();
7042 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
7043
7044 const auto *LivenessAA =
7045 A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: IRPosition::function(F: *F), DepClass: DepClassTy::NONE);
7046
7047 MustBeExecutedContextExplorer *Explorer =
7048 A.getInfoCache().getMustBeExecutedContextExplorer();
7049
7050 bool StackIsAccessibleByOtherThreads =
7051 A.getInfoCache().stackIsAccessibleByOtherThreads();
7052
7053 LoopInfo *LI =
7054 A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F: *F);
7055 std::optional<bool> MayContainIrreducibleControl;
7056 auto IsInLoop = [&](BasicBlock &BB) {
7057 if (&F->getEntryBlock() == &BB)
7058 return false;
7059 if (!MayContainIrreducibleControl.has_value())
7060 MayContainIrreducibleControl = mayContainIrreducibleControl(F: *F, LI);
7061 if (*MayContainIrreducibleControl)
7062 return true;
7063 if (!LI)
7064 return true;
7065 return LI->getLoopFor(BB: &BB) != nullptr;
7066 };
7067
7068 // Flag to ensure we update our deallocation information at most once per
7069 // updateImpl call and only if we use the free check reasoning.
7070 bool HasUpdatedFrees = false;
7071
7072 auto UpdateFrees = [&]() {
7073 HasUpdatedFrees = true;
7074
7075 for (auto &It : DeallocationInfos) {
7076 DeallocationInfo &DI = *It.second;
7077 // For now we cannot use deallocations that have unknown inputs, skip
7078 // them.
7079 if (DI.MightFreeUnknownObjects)
7080 continue;
7081
7082 // No need to analyze dead calls, ignore them instead.
7083 bool UsedAssumedInformation = false;
7084 if (A.isAssumedDead(I: *DI.CB, QueryingAA: this, LivenessAA, UsedAssumedInformation,
7085 /* CheckBBLivenessOnly */ true))
7086 continue;
7087
7088 // Use the non-optimistic version to get the freed object.
7089 Value *Obj = getUnderlyingObject(V: DI.FreedOp);
7090 if (!Obj) {
7091 LLVM_DEBUG(dbgs() << "[H2S] Unknown underlying object for free!\n");
7092 DI.MightFreeUnknownObjects = true;
7093 continue;
7094 }
7095
7096 // Free of null and undef can be ignored as no-ops (or UB in the latter
7097 // case).
7098 if (isa<ConstantPointerNull>(Val: Obj) || isa<UndefValue>(Val: Obj))
7099 continue;
7100
7101 CallBase *ObjCB = dyn_cast<CallBase>(Val: Obj);
7102 if (!ObjCB) {
7103 LLVM_DEBUG(dbgs() << "[H2S] Free of a non-call object: " << *Obj
7104 << "\n");
7105 DI.MightFreeUnknownObjects = true;
7106 continue;
7107 }
7108
7109 AllocationInfo *AI = AllocationInfos.lookup(Key: ObjCB);
7110 if (!AI) {
7111 LLVM_DEBUG(dbgs() << "[H2S] Free of a non-allocation object: " << *Obj
7112 << "\n");
7113 DI.MightFreeUnknownObjects = true;
7114 continue;
7115 }
7116
7117 DI.PotentialAllocationCalls.insert(X: ObjCB);
7118 }
7119 };
7120
7121 auto FreeCheck = [&](AllocationInfo &AI) {
7122 // If the stack is not accessible by other threads, the "must-free" logic
7123 // doesn't apply as the pointer could be shared and needs to be places in
7124 // "shareable" memory.
7125 if (!StackIsAccessibleByOtherThreads) {
7126 bool IsKnownNoSycn;
7127 if (!AA::hasAssumedIRAttr<Attribute::NoSync>(
7128 A, QueryingAA: this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoSycn)) {
7129 LLVM_DEBUG(
7130 dbgs() << "[H2S] found an escaping use, stack is not accessible by "
7131 "other threads and function is not nosync:\n");
7132 return false;
7133 }
7134 }
7135 if (!HasUpdatedFrees)
7136 UpdateFrees();
7137
7138 // TODO: Allow multi exit functions that have different free calls.
7139 if (AI.PotentialFreeCalls.size() != 1) {
7140 LLVM_DEBUG(dbgs() << "[H2S] did not find one free call but "
7141 << AI.PotentialFreeCalls.size() << "\n");
7142 return false;
7143 }
7144 CallBase *UniqueFree = *AI.PotentialFreeCalls.begin();
7145 DeallocationInfo *DI = DeallocationInfos.lookup(Key: UniqueFree);
7146 if (!DI) {
7147 LLVM_DEBUG(
7148 dbgs() << "[H2S] unique free call was not known as deallocation call "
7149 << *UniqueFree << "\n");
7150 return false;
7151 }
7152 if (DI->MightFreeUnknownObjects) {
7153 LLVM_DEBUG(
7154 dbgs() << "[H2S] unique free call might free unknown allocations\n");
7155 return false;
7156 }
7157 if (DI->PotentialAllocationCalls.empty())
7158 return true;
7159 if (DI->PotentialAllocationCalls.size() > 1) {
7160 LLVM_DEBUG(dbgs() << "[H2S] unique free call might free "
7161 << DI->PotentialAllocationCalls.size()
7162 << " different allocations\n");
7163 return false;
7164 }
7165 if (*DI->PotentialAllocationCalls.begin() != AI.CB) {
7166 LLVM_DEBUG(
7167 dbgs()
7168 << "[H2S] unique free call not known to free this allocation but "
7169 << **DI->PotentialAllocationCalls.begin() << "\n");
7170 return false;
7171 }
7172
7173 // __kmpc_alloc_shared and __kmpc_free_shared are by construction matched.
7174 if (!AI.IsGlobalizedLocal) {
7175 Instruction *CtxI = isa<InvokeInst>(Val: AI.CB) ? AI.CB : AI.CB->getNextNode();
7176 if (!Explorer || !Explorer->findInContextOf(I: UniqueFree, PP: CtxI)) {
7177 LLVM_DEBUG(dbgs() << "[H2S] unique free call might not be executed "
7178 "with the allocation "
7179 << *UniqueFree << "\n");
7180 return false;
7181 }
7182 }
7183 return true;
7184 };
7185
7186 auto UsesCheck = [&](AllocationInfo &AI) {
7187 bool ValidUsesOnly = true;
7188
7189 auto Pred = [&](const Use &U, bool &Follow) -> bool {
7190 Instruction *UserI = cast<Instruction>(Val: U.getUser());
7191 if (isa<LoadInst>(Val: UserI))
7192 return true;
7193 if (auto *SI = dyn_cast<StoreInst>(Val: UserI)) {
7194 if (SI->getValueOperand() == U.get()) {
7195 LLVM_DEBUG(dbgs()
7196 << "[H2S] escaping store to memory: " << *UserI << "\n");
7197 ValidUsesOnly = false;
7198 } else {
7199 // A store into the malloc'ed memory is fine.
7200 }
7201 return true;
7202 }
7203 if (auto *CB = dyn_cast<CallBase>(Val: UserI)) {
7204 if (!CB->isArgOperand(U: &U) || CB->isLifetimeStartOrEnd())
7205 return true;
7206 if (DeallocationInfos.count(Key: CB)) {
7207 AI.PotentialFreeCalls.insert(X: CB);
7208 return true;
7209 }
7210
7211 unsigned ArgNo = CB->getArgOperandNo(U: &U);
7212 auto CBIRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
7213
7214 bool IsKnownNoCapture;
7215 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
7216 A, QueryingAA: this, IRP: CBIRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture);
7217
7218 // If a call site argument use is nofree, we are fine.
7219 bool IsKnownNoFree;
7220 bool IsAssumedNoFree = AA::hasAssumedIRAttr<Attribute::NoFree>(
7221 A, QueryingAA: this, IRP: CBIRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoFree);
7222
7223 if (!IsAssumedNoCapture ||
7224 (!AI.IsGlobalizedLocal && !IsAssumedNoFree)) {
7225 AI.HasPotentiallyFreeingUnknownUses |= !IsAssumedNoFree;
7226
7227 // Emit a missed remark if this is missed OpenMP globalization.
7228 auto Remark = [&](OptimizationRemarkMissed ORM) {
7229 return ORM
7230 << "Could not move globalized variable to the stack. "
7231 "Variable is potentially captured in call. Mark "
7232 "parameter as `__attribute__((noescape))` to override.";
7233 };
7234
7235 if (ValidUsesOnly && AI.IsGlobalizedLocal)
7236 A.emitRemark<OptimizationRemarkMissed>(I: CB, RemarkName: "OMP113", RemarkCB&: Remark);
7237
7238 LLVM_DEBUG(dbgs() << "[H2S] Bad user: " << *UserI << "\n");
7239 ValidUsesOnly = false;
7240 }
7241 return true;
7242 }
7243
7244 if (isa<GetElementPtrInst>(Val: UserI) || isa<BitCastInst>(Val: UserI) ||
7245 isa<PHINode>(Val: UserI) || isa<SelectInst>(Val: UserI)) {
7246 Follow = true;
7247 return true;
7248 }
7249 // Unknown user for which we can not track uses further (in a way that
7250 // makes sense).
7251 LLVM_DEBUG(dbgs() << "[H2S] Unknown user: " << *UserI << "\n");
7252 ValidUsesOnly = false;
7253 return true;
7254 };
7255 if (!A.checkForAllUses(Pred, QueryingAA: *this, V: *AI.CB, /* CheckBBLivenessOnly */ false,
7256 LivenessDepClass: DepClassTy::OPTIONAL, /* IgnoreDroppableUses */ true,
7257 EquivalentUseCB: [&](const Use &OldU, const Use &NewU) {
7258 auto *SI = dyn_cast<StoreInst>(Val: OldU.getUser());
7259 return !SI || StackIsAccessibleByOtherThreads ||
7260 AA::isAssumedThreadLocalObject(
7261 A, Obj&: *SI->getPointerOperand(), QueryingAA: *this);
7262 }))
7263 return false;
7264 return ValidUsesOnly;
7265 };
7266
7267 // The actual update starts here. We look at all allocations and depending on
7268 // their status perform the appropriate check(s).
7269 for (auto &It : AllocationInfos) {
7270 AllocationInfo &AI = *It.second;
7271 if (AI.Status == AllocationInfo::INVALID)
7272 continue;
7273
7274 if (Value *Align = getAllocAlignment(V: AI.CB, TLI)) {
7275 std::optional<APInt> APAlign = getAPInt(A, AA: *this, V&: *Align);
7276 if (!APAlign) {
7277 // Can't generate an alloca which respects the required alignment
7278 // on the allocation.
7279 LLVM_DEBUG(dbgs() << "[H2S] Unknown allocation alignment: " << *AI.CB
7280 << "\n");
7281 AI.Status = AllocationInfo::INVALID;
7282 Changed = ChangeStatus::CHANGED;
7283 continue;
7284 }
7285 if (APAlign->ugt(RHS: llvm::Value::MaximumAlignment) ||
7286 !APAlign->isPowerOf2()) {
7287 LLVM_DEBUG(dbgs() << "[H2S] Invalid allocation alignment: " << APAlign
7288 << "\n");
7289 AI.Status = AllocationInfo::INVALID;
7290 Changed = ChangeStatus::CHANGED;
7291 continue;
7292 }
7293 }
7294
7295 std::optional<APInt> Size = getSize(A, AA: *this, AI);
7296 if (!AI.IsGlobalizedLocal && MaxHeapToStackSize != -1) {
7297 if (!Size || Size->ugt(RHS: MaxHeapToStackSize)) {
7298 LLVM_DEBUG({
7299 if (!Size)
7300 dbgs() << "[H2S] Unknown allocation size: " << *AI.CB << "\n";
7301 else
7302 dbgs() << "[H2S] Allocation size too large: " << *AI.CB << " vs. "
7303 << MaxHeapToStackSize << "\n";
7304 });
7305
7306 AI.Status = AllocationInfo::INVALID;
7307 Changed = ChangeStatus::CHANGED;
7308 continue;
7309 }
7310 }
7311
7312 switch (AI.Status) {
7313 case AllocationInfo::STACK_DUE_TO_USE:
7314 if (UsesCheck(AI))
7315 break;
7316 AI.Status = AllocationInfo::STACK_DUE_TO_FREE;
7317 [[fallthrough]];
7318 case AllocationInfo::STACK_DUE_TO_FREE:
7319 if (FreeCheck(AI))
7320 break;
7321 AI.Status = AllocationInfo::INVALID;
7322 Changed = ChangeStatus::CHANGED;
7323 break;
7324 case AllocationInfo::INVALID:
7325 llvm_unreachable("Invalid allocations should never reach this point!");
7326 };
7327
7328 // Check if we still think we can move it into the entry block. If the
7329 // alloca comes from a converted __kmpc_alloc_shared then we can usually
7330 // ignore the potential complications associated with loops.
7331 bool IsGlobalizedLocal = AI.IsGlobalizedLocal;
7332 if (AI.MoveAllocaIntoEntry &&
7333 (!Size.has_value() ||
7334 (!IsGlobalizedLocal && IsInLoop(*AI.CB->getParent()))))
7335 AI.MoveAllocaIntoEntry = false;
7336 }
7337
7338 return Changed;
7339}
7340} // namespace
7341
7342/// ----------------------- Privatizable Pointers ------------------------------
7343namespace {
7344struct AAPrivatizablePtrImpl : public AAPrivatizablePtr {
7345 AAPrivatizablePtrImpl(const IRPosition &IRP, Attributor &A)
7346 : AAPrivatizablePtr(IRP, A), PrivatizableType(std::nullopt) {}
7347
7348 ChangeStatus indicatePessimisticFixpoint() override {
7349 AAPrivatizablePtr::indicatePessimisticFixpoint();
7350 PrivatizableType = nullptr;
7351 return ChangeStatus::CHANGED;
7352 }
7353
7354 /// Identify the type we can chose for a private copy of the underlying
7355 /// argument. std::nullopt means it is not clear yet, nullptr means there is
7356 /// none.
7357 virtual std::optional<Type *> identifyPrivatizableType(Attributor &A) = 0;
7358
7359 /// Return a privatizable type that encloses both T0 and T1.
7360 /// TODO: This is merely a stub for now as we should manage a mapping as well.
7361 std::optional<Type *> combineTypes(std::optional<Type *> T0,
7362 std::optional<Type *> T1) {
7363 if (!T0)
7364 return T1;
7365 if (!T1)
7366 return T0;
7367 if (T0 == T1)
7368 return T0;
7369 return nullptr;
7370 }
7371
7372 std::optional<Type *> getPrivatizableType() const override {
7373 return PrivatizableType;
7374 }
7375
7376 const std::string getAsStr(Attributor *A) const override {
7377 return isAssumedPrivatizablePtr() ? "[priv]" : "[no-priv]";
7378 }
7379
7380protected:
7381 std::optional<Type *> PrivatizableType;
7382};
7383
7384// TODO: Do this for call site arguments (probably also other values) as well.
7385
7386struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl {
7387 AAPrivatizablePtrArgument(const IRPosition &IRP, Attributor &A)
7388 : AAPrivatizablePtrImpl(IRP, A) {}
7389
7390 /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
7391 std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7392 // If this is a byval argument and we know all the call sites (so we can
7393 // rewrite them), there is no need to check them explicitly.
7394 bool UsedAssumedInformation = false;
7395 SmallVector<Attribute, 1> Attrs;
7396 A.getAttrs(IRP: getIRPosition(), AKs: {Attribute::ByVal}, Attrs,
7397 /* IgnoreSubsumingPositions */ true);
7398 if (!Attrs.empty() &&
7399 A.checkForAllCallSites(Pred: [](AbstractCallSite ACS) { return true; }, QueryingAA: *this,
7400 RequireAllCallSites: true, UsedAssumedInformation))
7401 return Attrs[0].getValueAsType();
7402
7403 std::optional<Type *> Ty;
7404 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
7405
7406 // Make sure the associated call site argument has the same type at all call
7407 // sites and it is an allocation we know is safe to privatize, for now that
7408 // means we only allow alloca instructions.
7409 // TODO: We can additionally analyze the accesses in the callee to create
7410 // the type from that information instead. That is a little more
7411 // involved and will be done in a follow up patch.
7412 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7413 IRPosition ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
7414 // Check if a coresponding argument was found or if it is one not
7415 // associated (which can happen for callback calls).
7416 if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
7417 return false;
7418
7419 // Check that all call sites agree on a type.
7420 auto *PrivCSArgAA =
7421 A.getAAFor<AAPrivatizablePtr>(QueryingAA: *this, IRP: ACSArgPos, DepClass: DepClassTy::REQUIRED);
7422 if (!PrivCSArgAA)
7423 return false;
7424 std::optional<Type *> CSTy = PrivCSArgAA->getPrivatizableType();
7425
7426 LLVM_DEBUG({
7427 dbgs() << "[AAPrivatizablePtr] ACSPos: " << ACSArgPos << ", CSTy: ";
7428 if (CSTy && *CSTy)
7429 (*CSTy)->print(dbgs());
7430 else if (CSTy)
7431 dbgs() << "<nullptr>";
7432 else
7433 dbgs() << "<none>";
7434 });
7435
7436 Ty = combineTypes(T0: Ty, T1: CSTy);
7437
7438 LLVM_DEBUG({
7439 dbgs() << " : New Type: ";
7440 if (Ty && *Ty)
7441 (*Ty)->print(dbgs());
7442 else if (Ty)
7443 dbgs() << "<nullptr>";
7444 else
7445 dbgs() << "<none>";
7446 dbgs() << "\n";
7447 });
7448
7449 return !Ty || *Ty;
7450 };
7451
7452 if (!A.checkForAllCallSites(Pred: CallSiteCheck, QueryingAA: *this, RequireAllCallSites: true,
7453 UsedAssumedInformation))
7454 return nullptr;
7455 return Ty;
7456 }
7457
7458 /// See AbstractAttribute::updateImpl(...).
7459 ChangeStatus updateImpl(Attributor &A) override {
7460 PrivatizableType = identifyPrivatizableType(A);
7461 if (!PrivatizableType)
7462 return ChangeStatus::UNCHANGED;
7463 if (!*PrivatizableType)
7464 return indicatePessimisticFixpoint();
7465
7466 // The dependence is optional so we don't give up once we give up on the
7467 // alignment.
7468 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: getAssociatedValue()),
7469 DepClass: DepClassTy::OPTIONAL);
7470
7471 // Avoid arguments with padding for now.
7472 if (!A.hasAttr(IRP: getIRPosition(), AKs: Attribute::ByVal) &&
7473 !isDenselyPacked(Ty: *PrivatizableType, DL: A.getInfoCache().getDL())) {
7474 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Padding detected\n");
7475 return indicatePessimisticFixpoint();
7476 }
7477
7478 // Collect the types that will replace the privatizable type in the function
7479 // signature.
7480 SmallVector<Type *, 16> ReplacementTypes;
7481 identifyReplacementTypes(PrivType: *PrivatizableType, ReplacementTypes);
7482
7483 // Verify callee and caller agree on how the promoted argument would be
7484 // passed.
7485 Function &Fn = *getIRPosition().getAnchorScope();
7486 const auto *TTI =
7487 A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(F: Fn);
7488 if (!TTI) {
7489 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Missing TTI for function "
7490 << Fn.getName() << "\n");
7491 return indicatePessimisticFixpoint();
7492 }
7493
7494 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7495 CallBase *CB = ACS.getInstruction();
7496 return TTI->areTypesABICompatible(
7497 Caller: CB->getCaller(),
7498 Callee: dyn_cast_if_present<Function>(Val: CB->getCalledOperand()),
7499 Types: ReplacementTypes);
7500 };
7501 bool UsedAssumedInformation = false;
7502 if (!A.checkForAllCallSites(Pred: CallSiteCheck, QueryingAA: *this, RequireAllCallSites: true,
7503 UsedAssumedInformation)) {
7504 LLVM_DEBUG(
7505 dbgs() << "[AAPrivatizablePtr] ABI incompatibility detected for "
7506 << Fn.getName() << "\n");
7507 return indicatePessimisticFixpoint();
7508 }
7509
7510 // Register a rewrite of the argument.
7511 Argument *Arg = getAssociatedArgument();
7512 if (!A.isValidFunctionSignatureRewrite(Arg&: *Arg, ReplacementTypes)) {
7513 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Rewrite not valid\n");
7514 return indicatePessimisticFixpoint();
7515 }
7516
7517 unsigned ArgNo = Arg->getArgNo();
7518
7519 // Helper to check if for the given call site the associated argument is
7520 // passed to a callback where the privatization would be different.
7521 auto IsCompatiblePrivArgOfCallback = [&](CallBase &CB) {
7522 SmallVector<const Use *, 4> CallbackUses;
7523 AbstractCallSite::getCallbackUses(CB, CallbackUses);
7524 for (const Use *U : CallbackUses) {
7525 AbstractCallSite CBACS(U);
7526 assert(CBACS && CBACS.isCallbackCall());
7527 for (Argument &CBArg : CBACS.getCalledFunction()->args()) {
7528 int CBArgNo = CBACS.getCallArgOperandNo(Arg&: CBArg);
7529
7530 LLVM_DEBUG({
7531 dbgs()
7532 << "[AAPrivatizablePtr] Argument " << *Arg
7533 << "check if can be privatized in the context of its parent ("
7534 << Arg->getParent()->getName()
7535 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7536 "callback ("
7537 << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7538 << ")\n[AAPrivatizablePtr] " << CBArg << " : "
7539 << CBACS.getCallArgOperand(CBArg) << " vs "
7540 << CB.getArgOperand(ArgNo) << "\n"
7541 << "[AAPrivatizablePtr] " << CBArg << " : "
7542 << CBACS.getCallArgOperandNo(CBArg) << " vs " << ArgNo << "\n";
7543 });
7544
7545 if (CBArgNo != int(ArgNo))
7546 continue;
7547 const auto *CBArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7548 QueryingAA: *this, IRP: IRPosition::argument(Arg: CBArg), DepClass: DepClassTy::REQUIRED);
7549 if (CBArgPrivAA && CBArgPrivAA->isValidState()) {
7550 auto CBArgPrivTy = CBArgPrivAA->getPrivatizableType();
7551 if (!CBArgPrivTy)
7552 continue;
7553 if (*CBArgPrivTy == PrivatizableType)
7554 continue;
7555 }
7556
7557 LLVM_DEBUG({
7558 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7559 << " cannot be privatized in the context of its parent ("
7560 << Arg->getParent()->getName()
7561 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7562 "callback ("
7563 << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
7564 << ").\n[AAPrivatizablePtr] for which the argument "
7565 "privatization is not compatible.\n";
7566 });
7567 return false;
7568 }
7569 }
7570 return true;
7571 };
7572
7573 // Helper to check if for the given call site the associated argument is
7574 // passed to a direct call where the privatization would be different.
7575 auto IsCompatiblePrivArgOfDirectCS = [&](AbstractCallSite ACS) {
7576 CallBase *DC = cast<CallBase>(Val: ACS.getInstruction());
7577 int DCArgNo = ACS.getCallArgOperandNo(ArgNo);
7578 assert(DCArgNo >= 0 && unsigned(DCArgNo) < DC->arg_size() &&
7579 "Expected a direct call operand for callback call operand");
7580
7581 Function *DCCallee =
7582 dyn_cast_if_present<Function>(Val: DC->getCalledOperand());
7583 LLVM_DEBUG({
7584 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7585 << " check if be privatized in the context of its parent ("
7586 << Arg->getParent()->getName()
7587 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7588 "direct call of ("
7589 << DCArgNo << "@" << DCCallee->getName() << ").\n";
7590 });
7591
7592 if (unsigned(DCArgNo) < DCCallee->arg_size()) {
7593 const auto *DCArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
7594 QueryingAA: *this, IRP: IRPosition::argument(Arg: *DCCallee->getArg(i: DCArgNo)),
7595 DepClass: DepClassTy::REQUIRED);
7596 if (DCArgPrivAA && DCArgPrivAA->isValidState()) {
7597 auto DCArgPrivTy = DCArgPrivAA->getPrivatizableType();
7598 if (!DCArgPrivTy)
7599 return true;
7600 if (*DCArgPrivTy == PrivatizableType)
7601 return true;
7602 }
7603 }
7604
7605 LLVM_DEBUG({
7606 dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
7607 << " cannot be privatized in the context of its parent ("
7608 << Arg->getParent()->getName()
7609 << ")\n[AAPrivatizablePtr] because it is an argument in a "
7610 "direct call of ("
7611 << ACS.getInstruction()->getCalledOperand()->getName()
7612 << ").\n[AAPrivatizablePtr] for which the argument "
7613 "privatization is not compatible.\n";
7614 });
7615 return false;
7616 };
7617
7618 // Helper to check if the associated argument is used at the given abstract
7619 // call site in a way that is incompatible with the privatization assumed
7620 // here.
7621 auto IsCompatiblePrivArgOfOtherCallSite = [&](AbstractCallSite ACS) {
7622 if (ACS.isDirectCall())
7623 return IsCompatiblePrivArgOfCallback(*ACS.getInstruction());
7624 if (ACS.isCallbackCall())
7625 return IsCompatiblePrivArgOfDirectCS(ACS);
7626 return false;
7627 };
7628
7629 if (!A.checkForAllCallSites(Pred: IsCompatiblePrivArgOfOtherCallSite, QueryingAA: *this, RequireAllCallSites: true,
7630 UsedAssumedInformation))
7631 return indicatePessimisticFixpoint();
7632
7633 return ChangeStatus::UNCHANGED;
7634 }
7635
7636 /// Given a type to private \p PrivType, collect the constituates (which are
7637 /// used) in \p ReplacementTypes.
7638 static void
7639 identifyReplacementTypes(Type *PrivType,
7640 SmallVectorImpl<Type *> &ReplacementTypes) {
7641 // TODO: For now we expand the privatization type to the fullest which can
7642 // lead to dead arguments that need to be removed later.
7643 assert(PrivType && "Expected privatizable type!");
7644
7645 // Traverse the type, extract constituate types on the outermost level.
7646 if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7647 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++)
7648 ReplacementTypes.push_back(Elt: PrivStructType->getElementType(N: u));
7649 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7650 ReplacementTypes.append(NumInputs: PrivArrayType->getNumElements(),
7651 Elt: PrivArrayType->getElementType());
7652 } else {
7653 ReplacementTypes.push_back(Elt: PrivType);
7654 }
7655 }
7656
7657 /// Initialize \p Base according to the type \p PrivType at position \p IP.
7658 /// The values needed are taken from the arguments of \p F starting at
7659 /// position \p ArgNo.
7660 static void createInitialization(Type *PrivType, Value &Base, Function &F,
7661 unsigned ArgNo, BasicBlock::iterator IP) {
7662 assert(PrivType && "Expected privatizable type!");
7663
7664 IRBuilder<NoFolder> IRB(IP);
7665 const DataLayout &DL = F.getDataLayout();
7666
7667 // Traverse the type, build GEPs and stores.
7668 if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7669 const StructLayout *PrivStructLayout = DL.getStructLayout(Ty: PrivStructType);
7670 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7671 Value *Ptr =
7672 constructPointer(Ptr: &Base, Offset: PrivStructLayout->getElementOffset(Idx: u), IRB);
7673 new StoreInst(F.getArg(i: ArgNo + u), Ptr, IP);
7674 }
7675 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7676 Type *PointeeTy = PrivArrayType->getElementType();
7677 uint64_t PointeeTySize = DL.getTypeStoreSize(Ty: PointeeTy);
7678 for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7679 Value *Ptr = constructPointer(Ptr: &Base, Offset: u * PointeeTySize, IRB);
7680 new StoreInst(F.getArg(i: ArgNo + u), Ptr, IP);
7681 }
7682 } else {
7683 new StoreInst(F.getArg(i: ArgNo), &Base, IP);
7684 }
7685 }
7686
7687 /// Extract values from \p Base according to the type \p PrivType at the
7688 /// call position \p ACS. The values are appended to \p ReplacementValues.
7689 void createReplacementValues(Align Alignment, Type *PrivType,
7690 AbstractCallSite ACS, Value *Base,
7691 SmallVectorImpl<Value *> &ReplacementValues) {
7692 assert(Base && "Expected base value!");
7693 assert(PrivType && "Expected privatizable type!");
7694 Instruction *IP = ACS.getInstruction();
7695
7696 IRBuilder<NoFolder> IRB(IP);
7697 const DataLayout &DL = IP->getDataLayout();
7698
7699 // Traverse the type, build GEPs and loads.
7700 if (auto *PrivStructType = dyn_cast<StructType>(Val: PrivType)) {
7701 const StructLayout *PrivStructLayout = DL.getStructLayout(Ty: PrivStructType);
7702 for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7703 Type *PointeeTy = PrivStructType->getElementType(N: u);
7704 Value *Ptr =
7705 constructPointer(Ptr: Base, Offset: PrivStructLayout->getElementOffset(Idx: u), IRB);
7706 LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7707 L->setAlignment(Alignment);
7708 ReplacementValues.push_back(Elt: L);
7709 }
7710 } else if (auto *PrivArrayType = dyn_cast<ArrayType>(Val: PrivType)) {
7711 Type *PointeeTy = PrivArrayType->getElementType();
7712 uint64_t PointeeTySize = DL.getTypeStoreSize(Ty: PointeeTy);
7713 for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7714 Value *Ptr = constructPointer(Ptr: Base, Offset: u * PointeeTySize, IRB);
7715 LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
7716 L->setAlignment(Alignment);
7717 ReplacementValues.push_back(Elt: L);
7718 }
7719 } else {
7720 LoadInst *L = new LoadInst(PrivType, Base, "", IP->getIterator());
7721 L->setAlignment(Alignment);
7722 ReplacementValues.push_back(Elt: L);
7723 }
7724 }
7725
7726 /// See AbstractAttribute::manifest(...)
7727 ChangeStatus manifest(Attributor &A) override {
7728 if (!PrivatizableType)
7729 return ChangeStatus::UNCHANGED;
7730 assert(*PrivatizableType && "Expected privatizable type!");
7731
7732 // Collect all tail calls in the function as we cannot allow new allocas to
7733 // escape into tail recursion.
7734 // TODO: Be smarter about new allocas escaping into tail calls.
7735 SmallVector<CallInst *, 16> TailCalls;
7736 bool UsedAssumedInformation = false;
7737 if (!A.checkForAllInstructions(
7738 Pred: [&](Instruction &I) {
7739 CallInst &CI = cast<CallInst>(Val&: I);
7740 if (CI.isTailCall())
7741 TailCalls.push_back(Elt: &CI);
7742 return true;
7743 },
7744 QueryingAA: *this, Opcodes: {Instruction::Call}, UsedAssumedInformation))
7745 return ChangeStatus::UNCHANGED;
7746
7747 Argument *Arg = getAssociatedArgument();
7748 // Query AAAlign attribute for alignment of associated argument to
7749 // determine the best alignment of loads.
7750 const auto *AlignAA =
7751 A.getAAFor<AAAlign>(QueryingAA: *this, IRP: IRPosition::value(V: *Arg), DepClass: DepClassTy::NONE);
7752
7753 // Callback to repair the associated function. A new alloca is placed at the
7754 // beginning and initialized with the values passed through arguments. The
7755 // new alloca replaces the use of the old pointer argument.
7756 Attributor::ArgumentReplacementInfo::CalleeRepairCBTy FnRepairCB =
7757 [=](const Attributor::ArgumentReplacementInfo &ARI,
7758 Function &ReplacementFn, Function::arg_iterator ArgIt) {
7759 BasicBlock &EntryBB = ReplacementFn.getEntryBlock();
7760 BasicBlock::iterator IP = EntryBB.getFirstInsertionPt();
7761 const DataLayout &DL = IP->getDataLayout();
7762 unsigned AS = DL.getAllocaAddrSpace();
7763 Instruction *AI = new AllocaInst(*PrivatizableType, AS,
7764 Arg->getName() + ".priv", IP);
7765 createInitialization(PrivType: *PrivatizableType, Base&: *AI, F&: ReplacementFn,
7766 ArgNo: ArgIt->getArgNo(), IP);
7767
7768 if (AI->getType() != Arg->getType())
7769 AI = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
7770 S: AI, Ty: Arg->getType(), Name: "", InsertBefore: IP);
7771 Arg->replaceAllUsesWith(V: AI);
7772
7773 for (CallInst *CI : TailCalls)
7774 CI->setTailCall(false);
7775 };
7776
7777 // Callback to repair a call site of the associated function. The elements
7778 // of the privatizable type are loaded prior to the call and passed to the
7779 // new function version.
7780 Attributor::ArgumentReplacementInfo::ACSRepairCBTy ACSRepairCB =
7781 [=](const Attributor::ArgumentReplacementInfo &ARI,
7782 AbstractCallSite ACS, SmallVectorImpl<Value *> &NewArgOperands) {
7783 // When no alignment is specified for the load instruction,
7784 // natural alignment is assumed.
7785 createReplacementValues(
7786 Alignment: AlignAA ? AlignAA->getAssumedAlign() : Align(0),
7787 PrivType: *PrivatizableType, ACS,
7788 Base: ACS.getCallArgOperand(ArgNo: ARI.getReplacedArg().getArgNo()),
7789 ReplacementValues&: NewArgOperands);
7790 };
7791
7792 // Collect the types that will replace the privatizable type in the function
7793 // signature.
7794 SmallVector<Type *, 16> ReplacementTypes;
7795 identifyReplacementTypes(PrivType: *PrivatizableType, ReplacementTypes);
7796
7797 // Register a rewrite of the argument.
7798 if (A.registerFunctionSignatureRewrite(Arg&: *Arg, ReplacementTypes,
7799 CalleeRepairCB: std::move(FnRepairCB),
7800 ACSRepairCB: std::move(ACSRepairCB)))
7801 return ChangeStatus::CHANGED;
7802 return ChangeStatus::UNCHANGED;
7803 }
7804
7805 /// See AbstractAttribute::trackStatistics()
7806 void trackStatistics() const override {
7807 STATS_DECLTRACK_ARG_ATTR(privatizable_ptr);
7808 }
7809};
7810
7811struct AAPrivatizablePtrFloating : public AAPrivatizablePtrImpl {
7812 AAPrivatizablePtrFloating(const IRPosition &IRP, Attributor &A)
7813 : AAPrivatizablePtrImpl(IRP, A) {}
7814
7815 /// See AbstractAttribute::initialize(...).
7816 void initialize(Attributor &A) override {
7817 // TODO: We can privatize more than arguments.
7818 indicatePessimisticFixpoint();
7819 }
7820
7821 ChangeStatus updateImpl(Attributor &A) override {
7822 llvm_unreachable("AAPrivatizablePtr(Floating|Returned|CallSiteReturned)::"
7823 "updateImpl will not be called");
7824 }
7825
7826 /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
7827 std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
7828 Value *Obj = getUnderlyingObject(V: &getAssociatedValue());
7829 if (!Obj) {
7830 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] No underlying object found!\n");
7831 return nullptr;
7832 }
7833
7834 if (auto *AI = dyn_cast<AllocaInst>(Val: Obj))
7835 if (auto *CI = dyn_cast<ConstantInt>(Val: AI->getArraySize()))
7836 if (CI->isOne())
7837 return AI->getAllocatedType();
7838 if (auto *Arg = dyn_cast<Argument>(Val: Obj)) {
7839 auto *PrivArgAA = A.getAAFor<AAPrivatizablePtr>(
7840 QueryingAA: *this, IRP: IRPosition::argument(Arg: *Arg), DepClass: DepClassTy::REQUIRED);
7841 if (PrivArgAA && PrivArgAA->isAssumedPrivatizablePtr())
7842 return PrivArgAA->getPrivatizableType();
7843 }
7844
7845 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Underlying object neither valid "
7846 "alloca nor privatizable argument: "
7847 << *Obj << "!\n");
7848 return nullptr;
7849 }
7850
7851 /// See AbstractAttribute::trackStatistics()
7852 void trackStatistics() const override {
7853 STATS_DECLTRACK_FLOATING_ATTR(privatizable_ptr);
7854 }
7855};
7856
7857struct AAPrivatizablePtrCallSiteArgument final
7858 : public AAPrivatizablePtrFloating {
7859 AAPrivatizablePtrCallSiteArgument(const IRPosition &IRP, Attributor &A)
7860 : AAPrivatizablePtrFloating(IRP, A) {}
7861
7862 /// See AbstractAttribute::initialize(...).
7863 void initialize(Attributor &A) override {
7864 if (A.hasAttr(IRP: getIRPosition(), AKs: Attribute::ByVal))
7865 indicateOptimisticFixpoint();
7866 }
7867
7868 /// See AbstractAttribute::updateImpl(...).
7869 ChangeStatus updateImpl(Attributor &A) override {
7870 PrivatizableType = identifyPrivatizableType(A);
7871 if (!PrivatizableType)
7872 return ChangeStatus::UNCHANGED;
7873 if (!*PrivatizableType)
7874 return indicatePessimisticFixpoint();
7875
7876 const IRPosition &IRP = getIRPosition();
7877 bool IsKnownNoCapture;
7878 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
7879 A, QueryingAA: this, IRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoCapture);
7880 if (!IsAssumedNoCapture) {
7881 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might be captured!\n");
7882 return indicatePessimisticFixpoint();
7883 }
7884
7885 bool IsKnownNoAlias;
7886 if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
7887 A, QueryingAA: this, IRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoAlias)) {
7888 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might alias!\n");
7889 return indicatePessimisticFixpoint();
7890 }
7891
7892 bool IsKnown;
7893 if (!AA::isAssumedReadOnly(A, IRP, QueryingAA: *this, IsKnown)) {
7894 LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer is written!\n");
7895 return indicatePessimisticFixpoint();
7896 }
7897
7898 return ChangeStatus::UNCHANGED;
7899 }
7900
7901 /// See AbstractAttribute::trackStatistics()
7902 void trackStatistics() const override {
7903 STATS_DECLTRACK_CSARG_ATTR(privatizable_ptr);
7904 }
7905};
7906
7907struct AAPrivatizablePtrCallSiteReturned final
7908 : public AAPrivatizablePtrFloating {
7909 AAPrivatizablePtrCallSiteReturned(const IRPosition &IRP, Attributor &A)
7910 : AAPrivatizablePtrFloating(IRP, A) {}
7911
7912 /// See AbstractAttribute::initialize(...).
7913 void initialize(Attributor &A) override {
7914 // TODO: We can privatize more than arguments.
7915 indicatePessimisticFixpoint();
7916 }
7917
7918 /// See AbstractAttribute::trackStatistics()
7919 void trackStatistics() const override {
7920 STATS_DECLTRACK_CSRET_ATTR(privatizable_ptr);
7921 }
7922};
7923
7924struct AAPrivatizablePtrReturned final : public AAPrivatizablePtrFloating {
7925 AAPrivatizablePtrReturned(const IRPosition &IRP, Attributor &A)
7926 : AAPrivatizablePtrFloating(IRP, A) {}
7927
7928 /// See AbstractAttribute::initialize(...).
7929 void initialize(Attributor &A) override {
7930 // TODO: We can privatize more than arguments.
7931 indicatePessimisticFixpoint();
7932 }
7933
7934 /// See AbstractAttribute::trackStatistics()
7935 void trackStatistics() const override {
7936 STATS_DECLTRACK_FNRET_ATTR(privatizable_ptr);
7937 }
7938};
7939} // namespace
7940
7941/// -------------------- Memory Behavior Attributes ----------------------------
7942/// Includes read-none, read-only, and write-only.
7943/// ----------------------------------------------------------------------------
7944namespace {
7945struct AAMemoryBehaviorImpl : public AAMemoryBehavior {
7946 AAMemoryBehaviorImpl(const IRPosition &IRP, Attributor &A)
7947 : AAMemoryBehavior(IRP, A) {}
7948
7949 /// See AbstractAttribute::initialize(...).
7950 void initialize(Attributor &A) override {
7951 intersectAssumedBits(BitsEncoding: BEST_STATE);
7952 getKnownStateFromValue(A, IRP: getIRPosition(), State&: getState());
7953 AAMemoryBehavior::initialize(A);
7954 }
7955
7956 /// Return the memory behavior information encoded in the IR for \p IRP.
7957 static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
7958 BitIntegerState &State,
7959 bool IgnoreSubsumingPositions = false) {
7960 SmallVector<Attribute, 2> Attrs;
7961 A.getAttrs(IRP, AKs: AttrKinds, Attrs, IgnoreSubsumingPositions);
7962 for (const Attribute &Attr : Attrs) {
7963 switch (Attr.getKindAsEnum()) {
7964 case Attribute::ReadNone:
7965 State.addKnownBits(Bits: NO_ACCESSES);
7966 break;
7967 case Attribute::ReadOnly:
7968 State.addKnownBits(Bits: NO_WRITES);
7969 break;
7970 case Attribute::WriteOnly:
7971 State.addKnownBits(Bits: NO_READS);
7972 break;
7973 default:
7974 llvm_unreachable("Unexpected attribute!");
7975 }
7976 }
7977
7978 if (auto *I = dyn_cast<Instruction>(Val: &IRP.getAnchorValue())) {
7979 if (!I->mayReadFromMemory())
7980 State.addKnownBits(Bits: NO_READS);
7981 if (!I->mayWriteToMemory())
7982 State.addKnownBits(Bits: NO_WRITES);
7983 }
7984 }
7985
7986 /// See AbstractAttribute::getDeducedAttributes(...).
7987 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
7988 SmallVectorImpl<Attribute> &Attrs) const override {
7989 assert(Attrs.size() == 0);
7990 if (isAssumedReadNone())
7991 Attrs.push_back(Elt: Attribute::get(Context&: Ctx, Kind: Attribute::ReadNone));
7992 else if (isAssumedReadOnly())
7993 Attrs.push_back(Elt: Attribute::get(Context&: Ctx, Kind: Attribute::ReadOnly));
7994 else if (isAssumedWriteOnly())
7995 Attrs.push_back(Elt: Attribute::get(Context&: Ctx, Kind: Attribute::WriteOnly));
7996 assert(Attrs.size() <= 1);
7997 }
7998
7999 /// See AbstractAttribute::manifest(...).
8000 ChangeStatus manifest(Attributor &A) override {
8001 const IRPosition &IRP = getIRPosition();
8002
8003 if (A.hasAttr(IRP, AKs: Attribute::ReadNone,
8004 /* IgnoreSubsumingPositions */ true))
8005 return ChangeStatus::UNCHANGED;
8006
8007 // Check if we would improve the existing attributes first.
8008 SmallVector<Attribute, 4> DeducedAttrs;
8009 getDeducedAttributes(A, Ctx&: IRP.getAnchorValue().getContext(), Attrs&: DeducedAttrs);
8010 if (llvm::all_of(Range&: DeducedAttrs, P: [&](const Attribute &Attr) {
8011 return A.hasAttr(IRP, AKs: Attr.getKindAsEnum(),
8012 /* IgnoreSubsumingPositions */ true);
8013 }))
8014 return ChangeStatus::UNCHANGED;
8015
8016 // Clear existing attributes.
8017 A.removeAttrs(IRP, AttrKinds);
8018 // Clear conflicting writable attribute.
8019 if (isAssumedReadOnly())
8020 A.removeAttrs(IRP, AttrKinds: Attribute::Writable);
8021
8022 // Use the generic manifest method.
8023 return IRAttribute::manifest(A);
8024 }
8025
8026 /// See AbstractState::getAsStr().
8027 const std::string getAsStr(Attributor *A) const override {
8028 if (isAssumedReadNone())
8029 return "readnone";
8030 if (isAssumedReadOnly())
8031 return "readonly";
8032 if (isAssumedWriteOnly())
8033 return "writeonly";
8034 return "may-read/write";
8035 }
8036
8037 /// The set of IR attributes AAMemoryBehavior deals with.
8038 static const Attribute::AttrKind AttrKinds[3];
8039};
8040
8041const Attribute::AttrKind AAMemoryBehaviorImpl::AttrKinds[] = {
8042 Attribute::ReadNone, Attribute::ReadOnly, Attribute::WriteOnly};
8043
8044/// Memory behavior attribute for a floating value.
8045struct AAMemoryBehaviorFloating : AAMemoryBehaviorImpl {
8046 AAMemoryBehaviorFloating(const IRPosition &IRP, Attributor &A)
8047 : AAMemoryBehaviorImpl(IRP, A) {}
8048
8049 /// See AbstractAttribute::updateImpl(...).
8050 ChangeStatus updateImpl(Attributor &A) override;
8051
8052 /// See AbstractAttribute::trackStatistics()
8053 void trackStatistics() const override {
8054 if (isAssumedReadNone())
8055 STATS_DECLTRACK_FLOATING_ATTR(readnone)
8056 else if (isAssumedReadOnly())
8057 STATS_DECLTRACK_FLOATING_ATTR(readonly)
8058 else if (isAssumedWriteOnly())
8059 STATS_DECLTRACK_FLOATING_ATTR(writeonly)
8060 }
8061
8062private:
8063 /// Return true if users of \p UserI might access the underlying
8064 /// variable/location described by \p U and should therefore be analyzed.
8065 bool followUsersOfUseIn(Attributor &A, const Use &U,
8066 const Instruction *UserI);
8067
8068 /// Update the state according to the effect of use \p U in \p UserI.
8069 void analyzeUseIn(Attributor &A, const Use &U, const Instruction *UserI);
8070};
8071
8072/// Memory behavior attribute for function argument.
8073struct AAMemoryBehaviorArgument : AAMemoryBehaviorFloating {
8074 AAMemoryBehaviorArgument(const IRPosition &IRP, Attributor &A)
8075 : AAMemoryBehaviorFloating(IRP, A) {}
8076
8077 /// See AbstractAttribute::initialize(...).
8078 void initialize(Attributor &A) override {
8079 intersectAssumedBits(BitsEncoding: BEST_STATE);
8080 const IRPosition &IRP = getIRPosition();
8081 // TODO: Make IgnoreSubsumingPositions a property of an IRAttribute so we
8082 // can query it when we use has/getAttr. That would allow us to reuse the
8083 // initialize of the base class here.
8084 bool HasByVal = A.hasAttr(IRP, AKs: {Attribute::ByVal},
8085 /* IgnoreSubsumingPositions */ true);
8086 getKnownStateFromValue(A, IRP, State&: getState(),
8087 /* IgnoreSubsumingPositions */ HasByVal);
8088 }
8089
8090 ChangeStatus manifest(Attributor &A) override {
8091 // TODO: Pointer arguments are not supported on vectors of pointers yet.
8092 if (!getAssociatedValue().getType()->isPointerTy())
8093 return ChangeStatus::UNCHANGED;
8094
8095 // TODO: From readattrs.ll: "inalloca parameters are always
8096 // considered written"
8097 if (A.hasAttr(IRP: getIRPosition(),
8098 AKs: {Attribute::InAlloca, Attribute::Preallocated})) {
8099 removeKnownBits(BitsEncoding: NO_WRITES);
8100 removeAssumedBits(BitsEncoding: NO_WRITES);
8101 }
8102 A.removeAttrs(IRP: getIRPosition(), AttrKinds);
8103 return AAMemoryBehaviorFloating::manifest(A);
8104 }
8105
8106 /// See AbstractAttribute::trackStatistics()
8107 void trackStatistics() const override {
8108 if (isAssumedReadNone())
8109 STATS_DECLTRACK_ARG_ATTR(readnone)
8110 else if (isAssumedReadOnly())
8111 STATS_DECLTRACK_ARG_ATTR(readonly)
8112 else if (isAssumedWriteOnly())
8113 STATS_DECLTRACK_ARG_ATTR(writeonly)
8114 }
8115};
8116
8117struct AAMemoryBehaviorCallSiteArgument final : AAMemoryBehaviorArgument {
8118 AAMemoryBehaviorCallSiteArgument(const IRPosition &IRP, Attributor &A)
8119 : AAMemoryBehaviorArgument(IRP, A) {}
8120
8121 /// See AbstractAttribute::initialize(...).
8122 void initialize(Attributor &A) override {
8123 // If we don't have an associated attribute this is either a variadic call
8124 // or an indirect call, either way, nothing to do here.
8125 Argument *Arg = getAssociatedArgument();
8126 if (!Arg) {
8127 indicatePessimisticFixpoint();
8128 return;
8129 }
8130 if (Arg->hasByValAttr()) {
8131 addKnownBits(Bits: NO_WRITES);
8132 removeKnownBits(BitsEncoding: NO_READS);
8133 removeAssumedBits(BitsEncoding: NO_READS);
8134 }
8135 AAMemoryBehaviorArgument::initialize(A);
8136 if (getAssociatedFunction()->isDeclaration())
8137 indicatePessimisticFixpoint();
8138 }
8139
8140 /// See AbstractAttribute::updateImpl(...).
8141 ChangeStatus updateImpl(Attributor &A) override {
8142 // TODO: Once we have call site specific value information we can provide
8143 // call site specific liveness liveness information and then it makes
8144 // sense to specialize attributes for call sites arguments instead of
8145 // redirecting requests to the callee argument.
8146 Argument *Arg = getAssociatedArgument();
8147 const IRPosition &ArgPos = IRPosition::argument(Arg: *Arg);
8148 auto *ArgAA =
8149 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: ArgPos, DepClass: DepClassTy::REQUIRED);
8150 if (!ArgAA)
8151 return indicatePessimisticFixpoint();
8152 return clampStateAndIndicateChange(S&: getState(), R: ArgAA->getState());
8153 }
8154
8155 /// See AbstractAttribute::trackStatistics()
8156 void trackStatistics() const override {
8157 if (isAssumedReadNone())
8158 STATS_DECLTRACK_CSARG_ATTR(readnone)
8159 else if (isAssumedReadOnly())
8160 STATS_DECLTRACK_CSARG_ATTR(readonly)
8161 else if (isAssumedWriteOnly())
8162 STATS_DECLTRACK_CSARG_ATTR(writeonly)
8163 }
8164};
8165
8166/// Memory behavior attribute for a call site return position.
8167struct AAMemoryBehaviorCallSiteReturned final : AAMemoryBehaviorFloating {
8168 AAMemoryBehaviorCallSiteReturned(const IRPosition &IRP, Attributor &A)
8169 : AAMemoryBehaviorFloating(IRP, A) {}
8170
8171 /// See AbstractAttribute::initialize(...).
8172 void initialize(Attributor &A) override {
8173 AAMemoryBehaviorImpl::initialize(A);
8174 }
8175 /// See AbstractAttribute::manifest(...).
8176 ChangeStatus manifest(Attributor &A) override {
8177 // We do not annotate returned values.
8178 return ChangeStatus::UNCHANGED;
8179 }
8180
8181 /// See AbstractAttribute::trackStatistics()
8182 void trackStatistics() const override {}
8183};
8184
8185/// An AA to represent the memory behavior function attributes.
8186struct AAMemoryBehaviorFunction final : public AAMemoryBehaviorImpl {
8187 AAMemoryBehaviorFunction(const IRPosition &IRP, Attributor &A)
8188 : AAMemoryBehaviorImpl(IRP, A) {}
8189
8190 /// See AbstractAttribute::updateImpl(Attributor &A).
8191 ChangeStatus updateImpl(Attributor &A) override;
8192
8193 /// See AbstractAttribute::manifest(...).
8194 ChangeStatus manifest(Attributor &A) override {
8195 // TODO: It would be better to merge this with AAMemoryLocation, so that
8196 // we could determine read/write per location. This would also have the
8197 // benefit of only one place trying to manifest the memory attribute.
8198 Function &F = cast<Function>(Val&: getAnchorValue());
8199 MemoryEffects ME = MemoryEffects::unknown();
8200 if (isAssumedReadNone())
8201 ME = MemoryEffects::none();
8202 else if (isAssumedReadOnly())
8203 ME = MemoryEffects::readOnly();
8204 else if (isAssumedWriteOnly())
8205 ME = MemoryEffects::writeOnly();
8206
8207 A.removeAttrs(IRP: getIRPosition(), AttrKinds);
8208 // Clear conflicting writable attribute.
8209 if (ME.onlyReadsMemory())
8210 for (Argument &Arg : F.args())
8211 A.removeAttrs(IRP: IRPosition::argument(Arg), AttrKinds: Attribute::Writable);
8212 return A.manifestAttrs(IRP: getIRPosition(),
8213 DeducedAttrs: Attribute::getWithMemoryEffects(Context&: F.getContext(), ME));
8214 }
8215
8216 /// See AbstractAttribute::trackStatistics()
8217 void trackStatistics() const override {
8218 if (isAssumedReadNone())
8219 STATS_DECLTRACK_FN_ATTR(readnone)
8220 else if (isAssumedReadOnly())
8221 STATS_DECLTRACK_FN_ATTR(readonly)
8222 else if (isAssumedWriteOnly())
8223 STATS_DECLTRACK_FN_ATTR(writeonly)
8224 }
8225};
8226
8227/// AAMemoryBehavior attribute for call sites.
8228struct AAMemoryBehaviorCallSite final
8229 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl> {
8230 AAMemoryBehaviorCallSite(const IRPosition &IRP, Attributor &A)
8231 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl>(IRP, A) {}
8232
8233 /// See AbstractAttribute::manifest(...).
8234 ChangeStatus manifest(Attributor &A) override {
8235 // TODO: Deduplicate this with AAMemoryBehaviorFunction.
8236 CallBase &CB = cast<CallBase>(Val&: getAnchorValue());
8237 MemoryEffects ME = MemoryEffects::unknown();
8238 if (isAssumedReadNone())
8239 ME = MemoryEffects::none();
8240 else if (isAssumedReadOnly())
8241 ME = MemoryEffects::readOnly();
8242 else if (isAssumedWriteOnly())
8243 ME = MemoryEffects::writeOnly();
8244
8245 A.removeAttrs(IRP: getIRPosition(), AttrKinds);
8246 // Clear conflicting writable attribute.
8247 if (ME.onlyReadsMemory())
8248 for (Use &U : CB.args())
8249 A.removeAttrs(IRP: IRPosition::callsite_argument(CB, ArgNo: U.getOperandNo()),
8250 AttrKinds: Attribute::Writable);
8251 return A.manifestAttrs(
8252 IRP: getIRPosition(), DeducedAttrs: Attribute::getWithMemoryEffects(Context&: CB.getContext(), ME));
8253 }
8254
8255 /// See AbstractAttribute::trackStatistics()
8256 void trackStatistics() const override {
8257 if (isAssumedReadNone())
8258 STATS_DECLTRACK_CS_ATTR(readnone)
8259 else if (isAssumedReadOnly())
8260 STATS_DECLTRACK_CS_ATTR(readonly)
8261 else if (isAssumedWriteOnly())
8262 STATS_DECLTRACK_CS_ATTR(writeonly)
8263 }
8264};
8265
8266ChangeStatus AAMemoryBehaviorFunction::updateImpl(Attributor &A) {
8267
8268 // The current assumed state used to determine a change.
8269 auto AssumedState = getAssumed();
8270
8271 auto CheckRWInst = [&](Instruction &I) {
8272 // If the instruction has an own memory behavior state, use it to restrict
8273 // the local state. No further analysis is required as the other memory
8274 // state is as optimistic as it gets.
8275 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
8276 const auto *MemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
8277 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::REQUIRED);
8278 if (MemBehaviorAA) {
8279 intersectAssumedBits(BitsEncoding: MemBehaviorAA->getAssumed());
8280 return !isAtFixpoint();
8281 }
8282 }
8283
8284 // Remove access kind modifiers if necessary.
8285 if (I.mayReadFromMemory())
8286 removeAssumedBits(BitsEncoding: NO_READS);
8287 if (I.mayWriteToMemory())
8288 removeAssumedBits(BitsEncoding: NO_WRITES);
8289 return !isAtFixpoint();
8290 };
8291
8292 bool UsedAssumedInformation = false;
8293 if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInst, QueryingAA&: *this,
8294 UsedAssumedInformation))
8295 return indicatePessimisticFixpoint();
8296
8297 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8298 : ChangeStatus::UNCHANGED;
8299}
8300
8301ChangeStatus AAMemoryBehaviorFloating::updateImpl(Attributor &A) {
8302
8303 const IRPosition &IRP = getIRPosition();
8304 const IRPosition &FnPos = IRPosition::function_scope(IRP);
8305 AAMemoryBehavior::StateType &S = getState();
8306
8307 // First, check the function scope. We take the known information and we avoid
8308 // work if the assumed information implies the current assumed information for
8309 // this attribute. This is a valid for all but byval arguments.
8310 Argument *Arg = IRP.getAssociatedArgument();
8311 AAMemoryBehavior::base_t FnMemAssumedState =
8312 AAMemoryBehavior::StateType::getWorstState();
8313 if (!Arg || !Arg->hasByValAttr()) {
8314 const auto *FnMemAA =
8315 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::OPTIONAL);
8316 if (FnMemAA) {
8317 FnMemAssumedState = FnMemAA->getAssumed();
8318 S.addKnownBits(Bits: FnMemAA->getKnown());
8319 if ((S.getAssumed() & FnMemAA->getAssumed()) == S.getAssumed())
8320 return ChangeStatus::UNCHANGED;
8321 }
8322 }
8323
8324 // The current assumed state used to determine a change.
8325 auto AssumedState = S.getAssumed();
8326
8327 // Make sure the value is not captured (except through "return"), if
8328 // it is, any information derived would be irrelevant anyway as we cannot
8329 // check the potential aliases introduced by the capture. However, no need
8330 // to fall back to anythign less optimistic than the function state.
8331 bool IsKnownNoCapture;
8332 const AANoCapture *ArgNoCaptureAA = nullptr;
8333 bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::Captures>(
8334 A, QueryingAA: this, IRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture, IgnoreSubsumingPositions: false,
8335 AAPtr: &ArgNoCaptureAA);
8336
8337 if (!IsAssumedNoCapture &&
8338 (!ArgNoCaptureAA || !ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
8339 S.intersectAssumedBits(BitsEncoding: FnMemAssumedState);
8340 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8341 : ChangeStatus::UNCHANGED;
8342 }
8343
8344 // Visit and expand uses until all are analyzed or a fixpoint is reached.
8345 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
8346 Instruction *UserI = cast<Instruction>(Val: U.getUser());
8347 LLVM_DEBUG(dbgs() << "[AAMemoryBehavior] Use: " << *U << " in " << *UserI
8348 << " \n");
8349
8350 // Droppable users, e.g., llvm::assume does not actually perform any action.
8351 if (UserI->isDroppable())
8352 return true;
8353
8354 // Check if the users of UserI should also be visited.
8355 Follow = followUsersOfUseIn(A, U, UserI);
8356
8357 // If UserI might touch memory we analyze the use in detail.
8358 if (UserI->mayReadOrWriteMemory())
8359 analyzeUseIn(A, U, UserI);
8360
8361 return !isAtFixpoint();
8362 };
8363
8364 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: getAssociatedValue()))
8365 return indicatePessimisticFixpoint();
8366
8367 return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
8368 : ChangeStatus::UNCHANGED;
8369}
8370
8371bool AAMemoryBehaviorFloating::followUsersOfUseIn(Attributor &A, const Use &U,
8372 const Instruction *UserI) {
8373 // The loaded value is unrelated to the pointer argument, no need to
8374 // follow the users of the load.
8375 if (isa<LoadInst>(Val: UserI) || isa<ReturnInst>(Val: UserI))
8376 return false;
8377
8378 // By default we follow all uses assuming UserI might leak information on U,
8379 // we have special handling for call sites operands though.
8380 const auto *CB = dyn_cast<CallBase>(Val: UserI);
8381 if (!CB || !CB->isArgOperand(U: &U))
8382 return true;
8383
8384 // If the use is a call argument known not to be captured, the users of
8385 // the call do not need to be visited because they have to be unrelated to
8386 // the input. Note that this check is not trivial even though we disallow
8387 // general capturing of the underlying argument. The reason is that the
8388 // call might the argument "through return", which we allow and for which we
8389 // need to check call users.
8390 if (U.get()->getType()->isPointerTy()) {
8391 unsigned ArgNo = CB->getArgOperandNo(U: &U);
8392 bool IsKnownNoCapture;
8393 return !AA::hasAssumedIRAttr<Attribute::Captures>(
8394 A, QueryingAA: this, IRP: IRPosition::callsite_argument(CB: *CB, ArgNo),
8395 DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture);
8396 }
8397
8398 return true;
8399}
8400
8401void AAMemoryBehaviorFloating::analyzeUseIn(Attributor &A, const Use &U,
8402 const Instruction *UserI) {
8403 assert(UserI->mayReadOrWriteMemory());
8404
8405 switch (UserI->getOpcode()) {
8406 default:
8407 // TODO: Handle all atomics and other side-effect operations we know of.
8408 break;
8409 case Instruction::Load:
8410 // Loads cause the NO_READS property to disappear.
8411 removeAssumedBits(BitsEncoding: NO_READS);
8412 return;
8413
8414 case Instruction::Store:
8415 // Stores cause the NO_WRITES property to disappear if the use is the
8416 // pointer operand. Note that while capturing was taken care of somewhere
8417 // else we need to deal with stores of the value that is not looked through.
8418 if (cast<StoreInst>(Val: UserI)->getPointerOperand() == U.get())
8419 removeAssumedBits(BitsEncoding: NO_WRITES);
8420 else
8421 indicatePessimisticFixpoint();
8422 return;
8423
8424 case Instruction::Call:
8425 case Instruction::CallBr:
8426 case Instruction::Invoke: {
8427 // For call sites we look at the argument memory behavior attribute (this
8428 // could be recursive!) in order to restrict our own state.
8429 const auto *CB = cast<CallBase>(Val: UserI);
8430
8431 // Give up on operand bundles.
8432 if (CB->isBundleOperand(U: &U)) {
8433 indicatePessimisticFixpoint();
8434 return;
8435 }
8436
8437 // Calling a function does read the function pointer, maybe write it if the
8438 // function is self-modifying.
8439 if (CB->isCallee(U: &U)) {
8440 removeAssumedBits(BitsEncoding: NO_READS);
8441 break;
8442 }
8443
8444 // Adjust the possible access behavior based on the information on the
8445 // argument.
8446 IRPosition Pos;
8447 if (U.get()->getType()->isPointerTy())
8448 Pos = IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U));
8449 else
8450 Pos = IRPosition::callsite_function(CB: *CB);
8451 const auto *MemBehaviorAA =
8452 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: Pos, DepClass: DepClassTy::OPTIONAL);
8453 if (!MemBehaviorAA)
8454 break;
8455 // "assumed" has at most the same bits as the MemBehaviorAA assumed
8456 // and at least "known".
8457 intersectAssumedBits(BitsEncoding: MemBehaviorAA->getAssumed());
8458 return;
8459 }
8460 };
8461
8462 // Generally, look at the "may-properties" and adjust the assumed state if we
8463 // did not trigger special handling before.
8464 if (UserI->mayReadFromMemory())
8465 removeAssumedBits(BitsEncoding: NO_READS);
8466 if (UserI->mayWriteToMemory())
8467 removeAssumedBits(BitsEncoding: NO_WRITES);
8468}
8469} // namespace
8470
8471/// -------------------- Memory Locations Attributes ---------------------------
8472/// Includes read-none, argmemonly, inaccessiblememonly,
8473/// inaccessiblememorargmemonly
8474/// ----------------------------------------------------------------------------
8475
8476std::string AAMemoryLocation::getMemoryLocationsAsStr(
8477 AAMemoryLocation::MemoryLocationsKind MLK) {
8478 if (0 == (MLK & AAMemoryLocation::NO_LOCATIONS))
8479 return "all memory";
8480 if (MLK == AAMemoryLocation::NO_LOCATIONS)
8481 return "no memory";
8482 std::string S = "memory:";
8483 if (0 == (MLK & AAMemoryLocation::NO_LOCAL_MEM))
8484 S += "stack,";
8485 if (0 == (MLK & AAMemoryLocation::NO_CONST_MEM))
8486 S += "constant,";
8487 if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_INTERNAL_MEM))
8488 S += "internal global,";
8489 if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_EXTERNAL_MEM))
8490 S += "external global,";
8491 if (0 == (MLK & AAMemoryLocation::NO_ARGUMENT_MEM))
8492 S += "argument,";
8493 if (0 == (MLK & AAMemoryLocation::NO_INACCESSIBLE_MEM))
8494 S += "inaccessible,";
8495 if (0 == (MLK & AAMemoryLocation::NO_MALLOCED_MEM))
8496 S += "malloced,";
8497 if (0 == (MLK & AAMemoryLocation::NO_UNKOWN_MEM))
8498 S += "unknown,";
8499 S.pop_back();
8500 return S;
8501}
8502
8503namespace {
8504struct AAMemoryLocationImpl : public AAMemoryLocation {
8505
8506 AAMemoryLocationImpl(const IRPosition &IRP, Attributor &A)
8507 : AAMemoryLocation(IRP, A), Allocator(A.Allocator) {
8508 AccessKind2Accesses.fill(u: nullptr);
8509 }
8510
8511 ~AAMemoryLocationImpl() override {
8512 // The AccessSets are allocated via a BumpPtrAllocator, we call
8513 // the destructor manually.
8514 for (AccessSet *AS : AccessKind2Accesses)
8515 if (AS)
8516 AS->~AccessSet();
8517 }
8518
8519 /// See AbstractAttribute::initialize(...).
8520 void initialize(Attributor &A) override {
8521 intersectAssumedBits(BitsEncoding: BEST_STATE);
8522 getKnownStateFromValue(A, IRP: getIRPosition(), State&: getState());
8523 AAMemoryLocation::initialize(A);
8524 }
8525
8526 /// Return the memory behavior information encoded in the IR for \p IRP.
8527 static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
8528 BitIntegerState &State,
8529 bool IgnoreSubsumingPositions = false) {
8530 // For internal functions we ignore `argmemonly` and
8531 // `inaccessiblememorargmemonly` as we might break it via interprocedural
8532 // constant propagation. It is unclear if this is the best way but it is
8533 // unlikely this will cause real performance problems. If we are deriving
8534 // attributes for the anchor function we even remove the attribute in
8535 // addition to ignoring it.
8536 // TODO: A better way to handle this would be to add ~NO_GLOBAL_MEM /
8537 // MemoryEffects::Other as a possible location.
8538 bool UseArgMemOnly = true;
8539 Function *AnchorFn = IRP.getAnchorScope();
8540 if (AnchorFn && A.isRunOn(Fn&: *AnchorFn))
8541 UseArgMemOnly = !AnchorFn->hasLocalLinkage();
8542
8543 SmallVector<Attribute, 2> Attrs;
8544 A.getAttrs(IRP, AKs: {Attribute::Memory}, Attrs, IgnoreSubsumingPositions);
8545 for (const Attribute &Attr : Attrs) {
8546 // TODO: We can map MemoryEffects to Attributor locations more precisely.
8547 MemoryEffects ME = Attr.getMemoryEffects();
8548 if (ME.doesNotAccessMemory()) {
8549 State.addKnownBits(Bits: NO_LOCAL_MEM | NO_CONST_MEM);
8550 continue;
8551 }
8552 if (ME.onlyAccessesInaccessibleMem()) {
8553 State.addKnownBits(Bits: inverseLocation(Loc: NO_INACCESSIBLE_MEM, AndLocalMem: true, AndConstMem: true));
8554 continue;
8555 }
8556 if (ME.onlyAccessesArgPointees()) {
8557 if (UseArgMemOnly)
8558 State.addKnownBits(Bits: inverseLocation(Loc: NO_ARGUMENT_MEM, AndLocalMem: true, AndConstMem: true));
8559 else {
8560 // Remove location information, only keep read/write info.
8561 ME = MemoryEffects(ME.getModRef());
8562 A.manifestAttrs(IRP,
8563 DeducedAttrs: Attribute::getWithMemoryEffects(
8564 Context&: IRP.getAnchorValue().getContext(), ME),
8565 /*ForceReplace*/ true);
8566 }
8567 continue;
8568 }
8569 if (ME.onlyAccessesInaccessibleOrArgMem()) {
8570 if (UseArgMemOnly)
8571 State.addKnownBits(Bits: inverseLocation(
8572 Loc: NO_INACCESSIBLE_MEM | NO_ARGUMENT_MEM, AndLocalMem: true, AndConstMem: true));
8573 else {
8574 // Remove location information, only keep read/write info.
8575 ME = MemoryEffects(ME.getModRef());
8576 A.manifestAttrs(IRP,
8577 DeducedAttrs: Attribute::getWithMemoryEffects(
8578 Context&: IRP.getAnchorValue().getContext(), ME),
8579 /*ForceReplace*/ true);
8580 }
8581 continue;
8582 }
8583 }
8584 }
8585
8586 /// See AbstractAttribute::getDeducedAttributes(...).
8587 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
8588 SmallVectorImpl<Attribute> &Attrs) const override {
8589 // TODO: We can map Attributor locations to MemoryEffects more precisely.
8590 assert(Attrs.size() == 0);
8591 if (getIRPosition().getPositionKind() == IRPosition::IRP_FUNCTION) {
8592 if (isAssumedReadNone())
8593 Attrs.push_back(
8594 Elt: Attribute::getWithMemoryEffects(Context&: Ctx, ME: MemoryEffects::none()));
8595 else if (isAssumedInaccessibleMemOnly())
8596 Attrs.push_back(Elt: Attribute::getWithMemoryEffects(
8597 Context&: Ctx, ME: MemoryEffects::inaccessibleMemOnly()));
8598 else if (isAssumedArgMemOnly())
8599 Attrs.push_back(
8600 Elt: Attribute::getWithMemoryEffects(Context&: Ctx, ME: MemoryEffects::argMemOnly()));
8601 else if (isAssumedInaccessibleOrArgMemOnly())
8602 Attrs.push_back(Elt: Attribute::getWithMemoryEffects(
8603 Context&: Ctx, ME: MemoryEffects::inaccessibleOrArgMemOnly()));
8604 }
8605 assert(Attrs.size() <= 1);
8606 }
8607
8608 /// See AbstractAttribute::manifest(...).
8609 ChangeStatus manifest(Attributor &A) override {
8610 // TODO: If AAMemoryLocation and AAMemoryBehavior are merged, we could
8611 // provide per-location modref information here.
8612 const IRPosition &IRP = getIRPosition();
8613
8614 SmallVector<Attribute, 1> DeducedAttrs;
8615 getDeducedAttributes(A, Ctx&: IRP.getAnchorValue().getContext(), Attrs&: DeducedAttrs);
8616 if (DeducedAttrs.size() != 1)
8617 return ChangeStatus::UNCHANGED;
8618 MemoryEffects ME = DeducedAttrs[0].getMemoryEffects();
8619
8620 return A.manifestAttrs(IRP, DeducedAttrs: Attribute::getWithMemoryEffects(
8621 Context&: IRP.getAnchorValue().getContext(), ME));
8622 }
8623
8624 /// See AAMemoryLocation::checkForAllAccessesToMemoryKind(...).
8625 bool checkForAllAccessesToMemoryKind(
8626 function_ref<bool(const Instruction *, const Value *, AccessKind,
8627 MemoryLocationsKind)>
8628 Pred,
8629 MemoryLocationsKind RequestedMLK) const override {
8630 if (!isValidState())
8631 return false;
8632
8633 MemoryLocationsKind AssumedMLK = getAssumedNotAccessedLocation();
8634 if (AssumedMLK == NO_LOCATIONS)
8635 return true;
8636
8637 unsigned Idx = 0;
8638 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS;
8639 CurMLK *= 2, ++Idx) {
8640 if (CurMLK & RequestedMLK)
8641 continue;
8642
8643 if (const AccessSet *Accesses = AccessKind2Accesses[Idx])
8644 for (const AccessInfo &AI : *Accesses)
8645 if (!Pred(AI.I, AI.Ptr, AI.Kind, CurMLK))
8646 return false;
8647 }
8648
8649 return true;
8650 }
8651
8652 ChangeStatus indicatePessimisticFixpoint() override {
8653 // If we give up and indicate a pessimistic fixpoint this instruction will
8654 // become an access for all potential access kinds:
8655 // TODO: Add pointers for argmemonly and globals to improve the results of
8656 // checkForAllAccessesToMemoryKind.
8657 bool Changed = false;
8658 MemoryLocationsKind KnownMLK = getKnown();
8659 Instruction *I = dyn_cast<Instruction>(Val: &getAssociatedValue());
8660 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2)
8661 if (!(CurMLK & KnownMLK))
8662 updateStateAndAccessesMap(State&: getState(), MLK: CurMLK, I, Ptr: nullptr, Changed,
8663 AK: getAccessKindFromInst(I));
8664 return AAMemoryLocation::indicatePessimisticFixpoint();
8665 }
8666
8667protected:
8668 /// Helper struct to tie together an instruction that has a read or write
8669 /// effect with the pointer it accesses (if any).
8670 struct AccessInfo {
8671
8672 /// The instruction that caused the access.
8673 const Instruction *I;
8674
8675 /// The base pointer that is accessed, or null if unknown.
8676 const Value *Ptr;
8677
8678 /// The kind of access (read/write/read+write).
8679 AccessKind Kind;
8680
8681 bool operator==(const AccessInfo &RHS) const {
8682 return I == RHS.I && Ptr == RHS.Ptr && Kind == RHS.Kind;
8683 }
8684 bool operator()(const AccessInfo &LHS, const AccessInfo &RHS) const {
8685 if (LHS.I != RHS.I)
8686 return LHS.I < RHS.I;
8687 if (LHS.Ptr != RHS.Ptr)
8688 return LHS.Ptr < RHS.Ptr;
8689 if (LHS.Kind != RHS.Kind)
8690 return LHS.Kind < RHS.Kind;
8691 return false;
8692 }
8693 };
8694
8695 /// Mapping from *single* memory location kinds, e.g., LOCAL_MEM with the
8696 /// value of NO_LOCAL_MEM, to the accesses encountered for this memory kind.
8697 using AccessSet = SmallSet<AccessInfo, 2, AccessInfo>;
8698 std::array<AccessSet *, llvm::ConstantLog2<VALID_STATE>()>
8699 AccessKind2Accesses;
8700
8701 /// Categorize the pointer arguments of CB that might access memory in
8702 /// AccessedLoc and update the state and access map accordingly.
8703 void
8704 categorizeArgumentPointerLocations(Attributor &A, CallBase &CB,
8705 AAMemoryLocation::StateType &AccessedLocs,
8706 bool &Changed);
8707
8708 /// Return the kind(s) of location that may be accessed by \p V.
8709 AAMemoryLocation::MemoryLocationsKind
8710 categorizeAccessedLocations(Attributor &A, Instruction &I, bool &Changed);
8711
8712 /// Return the access kind as determined by \p I.
8713 AccessKind getAccessKindFromInst(const Instruction *I) {
8714 AccessKind AK = READ_WRITE;
8715 if (I) {
8716 AK = I->mayReadFromMemory() ? READ : NONE;
8717 AK = AccessKind(AK | (I->mayWriteToMemory() ? WRITE : NONE));
8718 }
8719 return AK;
8720 }
8721
8722 /// Update the state \p State and the AccessKind2Accesses given that \p I is
8723 /// an access of kind \p AK to a \p MLK memory location with the access
8724 /// pointer \p Ptr.
8725 void updateStateAndAccessesMap(AAMemoryLocation::StateType &State,
8726 MemoryLocationsKind MLK, const Instruction *I,
8727 const Value *Ptr, bool &Changed,
8728 AccessKind AK = READ_WRITE) {
8729
8730 assert(isPowerOf2_32(MLK) && "Expected a single location set!");
8731 auto *&Accesses = AccessKind2Accesses[llvm::Log2_32(Value: MLK)];
8732 if (!Accesses)
8733 Accesses = new (Allocator) AccessSet();
8734 Changed |= Accesses->insert(V: AccessInfo{.I: I, .Ptr: Ptr, .Kind: AK}).second;
8735 if (MLK == NO_UNKOWN_MEM)
8736 MLK = NO_LOCATIONS;
8737 State.removeAssumedBits(BitsEncoding: MLK);
8738 }
8739
8740 /// Determine the underlying locations kinds for \p Ptr, e.g., globals or
8741 /// arguments, and update the state and access map accordingly.
8742 void categorizePtrValue(Attributor &A, const Instruction &I, const Value &Ptr,
8743 AAMemoryLocation::StateType &State, bool &Changed,
8744 unsigned AccessAS = 0);
8745
8746 /// Used to allocate access sets.
8747 BumpPtrAllocator &Allocator;
8748};
8749
8750void AAMemoryLocationImpl::categorizePtrValue(
8751 Attributor &A, const Instruction &I, const Value &Ptr,
8752 AAMemoryLocation::StateType &State, bool &Changed, unsigned AccessAS) {
8753 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize pointer locations for "
8754 << Ptr << " ["
8755 << getMemoryLocationsAsStr(State.getAssumed()) << "]\n");
8756
8757 auto Pred = [&](Value &Obj) {
8758 unsigned ObjectAS = Obj.getType()->getPointerAddressSpace();
8759 // TODO: recognize the TBAA used for constant accesses.
8760 MemoryLocationsKind MLK = NO_LOCATIONS;
8761
8762 // Filter accesses to constant (GPU) memory if we have an AS at the access
8763 // site or the object is known to actually have the associated AS.
8764 if (AA::isGPU(M: A.getModule())) {
8765 if (AA::isGPUConstantAddressSpace(M: A.getModule(), AS: AccessAS) ||
8766 (AA::isGPUConstantAddressSpace(M: A.getModule(), AS: ObjectAS) &&
8767 isIdentifiedObject(V: &Obj)))
8768 return true;
8769 }
8770
8771 if (isa<UndefValue>(Val: &Obj))
8772 return true;
8773 if (isa<Argument>(Val: &Obj)) {
8774 // TODO: For now we do not treat byval arguments as local copies performed
8775 // on the call edge, though, we should. To make that happen we need to
8776 // teach various passes, e.g., DSE, about the copy effect of a byval. That
8777 // would also allow us to mark functions only accessing byval arguments as
8778 // readnone again, arguably their accesses have no effect outside of the
8779 // function, like accesses to allocas.
8780 MLK = NO_ARGUMENT_MEM;
8781 } else if (auto *GV = dyn_cast<GlobalValue>(Val: &Obj)) {
8782 // Reading constant memory is not treated as a read "effect" by the
8783 // function attr pass so we won't neither. Constants defined by TBAA are
8784 // similar. (We know we do not write it because it is constant.)
8785 if (auto *GVar = dyn_cast<GlobalVariable>(Val: GV))
8786 if (GVar->isConstant())
8787 return true;
8788
8789 if (GV->hasLocalLinkage())
8790 MLK = NO_GLOBAL_INTERNAL_MEM;
8791 else
8792 MLK = NO_GLOBAL_EXTERNAL_MEM;
8793 } else if (isa<ConstantPointerNull>(Val: &Obj) && ObjectAS == AccessAS &&
8794 !NullPointerIsDefined(F: getAssociatedFunction(), AS: AccessAS)) {
8795 // Accessing null is undefined, but not accessing a null pointer cast to
8796 // another address space, which may be a valid address there.
8797 return true;
8798 } else if (isa<AllocaInst>(Val: &Obj)) {
8799 MLK = NO_LOCAL_MEM;
8800 } else if (const auto *CB = dyn_cast<CallBase>(Val: &Obj)) {
8801 bool IsKnownNoAlias;
8802 if (AA::hasAssumedIRAttr<Attribute::NoAlias>(
8803 A, QueryingAA: this, IRP: IRPosition::callsite_returned(CB: *CB), DepClass: DepClassTy::OPTIONAL,
8804 IsKnown&: IsKnownNoAlias))
8805 MLK = NO_MALLOCED_MEM;
8806 else
8807 MLK = NO_UNKOWN_MEM;
8808 } else {
8809 MLK = NO_UNKOWN_MEM;
8810 }
8811
8812 assert(MLK != NO_LOCATIONS && "No location specified!");
8813 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Ptr value can be categorized: "
8814 << Obj << " -> " << getMemoryLocationsAsStr(MLK) << "\n");
8815 updateStateAndAccessesMap(State, MLK, I: &I, Ptr: &Obj, Changed,
8816 AK: getAccessKindFromInst(I: &I));
8817
8818 return true;
8819 };
8820
8821 const auto *AA = A.getAAFor<AAUnderlyingObjects>(
8822 QueryingAA: *this, IRP: IRPosition::value(V: Ptr), DepClass: DepClassTy::OPTIONAL);
8823 if (!AA || !AA->forallUnderlyingObjects(Pred, Scope: AA::Intraprocedural)) {
8824 LLVM_DEBUG(
8825 dbgs() << "[AAMemoryLocation] Pointer locations not categorized\n");
8826 updateStateAndAccessesMap(State, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr, Changed,
8827 AK: getAccessKindFromInst(I: &I));
8828 return;
8829 }
8830
8831 LLVM_DEBUG(
8832 dbgs() << "[AAMemoryLocation] Accessed locations with pointer locations: "
8833 << getMemoryLocationsAsStr(State.getAssumed()) << "\n");
8834}
8835
8836void AAMemoryLocationImpl::categorizeArgumentPointerLocations(
8837 Attributor &A, CallBase &CB, AAMemoryLocation::StateType &AccessedLocs,
8838 bool &Changed) {
8839 for (unsigned ArgNo = 0, E = CB.arg_size(); ArgNo < E; ++ArgNo) {
8840
8841 // Skip non-pointer arguments.
8842 const Value *ArgOp = CB.getArgOperand(i: ArgNo);
8843 if (!ArgOp->getType()->isPtrOrPtrVectorTy())
8844 continue;
8845
8846 // Skip readnone arguments.
8847 const IRPosition &ArgOpIRP = IRPosition::callsite_argument(CB, ArgNo);
8848 const auto *ArgOpMemLocationAA =
8849 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: ArgOpIRP, DepClass: DepClassTy::OPTIONAL);
8850
8851 if (ArgOpMemLocationAA && ArgOpMemLocationAA->isAssumedReadNone())
8852 continue;
8853
8854 // Categorize potentially accessed pointer arguments as if there was an
8855 // access instruction with them as pointer.
8856 categorizePtrValue(A, I: CB, Ptr: *ArgOp, State&: AccessedLocs, Changed);
8857 }
8858}
8859
8860AAMemoryLocation::MemoryLocationsKind
8861AAMemoryLocationImpl::categorizeAccessedLocations(Attributor &A, Instruction &I,
8862 bool &Changed) {
8863 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize accessed locations for "
8864 << I << "\n");
8865
8866 AAMemoryLocation::StateType AccessedLocs;
8867 AccessedLocs.intersectAssumedBits(BitsEncoding: NO_LOCATIONS);
8868
8869 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
8870
8871 // First check if we assume any memory is access is visible.
8872 const auto *CBMemLocationAA = A.getAAFor<AAMemoryLocation>(
8873 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL);
8874 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize call site: " << I
8875 << " [" << CBMemLocationAA << "]\n");
8876 if (!CBMemLocationAA) {
8877 updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr,
8878 Changed, AK: getAccessKindFromInst(I: &I));
8879 return NO_UNKOWN_MEM;
8880 }
8881
8882 if (CBMemLocationAA->isAssumedReadNone())
8883 return NO_LOCATIONS;
8884
8885 if (CBMemLocationAA->isAssumedInaccessibleMemOnly()) {
8886 updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_INACCESSIBLE_MEM, I: &I, Ptr: nullptr,
8887 Changed, AK: getAccessKindFromInst(I: &I));
8888 return AccessedLocs.getAssumed();
8889 }
8890
8891 uint32_t CBAssumedNotAccessedLocs =
8892 CBMemLocationAA->getAssumedNotAccessedLocation();
8893
8894 // Set the argmemonly and global bit as we handle them separately below.
8895 uint32_t CBAssumedNotAccessedLocsNoArgMem =
8896 CBAssumedNotAccessedLocs | NO_ARGUMENT_MEM | NO_GLOBAL_MEM;
8897
8898 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2) {
8899 if (CBAssumedNotAccessedLocsNoArgMem & CurMLK)
8900 continue;
8901 updateStateAndAccessesMap(State&: AccessedLocs, MLK: CurMLK, I: &I, Ptr: nullptr, Changed,
8902 AK: getAccessKindFromInst(I: &I));
8903 }
8904
8905 // Now handle global memory if it might be accessed. This is slightly tricky
8906 // as NO_GLOBAL_MEM has multiple bits set.
8907 bool HasGlobalAccesses = ((~CBAssumedNotAccessedLocs) & NO_GLOBAL_MEM);
8908 if (HasGlobalAccesses) {
8909 auto AccessPred = [&](const Instruction *, const Value *Ptr,
8910 AccessKind Kind, MemoryLocationsKind MLK) {
8911 updateStateAndAccessesMap(State&: AccessedLocs, MLK, I: &I, Ptr, Changed,
8912 AK: getAccessKindFromInst(I: &I));
8913 return true;
8914 };
8915 if (!CBMemLocationAA->checkForAllAccessesToMemoryKind(
8916 Pred: AccessPred, MLK: inverseLocation(Loc: NO_GLOBAL_MEM, AndLocalMem: false, AndConstMem: false)))
8917 return AccessedLocs.getWorstState();
8918 }
8919
8920 LLVM_DEBUG(
8921 dbgs() << "[AAMemoryLocation] Accessed state before argument handling: "
8922 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8923
8924 // Now handle argument memory if it might be accessed.
8925 bool HasArgAccesses = ((~CBAssumedNotAccessedLocs) & NO_ARGUMENT_MEM);
8926 if (HasArgAccesses)
8927 categorizeArgumentPointerLocations(A, CB&: *CB, AccessedLocs, Changed);
8928
8929 LLVM_DEBUG(
8930 dbgs() << "[AAMemoryLocation] Accessed state after argument handling: "
8931 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
8932
8933 return AccessedLocs.getAssumed();
8934 }
8935
8936 if (const Value *Ptr = getPointerOperand(I: &I, /* AllowVolatile */ true)) {
8937 LLVM_DEBUG(
8938 dbgs() << "[AAMemoryLocation] Categorize memory access with pointer: "
8939 << I << " [" << *Ptr << "]\n");
8940 categorizePtrValue(A, I, Ptr: *Ptr, State&: AccessedLocs, Changed,
8941 AccessAS: Ptr->getType()->getPointerAddressSpace());
8942 return AccessedLocs.getAssumed();
8943 }
8944
8945 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Failed to categorize instruction: "
8946 << I << "\n");
8947 updateStateAndAccessesMap(State&: AccessedLocs, MLK: NO_UNKOWN_MEM, I: &I, Ptr: nullptr, Changed,
8948 AK: getAccessKindFromInst(I: &I));
8949 return AccessedLocs.getAssumed();
8950}
8951
8952/// An AA to represent the memory behavior function attributes.
8953struct AAMemoryLocationFunction final : public AAMemoryLocationImpl {
8954 AAMemoryLocationFunction(const IRPosition &IRP, Attributor &A)
8955 : AAMemoryLocationImpl(IRP, A) {}
8956
8957 /// See AbstractAttribute::updateImpl(Attributor &A).
8958 ChangeStatus updateImpl(Attributor &A) override {
8959
8960 const auto *MemBehaviorAA =
8961 A.getAAFor<AAMemoryBehavior>(QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::NONE);
8962 if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
8963 if (MemBehaviorAA->isKnownReadNone())
8964 return indicateOptimisticFixpoint();
8965 assert(isAssumedReadNone() &&
8966 "AAMemoryLocation was not read-none but AAMemoryBehavior was!");
8967 A.recordDependence(FromAA: *MemBehaviorAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
8968 return ChangeStatus::UNCHANGED;
8969 }
8970
8971 // The current assumed state used to determine a change.
8972 auto AssumedState = getAssumed();
8973 bool Changed = false;
8974
8975 auto CheckRWInst = [&](Instruction &I) {
8976 MemoryLocationsKind MLK = categorizeAccessedLocations(A, I, Changed);
8977 LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Accessed locations for " << I
8978 << ": " << getMemoryLocationsAsStr(MLK) << "\n");
8979 removeAssumedBits(BitsEncoding: inverseLocation(Loc: MLK, AndLocalMem: false, AndConstMem: false));
8980 // Stop once only the valid bit set in the *not assumed location*, thus
8981 // once we don't actually exclude any memory locations in the state.
8982 return getAssumedNotAccessedLocation() != VALID_STATE;
8983 };
8984
8985 bool UsedAssumedInformation = false;
8986 if (!A.checkForAllReadWriteInstructions(Pred: CheckRWInst, QueryingAA&: *this,
8987 UsedAssumedInformation))
8988 return indicatePessimisticFixpoint();
8989
8990 Changed |= AssumedState != getAssumed();
8991 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
8992 }
8993
8994 /// See AbstractAttribute::trackStatistics()
8995 void trackStatistics() const override {
8996 if (isAssumedReadNone())
8997 STATS_DECLTRACK_FN_ATTR(readnone)
8998 else if (isAssumedArgMemOnly())
8999 STATS_DECLTRACK_FN_ATTR(argmemonly)
9000 else if (isAssumedInaccessibleMemOnly())
9001 STATS_DECLTRACK_FN_ATTR(inaccessiblememonly)
9002 else if (isAssumedInaccessibleOrArgMemOnly())
9003 STATS_DECLTRACK_FN_ATTR(inaccessiblememorargmemonly)
9004 }
9005};
9006
9007/// AAMemoryLocation attribute for call sites.
9008struct AAMemoryLocationCallSite final : AAMemoryLocationImpl {
9009 AAMemoryLocationCallSite(const IRPosition &IRP, Attributor &A)
9010 : AAMemoryLocationImpl(IRP, A) {}
9011
9012 /// See AbstractAttribute::updateImpl(...).
9013 ChangeStatus updateImpl(Attributor &A) override {
9014 // TODO: Once we have call site specific value information we can provide
9015 // call site specific liveness liveness information and then it makes
9016 // sense to specialize attributes for call sites arguments instead of
9017 // redirecting requests to the callee argument.
9018 Function *F = getAssociatedFunction();
9019 const IRPosition &FnPos = IRPosition::function(F: *F);
9020 auto *FnAA =
9021 A.getAAFor<AAMemoryLocation>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
9022 if (!FnAA)
9023 return indicatePessimisticFixpoint();
9024 bool Changed = false;
9025 auto AccessPred = [&](const Instruction *I, const Value *Ptr,
9026 AccessKind Kind, MemoryLocationsKind MLK) {
9027 updateStateAndAccessesMap(State&: getState(), MLK, I, Ptr, Changed,
9028 AK: getAccessKindFromInst(I));
9029 return true;
9030 };
9031 if (!FnAA->checkForAllAccessesToMemoryKind(Pred: AccessPred, MLK: ALL_LOCATIONS))
9032 return indicatePessimisticFixpoint();
9033 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
9034 }
9035
9036 /// See AbstractAttribute::trackStatistics()
9037 void trackStatistics() const override {
9038 if (isAssumedReadNone())
9039 STATS_DECLTRACK_CS_ATTR(readnone)
9040 }
9041};
9042} // namespace
9043
9044/// ------------------ denormal-fp-math Attribute -------------------------
9045
9046namespace {
9047struct AADenormalFPMathImpl : public AADenormalFPMath {
9048 AADenormalFPMathImpl(const IRPosition &IRP, Attributor &A)
9049 : AADenormalFPMath(IRP, A) {}
9050
9051 const std::string getAsStr(Attributor *A) const override {
9052 std::string Str("AADenormalFPMath[");
9053 raw_string_ostream OS(Str);
9054
9055 DenormalState Known = getKnown();
9056 if (Known.Mode.isValid())
9057 OS << "denormal-fp-math=" << Known.Mode;
9058 else
9059 OS << "invalid";
9060
9061 if (Known.ModeF32.isValid())
9062 OS << " denormal-fp-math-f32=" << Known.ModeF32;
9063 OS << ']';
9064 return Str;
9065 }
9066};
9067
9068struct AADenormalFPMathFunction final : AADenormalFPMathImpl {
9069 AADenormalFPMathFunction(const IRPosition &IRP, Attributor &A)
9070 : AADenormalFPMathImpl(IRP, A) {}
9071
9072 void initialize(Attributor &A) override {
9073 const Function *F = getAnchorScope();
9074 DenormalFPEnv DenormEnv = F->getDenormalFPEnv();
9075
9076 Known = DenormalState{.Mode: DenormEnv.DefaultMode, .ModeF32: DenormEnv.F32Mode};
9077 if (isModeFixed())
9078 indicateFixpoint();
9079 }
9080
9081 ChangeStatus updateImpl(Attributor &A) override {
9082 ChangeStatus Change = ChangeStatus::UNCHANGED;
9083
9084 auto CheckCallSite = [=, &Change, &A](AbstractCallSite CS) {
9085 Function *Caller = CS.getInstruction()->getFunction();
9086 LLVM_DEBUG(dbgs() << "[AADenormalFPMath] Call " << Caller->getName()
9087 << "->" << getAssociatedFunction()->getName() << '\n');
9088
9089 const auto *CallerInfo = A.getAAFor<AADenormalFPMath>(
9090 QueryingAA: *this, IRP: IRPosition::function(F: *Caller), DepClass: DepClassTy::REQUIRED);
9091 if (!CallerInfo)
9092 return false;
9093
9094 Change = Change | clampStateAndIndicateChange(S&: this->getState(),
9095 R: CallerInfo->getState());
9096 return true;
9097 };
9098
9099 bool AllCallSitesKnown = true;
9100 if (!A.checkForAllCallSites(Pred: CheckCallSite, QueryingAA: *this, RequireAllCallSites: true, UsedAssumedInformation&: AllCallSitesKnown))
9101 return indicatePessimisticFixpoint();
9102
9103 if (Change == ChangeStatus::CHANGED && isModeFixed())
9104 indicateFixpoint();
9105 return Change;
9106 }
9107
9108 ChangeStatus manifest(Attributor &A) override {
9109 LLVMContext &Ctx = getAssociatedFunction()->getContext();
9110
9111 SmallVector<Attribute, 2> AttrToAdd;
9112 SmallVector<Attribute::AttrKind, 2> AttrToRemove;
9113
9114 // TODO: Change to use DenormalFPEnv everywhere.
9115 DenormalFPEnv KnownEnv(Known.Mode, Known.ModeF32);
9116
9117 if (KnownEnv == DenormalFPEnv::getDefault()) {
9118 AttrToRemove.push_back(Elt: Attribute::DenormalFPEnv);
9119 } else {
9120 AttrToAdd.push_back(Elt: Attribute::get(
9121 Context&: Ctx, Kind: Attribute::DenormalFPEnv,
9122 Val: DenormalFPEnv(Known.Mode, Known.ModeF32).toIntValue()));
9123 }
9124
9125 auto &IRP = getIRPosition();
9126
9127 // TODO: There should be a combined add and remove API.
9128 return A.removeAttrs(IRP, AttrKinds: AttrToRemove) |
9129 A.manifestAttrs(IRP, DeducedAttrs: AttrToAdd, /*ForceReplace=*/true);
9130 }
9131
9132 void trackStatistics() const override {
9133 STATS_DECLTRACK_FN_ATTR(denormal_fpenv)
9134 }
9135};
9136} // namespace
9137
9138/// ------------------ Value Constant Range Attribute -------------------------
9139
9140namespace {
9141struct AAValueConstantRangeImpl : AAValueConstantRange {
9142 using StateType = IntegerRangeState;
9143 AAValueConstantRangeImpl(const IRPosition &IRP, Attributor &A)
9144 : AAValueConstantRange(IRP, A) {}
9145
9146 /// See AbstractAttribute::initialize(..).
9147 void initialize(Attributor &A) override {
9148 if (A.hasSimplificationCallback(IRP: getIRPosition())) {
9149 indicatePessimisticFixpoint();
9150 return;
9151 }
9152
9153 // Intersect a range given by SCEV.
9154 intersectKnown(R: getConstantRangeFromSCEV(A, I: getCtxI()));
9155
9156 // Intersect a range given by LVI.
9157 intersectKnown(R: getConstantRangeFromLVI(A, CtxI: getCtxI()));
9158 }
9159
9160 /// See AbstractAttribute::getAsStr().
9161 const std::string getAsStr(Attributor *A) const override {
9162 std::string Str;
9163 llvm::raw_string_ostream OS(Str);
9164 OS << "range(" << getBitWidth() << ")<";
9165 getKnown().print(OS);
9166 OS << " / ";
9167 getAssumed().print(OS);
9168 OS << ">";
9169 return Str;
9170 }
9171
9172 /// Helper function to get a SCEV expr for the associated value at program
9173 /// point \p I.
9174 const SCEV *getSCEV(Attributor &A, const Instruction *I = nullptr) const {
9175 if (!getAnchorScope())
9176 return nullptr;
9177
9178 ScalarEvolution *SE =
9179 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
9180 F: *getAnchorScope());
9181
9182 LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(
9183 F: *getAnchorScope());
9184
9185 if (!SE || !LI)
9186 return nullptr;
9187
9188 const SCEV *S = SE->getSCEV(V: &getAssociatedValue());
9189 if (!I)
9190 return S;
9191
9192 return SE->getSCEVAtScope(S, L: LI->getLoopFor(BB: I->getParent()));
9193 }
9194
9195 /// Helper function to get a range from SCEV for the associated value at
9196 /// program point \p I.
9197 ConstantRange getConstantRangeFromSCEV(Attributor &A,
9198 const Instruction *I = nullptr) const {
9199 if (!getAnchorScope())
9200 return getWorstState(BitWidth: getBitWidth());
9201
9202 ScalarEvolution *SE =
9203 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
9204 F: *getAnchorScope());
9205
9206 const SCEV *S = getSCEV(A, I);
9207 if (!SE || !S)
9208 return getWorstState(BitWidth: getBitWidth());
9209
9210 return SE->getUnsignedRange(S);
9211 }
9212
9213 /// Helper function to get a range from LVI for the associated value at
9214 /// program point \p I.
9215 ConstantRange
9216 getConstantRangeFromLVI(Attributor &A,
9217 const Instruction *CtxI = nullptr) const {
9218 if (!getAnchorScope())
9219 return getWorstState(BitWidth: getBitWidth());
9220
9221 LazyValueInfo *LVI =
9222 A.getInfoCache().getAnalysisResultForFunction<LazyValueAnalysis>(
9223 F: *getAnchorScope());
9224
9225 if (!LVI || !CtxI)
9226 return getWorstState(BitWidth: getBitWidth());
9227 return LVI->getConstantRange(V: &getAssociatedValue(),
9228 CtxI: const_cast<Instruction *>(CtxI),
9229 /*UndefAllowed*/ false);
9230 }
9231
9232 /// Return true if \p CtxI is valid for querying outside analyses.
9233 /// This basically makes sure we do not ask intra-procedural analysis
9234 /// about a context in the wrong function or a context that violates
9235 /// dominance assumptions they might have. The \p AllowAACtxI flag indicates
9236 /// if the original context of this AA is OK or should be considered invalid.
9237 bool isValidCtxInstructionForOutsideAnalysis(Attributor &A,
9238 const Instruction *CtxI,
9239 bool AllowAACtxI) const {
9240 if (!CtxI || (!AllowAACtxI && CtxI == getCtxI()))
9241 return false;
9242
9243 // Our context might be in a different function, neither intra-procedural
9244 // analysis (ScalarEvolution nor LazyValueInfo) can handle that.
9245 if (!AA::isValidInScope(V: getAssociatedValue(), Scope: CtxI->getFunction()))
9246 return false;
9247
9248 // If the context is not dominated by the value there are paths to the
9249 // context that do not define the value. This cannot be handled by
9250 // LazyValueInfo so we need to bail.
9251 if (auto *I = dyn_cast<Instruction>(Val: &getAssociatedValue())) {
9252 InformationCache &InfoCache = A.getInfoCache();
9253 const DominatorTree *DT =
9254 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(
9255 F: *I->getFunction());
9256 return DT && DT->dominates(Def: I, User: CtxI);
9257 }
9258
9259 return true;
9260 }
9261
9262 /// See AAValueConstantRange::getAssumedConstantRange(..).
9263 ConstantRange
9264 getAssumedConstantRange(Attributor &A,
9265 const Instruction *CtxI = nullptr) const override {
9266 // TODO: Make SCEV use Attributor assumption.
9267 // We may be able to bound a variable range via assumptions in
9268 // Attributor. ex.) If x is assumed to be in [1, 3] and y is known to
9269 // evolve to x^2 + x, then we can say that y is in [2, 12].
9270 if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
9271 /* AllowAACtxI */ false))
9272 return getAssumed();
9273
9274 ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
9275 ConstantRange SCEVR = getConstantRangeFromSCEV(A, I: CtxI);
9276 return getAssumed().intersectWith(CR: SCEVR).intersectWith(CR: LVIR);
9277 }
9278
9279 /// Helper function to create MDNode for range metadata.
9280 static MDNode *
9281 getMDNodeForConstantRange(Type *Ty, LLVMContext &Ctx,
9282 const ConstantRange &AssumedConstantRange) {
9283 Metadata *LowAndHigh[] = {ConstantAsMetadata::get(C: ConstantInt::get(
9284 Ty, V: AssumedConstantRange.getLower())),
9285 ConstantAsMetadata::get(C: ConstantInt::get(
9286 Ty, V: AssumedConstantRange.getUpper()))};
9287 return MDNode::get(Context&: Ctx, MDs: LowAndHigh);
9288 }
9289
9290 /// Return true if \p Assumed is included in ranges from instruction \p I.
9291 static bool isBetterRange(const ConstantRange &Assumed,
9292 const Instruction &I) {
9293 if (Assumed.isFullSet())
9294 return false;
9295
9296 std::optional<ConstantRange> Known;
9297
9298 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
9299 Known = CB->getRange();
9300 } else if (MDNode *KnownRanges = I.getMetadata(KindID: LLVMContext::MD_range)) {
9301 // If multiple ranges are annotated in IR, we give up to annotate assumed
9302 // range for now.
9303
9304 // TODO: If there exists a known range which containts assumed range, we
9305 // can say assumed range is better.
9306 if (KnownRanges->getNumOperands() > 2)
9307 return false;
9308
9309 ConstantInt *Lower =
9310 mdconst::extract<ConstantInt>(MD: KnownRanges->getOperand(I: 0));
9311 ConstantInt *Upper =
9312 mdconst::extract<ConstantInt>(MD: KnownRanges->getOperand(I: 1));
9313
9314 Known.emplace(args: Lower->getValue(), args: Upper->getValue());
9315 }
9316 return !Known || (*Known != Assumed && Known->contains(CR: Assumed));
9317 }
9318
9319 /// Helper function to set range metadata.
9320 static bool
9321 setRangeMetadataIfisBetterRange(Instruction *I,
9322 const ConstantRange &AssumedConstantRange) {
9323 if (isBetterRange(Assumed: AssumedConstantRange, I: *I)) {
9324 I->setMetadata(KindID: LLVMContext::MD_range,
9325 Node: getMDNodeForConstantRange(Ty: I->getType(), Ctx&: I->getContext(),
9326 AssumedConstantRange));
9327 return true;
9328 }
9329 return false;
9330 }
9331 /// Helper function to set range return attribute.
9332 static bool
9333 setRangeRetAttrIfisBetterRange(Attributor &A, const IRPosition &IRP,
9334 Instruction *I,
9335 const ConstantRange &AssumedConstantRange) {
9336 if (isBetterRange(Assumed: AssumedConstantRange, I: *I)) {
9337 A.manifestAttrs(IRP,
9338 DeducedAttrs: Attribute::get(Context&: I->getContext(), Kind: Attribute::Range,
9339 CR: AssumedConstantRange),
9340 /*ForceReplace*/ true);
9341 return true;
9342 }
9343 return false;
9344 }
9345
9346 /// See AbstractAttribute::manifest()
9347 ChangeStatus manifest(Attributor &A) override {
9348 ChangeStatus Changed = ChangeStatus::UNCHANGED;
9349 ConstantRange AssumedConstantRange = getAssumedConstantRange(A);
9350 assert(!AssumedConstantRange.isFullSet() && "Invalid state");
9351
9352 auto &V = getAssociatedValue();
9353 if (!AssumedConstantRange.isEmptySet() &&
9354 !AssumedConstantRange.isSingleElement()) {
9355 if (Instruction *I = dyn_cast<Instruction>(Val: &V)) {
9356 assert(I == getCtxI() && "Should not annotate an instruction which is "
9357 "not the context instruction");
9358 if (isa<LoadInst>(Val: I))
9359 if (setRangeMetadataIfisBetterRange(I, AssumedConstantRange))
9360 Changed = ChangeStatus::CHANGED;
9361 if (isa<CallInst>(Val: I))
9362 if (setRangeRetAttrIfisBetterRange(A, IRP: getIRPosition(), I,
9363 AssumedConstantRange))
9364 Changed = ChangeStatus::CHANGED;
9365 }
9366 }
9367
9368 return Changed;
9369 }
9370};
9371
9372struct AAValueConstantRangeArgument final
9373 : AAArgumentFromCallSiteArguments<
9374 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9375 true /* BridgeCallBaseContext */> {
9376 using Base = AAArgumentFromCallSiteArguments<
9377 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9378 true /* BridgeCallBaseContext */>;
9379 AAValueConstantRangeArgument(const IRPosition &IRP, Attributor &A)
9380 : Base(IRP, A) {}
9381
9382 /// See AbstractAttribute::trackStatistics()
9383 void trackStatistics() const override {
9384 STATS_DECLTRACK_ARG_ATTR(value_range)
9385 }
9386};
9387
9388struct AAValueConstantRangeReturned
9389 : AAReturnedFromReturnedValues<AAValueConstantRange,
9390 AAValueConstantRangeImpl,
9391 AAValueConstantRangeImpl::StateType,
9392 /* PropagateCallBaseContext */ true> {
9393 using Base =
9394 AAReturnedFromReturnedValues<AAValueConstantRange,
9395 AAValueConstantRangeImpl,
9396 AAValueConstantRangeImpl::StateType,
9397 /* PropagateCallBaseContext */ true>;
9398 AAValueConstantRangeReturned(const IRPosition &IRP, Attributor &A)
9399 : Base(IRP, A) {}
9400
9401 /// See AbstractAttribute::initialize(...).
9402 void initialize(Attributor &A) override {
9403 if (!A.isFunctionIPOAmendable(F: *getAssociatedFunction()))
9404 indicatePessimisticFixpoint();
9405 }
9406
9407 /// See AbstractAttribute::trackStatistics()
9408 void trackStatistics() const override {
9409 STATS_DECLTRACK_FNRET_ATTR(value_range)
9410 }
9411};
9412
9413struct AAValueConstantRangeFloating : AAValueConstantRangeImpl {
9414 AAValueConstantRangeFloating(const IRPosition &IRP, Attributor &A)
9415 : AAValueConstantRangeImpl(IRP, A) {}
9416
9417 /// See AbstractAttribute::initialize(...).
9418 void initialize(Attributor &A) override {
9419 AAValueConstantRangeImpl::initialize(A);
9420 if (isAtFixpoint())
9421 return;
9422
9423 Value &V = getAssociatedValue();
9424
9425 if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
9426 unionAssumed(R: ConstantRange(C->getValue()));
9427 indicateOptimisticFixpoint();
9428 return;
9429 }
9430
9431 if (isa<UndefValue>(Val: &V)) {
9432 // Collapse the undef state to 0.
9433 unionAssumed(R: ConstantRange(APInt(getBitWidth(), 0)));
9434 indicateOptimisticFixpoint();
9435 return;
9436 }
9437
9438 if (isa<CallBase>(Val: &V))
9439 return;
9440
9441 if (isa<BinaryOperator>(Val: &V) || isa<CmpInst>(Val: &V) || isa<CastInst>(Val: &V))
9442 return;
9443
9444 // If it is a load instruction with range metadata, use it.
9445 if (LoadInst *LI = dyn_cast<LoadInst>(Val: &V))
9446 if (auto *RangeMD = LI->getMetadata(KindID: LLVMContext::MD_range)) {
9447 intersectKnown(R: getConstantRangeFromMetadata(RangeMD: *RangeMD));
9448 return;
9449 }
9450
9451 // We can work with PHI and select instruction as we traverse their operands
9452 // during update.
9453 if (isa<SelectInst>(Val: V) || isa<PHINode>(Val: V))
9454 return;
9455
9456 // Otherwise we give up.
9457 indicatePessimisticFixpoint();
9458
9459 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] We give up: "
9460 << getAssociatedValue() << "\n");
9461 }
9462
9463 bool calculateBinaryOperator(
9464 Attributor &A, BinaryOperator *BinOp, IntegerRangeState &T,
9465 const Instruction *CtxI,
9466 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9467 Value *LHS = BinOp->getOperand(i_nocapture: 0);
9468 Value *RHS = BinOp->getOperand(i_nocapture: 1);
9469
9470 // Simplify the operands first.
9471 bool UsedAssumedInformation = false;
9472 const auto &SimplifiedLHS = A.getAssumedSimplified(
9473 IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()), AA: *this,
9474 UsedAssumedInformation, S: AA::Interprocedural);
9475 if (!SimplifiedLHS.has_value())
9476 return true;
9477 if (!*SimplifiedLHS)
9478 return false;
9479 LHS = *SimplifiedLHS;
9480
9481 const auto &SimplifiedRHS = A.getAssumedSimplified(
9482 IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()), AA: *this,
9483 UsedAssumedInformation, S: AA::Interprocedural);
9484 if (!SimplifiedRHS.has_value())
9485 return true;
9486 if (!*SimplifiedRHS)
9487 return false;
9488 RHS = *SimplifiedRHS;
9489
9490 // TODO: Allow non integers as well.
9491 if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9492 return false;
9493
9494 auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9495 QueryingAA: *this, IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()),
9496 DepClass: DepClassTy::REQUIRED);
9497 if (!LHSAA)
9498 return false;
9499 QuerriedAAs.push_back(Elt: LHSAA);
9500 auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9501
9502 auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9503 QueryingAA: *this, IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()),
9504 DepClass: DepClassTy::REQUIRED);
9505 if (!RHSAA)
9506 return false;
9507 QuerriedAAs.push_back(Elt: RHSAA);
9508 auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9509
9510 auto AssumedRange = LHSAARange.binaryOp(BinOp: BinOp->getOpcode(), Other: RHSAARange);
9511
9512 T.unionAssumed(R: AssumedRange);
9513
9514 // TODO: Track a known state too.
9515
9516 return T.isValidState();
9517 }
9518
9519 bool calculateCastInst(
9520 Attributor &A, CastInst *CastI, IntegerRangeState &T,
9521 const Instruction *CtxI,
9522 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9523 assert(CastI->getNumOperands() == 1 && "Expected cast to be unary!");
9524 // TODO: Allow non integers as well.
9525 Value *OpV = CastI->getOperand(i_nocapture: 0);
9526
9527 // Simplify the operand first.
9528 bool UsedAssumedInformation = false;
9529 const auto &SimplifiedOpV = A.getAssumedSimplified(
9530 IRP: IRPosition::value(V: *OpV, CBContext: getCallBaseContext()), AA: *this,
9531 UsedAssumedInformation, S: AA::Interprocedural);
9532 if (!SimplifiedOpV.has_value())
9533 return true;
9534 if (!*SimplifiedOpV)
9535 return false;
9536 OpV = *SimplifiedOpV;
9537
9538 if (!OpV->getType()->isIntegerTy())
9539 return false;
9540
9541 auto *OpAA = A.getAAFor<AAValueConstantRange>(
9542 QueryingAA: *this, IRP: IRPosition::value(V: *OpV, CBContext: getCallBaseContext()),
9543 DepClass: DepClassTy::REQUIRED);
9544 if (!OpAA)
9545 return false;
9546 QuerriedAAs.push_back(Elt: OpAA);
9547 T.unionAssumed(R: OpAA->getAssumed().castOp(CastOp: CastI->getOpcode(),
9548 BitWidth: getState().getBitWidth()));
9549 return T.isValidState();
9550 }
9551
9552 bool
9553 calculateCmpInst(Attributor &A, CmpInst *CmpI, IntegerRangeState &T,
9554 const Instruction *CtxI,
9555 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9556 Value *LHS = CmpI->getOperand(i_nocapture: 0);
9557 Value *RHS = CmpI->getOperand(i_nocapture: 1);
9558
9559 // Simplify the operands first.
9560 bool UsedAssumedInformation = false;
9561 const auto &SimplifiedLHS = A.getAssumedSimplified(
9562 IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()), AA: *this,
9563 UsedAssumedInformation, S: AA::Interprocedural);
9564 if (!SimplifiedLHS.has_value())
9565 return true;
9566 if (!*SimplifiedLHS)
9567 return false;
9568 LHS = *SimplifiedLHS;
9569
9570 const auto &SimplifiedRHS = A.getAssumedSimplified(
9571 IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()), AA: *this,
9572 UsedAssumedInformation, S: AA::Interprocedural);
9573 if (!SimplifiedRHS.has_value())
9574 return true;
9575 if (!*SimplifiedRHS)
9576 return false;
9577 RHS = *SimplifiedRHS;
9578
9579 // TODO: Allow non integers as well.
9580 if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
9581 return false;
9582
9583 auto *LHSAA = A.getAAFor<AAValueConstantRange>(
9584 QueryingAA: *this, IRP: IRPosition::value(V: *LHS, CBContext: getCallBaseContext()),
9585 DepClass: DepClassTy::REQUIRED);
9586 if (!LHSAA)
9587 return false;
9588 QuerriedAAs.push_back(Elt: LHSAA);
9589 auto *RHSAA = A.getAAFor<AAValueConstantRange>(
9590 QueryingAA: *this, IRP: IRPosition::value(V: *RHS, CBContext: getCallBaseContext()),
9591 DepClass: DepClassTy::REQUIRED);
9592 if (!RHSAA)
9593 return false;
9594 QuerriedAAs.push_back(Elt: RHSAA);
9595 auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
9596 auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
9597
9598 // If one of them is empty set, we can't decide.
9599 if (LHSAARange.isEmptySet() || RHSAARange.isEmptySet())
9600 return true;
9601
9602 bool MustTrue = false, MustFalse = false;
9603
9604 auto AllowedRegion =
9605 ConstantRange::makeAllowedICmpRegion(Pred: CmpI->getPredicate(), Other: RHSAARange);
9606
9607 if (AllowedRegion.intersectWith(CR: LHSAARange).isEmptySet())
9608 MustFalse = true;
9609
9610 if (LHSAARange.icmp(Pred: CmpI->getPredicate(), Other: RHSAARange))
9611 MustTrue = true;
9612
9613 assert((!MustTrue || !MustFalse) &&
9614 "Either MustTrue or MustFalse should be false!");
9615
9616 if (MustTrue)
9617 T.unionAssumed(R: ConstantRange(APInt(/* numBits */ 1, /* val */ 1)));
9618 else if (MustFalse)
9619 T.unionAssumed(R: ConstantRange(APInt(/* numBits */ 1, /* val */ 0)));
9620 else
9621 T.unionAssumed(R: ConstantRange(/* BitWidth */ 1, /* isFullSet */ true));
9622
9623 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] " << *CmpI << " after "
9624 << (MustTrue ? "true" : (MustFalse ? "false" : "unknown"))
9625 << ": " << T << "\n\t" << *LHSAA << "\t<op>\n\t"
9626 << *RHSAA);
9627
9628 // TODO: Track a known state too.
9629 return T.isValidState();
9630 }
9631
9632 /// See AbstractAttribute::updateImpl(...).
9633 ChangeStatus updateImpl(Attributor &A) override {
9634
9635 IntegerRangeState T(getBitWidth());
9636 auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
9637 Instruction *I = dyn_cast<Instruction>(Val: &V);
9638 if (!I || isa<CallBase>(Val: I)) {
9639
9640 // Simplify the operand first.
9641 bool UsedAssumedInformation = false;
9642 const auto &SimplifiedOpV = A.getAssumedSimplified(
9643 IRP: IRPosition::value(V, CBContext: getCallBaseContext()), AA: *this,
9644 UsedAssumedInformation, S: AA::Interprocedural);
9645 if (!SimplifiedOpV.has_value())
9646 return true;
9647 if (!*SimplifiedOpV)
9648 return false;
9649 Value *VPtr = *SimplifiedOpV;
9650
9651 // If the value is not instruction, we query AA to Attributor.
9652 const auto *AA = A.getAAFor<AAValueConstantRange>(
9653 QueryingAA: *this, IRP: IRPosition::value(V: *VPtr, CBContext: getCallBaseContext()),
9654 DepClass: DepClassTy::REQUIRED);
9655
9656 // Clamp operator is not used to utilize a program point CtxI.
9657 if (AA)
9658 T.unionAssumed(R: AA->getAssumedConstantRange(A, CtxI));
9659 else
9660 return false;
9661
9662 return T.isValidState();
9663 }
9664
9665 SmallVector<const AAValueConstantRange *, 4> QuerriedAAs;
9666 if (auto *BinOp = dyn_cast<BinaryOperator>(Val: I)) {
9667 if (!calculateBinaryOperator(A, BinOp, T, CtxI, QuerriedAAs))
9668 return false;
9669 } else if (auto *CmpI = dyn_cast<CmpInst>(Val: I)) {
9670 if (!calculateCmpInst(A, CmpI, T, CtxI, QuerriedAAs))
9671 return false;
9672 } else if (auto *CastI = dyn_cast<CastInst>(Val: I)) {
9673 if (!calculateCastInst(A, CastI, T, CtxI, QuerriedAAs))
9674 return false;
9675 } else {
9676 // Give up with other instructions.
9677 // TODO: Add other instructions
9678
9679 T.indicatePessimisticFixpoint();
9680 return false;
9681 }
9682
9683 // Catch circular reasoning in a pessimistic way for now.
9684 // TODO: Check how the range evolves and if we stripped anything, see also
9685 // AADereferenceable or AAAlign for similar situations.
9686 for (const AAValueConstantRange *QueriedAA : QuerriedAAs) {
9687 if (QueriedAA != this)
9688 continue;
9689 // If we are in a stady state we do not need to worry.
9690 if (T.getAssumed() == getState().getAssumed())
9691 continue;
9692 T.indicatePessimisticFixpoint();
9693 }
9694
9695 return T.isValidState();
9696 };
9697
9698 if (!VisitValueCB(getAssociatedValue(), getCtxI()))
9699 return indicatePessimisticFixpoint();
9700
9701 // Ensure that long def-use chains can't cause circular reasoning either by
9702 // introducing a cutoff below.
9703 if (clampStateAndIndicateChange(S&: getState(), R: T) == ChangeStatus::UNCHANGED)
9704 return ChangeStatus::UNCHANGED;
9705 if (++NumChanges > MaxNumChanges) {
9706 LLVM_DEBUG(dbgs() << "[AAValueConstantRange] performed " << NumChanges
9707 << " but only " << MaxNumChanges
9708 << " are allowed to avoid cyclic reasoning.");
9709 return indicatePessimisticFixpoint();
9710 }
9711 return ChangeStatus::CHANGED;
9712 }
9713
9714 /// See AbstractAttribute::trackStatistics()
9715 void trackStatistics() const override {
9716 STATS_DECLTRACK_FLOATING_ATTR(value_range)
9717 }
9718
9719 /// Tracker to bail after too many widening steps of the constant range.
9720 int NumChanges = 0;
9721
9722 /// Upper bound for the number of allowed changes (=widening steps) for the
9723 /// constant range before we give up.
9724 static constexpr int MaxNumChanges = 5;
9725};
9726
9727struct AAValueConstantRangeFunction : AAValueConstantRangeImpl {
9728 AAValueConstantRangeFunction(const IRPosition &IRP, Attributor &A)
9729 : AAValueConstantRangeImpl(IRP, A) {}
9730
9731 /// See AbstractAttribute::initialize(...).
9732 ChangeStatus updateImpl(Attributor &A) override {
9733 llvm_unreachable("AAValueConstantRange(Function|CallSite)::updateImpl will "
9734 "not be called");
9735 }
9736
9737 /// See AbstractAttribute::trackStatistics()
9738 void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(value_range) }
9739};
9740
9741struct AAValueConstantRangeCallSite : AAValueConstantRangeFunction {
9742 AAValueConstantRangeCallSite(const IRPosition &IRP, Attributor &A)
9743 : AAValueConstantRangeFunction(IRP, A) {}
9744
9745 /// See AbstractAttribute::trackStatistics()
9746 void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(value_range) }
9747};
9748
9749struct AAValueConstantRangeCallSiteReturned
9750 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9751 AAValueConstantRangeImpl::StateType,
9752 /* IntroduceCallBaseContext */ true> {
9753 AAValueConstantRangeCallSiteReturned(const IRPosition &IRP, Attributor &A)
9754 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9755 AAValueConstantRangeImpl::StateType,
9756 /* IntroduceCallBaseContext */ true>(IRP, A) {}
9757
9758 /// See AbstractAttribute::initialize(...).
9759 void initialize(Attributor &A) override {
9760 // If it is a call instruction with range attribute, use the range.
9761 if (CallInst *CI = dyn_cast<CallInst>(Val: &getAssociatedValue())) {
9762 if (std::optional<ConstantRange> Range = CI->getRange())
9763 intersectKnown(R: *Range);
9764 }
9765
9766 AAValueConstantRangeImpl::initialize(A);
9767 }
9768
9769 /// See AbstractAttribute::trackStatistics()
9770 void trackStatistics() const override {
9771 STATS_DECLTRACK_CSRET_ATTR(value_range)
9772 }
9773};
9774struct AAValueConstantRangeCallSiteArgument : AAValueConstantRangeFloating {
9775 AAValueConstantRangeCallSiteArgument(const IRPosition &IRP, Attributor &A)
9776 : AAValueConstantRangeFloating(IRP, A) {}
9777
9778 /// See AbstractAttribute::manifest()
9779 ChangeStatus manifest(Attributor &A) override {
9780 return ChangeStatus::UNCHANGED;
9781 }
9782
9783 /// See AbstractAttribute::trackStatistics()
9784 void trackStatistics() const override {
9785 STATS_DECLTRACK_CSARG_ATTR(value_range)
9786 }
9787};
9788} // namespace
9789
9790/// ------------------ Potential Values Attribute -------------------------
9791
9792namespace {
9793struct AAPotentialConstantValuesImpl : AAPotentialConstantValues {
9794 using StateType = PotentialConstantIntValuesState;
9795
9796 AAPotentialConstantValuesImpl(const IRPosition &IRP, Attributor &A)
9797 : AAPotentialConstantValues(IRP, A) {}
9798
9799 /// See AbstractAttribute::initialize(..).
9800 void initialize(Attributor &A) override {
9801 if (A.hasSimplificationCallback(IRP: getIRPosition()))
9802 indicatePessimisticFixpoint();
9803 else
9804 AAPotentialConstantValues::initialize(A);
9805 }
9806
9807 bool fillSetWithConstantValues(Attributor &A, const IRPosition &IRP, SetTy &S,
9808 bool &ContainsUndef, bool ForSelf) {
9809 SmallVector<AA::ValueAndContext> Values;
9810 bool UsedAssumedInformation = false;
9811 if (!A.getAssumedSimplifiedValues(IRP, AA: *this, Values, S: AA::Interprocedural,
9812 UsedAssumedInformation)) {
9813 // Avoid recursion when the caller is computing constant values for this
9814 // IRP itself.
9815 if (ForSelf)
9816 return false;
9817 if (!IRP.getAssociatedType()->isIntegerTy())
9818 return false;
9819 auto *PotentialValuesAA = A.getAAFor<AAPotentialConstantValues>(
9820 QueryingAA: *this, IRP, DepClass: DepClassTy::REQUIRED);
9821 if (!PotentialValuesAA || !PotentialValuesAA->getState().isValidState())
9822 return false;
9823 ContainsUndef = PotentialValuesAA->getState().undefIsContained();
9824 S = PotentialValuesAA->getState().getAssumedSet();
9825 return true;
9826 }
9827
9828 // Copy all the constant values, except UndefValue. ContainsUndef is true
9829 // iff Values contains only UndefValue instances. If there are other known
9830 // constants, then UndefValue is dropped.
9831 ContainsUndef = false;
9832 for (auto &It : Values) {
9833 if (isa<UndefValue>(Val: It.getValue())) {
9834 ContainsUndef = true;
9835 continue;
9836 }
9837 auto *CI = dyn_cast<ConstantInt>(Val: It.getValue());
9838 if (!CI)
9839 return false;
9840 S.insert(X: CI->getValue());
9841 }
9842 ContainsUndef &= S.empty();
9843
9844 return true;
9845 }
9846
9847 /// See AbstractAttribute::getAsStr().
9848 const std::string getAsStr(Attributor *A) const override {
9849 std::string Str;
9850 llvm::raw_string_ostream OS(Str);
9851 OS << getState();
9852 return Str;
9853 }
9854
9855 /// See AbstractAttribute::updateImpl(...).
9856 ChangeStatus updateImpl(Attributor &A) override {
9857 return indicatePessimisticFixpoint();
9858 }
9859};
9860
9861struct AAPotentialConstantValuesArgument final
9862 : AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9863 AAPotentialConstantValuesImpl,
9864 PotentialConstantIntValuesState> {
9865 using Base = AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9866 AAPotentialConstantValuesImpl,
9867 PotentialConstantIntValuesState>;
9868 AAPotentialConstantValuesArgument(const IRPosition &IRP, Attributor &A)
9869 : Base(IRP, A) {}
9870
9871 /// See AbstractAttribute::trackStatistics()
9872 void trackStatistics() const override {
9873 STATS_DECLTRACK_ARG_ATTR(potential_values)
9874 }
9875};
9876
9877struct AAPotentialConstantValuesReturned
9878 : AAReturnedFromReturnedValues<AAPotentialConstantValues,
9879 AAPotentialConstantValuesImpl> {
9880 using Base = AAReturnedFromReturnedValues<AAPotentialConstantValues,
9881 AAPotentialConstantValuesImpl>;
9882 AAPotentialConstantValuesReturned(const IRPosition &IRP, Attributor &A)
9883 : Base(IRP, A) {}
9884
9885 void initialize(Attributor &A) override {
9886 if (!A.isFunctionIPOAmendable(F: *getAssociatedFunction()))
9887 indicatePessimisticFixpoint();
9888 Base::initialize(A);
9889 }
9890
9891 /// See AbstractAttribute::trackStatistics()
9892 void trackStatistics() const override {
9893 STATS_DECLTRACK_FNRET_ATTR(potential_values)
9894 }
9895};
9896
9897struct AAPotentialConstantValuesFloating : AAPotentialConstantValuesImpl {
9898 AAPotentialConstantValuesFloating(const IRPosition &IRP, Attributor &A)
9899 : AAPotentialConstantValuesImpl(IRP, A) {}
9900
9901 /// See AbstractAttribute::initialize(..).
9902 void initialize(Attributor &A) override {
9903 AAPotentialConstantValuesImpl::initialize(A);
9904 if (isAtFixpoint())
9905 return;
9906
9907 Value &V = getAssociatedValue();
9908
9909 if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
9910 unionAssumed(C: C->getValue());
9911 indicateOptimisticFixpoint();
9912 return;
9913 }
9914
9915 if (isa<UndefValue>(Val: &V)) {
9916 unionAssumedWithUndef();
9917 indicateOptimisticFixpoint();
9918 return;
9919 }
9920
9921 if (isa<BinaryOperator>(Val: &V) || isa<ICmpInst>(Val: &V) || isa<CastInst>(Val: &V))
9922 return;
9923
9924 if (isa<SelectInst>(Val: V) || isa<PHINode>(Val: V) || isa<LoadInst>(Val: V))
9925 return;
9926
9927 indicatePessimisticFixpoint();
9928
9929 LLVM_DEBUG(dbgs() << "[AAPotentialConstantValues] We give up: "
9930 << getAssociatedValue() << "\n");
9931 }
9932
9933 static bool calculateICmpInst(const ICmpInst *ICI, const APInt &LHS,
9934 const APInt &RHS) {
9935 return ICmpInst::compare(LHS, RHS, Pred: ICI->getPredicate());
9936 }
9937
9938 static APInt calculateCastInst(const CastInst *CI, const APInt &Src,
9939 uint32_t ResultBitWidth) {
9940 Instruction::CastOps CastOp = CI->getOpcode();
9941 switch (CastOp) {
9942 default:
9943 llvm_unreachable("unsupported or not integer cast");
9944 case Instruction::Trunc:
9945 return Src.trunc(width: ResultBitWidth);
9946 case Instruction::SExt:
9947 return Src.sext(width: ResultBitWidth);
9948 case Instruction::ZExt:
9949 return Src.zext(width: ResultBitWidth);
9950 case Instruction::BitCast:
9951 return Src;
9952 }
9953 }
9954
9955 static APInt calculateBinaryOperator(const BinaryOperator *BinOp,
9956 const APInt &LHS, const APInt &RHS,
9957 bool &SkipOperation, bool &Unsupported) {
9958 Instruction::BinaryOps BinOpcode = BinOp->getOpcode();
9959 // Unsupported is set to true when the binary operator is not supported.
9960 // SkipOperation is set to true when UB occur with the given operand pair
9961 // (LHS, RHS).
9962 // TODO: we should look at nsw and nuw keywords to handle operations
9963 // that create poison or undef value.
9964 switch (BinOpcode) {
9965 default:
9966 Unsupported = true;
9967 return LHS;
9968 case Instruction::Add:
9969 return LHS + RHS;
9970 case Instruction::Sub:
9971 return LHS - RHS;
9972 case Instruction::Mul:
9973 return LHS * RHS;
9974 case Instruction::UDiv:
9975 if (RHS.isZero()) {
9976 SkipOperation = true;
9977 return LHS;
9978 }
9979 return LHS.udiv(RHS);
9980 case Instruction::SDiv:
9981 if (RHS.isZero()) {
9982 SkipOperation = true;
9983 return LHS;
9984 }
9985 return LHS.sdiv(RHS);
9986 case Instruction::URem:
9987 if (RHS.isZero()) {
9988 SkipOperation = true;
9989 return LHS;
9990 }
9991 return LHS.urem(RHS);
9992 case Instruction::SRem:
9993 if (RHS.isZero()) {
9994 SkipOperation = true;
9995 return LHS;
9996 }
9997 return LHS.srem(RHS);
9998 case Instruction::Shl:
9999 return LHS.shl(ShiftAmt: RHS);
10000 case Instruction::LShr:
10001 return LHS.lshr(ShiftAmt: RHS);
10002 case Instruction::AShr:
10003 return LHS.ashr(ShiftAmt: RHS);
10004 case Instruction::And:
10005 return LHS & RHS;
10006 case Instruction::Or:
10007 return LHS | RHS;
10008 case Instruction::Xor:
10009 return LHS ^ RHS;
10010 }
10011 }
10012
10013 bool calculateBinaryOperatorAndTakeUnion(const BinaryOperator *BinOp,
10014 const APInt &LHS, const APInt &RHS) {
10015 bool SkipOperation = false;
10016 bool Unsupported = false;
10017 APInt Result =
10018 calculateBinaryOperator(BinOp, LHS, RHS, SkipOperation, Unsupported);
10019 if (Unsupported)
10020 return false;
10021 // If SkipOperation is true, we can ignore this operand pair (L, R).
10022 if (!SkipOperation)
10023 unionAssumed(C: Result);
10024 return isValidState();
10025 }
10026
10027 ChangeStatus updateWithICmpInst(Attributor &A, ICmpInst *ICI) {
10028 auto AssumedBefore = getAssumed();
10029 Value *LHS = ICI->getOperand(i_nocapture: 0);
10030 Value *RHS = ICI->getOperand(i_nocapture: 1);
10031
10032 bool LHSContainsUndef = false, RHSContainsUndef = false;
10033 SetTy LHSAAPVS, RHSAAPVS;
10034 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
10035 ContainsUndef&: LHSContainsUndef, /* ForSelf */ false) ||
10036 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
10037 ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
10038 return indicatePessimisticFixpoint();
10039
10040 // TODO: make use of undef flag to limit potential values aggressively.
10041 bool MaybeTrue = false, MaybeFalse = false;
10042 const APInt Zero(RHS->getType()->getIntegerBitWidth(), 0);
10043 if (LHSContainsUndef && RHSContainsUndef) {
10044 // The result of any comparison between undefs can be soundly replaced
10045 // with undef.
10046 unionAssumedWithUndef();
10047 } else if (LHSContainsUndef) {
10048 for (const APInt &R : RHSAAPVS) {
10049 bool CmpResult = calculateICmpInst(ICI, LHS: Zero, RHS: R);
10050 MaybeTrue |= CmpResult;
10051 MaybeFalse |= !CmpResult;
10052 if (MaybeTrue & MaybeFalse)
10053 return indicatePessimisticFixpoint();
10054 }
10055 } else if (RHSContainsUndef) {
10056 for (const APInt &L : LHSAAPVS) {
10057 bool CmpResult = calculateICmpInst(ICI, LHS: L, RHS: Zero);
10058 MaybeTrue |= CmpResult;
10059 MaybeFalse |= !CmpResult;
10060 if (MaybeTrue & MaybeFalse)
10061 return indicatePessimisticFixpoint();
10062 }
10063 } else {
10064 for (const APInt &L : LHSAAPVS) {
10065 for (const APInt &R : RHSAAPVS) {
10066 bool CmpResult = calculateICmpInst(ICI, LHS: L, RHS: R);
10067 MaybeTrue |= CmpResult;
10068 MaybeFalse |= !CmpResult;
10069 if (MaybeTrue & MaybeFalse)
10070 return indicatePessimisticFixpoint();
10071 }
10072 }
10073 }
10074 if (MaybeTrue)
10075 unionAssumed(C: APInt(/* numBits */ 1, /* val */ 1));
10076 if (MaybeFalse)
10077 unionAssumed(C: APInt(/* numBits */ 1, /* val */ 0));
10078 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10079 : ChangeStatus::CHANGED;
10080 }
10081
10082 ChangeStatus updateWithSelectInst(Attributor &A, SelectInst *SI) {
10083 auto AssumedBefore = getAssumed();
10084 Value *LHS = SI->getTrueValue();
10085 Value *RHS = SI->getFalseValue();
10086
10087 bool UsedAssumedInformation = false;
10088 std::optional<Constant *> C = A.getAssumedConstant(
10089 V: *SI->getCondition(), AA: *this, UsedAssumedInformation);
10090
10091 // Check if we only need one operand.
10092 bool OnlyLeft = false, OnlyRight = false;
10093 if (C && *C && (*C)->isOneValue())
10094 OnlyLeft = true;
10095 else if (C && *C && (*C)->isNullValue())
10096 OnlyRight = true;
10097
10098 bool LHSContainsUndef = false, RHSContainsUndef = false;
10099 SetTy LHSAAPVS, RHSAAPVS;
10100 if (!OnlyRight &&
10101 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
10102 ContainsUndef&: LHSContainsUndef, /* ForSelf */ false))
10103 return indicatePessimisticFixpoint();
10104
10105 if (!OnlyLeft &&
10106 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
10107 ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
10108 return indicatePessimisticFixpoint();
10109
10110 if (OnlyLeft || OnlyRight) {
10111 // select (true/false), lhs, rhs
10112 auto *OpAA = OnlyLeft ? &LHSAAPVS : &RHSAAPVS;
10113 auto Undef = OnlyLeft ? LHSContainsUndef : RHSContainsUndef;
10114
10115 if (Undef)
10116 unionAssumedWithUndef();
10117 else {
10118 for (const auto &It : *OpAA)
10119 unionAssumed(C: It);
10120 }
10121
10122 } else if (LHSContainsUndef && RHSContainsUndef) {
10123 // select i1 *, undef , undef => undef
10124 unionAssumedWithUndef();
10125 } else {
10126 for (const auto &It : LHSAAPVS)
10127 unionAssumed(C: It);
10128 for (const auto &It : RHSAAPVS)
10129 unionAssumed(C: It);
10130 }
10131 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10132 : ChangeStatus::CHANGED;
10133 }
10134
10135 ChangeStatus updateWithCastInst(Attributor &A, CastInst *CI) {
10136 auto AssumedBefore = getAssumed();
10137 if (!CI->isIntegerCast())
10138 return indicatePessimisticFixpoint();
10139 assert(CI->getNumOperands() == 1 && "Expected cast to be unary!");
10140 uint32_t ResultBitWidth = CI->getDestTy()->getIntegerBitWidth();
10141 Value *Src = CI->getOperand(i_nocapture: 0);
10142
10143 bool SrcContainsUndef = false;
10144 SetTy SrcPVS;
10145 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *Src), S&: SrcPVS,
10146 ContainsUndef&: SrcContainsUndef, /* ForSelf */ false))
10147 return indicatePessimisticFixpoint();
10148
10149 if (SrcContainsUndef)
10150 unionAssumedWithUndef();
10151 else {
10152 for (const APInt &S : SrcPVS) {
10153 APInt T = calculateCastInst(CI, Src: S, ResultBitWidth);
10154 unionAssumed(C: T);
10155 }
10156 }
10157 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10158 : ChangeStatus::CHANGED;
10159 }
10160
10161 ChangeStatus updateWithBinaryOperator(Attributor &A, BinaryOperator *BinOp) {
10162 auto AssumedBefore = getAssumed();
10163 Value *LHS = BinOp->getOperand(i_nocapture: 0);
10164 Value *RHS = BinOp->getOperand(i_nocapture: 1);
10165
10166 bool LHSContainsUndef = false, RHSContainsUndef = false;
10167 SetTy LHSAAPVS, RHSAAPVS;
10168 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *LHS), S&: LHSAAPVS,
10169 ContainsUndef&: LHSContainsUndef, /* ForSelf */ false) ||
10170 !fillSetWithConstantValues(A, IRP: IRPosition::value(V: *RHS), S&: RHSAAPVS,
10171 ContainsUndef&: RHSContainsUndef, /* ForSelf */ false))
10172 return indicatePessimisticFixpoint();
10173
10174 const APInt Zero = APInt(LHS->getType()->getIntegerBitWidth(), 0);
10175
10176 // TODO: make use of undef flag to limit potential values aggressively.
10177 if (LHSContainsUndef && RHSContainsUndef) {
10178 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: Zero, RHS: Zero))
10179 return indicatePessimisticFixpoint();
10180 } else if (LHSContainsUndef) {
10181 for (const APInt &R : RHSAAPVS) {
10182 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: Zero, RHS: R))
10183 return indicatePessimisticFixpoint();
10184 }
10185 } else if (RHSContainsUndef) {
10186 for (const APInt &L : LHSAAPVS) {
10187 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: L, RHS: Zero))
10188 return indicatePessimisticFixpoint();
10189 }
10190 } else {
10191 for (const APInt &L : LHSAAPVS) {
10192 for (const APInt &R : RHSAAPVS) {
10193 if (!calculateBinaryOperatorAndTakeUnion(BinOp, LHS: L, RHS: R))
10194 return indicatePessimisticFixpoint();
10195 }
10196 }
10197 }
10198 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10199 : ChangeStatus::CHANGED;
10200 }
10201
10202 ChangeStatus updateWithInstruction(Attributor &A, Instruction *Inst) {
10203 auto AssumedBefore = getAssumed();
10204 SetTy Incoming;
10205 bool ContainsUndef;
10206 if (!fillSetWithConstantValues(A, IRP: IRPosition::value(V: *Inst), S&: Incoming,
10207 ContainsUndef, /* ForSelf */ true))
10208 return indicatePessimisticFixpoint();
10209 if (ContainsUndef) {
10210 unionAssumedWithUndef();
10211 } else {
10212 for (const auto &It : Incoming)
10213 unionAssumed(C: It);
10214 }
10215 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10216 : ChangeStatus::CHANGED;
10217 }
10218
10219 /// See AbstractAttribute::updateImpl(...).
10220 ChangeStatus updateImpl(Attributor &A) override {
10221 Value &V = getAssociatedValue();
10222 Instruction *I = dyn_cast<Instruction>(Val: &V);
10223
10224 if (auto *ICI = dyn_cast<ICmpInst>(Val: I))
10225 return updateWithICmpInst(A, ICI);
10226
10227 if (auto *SI = dyn_cast<SelectInst>(Val: I))
10228 return updateWithSelectInst(A, SI);
10229
10230 if (auto *CI = dyn_cast<CastInst>(Val: I))
10231 return updateWithCastInst(A, CI);
10232
10233 if (auto *BinOp = dyn_cast<BinaryOperator>(Val: I))
10234 return updateWithBinaryOperator(A, BinOp);
10235
10236 if (isa<PHINode>(Val: I) || isa<LoadInst>(Val: I))
10237 return updateWithInstruction(A, Inst: I);
10238
10239 return indicatePessimisticFixpoint();
10240 }
10241
10242 /// See AbstractAttribute::trackStatistics()
10243 void trackStatistics() const override {
10244 STATS_DECLTRACK_FLOATING_ATTR(potential_values)
10245 }
10246};
10247
10248struct AAPotentialConstantValuesFunction : AAPotentialConstantValuesImpl {
10249 AAPotentialConstantValuesFunction(const IRPosition &IRP, Attributor &A)
10250 : AAPotentialConstantValuesImpl(IRP, A) {}
10251
10252 /// See AbstractAttribute::initialize(...).
10253 ChangeStatus updateImpl(Attributor &A) override {
10254 llvm_unreachable(
10255 "AAPotentialConstantValues(Function|CallSite)::updateImpl will "
10256 "not be called");
10257 }
10258
10259 /// See AbstractAttribute::trackStatistics()
10260 void trackStatistics() const override {
10261 STATS_DECLTRACK_FN_ATTR(potential_values)
10262 }
10263};
10264
10265struct AAPotentialConstantValuesCallSite : AAPotentialConstantValuesFunction {
10266 AAPotentialConstantValuesCallSite(const IRPosition &IRP, Attributor &A)
10267 : AAPotentialConstantValuesFunction(IRP, A) {}
10268
10269 /// See AbstractAttribute::trackStatistics()
10270 void trackStatistics() const override {
10271 STATS_DECLTRACK_CS_ATTR(potential_values)
10272 }
10273};
10274
10275struct AAPotentialConstantValuesCallSiteReturned
10276 : AACalleeToCallSite<AAPotentialConstantValues,
10277 AAPotentialConstantValuesImpl> {
10278 AAPotentialConstantValuesCallSiteReturned(const IRPosition &IRP,
10279 Attributor &A)
10280 : AACalleeToCallSite<AAPotentialConstantValues,
10281 AAPotentialConstantValuesImpl>(IRP, A) {}
10282
10283 /// See AbstractAttribute::trackStatistics()
10284 void trackStatistics() const override {
10285 STATS_DECLTRACK_CSRET_ATTR(potential_values)
10286 }
10287};
10288
10289struct AAPotentialConstantValuesCallSiteArgument
10290 : AAPotentialConstantValuesFloating {
10291 AAPotentialConstantValuesCallSiteArgument(const IRPosition &IRP,
10292 Attributor &A)
10293 : AAPotentialConstantValuesFloating(IRP, A) {}
10294
10295 /// See AbstractAttribute::initialize(..).
10296 void initialize(Attributor &A) override {
10297 AAPotentialConstantValuesImpl::initialize(A);
10298 if (isAtFixpoint())
10299 return;
10300
10301 Value &V = getAssociatedValue();
10302
10303 if (auto *C = dyn_cast<ConstantInt>(Val: &V)) {
10304 unionAssumed(C: C->getValue());
10305 indicateOptimisticFixpoint();
10306 return;
10307 }
10308
10309 if (isa<UndefValue>(Val: &V)) {
10310 unionAssumedWithUndef();
10311 indicateOptimisticFixpoint();
10312 return;
10313 }
10314 }
10315
10316 /// See AbstractAttribute::updateImpl(...).
10317 ChangeStatus updateImpl(Attributor &A) override {
10318 Value &V = getAssociatedValue();
10319 auto AssumedBefore = getAssumed();
10320 auto *AA = A.getAAFor<AAPotentialConstantValues>(
10321 QueryingAA: *this, IRP: IRPosition::value(V), DepClass: DepClassTy::REQUIRED);
10322 if (!AA)
10323 return indicatePessimisticFixpoint();
10324 const auto &S = AA->getAssumed();
10325 unionAssumed(PVS: S);
10326 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10327 : ChangeStatus::CHANGED;
10328 }
10329
10330 /// See AbstractAttribute::trackStatistics()
10331 void trackStatistics() const override {
10332 STATS_DECLTRACK_CSARG_ATTR(potential_values)
10333 }
10334};
10335} // namespace
10336
10337/// ------------------------ NoUndef Attribute ---------------------------------
10338bool AANoUndef::isImpliedByIR(Attributor &A, const IRPosition &IRP,
10339 Attribute::AttrKind ImpliedAttributeKind,
10340 bool IgnoreSubsumingPositions) {
10341 assert(ImpliedAttributeKind == Attribute::NoUndef &&
10342 "Unexpected attribute kind");
10343 if (A.hasAttr(IRP, AKs: {Attribute::NoUndef}, IgnoreSubsumingPositions,
10344 ImpliedAttributeKind: Attribute::NoUndef))
10345 return true;
10346
10347 Value &Val = IRP.getAssociatedValue();
10348 if (IRP.getPositionKind() != IRPosition::IRP_RETURNED &&
10349 isGuaranteedNotToBeUndefOrPoison(V: &Val)) {
10350 LLVMContext &Ctx = Val.getContext();
10351 A.manifestAttrs(IRP, DeducedAttrs: Attribute::get(Context&: Ctx, Kind: Attribute::NoUndef));
10352 return true;
10353 }
10354
10355 return false;
10356}
10357
10358namespace {
10359struct AANoUndefImpl : AANoUndef {
10360 AANoUndefImpl(const IRPosition &IRP, Attributor &A) : AANoUndef(IRP, A) {}
10361
10362 /// See AbstractAttribute::initialize(...).
10363 void initialize(Attributor &A) override {
10364 Value &V = getAssociatedValue();
10365 if (isa<UndefValue>(Val: V))
10366 indicatePessimisticFixpoint();
10367 assert(!isImpliedByIR(A, getIRPosition(), Attribute::NoUndef));
10368 }
10369
10370 /// See followUsesInMBEC
10371 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10372 AANoUndef::StateType &State) {
10373 const Value *UseV = U->get();
10374 const DominatorTree *DT = nullptr;
10375 AssumptionCache *AC = nullptr;
10376 InformationCache &InfoCache = A.getInfoCache();
10377 if (Function *F = getAnchorScope()) {
10378 DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
10379 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
10380 }
10381 State.setKnown(isGuaranteedNotToBeUndefOrPoison(V: UseV, AC, CtxI: I, DT));
10382 bool TrackUse = false;
10383 // Track use for instructions which must produce undef or poison bits when
10384 // at least one operand contains such bits.
10385 if (isa<CastInst>(Val: *I) || isa<GetElementPtrInst>(Val: *I))
10386 TrackUse = true;
10387 return TrackUse;
10388 }
10389
10390 /// See AbstractAttribute::getAsStr().
10391 const std::string getAsStr(Attributor *A) const override {
10392 return getAssumed() ? "noundef" : "may-undef-or-poison";
10393 }
10394
10395 ChangeStatus manifest(Attributor &A) override {
10396 // We don't manifest noundef attribute for dead positions because the
10397 // associated values with dead positions would be replaced with undef
10398 // values.
10399 bool UsedAssumedInformation = false;
10400 if (A.isAssumedDead(IRP: getIRPosition(), QueryingAA: nullptr, FnLivenessAA: nullptr,
10401 UsedAssumedInformation))
10402 return ChangeStatus::UNCHANGED;
10403 // A position whose simplified value does not have any value is
10404 // considered to be dead. We don't manifest noundef in such positions for
10405 // the same reason above.
10406 if (!A.getAssumedSimplified(IRP: getIRPosition(), AA: *this, UsedAssumedInformation,
10407 S: AA::Interprocedural)
10408 .has_value())
10409 return ChangeStatus::UNCHANGED;
10410 return AANoUndef::manifest(A);
10411 }
10412};
10413
10414struct AANoUndefFloating : public AANoUndefImpl {
10415 AANoUndefFloating(const IRPosition &IRP, Attributor &A)
10416 : AANoUndefImpl(IRP, A) {}
10417
10418 /// See AbstractAttribute::initialize(...).
10419 void initialize(Attributor &A) override {
10420 AANoUndefImpl::initialize(A);
10421 if (!getState().isAtFixpoint() && getAnchorScope() &&
10422 !getAnchorScope()->isDeclaration())
10423 if (Instruction *CtxI = getCtxI())
10424 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
10425 }
10426
10427 /// See AbstractAttribute::updateImpl(...).
10428 ChangeStatus updateImpl(Attributor &A) override {
10429 auto VisitValueCB = [&](const IRPosition &IRP) -> bool {
10430 bool IsKnownNoUndef;
10431 return AA::hasAssumedIRAttr<Attribute::NoUndef>(
10432 A, QueryingAA: this, IRP, DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNoUndef);
10433 };
10434
10435 bool Stripped;
10436 bool UsedAssumedInformation = false;
10437 Value *AssociatedValue = &getAssociatedValue();
10438 SmallVector<AA::ValueAndContext> Values;
10439 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
10440 S: AA::AnyScope, UsedAssumedInformation))
10441 Stripped = false;
10442 else
10443 Stripped =
10444 Values.size() != 1 || Values.front().getValue() != AssociatedValue;
10445
10446 if (!Stripped) {
10447 // If we haven't stripped anything we might still be able to use a
10448 // different AA, but only if the IRP changes. Effectively when we
10449 // interpret this not as a call site value but as a floating/argument
10450 // value.
10451 const IRPosition AVIRP = IRPosition::value(V: *AssociatedValue);
10452 if (AVIRP == getIRPosition() || !VisitValueCB(AVIRP))
10453 return indicatePessimisticFixpoint();
10454 return ChangeStatus::UNCHANGED;
10455 }
10456
10457 for (const auto &VAC : Values)
10458 if (!VisitValueCB(IRPosition::value(V: *VAC.getValue())))
10459 return indicatePessimisticFixpoint();
10460
10461 return ChangeStatus::UNCHANGED;
10462 }
10463
10464 /// See AbstractAttribute::trackStatistics()
10465 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10466};
10467
10468struct AANoUndefReturned final
10469 : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl> {
10470 AANoUndefReturned(const IRPosition &IRP, Attributor &A)
10471 : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl>(IRP, A) {}
10472
10473 /// See AbstractAttribute::trackStatistics()
10474 void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
10475};
10476
10477struct AANoUndefArgument final
10478 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl> {
10479 AANoUndefArgument(const IRPosition &IRP, Attributor &A)
10480 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl>(IRP, A) {}
10481
10482 /// See AbstractAttribute::trackStatistics()
10483 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noundef) }
10484};
10485
10486struct AANoUndefCallSiteArgument final : AANoUndefFloating {
10487 AANoUndefCallSiteArgument(const IRPosition &IRP, Attributor &A)
10488 : AANoUndefFloating(IRP, A) {}
10489
10490 /// See AbstractAttribute::trackStatistics()
10491 void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noundef) }
10492};
10493
10494struct AANoUndefCallSiteReturned final
10495 : AACalleeToCallSite<AANoUndef, AANoUndefImpl> {
10496 AANoUndefCallSiteReturned(const IRPosition &IRP, Attributor &A)
10497 : AACalleeToCallSite<AANoUndef, AANoUndefImpl>(IRP, A) {}
10498
10499 /// See AbstractAttribute::trackStatistics()
10500 void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noundef) }
10501};
10502
10503/// ------------------------ NoFPClass Attribute -------------------------------
10504
10505struct AANoFPClassImpl : AANoFPClass {
10506 AANoFPClassImpl(const IRPosition &IRP, Attributor &A) : AANoFPClass(IRP, A) {}
10507
10508 void initialize(Attributor &A) override {
10509 const IRPosition &IRP = getIRPosition();
10510
10511 Value &V = IRP.getAssociatedValue();
10512 if (isa<UndefValue>(Val: V)) {
10513 indicateOptimisticFixpoint();
10514 return;
10515 }
10516
10517 SmallVector<Attribute> Attrs;
10518 A.getAttrs(IRP: getIRPosition(), AKs: {Attribute::NoFPClass}, Attrs, IgnoreSubsumingPositions: false);
10519 for (const auto &Attr : Attrs) {
10520 addKnownBits(Bits: Attr.getNoFPClass());
10521 }
10522
10523 Instruction *CtxI = getCtxI();
10524
10525 if (getPositionKind() != IRPosition::IRP_RETURNED) {
10526 const DataLayout &DL = A.getDataLayout();
10527 InformationCache &InfoCache = A.getInfoCache();
10528
10529 const DominatorTree *DT = nullptr;
10530 AssumptionCache *AC = nullptr;
10531 const TargetLibraryInfo *TLI = nullptr;
10532 Function *F = getAnchorScope();
10533 if (F) {
10534 TLI = InfoCache.getTargetLibraryInfoForFunction(F: *F);
10535 if (!F->isDeclaration()) {
10536 DT =
10537 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
10538 AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
10539 }
10540 }
10541
10542 SimplifyQuery Q(DL, TLI, DT, AC, CtxI);
10543
10544 KnownFPClass KnownFPClass = computeKnownFPClass(V: &V, InterestedClasses: fcAllFlags, SQ: Q);
10545 addKnownBits(Bits: ~KnownFPClass.getKnownFPClasses());
10546 }
10547
10548 if (CtxI)
10549 followUsesInMBEC(AA&: *this, A, S&: getState(), CtxI&: *CtxI);
10550 }
10551
10552 /// See followUsesInMBEC
10553 bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
10554 AANoFPClass::StateType &State) {
10555 // TODO: Determine what instructions can be looked through.
10556 auto *CB = dyn_cast<CallBase>(Val: I);
10557 if (!CB)
10558 return false;
10559
10560 if (!CB->isArgOperand(U))
10561 return false;
10562
10563 unsigned ArgNo = CB->getArgOperandNo(U);
10564 IRPosition IRP = IRPosition::callsite_argument(CB: *CB, ArgNo);
10565 if (auto *NoFPAA = A.getAAFor<AANoFPClass>(QueryingAA: *this, IRP, DepClass: DepClassTy::NONE))
10566 State.addKnownBits(Bits: NoFPAA->getState().getKnown());
10567 return false;
10568 }
10569
10570 const std::string getAsStr(Attributor *A) const override {
10571 std::string Result = "nofpclass";
10572 raw_string_ostream OS(Result);
10573 OS << getKnownNoFPClass() << '/' << getAssumedNoFPClass();
10574 return Result;
10575 }
10576
10577 void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
10578 SmallVectorImpl<Attribute> &Attrs) const override {
10579 Attrs.emplace_back(Args: Attribute::getWithNoFPClass(Context&: Ctx, Mask: getAssumedNoFPClass()));
10580 }
10581};
10582
10583struct AANoFPClassFloating : public AANoFPClassImpl {
10584 AANoFPClassFloating(const IRPosition &IRP, Attributor &A)
10585 : AANoFPClassImpl(IRP, A) {}
10586
10587 /// See AbstractAttribute::updateImpl(...).
10588 ChangeStatus updateImpl(Attributor &A) override {
10589 SmallVector<AA::ValueAndContext> Values;
10590 bool UsedAssumedInformation = false;
10591 if (!A.getAssumedSimplifiedValues(IRP: getIRPosition(), AA: *this, Values,
10592 S: AA::AnyScope, UsedAssumedInformation)) {
10593 Values.push_back(Elt: {getAssociatedValue(), getCtxI()});
10594 }
10595
10596 StateType T;
10597 auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
10598 const auto *AA = A.getAAFor<AANoFPClass>(QueryingAA: *this, IRP: IRPosition::value(V),
10599 DepClass: DepClassTy::REQUIRED);
10600 if (!AA || this == AA) {
10601 T.indicatePessimisticFixpoint();
10602 } else {
10603 const AANoFPClass::StateType &S =
10604 static_cast<const AANoFPClass::StateType &>(AA->getState());
10605 T ^= S;
10606 }
10607 return T.isValidState();
10608 };
10609
10610 for (const auto &VAC : Values)
10611 if (!VisitValueCB(*VAC.getValue(), VAC.getCtxI()))
10612 return indicatePessimisticFixpoint();
10613
10614 return clampStateAndIndicateChange(S&: getState(), R: T);
10615 }
10616
10617 /// See AbstractAttribute::trackStatistics()
10618 void trackStatistics() const override {
10619 STATS_DECLTRACK_FNRET_ATTR(nofpclass)
10620 }
10621};
10622
10623struct AANoFPClassReturned final
10624 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10625 AANoFPClassImpl::StateType, false,
10626 Attribute::None, false> {
10627 AANoFPClassReturned(const IRPosition &IRP, Attributor &A)
10628 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10629 AANoFPClassImpl::StateType, false,
10630 Attribute::None, false>(IRP, A) {}
10631
10632 /// See AbstractAttribute::trackStatistics()
10633 void trackStatistics() const override {
10634 STATS_DECLTRACK_FNRET_ATTR(nofpclass)
10635 }
10636};
10637
10638struct AANoFPClassArgument final
10639 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl> {
10640 AANoFPClassArgument(const IRPosition &IRP, Attributor &A)
10641 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10642
10643 /// See AbstractAttribute::trackStatistics()
10644 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofpclass) }
10645};
10646
10647struct AANoFPClassCallSiteArgument final : AANoFPClassFloating {
10648 AANoFPClassCallSiteArgument(const IRPosition &IRP, Attributor &A)
10649 : AANoFPClassFloating(IRP, A) {}
10650
10651 /// See AbstractAttribute::trackStatistics()
10652 void trackStatistics() const override {
10653 STATS_DECLTRACK_CSARG_ATTR(nofpclass)
10654 }
10655};
10656
10657struct AANoFPClassCallSiteReturned final
10658 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl> {
10659 AANoFPClassCallSiteReturned(const IRPosition &IRP, Attributor &A)
10660 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
10661
10662 /// See AbstractAttribute::trackStatistics()
10663 void trackStatistics() const override {
10664 STATS_DECLTRACK_CSRET_ATTR(nofpclass)
10665 }
10666};
10667
10668struct AACallEdgesImpl : public AACallEdges {
10669 AACallEdgesImpl(const IRPosition &IRP, Attributor &A) : AACallEdges(IRP, A) {}
10670
10671 const SetVector<Function *> &getOptimisticEdges() const override {
10672 return CalledFunctions;
10673 }
10674
10675 bool hasUnknownCallee() const override { return HasUnknownCallee; }
10676
10677 bool hasNonAsmUnknownCallee() const override {
10678 return HasUnknownCalleeNonAsm;
10679 }
10680
10681 const std::string getAsStr(Attributor *A) const override {
10682 return "CallEdges[" + std::to_string(val: HasUnknownCallee) + "," +
10683 std::to_string(val: CalledFunctions.size()) + "]";
10684 }
10685
10686 void trackStatistics() const override {}
10687
10688protected:
10689 void addCalledFunction(Function *Fn, ChangeStatus &Change) {
10690 if (CalledFunctions.insert(X: Fn)) {
10691 Change = ChangeStatus::CHANGED;
10692 LLVM_DEBUG(dbgs() << "[AACallEdges] New call edge: " << Fn->getName()
10693 << "\n");
10694 }
10695 }
10696
10697 void setHasUnknownCallee(bool NonAsm, ChangeStatus &Change) {
10698 if (!HasUnknownCallee)
10699 Change = ChangeStatus::CHANGED;
10700 if (NonAsm && !HasUnknownCalleeNonAsm)
10701 Change = ChangeStatus::CHANGED;
10702 HasUnknownCalleeNonAsm |= NonAsm;
10703 HasUnknownCallee = true;
10704 }
10705
10706private:
10707 /// Optimistic set of functions that might be called by this position.
10708 SetVector<Function *> CalledFunctions;
10709
10710 /// Is there any call with a unknown callee.
10711 bool HasUnknownCallee = false;
10712
10713 /// Is there any call with a unknown callee, excluding any inline asm.
10714 bool HasUnknownCalleeNonAsm = false;
10715};
10716
10717struct AACallEdgesCallSite : public AACallEdgesImpl {
10718 AACallEdgesCallSite(const IRPosition &IRP, Attributor &A)
10719 : AACallEdgesImpl(IRP, A) {}
10720 /// See AbstractAttribute::updateImpl(...).
10721 ChangeStatus updateImpl(Attributor &A) override {
10722 ChangeStatus Change = ChangeStatus::UNCHANGED;
10723
10724 auto VisitValue = [&](Value &V, const Instruction *CtxI) -> bool {
10725 if (Function *Fn = dyn_cast<Function>(Val: &V)) {
10726 addCalledFunction(Fn, Change);
10727 } else {
10728 LLVM_DEBUG(dbgs() << "[AACallEdges] Unrecognized value: " << V << "\n");
10729 setHasUnknownCallee(NonAsm: true, Change);
10730 }
10731
10732 // Explore all values.
10733 return true;
10734 };
10735
10736 SmallVector<AA::ValueAndContext> Values;
10737 // Process any value that we might call.
10738 auto ProcessCalledOperand = [&](Value *V, Instruction *CtxI) {
10739 if (isa<Constant>(Val: V)) {
10740 VisitValue(*V, CtxI);
10741 return;
10742 }
10743
10744 bool UsedAssumedInformation = false;
10745 Values.clear();
10746 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *V), AA: *this, Values,
10747 S: AA::AnyScope, UsedAssumedInformation)) {
10748 Values.push_back(Elt: {*V, CtxI});
10749 }
10750 for (auto &VAC : Values)
10751 VisitValue(*VAC.getValue(), VAC.getCtxI());
10752 };
10753
10754 CallBase *CB = cast<CallBase>(Val: getCtxI());
10755
10756 if (auto *IA = dyn_cast<InlineAsm>(Val: CB->getCalledOperand())) {
10757 if (IA->hasSideEffects() &&
10758 !hasAssumption(F: *CB->getCaller(), AssumptionStr: "ompx_no_call_asm") &&
10759 !hasAssumption(CB: *CB, AssumptionStr: "ompx_no_call_asm")) {
10760 setHasUnknownCallee(NonAsm: false, Change);
10761 }
10762 return Change;
10763 }
10764
10765 if (CB->isIndirectCall())
10766 if (auto *IndirectCallAA = A.getAAFor<AAIndirectCallInfo>(
10767 QueryingAA: *this, IRP: getIRPosition(), DepClass: DepClassTy::OPTIONAL))
10768 if (IndirectCallAA->foreachCallee(
10769 CB: [&](Function *Fn) { return VisitValue(*Fn, CB); }))
10770 return Change;
10771
10772 // The most simple case.
10773 ProcessCalledOperand(CB->getCalledOperand(), CB);
10774
10775 // Process callback functions.
10776 SmallVector<const Use *, 4u> CallbackUses;
10777 AbstractCallSite::getCallbackUses(CB: *CB, CallbackUses);
10778 for (const Use *U : CallbackUses)
10779 ProcessCalledOperand(U->get(), CB);
10780
10781 return Change;
10782 }
10783};
10784
10785struct AACallEdgesFunction : public AACallEdgesImpl {
10786 AACallEdgesFunction(const IRPosition &IRP, Attributor &A)
10787 : AACallEdgesImpl(IRP, A) {}
10788
10789 /// See AbstractAttribute::updateImpl(...).
10790 ChangeStatus updateImpl(Attributor &A) override {
10791 ChangeStatus Change = ChangeStatus::UNCHANGED;
10792
10793 auto ProcessCallInst = [&](Instruction &Inst) {
10794 CallBase &CB = cast<CallBase>(Val&: Inst);
10795
10796 auto *CBEdges = A.getAAFor<AACallEdges>(
10797 QueryingAA: *this, IRP: IRPosition::callsite_function(CB), DepClass: DepClassTy::REQUIRED);
10798 if (!CBEdges)
10799 return false;
10800 if (CBEdges->hasNonAsmUnknownCallee())
10801 setHasUnknownCallee(NonAsm: true, Change);
10802 if (CBEdges->hasUnknownCallee())
10803 setHasUnknownCallee(NonAsm: false, Change);
10804
10805 for (Function *F : CBEdges->getOptimisticEdges())
10806 addCalledFunction(Fn: F, Change);
10807
10808 return true;
10809 };
10810
10811 // Visit all callable instructions.
10812 bool UsedAssumedInformation = false;
10813 if (!A.checkForAllCallLikeInstructions(Pred: ProcessCallInst, QueryingAA: *this,
10814 UsedAssumedInformation,
10815 /* CheckBBLivenessOnly */ true)) {
10816 // If we haven't looked at all call like instructions, assume that there
10817 // are unknown callees.
10818 setHasUnknownCallee(NonAsm: true, Change);
10819 }
10820
10821 return Change;
10822 }
10823};
10824
10825/// -------------------AAInterFnReachability Attribute--------------------------
10826
10827struct AAInterFnReachabilityFunction
10828 : public CachedReachabilityAA<AAInterFnReachability, Function> {
10829 using Base = CachedReachabilityAA<AAInterFnReachability, Function>;
10830 AAInterFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
10831 : Base(IRP, A) {}
10832
10833 bool instructionCanReach(
10834 Attributor &A, const Instruction &From, const Function &To,
10835 const AA::InstExclusionSetTy *ExclusionSet) const override {
10836 assert(From.getFunction() == getAnchorScope() && "Queried the wrong AA!");
10837 auto *NonConstThis = const_cast<AAInterFnReachabilityFunction *>(this);
10838
10839 RQITy StackRQI(A, From, To, ExclusionSet, false);
10840 RQITy::Reachable Result;
10841 if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
10842 return NonConstThis->isReachableImpl(A, RQI&: StackRQI,
10843 /*IsTemporaryRQI=*/true);
10844 return Result == RQITy::Reachable::Yes;
10845 }
10846
10847 bool isReachableImpl(Attributor &A, RQITy &RQI,
10848 bool IsTemporaryRQI) override {
10849 const Instruction *EntryI =
10850 &RQI.From->getFunction()->getEntryBlock().front();
10851 if (EntryI != RQI.From &&
10852 !instructionCanReach(A, From: *EntryI, To: *RQI.To, ExclusionSet: nullptr))
10853 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet: false,
10854 IsTemporaryRQI);
10855
10856 auto CheckReachableCallBase = [&](CallBase *CB) {
10857 auto *CBEdges = A.getAAFor<AACallEdges>(
10858 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *CB), DepClass: DepClassTy::OPTIONAL);
10859 if (!CBEdges || !CBEdges->getState().isValidState())
10860 return false;
10861 // TODO Check To backwards in this case.
10862 if (CBEdges->hasUnknownCallee())
10863 return false;
10864
10865 for (Function *Fn : CBEdges->getOptimisticEdges()) {
10866 if (Fn == RQI.To)
10867 return false;
10868
10869 if (Fn->isDeclaration()) {
10870 if (Fn->hasFnAttribute(Kind: Attribute::NoCallback))
10871 continue;
10872 // TODO Check To backwards in this case.
10873 return false;
10874 }
10875
10876 if (Fn == getAnchorScope()) {
10877 if (EntryI == RQI.From)
10878 continue;
10879 return false;
10880 }
10881
10882 const AAInterFnReachability *InterFnReachability =
10883 A.getAAFor<AAInterFnReachability>(QueryingAA: *this, IRP: IRPosition::function(F: *Fn),
10884 DepClass: DepClassTy::OPTIONAL);
10885
10886 const Instruction &FnFirstInst = Fn->getEntryBlock().front();
10887 if (!InterFnReachability ||
10888 InterFnReachability->instructionCanReach(A, Inst: FnFirstInst, Fn: *RQI.To,
10889 ExclusionSet: RQI.ExclusionSet))
10890 return false;
10891 }
10892 return true;
10893 };
10894
10895 const auto *IntraFnReachability = A.getAAFor<AAIntraFnReachability>(
10896 QueryingAA: *this, IRP: IRPosition::function(F: *RQI.From->getFunction()),
10897 DepClass: DepClassTy::OPTIONAL);
10898
10899 // Determine call like instructions that we can reach from the inst.
10900 auto CheckCallBase = [&](Instruction &CBInst) {
10901 // There are usually less nodes in the call graph, check inter function
10902 // reachability first.
10903 if (CheckReachableCallBase(cast<CallBase>(Val: &CBInst)))
10904 return true;
10905 return IntraFnReachability && !IntraFnReachability->isAssumedReachable(
10906 A, From: *RQI.From, To: CBInst, ExclusionSet: RQI.ExclusionSet);
10907 };
10908
10909 bool UsedExclusionSet = /* conservative */ true;
10910 bool UsedAssumedInformation = false;
10911 if (!A.checkForAllCallLikeInstructions(Pred: CheckCallBase, QueryingAA: *this,
10912 UsedAssumedInformation,
10913 /* CheckBBLivenessOnly */ true))
10914 return rememberResult(A, Result: RQITy::Reachable::Yes, RQI, UsedExclusionSet,
10915 IsTemporaryRQI);
10916
10917 return rememberResult(A, Result: RQITy::Reachable::No, RQI, UsedExclusionSet,
10918 IsTemporaryRQI);
10919 }
10920
10921 void trackStatistics() const override {}
10922};
10923} // namespace
10924
10925template <typename AAType>
10926static std::optional<Constant *>
10927askForAssumedConstant(Attributor &A, const AbstractAttribute &QueryingAA,
10928 const IRPosition &IRP, Type &Ty) {
10929 if (!Ty.isIntegerTy())
10930 return nullptr;
10931
10932 // This will also pass the call base context.
10933 const auto *AA = A.getAAFor<AAType>(QueryingAA, IRP, DepClassTy::NONE);
10934 if (!AA)
10935 return nullptr;
10936
10937 std::optional<Constant *> COpt = AA->getAssumedConstant(A);
10938
10939 if (!COpt.has_value()) {
10940 A.recordDependence(FromAA: *AA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
10941 return std::nullopt;
10942 }
10943 if (auto *C = *COpt) {
10944 A.recordDependence(FromAA: *AA, ToAA: QueryingAA, DepClass: DepClassTy::OPTIONAL);
10945 return C;
10946 }
10947 return nullptr;
10948}
10949
10950Value *AAPotentialValues::getSingleValue(
10951 Attributor &A, const AbstractAttribute &AA, const IRPosition &IRP,
10952 SmallVectorImpl<AA::ValueAndContext> &Values) {
10953 Type &Ty = *IRP.getAssociatedType();
10954 std::optional<Value *> V;
10955 for (auto &It : Values) {
10956 V = AA::combineOptionalValuesInAAValueLatice(A: V, B: It.getValue(), Ty: &Ty);
10957 if (V.has_value() && !*V)
10958 break;
10959 }
10960 if (!V.has_value())
10961 return UndefValue::get(T: &Ty);
10962 return *V;
10963}
10964
10965namespace {
10966struct AAPotentialValuesImpl : AAPotentialValues {
10967 using StateType = PotentialLLVMValuesState;
10968
10969 AAPotentialValuesImpl(const IRPosition &IRP, Attributor &A)
10970 : AAPotentialValues(IRP, A) {}
10971
10972 /// See AbstractAttribute::initialize(..).
10973 void initialize(Attributor &A) override {
10974 if (A.hasSimplificationCallback(IRP: getIRPosition())) {
10975 indicatePessimisticFixpoint();
10976 return;
10977 }
10978 Value *Stripped = getAssociatedValue().stripPointerCasts();
10979 if (isa<Constant>(Val: Stripped) && !isa<ConstantExpr>(Val: Stripped)) {
10980 addValue(A, State&: getState(), V&: *Stripped, CtxI: getCtxI(), S: AA::AnyScope,
10981 AnchorScope: getAnchorScope());
10982 indicateOptimisticFixpoint();
10983 return;
10984 }
10985 AAPotentialValues::initialize(A);
10986 }
10987
10988 /// See AbstractAttribute::getAsStr().
10989 const std::string getAsStr(Attributor *A) const override {
10990 std::string Str;
10991 llvm::raw_string_ostream OS(Str);
10992 OS << getState();
10993 return Str;
10994 }
10995
10996 template <typename AAType>
10997 static std::optional<Value *> askOtherAA(Attributor &A,
10998 const AbstractAttribute &AA,
10999 const IRPosition &IRP, Type &Ty) {
11000 if (isa<Constant>(Val: IRP.getAssociatedValue()))
11001 return &IRP.getAssociatedValue();
11002 std::optional<Constant *> C = askForAssumedConstant<AAType>(A, AA, IRP, Ty);
11003 if (!C)
11004 return std::nullopt;
11005 if (*C)
11006 if (auto *CC = AA::getWithType(V&: **C, Ty))
11007 return CC;
11008 return nullptr;
11009 }
11010
11011 virtual void addValue(Attributor &A, StateType &State, Value &V,
11012 const Instruction *CtxI, AA::ValueScope S,
11013 Function *AnchorScope) const {
11014
11015 IRPosition ValIRP = IRPosition::value(V);
11016 if (auto *CB = dyn_cast_or_null<CallBase>(Val: CtxI)) {
11017 for (const auto &U : CB->args()) {
11018 if (U.get() != &V)
11019 continue;
11020 ValIRP = IRPosition::callsite_argument(CB: *CB, ArgNo: CB->getArgOperandNo(U: &U));
11021 break;
11022 }
11023 }
11024
11025 Value *VPtr = &V;
11026 if (ValIRP.getAssociatedType()->isIntegerTy()) {
11027 Type &Ty = *getAssociatedType();
11028 std::optional<Value *> SimpleV =
11029 askOtherAA<AAValueConstantRange>(A, AA: *this, IRP: ValIRP, Ty);
11030 if (SimpleV.has_value() && !*SimpleV) {
11031 auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
11032 QueryingAA: *this, IRP: ValIRP, DepClass: DepClassTy::OPTIONAL);
11033 if (PotentialConstantsAA && PotentialConstantsAA->isValidState()) {
11034 for (const auto &It : PotentialConstantsAA->getAssumedSet())
11035 State.unionAssumed(C: {{*ConstantInt::get(Ty: &Ty, V: It), nullptr}, S});
11036 if (PotentialConstantsAA->undefIsContained())
11037 State.unionAssumed(C: {{*UndefValue::get(T: &Ty), nullptr}, S});
11038 return;
11039 }
11040 }
11041 if (!SimpleV.has_value())
11042 return;
11043
11044 if (*SimpleV)
11045 VPtr = *SimpleV;
11046 }
11047
11048 if (isa<ConstantInt>(Val: VPtr))
11049 CtxI = nullptr;
11050 if (!AA::isValidInScope(V: *VPtr, Scope: AnchorScope))
11051 S = AA::ValueScope(S | AA::Interprocedural);
11052
11053 State.unionAssumed(C: {{*VPtr, CtxI}, S});
11054 }
11055
11056 /// Helper struct to tie a value+context pair together with the scope for
11057 /// which this is the simplified version.
11058 struct ItemInfo {
11059 AA::ValueAndContext I;
11060 AA::ValueScope S;
11061
11062 bool operator==(const ItemInfo &II) const {
11063 return II.I == I && II.S == S;
11064 };
11065 bool operator<(const ItemInfo &II) const {
11066 return std::tie(args: I, args: S) < std::tie(args: II.I, args: II.S);
11067 };
11068 };
11069
11070 bool recurseForValue(Attributor &A, const IRPosition &IRP, AA::ValueScope S) {
11071 SmallMapVector<AA::ValueAndContext, int, 8> ValueScopeMap;
11072 for (auto CS : {AA::Intraprocedural, AA::Interprocedural}) {
11073 if (!(CS & S))
11074 continue;
11075
11076 bool UsedAssumedInformation = false;
11077 SmallVector<AA::ValueAndContext> Values;
11078 if (!A.getAssumedSimplifiedValues(IRP, AA: this, Values, S: CS,
11079 UsedAssumedInformation))
11080 return false;
11081
11082 for (auto &It : Values)
11083 ValueScopeMap[It] += CS;
11084 }
11085 for (auto &It : ValueScopeMap)
11086 addValue(A, State&: getState(), V&: *It.first.getValue(), CtxI: It.first.getCtxI(),
11087 S: AA::ValueScope(It.second), AnchorScope: getAnchorScope());
11088
11089 return true;
11090 }
11091
11092 void giveUpOnIntraprocedural(Attributor &A) {
11093 auto NewS = StateType::getBestState(PVS: getState());
11094 for (const auto &It : getAssumedSet()) {
11095 if (It.second == AA::Intraprocedural)
11096 continue;
11097 addValue(A, State&: NewS, V&: *It.first.getValue(), CtxI: It.first.getCtxI(),
11098 S: AA::Interprocedural, AnchorScope: getAnchorScope());
11099 }
11100 assert(!undefIsContained() && "Undef should be an explicit value!");
11101 addValue(A, State&: NewS, V&: getAssociatedValue(), CtxI: getCtxI(), S: AA::Intraprocedural,
11102 AnchorScope: getAnchorScope());
11103 getState() = NewS;
11104 }
11105
11106 /// See AbstractState::indicatePessimisticFixpoint(...).
11107 ChangeStatus indicatePessimisticFixpoint() override {
11108 getState() = StateType::getBestState(PVS: getState());
11109 getState().unionAssumed(C: {{getAssociatedValue(), getCtxI()}, AA::AnyScope});
11110 AAPotentialValues::indicateOptimisticFixpoint();
11111 return ChangeStatus::CHANGED;
11112 }
11113
11114 /// See AbstractAttribute::updateImpl(...).
11115 ChangeStatus updateImpl(Attributor &A) override {
11116 return indicatePessimisticFixpoint();
11117 }
11118
11119 /// See AbstractAttribute::manifest(...).
11120 ChangeStatus manifest(Attributor &A) override {
11121 SmallVector<AA::ValueAndContext> Values;
11122 for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {
11123 Values.clear();
11124 if (!getAssumedSimplifiedValues(A, Values, S))
11125 continue;
11126 Value &OldV = getAssociatedValue();
11127 if (isa<UndefValue>(Val: OldV))
11128 continue;
11129 Value *NewV = getSingleValue(A, AA: *this, IRP: getIRPosition(), Values);
11130 if (!NewV || NewV == &OldV)
11131 continue;
11132 if (getCtxI() &&
11133 !AA::isValidAtPosition(VAC: {*NewV, *getCtxI()}, InfoCache&: A.getInfoCache()))
11134 continue;
11135 if (A.changeAfterManifest(IRP: getIRPosition(), NV&: *NewV))
11136 return ChangeStatus::CHANGED;
11137 }
11138 return ChangeStatus::UNCHANGED;
11139 }
11140
11141 bool getAssumedSimplifiedValues(
11142 Attributor &A, SmallVectorImpl<AA::ValueAndContext> &Values,
11143 AA::ValueScope S, bool RecurseForSelectAndPHI = false) const override {
11144 if (!isValidState())
11145 return false;
11146 bool UsedAssumedInformation = false;
11147 for (const auto &It : getAssumedSet())
11148 if (It.second & S) {
11149 if (RecurseForSelectAndPHI && (isa<PHINode>(Val: It.first.getValue()) ||
11150 isa<SelectInst>(Val: It.first.getValue()))) {
11151 if (A.getAssumedSimplifiedValues(
11152 IRP: IRPosition::inst(I: *cast<Instruction>(Val: It.first.getValue())),
11153 AA: this, Values, S, UsedAssumedInformation))
11154 continue;
11155 }
11156 Values.push_back(Elt: It.first);
11157 }
11158 assert(!undefIsContained() && "Undef should be an explicit value!");
11159 return true;
11160 }
11161};
11162
11163struct AAPotentialValuesFloating : AAPotentialValuesImpl {
11164 AAPotentialValuesFloating(const IRPosition &IRP, Attributor &A)
11165 : AAPotentialValuesImpl(IRP, A) {}
11166
11167 /// See AbstractAttribute::updateImpl(...).
11168 ChangeStatus updateImpl(Attributor &A) override {
11169 auto AssumedBefore = getAssumed();
11170
11171 genericValueTraversal(A, InitialV: &getAssociatedValue());
11172
11173 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11174 : ChangeStatus::CHANGED;
11175 }
11176
11177 /// Helper struct to remember which AAIsDead instances we actually used.
11178 struct LivenessInfo {
11179 const AAIsDead *LivenessAA = nullptr;
11180 bool AnyDead = false;
11181 };
11182
11183 /// Check if \p Cmp is a comparison we can simplify.
11184 ///
11185 /// We handle multiple cases, one in which at least one operand is an
11186 /// (assumed) nullptr. If so, try to simplify it using AANonNull on the other
11187 /// operand. Return true if successful, in that case Worklist will be updated.
11188 bool handleCmp(Attributor &A, Value &Cmp, Value *LHS, Value *RHS,
11189 CmpInst::Predicate Pred, ItemInfo II,
11190 SmallVectorImpl<ItemInfo> &Worklist) {
11191
11192 // Simplify the operands first.
11193 bool UsedAssumedInformation = false;
11194 SmallVector<AA::ValueAndContext> LHSValues, RHSValues;
11195 auto GetSimplifiedValues = [&](Value &V,
11196 SmallVector<AA::ValueAndContext> &Values) {
11197 if (!A.getAssumedSimplifiedValues(
11198 IRP: IRPosition::value(V, CBContext: getCallBaseContext()), AA: this, Values,
11199 S: AA::Intraprocedural, UsedAssumedInformation)) {
11200 Values.clear();
11201 Values.push_back(Elt: AA::ValueAndContext{V, II.I.getCtxI()});
11202 }
11203 return Values.empty();
11204 };
11205 if (GetSimplifiedValues(*LHS, LHSValues))
11206 return true;
11207 if (GetSimplifiedValues(*RHS, RHSValues))
11208 return true;
11209
11210 LLVMContext &Ctx = LHS->getContext();
11211
11212 InformationCache &InfoCache = A.getInfoCache();
11213 Instruction *CmpI = dyn_cast<Instruction>(Val: &Cmp);
11214 Function *F = CmpI ? CmpI->getFunction() : nullptr;
11215 const auto *DT =
11216 F ? InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F)
11217 : nullptr;
11218 const auto *TLI =
11219 F ? A.getInfoCache().getTargetLibraryInfoForFunction(F: *F) : nullptr;
11220 auto *AC =
11221 F ? InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F)
11222 : nullptr;
11223
11224 const DataLayout &DL = A.getDataLayout();
11225 SimplifyQuery Q(DL, TLI, DT, AC, CmpI);
11226
11227 auto CheckPair = [&](Value &LHSV, Value &RHSV) {
11228 if (isa<UndefValue>(Val: LHSV) || isa<UndefValue>(Val: RHSV)) {
11229 addValue(A, State&: getState(), V&: *UndefValue::get(T: Cmp.getType()),
11230 /* CtxI */ nullptr, S: II.S, AnchorScope: getAnchorScope());
11231 return true;
11232 }
11233
11234 // Handle the trivial case first in which we don't even need to think
11235 // about null or non-null.
11236 if (&LHSV == &RHSV &&
11237 (CmpInst::isTrueWhenEqual(predicate: Pred) || CmpInst::isFalseWhenEqual(predicate: Pred))) {
11238 Constant *NewV = ConstantInt::get(Ty: Type::getInt1Ty(C&: Ctx),
11239 V: CmpInst::isTrueWhenEqual(predicate: Pred));
11240 addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11241 AnchorScope: getAnchorScope());
11242 return true;
11243 }
11244
11245 auto *TypedLHS = AA::getWithType(V&: LHSV, Ty&: *LHS->getType());
11246 auto *TypedRHS = AA::getWithType(V&: RHSV, Ty&: *RHS->getType());
11247 if (TypedLHS && TypedRHS) {
11248 Value *NewV = simplifyCmpInst(Predicate: Pred, LHS: TypedLHS, RHS: TypedRHS, Q);
11249 if (NewV && NewV != &Cmp) {
11250 addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11251 AnchorScope: getAnchorScope());
11252 return true;
11253 }
11254 }
11255
11256 // From now on we only handle equalities (==, !=).
11257 if (!CmpInst::isEquality(pred: Pred))
11258 return false;
11259
11260 bool LHSIsNull = isa<ConstantPointerNull>(Val: LHSV);
11261 bool RHSIsNull = isa<ConstantPointerNull>(Val: RHSV);
11262 if (!LHSIsNull && !RHSIsNull)
11263 return false;
11264
11265 // Left is the nullptr ==/!= non-nullptr case. We'll use AANonNull on the
11266 // non-nullptr operand and if we assume it's non-null we can conclude the
11267 // result of the comparison.
11268 assert((LHSIsNull || RHSIsNull) &&
11269 "Expected nullptr versus non-nullptr comparison at this point");
11270
11271 // The index is the operand that we assume is not null.
11272 unsigned PtrIdx = LHSIsNull;
11273 bool IsKnownNonNull;
11274 bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
11275 A, QueryingAA: this, IRP: IRPosition::value(V: *(PtrIdx ? &RHSV : &LHSV)),
11276 DepClass: DepClassTy::REQUIRED, IsKnown&: IsKnownNonNull);
11277 if (!IsAssumedNonNull)
11278 return false;
11279
11280 // The new value depends on the predicate, true for != and false for ==.
11281 Constant *NewV =
11282 ConstantInt::get(Ty: Type::getInt1Ty(C&: Ctx), V: Pred == CmpInst::ICMP_NE);
11283 addValue(A, State&: getState(), V&: *NewV, /* CtxI */ nullptr, S: II.S,
11284 AnchorScope: getAnchorScope());
11285 return true;
11286 };
11287
11288 for (auto &LHSValue : LHSValues)
11289 for (auto &RHSValue : RHSValues)
11290 if (!CheckPair(*LHSValue.getValue(), *RHSValue.getValue()))
11291 return false;
11292 return true;
11293 }
11294
11295 bool handleSelectInst(Attributor &A, SelectInst &SI, ItemInfo II,
11296 SmallVectorImpl<ItemInfo> &Worklist) {
11297 const Instruction *CtxI = II.I.getCtxI();
11298 bool UsedAssumedInformation = false;
11299
11300 std::optional<Constant *> C =
11301 A.getAssumedConstant(V: *SI.getCondition(), AA: *this, UsedAssumedInformation);
11302 bool NoValueYet = !C.has_value();
11303 if (NoValueYet || isa_and_nonnull<UndefValue>(Val: *C))
11304 return true;
11305 if (auto *CI = dyn_cast_or_null<ConstantInt>(Val: *C)) {
11306 if (CI->isZero())
11307 Worklist.push_back(Elt: {.I: {*SI.getFalseValue(), CtxI}, .S: II.S});
11308 else
11309 Worklist.push_back(Elt: {.I: {*SI.getTrueValue(), CtxI}, .S: II.S});
11310 } else if (&SI == &getAssociatedValue()) {
11311 // We could not simplify the condition, assume both values.
11312 Worklist.push_back(Elt: {.I: {*SI.getTrueValue(), CtxI}, .S: II.S});
11313 Worklist.push_back(Elt: {.I: {*SI.getFalseValue(), CtxI}, .S: II.S});
11314 } else {
11315 std::optional<Value *> SimpleV = A.getAssumedSimplified(
11316 IRP: IRPosition::inst(I: SI), AA: *this, UsedAssumedInformation, S: II.S);
11317 if (!SimpleV.has_value())
11318 return true;
11319 if (*SimpleV) {
11320 addValue(A, State&: getState(), V&: **SimpleV, CtxI, S: II.S, AnchorScope: getAnchorScope());
11321 return true;
11322 }
11323 return false;
11324 }
11325 return true;
11326 }
11327
11328 bool handleLoadInst(Attributor &A, LoadInst &LI, ItemInfo II,
11329 SmallVectorImpl<ItemInfo> &Worklist) {
11330 SmallSetVector<Value *, 4> PotentialCopies;
11331 SmallSetVector<Instruction *, 4> PotentialValueOrigins;
11332 bool UsedAssumedInformation = false;
11333 if (!AA::getPotentiallyLoadedValues(A, LI, PotentialValues&: PotentialCopies,
11334 PotentialValueOrigins, QueryingAA: *this,
11335 UsedAssumedInformation,
11336 /* OnlyExact */ true)) {
11337 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Failed to get potentially "
11338 "loaded values for load instruction "
11339 << LI << "\n");
11340 return false;
11341 }
11342
11343 // Do not simplify loads that are only used in llvm.assume if we cannot also
11344 // remove all stores that may feed into the load. The reason is that the
11345 // assume is probably worth something as long as the stores are around.
11346 InformationCache &InfoCache = A.getInfoCache();
11347 if (InfoCache.isOnlyUsedByAssume(I: LI)) {
11348 if (!llvm::all_of(Range&: PotentialValueOrigins, P: [&](Instruction *I) {
11349 if (!I || isa<AssumeInst>(Val: I))
11350 return true;
11351 if (auto *SI = dyn_cast<StoreInst>(Val: I))
11352 return A.isAssumedDead(U: SI->getOperandUse(i: 0), QueryingAA: this,
11353 /* LivenessAA */ FnLivenessAA: nullptr,
11354 UsedAssumedInformation,
11355 /* CheckBBLivenessOnly */ false);
11356 return A.isAssumedDead(I: *I, QueryingAA: this, /* LivenessAA */ nullptr,
11357 UsedAssumedInformation,
11358 /* CheckBBLivenessOnly */ false);
11359 })) {
11360 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Load is onl used by assumes "
11361 "and we cannot delete all the stores: "
11362 << LI << "\n");
11363 return false;
11364 }
11365 }
11366
11367 // Values have to be dynamically unique or we loose the fact that a
11368 // single llvm::Value might represent two runtime values (e.g.,
11369 // stack locations in different recursive calls).
11370 const Instruction *CtxI = II.I.getCtxI();
11371 bool ScopeIsLocal = (II.S & AA::Intraprocedural);
11372 bool AllLocal = ScopeIsLocal;
11373 bool DynamicallyUnique = llvm::all_of(Range&: PotentialCopies, P: [&](Value *PC) {
11374 AllLocal &= AA::isValidInScope(V: *PC, Scope: getAnchorScope());
11375 return AA::isDynamicallyUnique(A, QueryingAA: *this, V: *PC);
11376 });
11377 if (!DynamicallyUnique) {
11378 LLVM_DEBUG(dbgs() << "[AAPotentialValues] Not all potentially loaded "
11379 "values are dynamically unique: "
11380 << LI << "\n");
11381 return false;
11382 }
11383
11384 for (auto *PotentialCopy : PotentialCopies) {
11385 if (AllLocal) {
11386 Worklist.push_back(Elt: {.I: {*PotentialCopy, CtxI}, .S: II.S});
11387 } else {
11388 Worklist.push_back(Elt: {.I: {*PotentialCopy, CtxI}, .S: AA::Interprocedural});
11389 }
11390 }
11391 if (!AllLocal && ScopeIsLocal)
11392 addValue(A, State&: getState(), V&: LI, CtxI, S: AA::Intraprocedural, AnchorScope: getAnchorScope());
11393 return true;
11394 }
11395
11396 bool handlePHINode(
11397 Attributor &A, PHINode &PHI, ItemInfo II,
11398 SmallVectorImpl<ItemInfo> &Worklist,
11399 SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
11400 auto GetLivenessInfo = [&](const Function &F) -> LivenessInfo & {
11401 LivenessInfo &LI = LivenessAAs[&F];
11402 if (!LI.LivenessAA)
11403 LI.LivenessAA = A.getAAFor<AAIsDead>(QueryingAA: *this, IRP: IRPosition::function(F),
11404 DepClass: DepClassTy::NONE);
11405 return LI;
11406 };
11407
11408 if (&PHI == &getAssociatedValue()) {
11409 LivenessInfo &LI = GetLivenessInfo(*PHI.getFunction());
11410 const auto *CI =
11411 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
11412 F: *PHI.getFunction());
11413
11414 CycleRef C;
11415 bool CyclePHI = mayBeInCycle(CI, I: &PHI, /* HeaderOnly */ true, CPtr: &C);
11416 for (unsigned u = 0, e = PHI.getNumIncomingValues(); u < e; u++) {
11417 BasicBlock *IncomingBB = PHI.getIncomingBlock(i: u);
11418 if (LI.LivenessAA &&
11419 LI.LivenessAA->isEdgeDead(From: IncomingBB, To: PHI.getParent())) {
11420 LI.AnyDead = true;
11421 continue;
11422 }
11423 Value *V = PHI.getIncomingValue(i: u);
11424 if (V == &PHI)
11425 continue;
11426
11427 // If the incoming value is not the PHI but an instruction in the same
11428 // cycle we might have multiple versions of it flying around.
11429 if (CyclePHI && isa<Instruction>(Val: V) &&
11430 (!C || CI->contains(C, Block: cast<Instruction>(Val: V)->getParent())))
11431 return false;
11432
11433 Worklist.push_back(Elt: {.I: {*V, IncomingBB->getTerminator()}, .S: II.S});
11434 }
11435 return true;
11436 }
11437
11438 bool UsedAssumedInformation = false;
11439 std::optional<Value *> SimpleV = A.getAssumedSimplified(
11440 IRP: IRPosition::inst(I: PHI), AA: *this, UsedAssumedInformation, S: II.S);
11441 if (!SimpleV.has_value())
11442 return true;
11443 if (!(*SimpleV))
11444 return false;
11445 addValue(A, State&: getState(), V&: **SimpleV, CtxI: &PHI, S: II.S, AnchorScope: getAnchorScope());
11446 return true;
11447 }
11448
11449 /// Use the generic, non-optimistic InstSimplfy functionality if we managed to
11450 /// simplify any operand of the instruction \p I. Return true if successful,
11451 /// in that case Worklist will be updated.
11452 bool handleGenericInst(Attributor &A, Instruction &I, ItemInfo II,
11453 SmallVectorImpl<ItemInfo> &Worklist) {
11454 bool SomeSimplified = false;
11455 bool UsedAssumedInformation = false;
11456
11457 SmallVector<Value *, 8> NewOps(I.getNumOperands());
11458 int Idx = 0;
11459 for (Value *Op : I.operands()) {
11460 const auto &SimplifiedOp = A.getAssumedSimplified(
11461 IRP: IRPosition::value(V: *Op, CBContext: getCallBaseContext()), AA: *this,
11462 UsedAssumedInformation, S: AA::Intraprocedural);
11463 // If we are not sure about any operand we are not sure about the entire
11464 // instruction, we'll wait.
11465 if (!SimplifiedOp.has_value())
11466 return true;
11467
11468 if (*SimplifiedOp)
11469 NewOps[Idx] = *SimplifiedOp;
11470 else
11471 NewOps[Idx] = Op;
11472
11473 SomeSimplified |= (NewOps[Idx] != Op);
11474 ++Idx;
11475 }
11476
11477 // We won't bother with the InstSimplify interface if we didn't simplify any
11478 // operand ourselves.
11479 if (!SomeSimplified)
11480 return false;
11481
11482 InformationCache &InfoCache = A.getInfoCache();
11483 Function *F = I.getFunction();
11484 const auto *DT =
11485 InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(F: *F);
11486 const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(F: *F);
11487 auto *AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(F: *F);
11488
11489 const DataLayout &DL = I.getDataLayout();
11490 SimplifyQuery Q(DL, TLI, DT, AC, &I);
11491 Value *NewV = simplifyInstructionWithOperands(I: &I, NewOps, Q);
11492 if (!NewV || NewV == &I)
11493 return false;
11494
11495 LLVM_DEBUG(dbgs() << "Generic inst " << I << " assumed simplified to "
11496 << *NewV << "\n");
11497 Worklist.push_back(Elt: {.I: {*NewV, II.I.getCtxI()}, .S: II.S});
11498 return true;
11499 }
11500
11501 bool simplifyInstruction(
11502 Attributor &A, Instruction &I, ItemInfo II,
11503 SmallVectorImpl<ItemInfo> &Worklist,
11504 SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
11505 if (auto *CI = dyn_cast<CmpInst>(Val: &I))
11506 return handleCmp(A, Cmp&: *CI, LHS: CI->getOperand(i_nocapture: 0), RHS: CI->getOperand(i_nocapture: 1),
11507 Pred: CI->getPredicate(), II, Worklist);
11508
11509 switch (I.getOpcode()) {
11510 case Instruction::Select:
11511 return handleSelectInst(A, SI&: cast<SelectInst>(Val&: I), II, Worklist);
11512 case Instruction::PHI:
11513 return handlePHINode(A, PHI&: cast<PHINode>(Val&: I), II, Worklist, LivenessAAs);
11514 case Instruction::Load:
11515 return handleLoadInst(A, LI&: cast<LoadInst>(Val&: I), II, Worklist);
11516 default:
11517 return handleGenericInst(A, I, II, Worklist);
11518 };
11519 return false;
11520 }
11521
11522 void genericValueTraversal(Attributor &A, Value *InitialV) {
11523 SmallMapVector<const Function *, LivenessInfo, 4> LivenessAAs;
11524
11525 SmallSet<ItemInfo, 16> Visited;
11526 SmallVector<ItemInfo, 16> Worklist;
11527 Worklist.push_back(Elt: {.I: {*InitialV, getCtxI()}, .S: AA::AnyScope});
11528
11529 int Iteration = 0;
11530 do {
11531 ItemInfo II = Worklist.pop_back_val();
11532 Value *V = II.I.getValue();
11533 assert(V);
11534 const Instruction *CtxI = II.I.getCtxI();
11535 AA::ValueScope S = II.S;
11536
11537 // Check if we should process the current value. To prevent endless
11538 // recursion keep a record of the values we followed!
11539 if (!Visited.insert(V: II).second)
11540 continue;
11541
11542 // Make sure we limit the compile time for complex expressions.
11543 if (Iteration++ >= MaxPotentialValuesIterations) {
11544 LLVM_DEBUG(dbgs() << "Generic value traversal reached iteration limit: "
11545 << Iteration << "!\n");
11546 addValue(A, State&: getState(), V&: *V, CtxI, S, AnchorScope: getAnchorScope());
11547 continue;
11548 }
11549
11550 // Explicitly look through calls with a "returned" attribute if we do
11551 // not have a pointer as stripPointerCasts only works on them.
11552 Value *NewV = nullptr;
11553 if (V->getType()->isPointerTy()) {
11554 NewV = AA::getWithType(V&: *V->stripPointerCasts(), Ty&: *V->getType());
11555 } else {
11556 if (auto *CB = dyn_cast<CallBase>(Val: V))
11557 if (auto *Callee =
11558 dyn_cast_if_present<Function>(Val: CB->getCalledOperand())) {
11559 for (Argument &Arg : Callee->args())
11560 if (Arg.hasReturnedAttr()) {
11561 NewV = CB->getArgOperand(i: Arg.getArgNo());
11562 break;
11563 }
11564 }
11565 }
11566 if (NewV && NewV != V) {
11567 Worklist.push_back(Elt: {.I: {*NewV, CtxI}, .S: S});
11568 continue;
11569 }
11570
11571 if (auto *I = dyn_cast<Instruction>(Val: V)) {
11572 if (simplifyInstruction(A, I&: *I, II, Worklist, LivenessAAs))
11573 continue;
11574 }
11575
11576 if (V != InitialV || isa<Argument>(Val: V))
11577 if (recurseForValue(A, IRP: IRPosition::value(V: *V), S: II.S))
11578 continue;
11579
11580 // If we haven't stripped anything we give up.
11581 if (V == InitialV && CtxI == getCtxI()) {
11582 indicatePessimisticFixpoint();
11583 return;
11584 }
11585
11586 addValue(A, State&: getState(), V&: *V, CtxI, S, AnchorScope: getAnchorScope());
11587 } while (!Worklist.empty());
11588
11589 // If we actually used liveness information so we have to record a
11590 // dependence.
11591 for (auto &It : LivenessAAs)
11592 if (It.second.AnyDead)
11593 A.recordDependence(FromAA: *It.second.LivenessAA, ToAA: *this, DepClass: DepClassTy::OPTIONAL);
11594 }
11595
11596 /// See AbstractAttribute::trackStatistics()
11597 void trackStatistics() const override {
11598 STATS_DECLTRACK_FLOATING_ATTR(potential_values)
11599 }
11600};
11601
11602struct AAPotentialValuesArgument final : AAPotentialValuesImpl {
11603 using Base = AAPotentialValuesImpl;
11604 AAPotentialValuesArgument(const IRPosition &IRP, Attributor &A)
11605 : Base(IRP, A) {}
11606
11607 /// See AbstractAttribute::initialize(..).
11608 void initialize(Attributor &A) override {
11609 auto &Arg = cast<Argument>(Val&: getAssociatedValue());
11610 if (Arg.hasPointeeInMemoryValueAttr())
11611 indicatePessimisticFixpoint();
11612 }
11613
11614 /// See AbstractAttribute::updateImpl(...).
11615 ChangeStatus updateImpl(Attributor &A) override {
11616 auto AssumedBefore = getAssumed();
11617
11618 unsigned ArgNo = getCalleeArgNo();
11619
11620 bool UsedAssumedInformation = false;
11621 SmallVector<AA::ValueAndContext> Values;
11622 auto CallSitePred = [&](AbstractCallSite ACS) {
11623 const auto CSArgIRP = IRPosition::callsite_argument(ACS, ArgNo);
11624 if (CSArgIRP.getPositionKind() == IRP_INVALID)
11625 return false;
11626
11627 if (!A.getAssumedSimplifiedValues(IRP: CSArgIRP, AA: this, Values,
11628 S: AA::Interprocedural,
11629 UsedAssumedInformation))
11630 return false;
11631
11632 return isValidState();
11633 };
11634
11635 if (!A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this,
11636 /* RequireAllCallSites */ true,
11637 UsedAssumedInformation))
11638 return indicatePessimisticFixpoint();
11639
11640 Function *Fn = getAssociatedFunction();
11641 bool AnyNonLocal = false;
11642 for (auto &It : Values) {
11643 if (isa<Constant>(Val: It.getValue())) {
11644 addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::AnyScope,
11645 AnchorScope: getAnchorScope());
11646 continue;
11647 }
11648 if (!AA::isDynamicallyUnique(A, QueryingAA: *this, V: *It.getValue()))
11649 return indicatePessimisticFixpoint();
11650
11651 if (auto *Arg = dyn_cast<Argument>(Val: It.getValue()))
11652 if (Arg->getParent() == Fn) {
11653 addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::AnyScope,
11654 AnchorScope: getAnchorScope());
11655 continue;
11656 }
11657 addValue(A, State&: getState(), V&: *It.getValue(), CtxI: It.getCtxI(), S: AA::Interprocedural,
11658 AnchorScope: getAnchorScope());
11659 AnyNonLocal = true;
11660 }
11661 assert(!undefIsContained() && "Undef should be an explicit value!");
11662 if (AnyNonLocal)
11663 giveUpOnIntraprocedural(A);
11664
11665 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11666 : ChangeStatus::CHANGED;
11667 }
11668
11669 /// See AbstractAttribute::trackStatistics()
11670 void trackStatistics() const override {
11671 STATS_DECLTRACK_ARG_ATTR(potential_values)
11672 }
11673};
11674
11675struct AAPotentialValuesReturned : public AAPotentialValuesFloating {
11676 using Base = AAPotentialValuesFloating;
11677 AAPotentialValuesReturned(const IRPosition &IRP, Attributor &A)
11678 : Base(IRP, A) {}
11679
11680 /// See AbstractAttribute::initialize(..).
11681 void initialize(Attributor &A) override {
11682 Function *F = getAssociatedFunction();
11683 if (!F || F->isDeclaration() || F->getReturnType()->isVoidTy()) {
11684 indicatePessimisticFixpoint();
11685 return;
11686 }
11687
11688 for (Argument &Arg : F->args())
11689 if (Arg.hasReturnedAttr()) {
11690 addValue(A, State&: getState(), V&: Arg, CtxI: nullptr, S: AA::AnyScope, AnchorScope: F);
11691 ReturnedArg = &Arg;
11692 break;
11693 }
11694 if (!A.isFunctionIPOAmendable(F: *F) ||
11695 A.hasSimplificationCallback(IRP: getIRPosition())) {
11696 if (!ReturnedArg)
11697 indicatePessimisticFixpoint();
11698 else
11699 indicateOptimisticFixpoint();
11700 }
11701 }
11702
11703 /// See AbstractAttribute::updateImpl(...).
11704 ChangeStatus updateImpl(Attributor &A) override {
11705 auto AssumedBefore = getAssumed();
11706 bool UsedAssumedInformation = false;
11707
11708 SmallVector<AA::ValueAndContext> Values;
11709 Function *AnchorScope = getAnchorScope();
11710 auto HandleReturnedValue = [&](Value &V, Instruction *CtxI,
11711 bool AddValues) {
11712 for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {
11713 Values.clear();
11714 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V), AA: this, Values, S,
11715 UsedAssumedInformation,
11716 /* RecurseForSelectAndPHI */ true))
11717 return false;
11718 if (!AddValues)
11719 continue;
11720
11721 bool AllInterAreIntra = false;
11722 if (S == AA::Interprocedural)
11723 AllInterAreIntra =
11724 llvm::all_of(Range&: Values, P: [&](const AA::ValueAndContext &VAC) {
11725 return AA::isValidInScope(V: *VAC.getValue(), Scope: AnchorScope);
11726 });
11727
11728 for (const AA::ValueAndContext &VAC : Values) {
11729 addValue(A, State&: getState(), V&: *VAC.getValue(),
11730 CtxI: VAC.getCtxI() ? VAC.getCtxI() : CtxI,
11731 S: AllInterAreIntra ? AA::AnyScope : S, AnchorScope);
11732 }
11733 if (AllInterAreIntra)
11734 break;
11735 }
11736 return true;
11737 };
11738
11739 if (ReturnedArg) {
11740 HandleReturnedValue(*ReturnedArg, nullptr, true);
11741 } else {
11742 auto RetInstPred = [&](Instruction &RetI) {
11743 bool AddValues = true;
11744 if (isa<PHINode>(Val: RetI.getOperand(i: 0)) ||
11745 isa<SelectInst>(Val: RetI.getOperand(i: 0))) {
11746 addValue(A, State&: getState(), V&: *RetI.getOperand(i: 0), CtxI: &RetI, S: AA::AnyScope,
11747 AnchorScope);
11748 AddValues = false;
11749 }
11750 return HandleReturnedValue(*RetI.getOperand(i: 0), &RetI, AddValues);
11751 };
11752
11753 if (!A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
11754 UsedAssumedInformation,
11755 /* CheckBBLivenessOnly */ true))
11756 return indicatePessimisticFixpoint();
11757 }
11758
11759 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11760 : ChangeStatus::CHANGED;
11761 }
11762
11763 ChangeStatus manifest(Attributor &A) override {
11764 if (ReturnedArg)
11765 return ChangeStatus::UNCHANGED;
11766 SmallVector<AA::ValueAndContext> Values;
11767 if (!getAssumedSimplifiedValues(A, Values, S: AA::ValueScope::Intraprocedural,
11768 /* RecurseForSelectAndPHI */ true))
11769 return ChangeStatus::UNCHANGED;
11770 Value *NewVal = getSingleValue(A, AA: *this, IRP: getIRPosition(), Values);
11771 if (!NewVal)
11772 return ChangeStatus::UNCHANGED;
11773
11774 ChangeStatus Changed = ChangeStatus::UNCHANGED;
11775 if (auto *Arg = dyn_cast<Argument>(Val: NewVal)) {
11776 STATS_DECLTRACK(UniqueReturnValue, FunctionReturn,
11777 "Number of function with unique return");
11778 Changed |= A.manifestAttrs(
11779 IRP: IRPosition::argument(Arg: *Arg),
11780 DeducedAttrs: {Attribute::get(Context&: Arg->getContext(), Kind: Attribute::Returned)});
11781 STATS_DECLTRACK_ARG_ATTR(returned);
11782 }
11783
11784 auto RetInstPred = [&](Instruction &RetI) {
11785 Value *RetOp = RetI.getOperand(i: 0);
11786 if (isa<UndefValue>(Val: RetOp) || RetOp == NewVal)
11787 return true;
11788 if (AA::isValidAtPosition(VAC: {*NewVal, RetI}, InfoCache&: A.getInfoCache()))
11789 if (A.changeUseAfterManifest(U&: RetI.getOperandUse(i: 0), NV&: *NewVal))
11790 Changed = ChangeStatus::CHANGED;
11791 return true;
11792 };
11793 bool UsedAssumedInformation = false;
11794 (void)A.checkForAllInstructions(Pred: RetInstPred, QueryingAA: *this, Opcodes: {Instruction::Ret},
11795 UsedAssumedInformation,
11796 /* CheckBBLivenessOnly */ true);
11797 return Changed;
11798 }
11799
11800 ChangeStatus indicatePessimisticFixpoint() override {
11801 return AAPotentialValues::indicatePessimisticFixpoint();
11802 }
11803
11804 /// See AbstractAttribute::trackStatistics()
11805 void trackStatistics() const override{
11806 STATS_DECLTRACK_FNRET_ATTR(potential_values)}
11807
11808 /// The argumented with an existing `returned` attribute.
11809 Argument *ReturnedArg = nullptr;
11810};
11811
11812struct AAPotentialValuesFunction : AAPotentialValuesImpl {
11813 AAPotentialValuesFunction(const IRPosition &IRP, Attributor &A)
11814 : AAPotentialValuesImpl(IRP, A) {}
11815
11816 /// See AbstractAttribute::updateImpl(...).
11817 ChangeStatus updateImpl(Attributor &A) override {
11818 llvm_unreachable("AAPotentialValues(Function|CallSite)::updateImpl will "
11819 "not be called");
11820 }
11821
11822 /// See AbstractAttribute::trackStatistics()
11823 void trackStatistics() const override {
11824 STATS_DECLTRACK_FN_ATTR(potential_values)
11825 }
11826};
11827
11828struct AAPotentialValuesCallSite : AAPotentialValuesFunction {
11829 AAPotentialValuesCallSite(const IRPosition &IRP, Attributor &A)
11830 : AAPotentialValuesFunction(IRP, A) {}
11831
11832 /// See AbstractAttribute::trackStatistics()
11833 void trackStatistics() const override {
11834 STATS_DECLTRACK_CS_ATTR(potential_values)
11835 }
11836};
11837
11838struct AAPotentialValuesCallSiteReturned : AAPotentialValuesImpl {
11839 AAPotentialValuesCallSiteReturned(const IRPosition &IRP, Attributor &A)
11840 : AAPotentialValuesImpl(IRP, A) {}
11841
11842 /// See AbstractAttribute::updateImpl(...).
11843 ChangeStatus updateImpl(Attributor &A) override {
11844 auto AssumedBefore = getAssumed();
11845
11846 Function *Callee = getAssociatedFunction();
11847 if (!Callee)
11848 return indicatePessimisticFixpoint();
11849
11850 bool UsedAssumedInformation = false;
11851 auto *CB = cast<CallBase>(Val: getCtxI());
11852 if (CB->isMustTailCall() &&
11853 !A.isAssumedDead(IRP: IRPosition::inst(I: *CB), QueryingAA: this, FnLivenessAA: nullptr,
11854 UsedAssumedInformation))
11855 return indicatePessimisticFixpoint();
11856
11857 Function *Caller = CB->getCaller();
11858
11859 auto AddScope = [&](AA::ValueScope S) {
11860 SmallVector<AA::ValueAndContext> Values;
11861 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::returned(F: *Callee), AA: this,
11862 Values, S, UsedAssumedInformation))
11863 return false;
11864
11865 for (auto &It : Values) {
11866 Value *V = It.getValue();
11867 std::optional<Value *> CallerV = A.translateArgumentToCallSiteContent(
11868 V, CB&: *CB, AA: *this, UsedAssumedInformation);
11869 if (!CallerV.has_value()) {
11870 // Nothing to do as long as no value was determined.
11871 continue;
11872 }
11873 V = *CallerV ? *CallerV : V;
11874 if (*CallerV && AA::isDynamicallyUnique(A, QueryingAA: *this, V: *V)) {
11875 if (recurseForValue(A, IRP: IRPosition::value(V: *V), S))
11876 continue;
11877 }
11878 if (S == AA::Intraprocedural && !AA::isValidInScope(V: *V, Scope: Caller)) {
11879 giveUpOnIntraprocedural(A);
11880 return true;
11881 }
11882 addValue(A, State&: getState(), V&: *V, CtxI: CB, S, AnchorScope: getAnchorScope());
11883 }
11884 return true;
11885 };
11886 if (!AddScope(AA::Intraprocedural))
11887 return indicatePessimisticFixpoint();
11888 if (!AddScope(AA::Interprocedural))
11889 return indicatePessimisticFixpoint();
11890 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11891 : ChangeStatus::CHANGED;
11892 }
11893
11894 ChangeStatus indicatePessimisticFixpoint() override {
11895 return AAPotentialValues::indicatePessimisticFixpoint();
11896 }
11897
11898 /// See AbstractAttribute::trackStatistics()
11899 void trackStatistics() const override {
11900 STATS_DECLTRACK_CSRET_ATTR(potential_values)
11901 }
11902};
11903
11904struct AAPotentialValuesCallSiteArgument : AAPotentialValuesFloating {
11905 AAPotentialValuesCallSiteArgument(const IRPosition &IRP, Attributor &A)
11906 : AAPotentialValuesFloating(IRP, A) {}
11907
11908 /// See AbstractAttribute::trackStatistics()
11909 void trackStatistics() const override {
11910 STATS_DECLTRACK_CSARG_ATTR(potential_values)
11911 }
11912};
11913} // namespace
11914
11915/// ---------------------- Assumption Propagation ------------------------------
11916namespace {
11917struct AAAssumptionInfoImpl : public AAAssumptionInfo {
11918 AAAssumptionInfoImpl(const IRPosition &IRP, Attributor &A,
11919 const DenseSet<StringRef> &Known)
11920 : AAAssumptionInfo(IRP, A, Known) {}
11921
11922 /// See AbstractAttribute::manifest(...).
11923 ChangeStatus manifest(Attributor &A) override {
11924 // Don't manifest a universal set if it somehow made it here.
11925 if (getKnown().isUniversal())
11926 return ChangeStatus::UNCHANGED;
11927
11928 const IRPosition &IRP = getIRPosition();
11929 SmallVector<StringRef, 0> Set(getAssumed().getSet().begin(),
11930 getAssumed().getSet().end());
11931 llvm::sort(C&: Set);
11932 return A.manifestAttrs(IRP,
11933 DeducedAttrs: Attribute::get(Context&: IRP.getAnchorValue().getContext(),
11934 Kind: AssumptionAttrKey,
11935 Val: llvm::join(R&: Set, Separator: ",")),
11936 /*ForceReplace=*/true);
11937 }
11938
11939 bool hasAssumption(const StringRef Assumption) const override {
11940 return isValidState() && setContains(Assumption);
11941 }
11942
11943 /// See AbstractAttribute::getAsStr()
11944 const std::string getAsStr(Attributor *A) const override {
11945 const SetContents &Known = getKnown();
11946 const SetContents &Assumed = getAssumed();
11947
11948 SmallVector<StringRef, 0> Set(Known.getSet().begin(), Known.getSet().end());
11949 llvm::sort(C&: Set);
11950 const std::string KnownStr = llvm::join(R&: Set, Separator: ",");
11951
11952 std::string AssumedStr = "Universal";
11953 if (!Assumed.isUniversal()) {
11954 Set.assign(in_start: Assumed.getSet().begin(), in_end: Assumed.getSet().end());
11955 AssumedStr = llvm::join(R&: Set, Separator: ",");
11956 }
11957 return "Known [" + KnownStr + "]," + " Assumed [" + AssumedStr + "]";
11958 }
11959};
11960
11961/// Propagates assumption information from parent functions to all of their
11962/// successors. An assumption can be propagated if the containing function
11963/// dominates the called function.
11964///
11965/// We start with a "known" set of assumptions already valid for the associated
11966/// function and an "assumed" set that initially contains all possible
11967/// assumptions. The assumed set is inter-procedurally updated by narrowing its
11968/// contents as concrete values are known. The concrete values are seeded by the
11969/// first nodes that are either entries into the call graph, or contains no
11970/// assumptions. Each node is updated as the intersection of the assumed state
11971/// with all of its predecessors.
11972struct AAAssumptionInfoFunction final : AAAssumptionInfoImpl {
11973 AAAssumptionInfoFunction(const IRPosition &IRP, Attributor &A)
11974 : AAAssumptionInfoImpl(IRP, A,
11975 getAssumptions(F: *IRP.getAssociatedFunction())) {}
11976
11977 /// See AbstractAttribute::updateImpl(...).
11978 ChangeStatus updateImpl(Attributor &A) override {
11979 bool Changed = false;
11980
11981 auto CallSitePred = [&](AbstractCallSite ACS) {
11982 const auto *AssumptionAA = A.getAAFor<AAAssumptionInfo>(
11983 QueryingAA: *this, IRP: IRPosition::callsite_function(CB: *ACS.getInstruction()),
11984 DepClass: DepClassTy::REQUIRED);
11985 if (!AssumptionAA)
11986 return false;
11987 // Get the set of assumptions shared by all of this function's callers.
11988 Changed |= getIntersection(RHS: AssumptionAA->getAssumed());
11989 return !getAssumed().empty() || !getKnown().empty();
11990 };
11991
11992 bool UsedAssumedInformation = false;
11993 // Get the intersection of all assumptions held by this node's predecessors.
11994 // If we don't know all the call sites then this is either an entry into the
11995 // call graph or an empty node. This node is known to only contain its own
11996 // assumptions and can be propagated to its successors.
11997 if (!A.checkForAllCallSites(Pred: CallSitePred, QueryingAA: *this, RequireAllCallSites: true,
11998 UsedAssumedInformation))
11999 return indicatePessimisticFixpoint();
12000
12001 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
12002 }
12003
12004 void trackStatistics() const override {}
12005};
12006
12007/// Assumption Info defined for call sites.
12008struct AAAssumptionInfoCallSite final : AAAssumptionInfoImpl {
12009
12010 AAAssumptionInfoCallSite(const IRPosition &IRP, Attributor &A)
12011 : AAAssumptionInfoImpl(IRP, A, getInitialAssumptions(IRP)) {}
12012
12013 /// See AbstractAttribute::initialize(...).
12014 void initialize(Attributor &A) override {
12015 const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
12016 A.getAAFor<AAAssumptionInfo>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
12017 }
12018
12019 /// See AbstractAttribute::updateImpl(...).
12020 ChangeStatus updateImpl(Attributor &A) override {
12021 const IRPosition &FnPos = IRPosition::function(F: *getAnchorScope());
12022 auto *AssumptionAA =
12023 A.getAAFor<AAAssumptionInfo>(QueryingAA: *this, IRP: FnPos, DepClass: DepClassTy::REQUIRED);
12024 if (!AssumptionAA)
12025 return indicatePessimisticFixpoint();
12026 bool Changed = getIntersection(RHS: AssumptionAA->getAssumed());
12027 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
12028 }
12029
12030 /// See AbstractAttribute::trackStatistics()
12031 void trackStatistics() const override {}
12032
12033private:
12034 /// Helper to initialized the known set as all the assumptions this call and
12035 /// the callee contain.
12036 DenseSet<StringRef> getInitialAssumptions(const IRPosition &IRP) {
12037 const CallBase &CB = cast<CallBase>(Val&: IRP.getAssociatedValue());
12038 auto Assumptions = getAssumptions(CB);
12039 if (const Function *F = CB.getCaller())
12040 set_union(S1&: Assumptions, S2: getAssumptions(F: *F));
12041 if (Function *F = IRP.getAssociatedFunction())
12042 set_union(S1&: Assumptions, S2: getAssumptions(F: *F));
12043 return Assumptions;
12044 }
12045};
12046} // namespace
12047
12048AACallGraphNode *AACallEdgeIterator::operator*() const {
12049 return static_cast<AACallGraphNode *>(const_cast<AACallEdges *>(
12050 A.getOrCreateAAFor<AACallEdges>(IRP: IRPosition::function(F: **I))));
12051}
12052
12053void AttributorCallGraph::print() { llvm::WriteGraph(O&: outs(), G: this); }
12054
12055/// ------------------------ UnderlyingObjects ---------------------------------
12056
12057namespace {
12058struct AAUnderlyingObjectsImpl
12059 : StateWrapper<BooleanState, AAUnderlyingObjects> {
12060 using BaseTy = StateWrapper<BooleanState, AAUnderlyingObjects>;
12061 AAUnderlyingObjectsImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
12062
12063 /// See AbstractAttribute::getAsStr().
12064 const std::string getAsStr(Attributor *A) const override {
12065 if (!isValidState())
12066 return "<invalid>";
12067 std::string Str;
12068 llvm::raw_string_ostream OS(Str);
12069 OS << "underlying objects: inter " << InterAssumedUnderlyingObjects.size()
12070 << " objects, intra " << IntraAssumedUnderlyingObjects.size()
12071 << " objects.\n";
12072 if (!InterAssumedUnderlyingObjects.empty()) {
12073 OS << "inter objects:\n";
12074 for (auto *Obj : InterAssumedUnderlyingObjects)
12075 OS << *Obj << '\n';
12076 }
12077 if (!IntraAssumedUnderlyingObjects.empty()) {
12078 OS << "intra objects:\n";
12079 for (auto *Obj : IntraAssumedUnderlyingObjects)
12080 OS << *Obj << '\n';
12081 }
12082 return Str;
12083 }
12084
12085 /// See AbstractAttribute::trackStatistics()
12086 void trackStatistics() const override {}
12087
12088 /// See AbstractAttribute::updateImpl(...).
12089 ChangeStatus updateImpl(Attributor &A) override {
12090 auto &Ptr = getAssociatedValue();
12091
12092 bool UsedAssumedInformation = false;
12093 auto DoUpdate = [&](SmallSetVector<Value *, 8> &UnderlyingObjects,
12094 AA::ValueScope Scope) {
12095 SmallPtrSet<Value *, 8> SeenObjects;
12096 SmallVector<AA::ValueAndContext> Values;
12097
12098 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: Ptr), AA: *this, Values,
12099 S: Scope, UsedAssumedInformation))
12100 return UnderlyingObjects.insert(X: &Ptr);
12101
12102 bool Changed = false;
12103
12104 for (unsigned I = 0; I < Values.size(); ++I) {
12105 auto &VAC = Values[I];
12106 auto *Obj = VAC.getValue();
12107 Value *UO = getUnderlyingObject(V: Obj);
12108 if (!SeenObjects.insert(Ptr: UO ? UO : Obj).second)
12109 continue;
12110 if (UO && UO != Obj) {
12111 if (isa<AllocaInst>(Val: UO) || isa<GlobalValue>(Val: UO)) {
12112 Changed |= UnderlyingObjects.insert(X: UO);
12113 continue;
12114 }
12115
12116 const auto *OtherAA = A.getAAFor<AAUnderlyingObjects>(
12117 QueryingAA: *this, IRP: IRPosition::value(V: *UO), DepClass: DepClassTy::OPTIONAL);
12118 auto Pred = [&](Value &V) {
12119 if (&V == UO)
12120 Changed |= UnderlyingObjects.insert(X: UO);
12121 else
12122 Values.emplace_back(Args&: V, Args: nullptr);
12123 return true;
12124 };
12125
12126 if (!OtherAA || !OtherAA->forallUnderlyingObjects(Pred, Scope))
12127 llvm_unreachable(
12128 "The forall call should not return false at this position");
12129 UsedAssumedInformation |= !OtherAA->getState().isAtFixpoint();
12130 continue;
12131 }
12132
12133 if (isa<SelectInst>(Val: Obj)) {
12134 Changed |= handleIndirect(A, V&: *Obj, UnderlyingObjects, Scope,
12135 UsedAssumedInformation);
12136 continue;
12137 }
12138 if (auto *PHI = dyn_cast<PHINode>(Val: Obj)) {
12139 // Explicitly look through PHIs as we do not care about dynamically
12140 // uniqueness.
12141 for (unsigned u = 0, e = PHI->getNumIncomingValues(); u < e; u++) {
12142 Changed |=
12143 handleIndirect(A, V&: *PHI->getIncomingValue(i: u), UnderlyingObjects,
12144 Scope, UsedAssumedInformation);
12145 }
12146 continue;
12147 }
12148
12149 Changed |= UnderlyingObjects.insert(X: Obj);
12150 }
12151
12152 return Changed;
12153 };
12154
12155 bool Changed = false;
12156 Changed |= DoUpdate(IntraAssumedUnderlyingObjects, AA::Intraprocedural);
12157 Changed |= DoUpdate(InterAssumedUnderlyingObjects, AA::Interprocedural);
12158 if (!UsedAssumedInformation)
12159 indicateOptimisticFixpoint();
12160 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
12161 }
12162
12163 bool forallUnderlyingObjects(
12164 function_ref<bool(Value &)> Pred,
12165 AA::ValueScope Scope = AA::Interprocedural) const override {
12166 if (!isValidState())
12167 return Pred(getAssociatedValue());
12168
12169 auto &AssumedUnderlyingObjects = Scope == AA::Intraprocedural
12170 ? IntraAssumedUnderlyingObjects
12171 : InterAssumedUnderlyingObjects;
12172 for (Value *Obj : AssumedUnderlyingObjects)
12173 if (!Pred(*Obj))
12174 return false;
12175
12176 return true;
12177 }
12178
12179private:
12180 /// Handle the case where the value is not the actual underlying value, such
12181 /// as a phi node or a select instruction.
12182 bool handleIndirect(Attributor &A, Value &V,
12183 SmallSetVector<Value *, 8> &UnderlyingObjects,
12184 AA::ValueScope Scope, bool &UsedAssumedInformation) {
12185 bool Changed = false;
12186 const auto *AA = A.getAAFor<AAUnderlyingObjects>(
12187 QueryingAA: *this, IRP: IRPosition::value(V), DepClass: DepClassTy::OPTIONAL);
12188 auto Pred = [&](Value &V) {
12189 Changed |= UnderlyingObjects.insert(X: &V);
12190 return true;
12191 };
12192 if (!AA || !AA->forallUnderlyingObjects(Pred, Scope))
12193 llvm_unreachable(
12194 "The forall call should not return false at this position");
12195 UsedAssumedInformation |= !AA->getState().isAtFixpoint();
12196 return Changed;
12197 }
12198
12199 /// All the underlying objects collected so far via intra procedural scope.
12200 SmallSetVector<Value *, 8> IntraAssumedUnderlyingObjects;
12201 /// All the underlying objects collected so far via inter procedural scope.
12202 SmallSetVector<Value *, 8> InterAssumedUnderlyingObjects;
12203};
12204
12205struct AAUnderlyingObjectsFloating final : AAUnderlyingObjectsImpl {
12206 AAUnderlyingObjectsFloating(const IRPosition &IRP, Attributor &A)
12207 : AAUnderlyingObjectsImpl(IRP, A) {}
12208};
12209
12210struct AAUnderlyingObjectsArgument final : AAUnderlyingObjectsImpl {
12211 AAUnderlyingObjectsArgument(const IRPosition &IRP, Attributor &A)
12212 : AAUnderlyingObjectsImpl(IRP, A) {}
12213};
12214
12215struct AAUnderlyingObjectsCallSite final : AAUnderlyingObjectsImpl {
12216 AAUnderlyingObjectsCallSite(const IRPosition &IRP, Attributor &A)
12217 : AAUnderlyingObjectsImpl(IRP, A) {}
12218};
12219
12220struct AAUnderlyingObjectsCallSiteArgument final : AAUnderlyingObjectsImpl {
12221 AAUnderlyingObjectsCallSiteArgument(const IRPosition &IRP, Attributor &A)
12222 : AAUnderlyingObjectsImpl(IRP, A) {}
12223};
12224
12225struct AAUnderlyingObjectsReturned final : AAUnderlyingObjectsImpl {
12226 AAUnderlyingObjectsReturned(const IRPosition &IRP, Attributor &A)
12227 : AAUnderlyingObjectsImpl(IRP, A) {}
12228};
12229
12230struct AAUnderlyingObjectsCallSiteReturned final : AAUnderlyingObjectsImpl {
12231 AAUnderlyingObjectsCallSiteReturned(const IRPosition &IRP, Attributor &A)
12232 : AAUnderlyingObjectsImpl(IRP, A) {}
12233};
12234
12235struct AAUnderlyingObjectsFunction final : AAUnderlyingObjectsImpl {
12236 AAUnderlyingObjectsFunction(const IRPosition &IRP, Attributor &A)
12237 : AAUnderlyingObjectsImpl(IRP, A) {}
12238};
12239} // namespace
12240
12241/// ------------------------ Global Value Info -------------------------------
12242namespace {
12243struct AAGlobalValueInfoFloating : public AAGlobalValueInfo {
12244 AAGlobalValueInfoFloating(const IRPosition &IRP, Attributor &A)
12245 : AAGlobalValueInfo(IRP, A) {}
12246
12247 /// See AbstractAttribute::initialize(...).
12248 void initialize(Attributor &A) override {}
12249
12250 bool checkUse(Attributor &A, const Use &U, bool &Follow,
12251 SmallVectorImpl<const Value *> &Worklist) {
12252 Instruction *UInst = dyn_cast<Instruction>(Val: U.getUser());
12253 if (!UInst) {
12254 // Outside a closed world, code outside the module can read an
12255 // externally visible global, so the value escapes through it.
12256 if (auto *GV = dyn_cast<GlobalValue>(Val: U.getUser()))
12257 if (!GV->hasLocalLinkage() && !A.isClosedWorldModule())
12258 return false;
12259 Follow = true;
12260 return true;
12261 }
12262
12263 LLVM_DEBUG(dbgs() << "[AAGlobalValueInfo] Check use: " << *U.get() << " in "
12264 << *UInst << "\n");
12265
12266 if (auto *Cmp = dyn_cast<ICmpInst>(Val: U.getUser())) {
12267 int Idx = &Cmp->getOperandUse(i: 0) == &U;
12268 if (isa<Constant>(Val: Cmp->getOperand(i_nocapture: Idx)))
12269 return true;
12270 return U == &getAnchorValue();
12271 }
12272
12273 // Explicitly catch return instructions.
12274 if (isa<ReturnInst>(Val: UInst)) {
12275 auto CallSitePred = [&](AbstractCallSite ACS) {
12276 Worklist.push_back(Elt: ACS.getInstruction());
12277 return true;
12278 };
12279 bool UsedAssumedInformation = false;
12280 // TODO: We should traverse the uses or add a "non-call-site" CB.
12281 if (!A.checkForAllCallSites(Pred: CallSitePred, Fn: *UInst->getFunction(),
12282 /*RequireAllCallSites=*/true, QueryingAA: this,
12283 UsedAssumedInformation))
12284 return false;
12285 return true;
12286 }
12287
12288 // For now we only use special logic for call sites. However, the tracker
12289 // itself knows about a lot of other non-capturing cases already.
12290 auto *CB = dyn_cast<CallBase>(Val: UInst);
12291 if (!CB)
12292 return false;
12293 // Direct calls are OK uses.
12294 if (CB->isCallee(U: &U))
12295 return true;
12296 // Non-argument uses are scary.
12297 if (!CB->isArgOperand(U: &U))
12298 return false;
12299 // TODO: Iterate callees.
12300 auto *Fn = dyn_cast<Function>(Val: CB->getCalledOperand());
12301 if (!Fn || !A.isFunctionIPOAmendable(F: *Fn))
12302 return false;
12303
12304 unsigned ArgNo = CB->getArgOperandNo(U: &U);
12305 Worklist.push_back(Elt: Fn->getArg(i: ArgNo));
12306 return true;
12307 }
12308
12309 ChangeStatus updateImpl(Attributor &A) override {
12310 unsigned NumUsesBefore = Uses.size();
12311
12312 SmallPtrSet<const Value *, 8> Visited;
12313 SmallVector<const Value *> Worklist;
12314 Worklist.push_back(Elt: &getAnchorValue());
12315
12316 auto UsePred = [&](const Use &U, bool &Follow) -> bool {
12317 Uses.insert(Ptr: &U);
12318 // TODO(captures): Make this more precise.
12319 UseCaptureInfo CI = DetermineUseCaptureKind(U, /*Base=*/nullptr);
12320 if (CI.isPassthrough()) {
12321 Follow = true;
12322 return true;
12323 }
12324 return checkUse(A, U, Follow, Worklist);
12325 };
12326 auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
12327 Uses.insert(Ptr: &OldU);
12328 return true;
12329 };
12330
12331 while (!Worklist.empty()) {
12332 const Value *V = Worklist.pop_back_val();
12333 if (!Visited.insert(Ptr: V).second)
12334 continue;
12335 if (!A.checkForAllUses(Pred: UsePred, QueryingAA: *this, V: *V,
12336 /* CheckBBLivenessOnly */ true,
12337 LivenessDepClass: DepClassTy::OPTIONAL,
12338 /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
12339 return indicatePessimisticFixpoint();
12340 }
12341 }
12342
12343 return Uses.size() == NumUsesBefore ? ChangeStatus::UNCHANGED
12344 : ChangeStatus::CHANGED;
12345 }
12346
12347 bool isPotentialUse(const Use &U) const override {
12348 return !isValidState() || Uses.contains(Ptr: &U);
12349 }
12350
12351 /// See AbstractAttribute::manifest(...).
12352 ChangeStatus manifest(Attributor &A) override {
12353 return ChangeStatus::UNCHANGED;
12354 }
12355
12356 /// See AbstractAttribute::getAsStr().
12357 const std::string getAsStr(Attributor *A) const override {
12358 return "[" + std::to_string(val: Uses.size()) + " uses]";
12359 }
12360
12361 void trackStatistics() const override {
12362 STATS_DECLTRACK_FLOATING_ATTR(GlobalValuesTracked);
12363 }
12364
12365private:
12366 /// Set of (transitive) uses of this GlobalValue.
12367 SmallPtrSet<const Use *, 8> Uses;
12368};
12369} // namespace
12370
12371/// ------------------------ Indirect Call Info -------------------------------
12372namespace {
12373struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo {
12374 AAIndirectCallInfoCallSite(const IRPosition &IRP, Attributor &A)
12375 : AAIndirectCallInfo(IRP, A) {}
12376
12377 /// See AbstractAttribute::initialize(...).
12378 void initialize(Attributor &A) override {
12379 auto *MD = getCtxI()->getMetadata(KindID: LLVMContext::MD_callees);
12380 if (!MD && !A.isClosedWorldModule())
12381 return;
12382
12383 if (MD) {
12384 for (const auto &Op : MD->operands())
12385 if (Function *Callee = mdconst::dyn_extract_or_null<Function>(MD: Op))
12386 PotentialCallees.insert(X: Callee);
12387 } else if (A.isClosedWorldModule()) {
12388 ArrayRef<Function *> IndirectlyCallableFunctions =
12389 A.getInfoCache().getIndirectlyCallableFunctions(A);
12390 PotentialCallees.insert_range(R&: IndirectlyCallableFunctions);
12391 }
12392
12393 if (PotentialCallees.empty())
12394 indicateOptimisticFixpoint();
12395 }
12396
12397 ChangeStatus updateImpl(Attributor &A) override {
12398 CallBase *CB = cast<CallBase>(Val: getCtxI());
12399 const Use &CalleeUse = CB->getCalledOperandUse();
12400 Value *FP = CB->getCalledOperand();
12401
12402 SmallSetVector<Function *, 4> AssumedCalleesNow;
12403 bool AllCalleesKnownNow = AllCalleesKnown;
12404
12405 auto CheckPotentialCalleeUse = [&](Function &PotentialCallee,
12406 bool &UsedAssumedInformation) {
12407 const auto *GIAA = A.getAAFor<AAGlobalValueInfo>(
12408 QueryingAA: *this, IRP: IRPosition::value(V: PotentialCallee), DepClass: DepClassTy::OPTIONAL);
12409 if (!GIAA || GIAA->isPotentialUse(U: CalleeUse))
12410 return true;
12411 UsedAssumedInformation = !GIAA->isAtFixpoint();
12412 return false;
12413 };
12414
12415 auto AddPotentialCallees = [&]() {
12416 for (auto *PotentialCallee : PotentialCallees) {
12417 bool UsedAssumedInformation = false;
12418 if (CheckPotentialCalleeUse(*PotentialCallee, UsedAssumedInformation))
12419 AssumedCalleesNow.insert(X: PotentialCallee);
12420 }
12421 };
12422
12423 // Use simplification to find potential callees, if !callees was present,
12424 // fallback to that set if necessary.
12425 bool UsedAssumedInformation = false;
12426 SmallVector<AA::ValueAndContext> Values;
12427 if (!A.getAssumedSimplifiedValues(IRP: IRPosition::value(V: *FP), AA: this, Values,
12428 S: AA::ValueScope::AnyScope,
12429 UsedAssumedInformation)) {
12430 if (PotentialCallees.empty())
12431 return indicatePessimisticFixpoint();
12432 AddPotentialCallees();
12433 }
12434
12435 // Try to find a reason for \p Fn not to be a potential callee. If none was
12436 // found, add it to the assumed callees set.
12437 auto CheckPotentialCallee = [&](Function &Fn) {
12438 if (!PotentialCallees.empty() && !PotentialCallees.count(key: &Fn))
12439 return false;
12440
12441 auto &CachedResult = FilterResults[&Fn];
12442 if (CachedResult.has_value())
12443 return CachedResult.value();
12444
12445 bool UsedAssumedInformation = false;
12446 if (!CheckPotentialCalleeUse(Fn, UsedAssumedInformation)) {
12447 if (!UsedAssumedInformation)
12448 CachedResult = false;
12449 return false;
12450 }
12451
12452 int NumFnArgs = Fn.arg_size();
12453 int NumCBArgs = CB->arg_size();
12454
12455 // Check if any excess argument (which we fill up with poison) is known to
12456 // be UB on undef.
12457 for (int I = NumCBArgs; I < NumFnArgs; ++I) {
12458 bool IsKnown = false;
12459 if (AA::hasAssumedIRAttr<Attribute::NoUndef>(
12460 A, QueryingAA: this, IRP: IRPosition::argument(Arg: *Fn.getArg(i: I)),
12461 DepClass: DepClassTy::OPTIONAL, IsKnown)) {
12462 if (IsKnown)
12463 CachedResult = false;
12464 return false;
12465 }
12466 }
12467
12468 CachedResult = true;
12469 return true;
12470 };
12471
12472 // Check simplification result, prune known UB callees, also restrict it to
12473 // the !callees set, if present.
12474 for (auto &VAC : Values) {
12475 if (isa<UndefValue>(Val: VAC.getValue()))
12476 continue;
12477 if (isa<ConstantPointerNull>(Val: VAC.getValue()) &&
12478 VAC.getValue()->getType()->getPointerAddressSpace() == 0)
12479 continue;
12480 // TODO: Check for known UB, e.g., poison + noundef.
12481 if (auto *VACFn = dyn_cast<Function>(Val: VAC.getValue())) {
12482 if (CheckPotentialCallee(*VACFn))
12483 AssumedCalleesNow.insert(X: VACFn);
12484 continue;
12485 }
12486 if (!PotentialCallees.empty()) {
12487 AddPotentialCallees();
12488 break;
12489 }
12490 AllCalleesKnownNow = false;
12491 }
12492
12493 if (AssumedCalleesNow == AssumedCallees &&
12494 AllCalleesKnown == AllCalleesKnownNow)
12495 return ChangeStatus::UNCHANGED;
12496
12497 std::swap(LHS&: AssumedCallees, RHS&: AssumedCalleesNow);
12498 AllCalleesKnown = AllCalleesKnownNow;
12499 return ChangeStatus::CHANGED;
12500 }
12501
12502 /// See AbstractAttribute::manifest(...).
12503 ChangeStatus manifest(Attributor &A) override {
12504 // If we can't specialize at all, give up now.
12505 if (!AllCalleesKnown && AssumedCallees.empty())
12506 return ChangeStatus::UNCHANGED;
12507
12508 CallBase *CB = cast<CallBase>(Val: getCtxI());
12509 bool UsedAssumedInformation = false;
12510 if (A.isAssumedDead(I: *CB, QueryingAA: this, /*LivenessAA=*/nullptr,
12511 UsedAssumedInformation))
12512 return ChangeStatus::UNCHANGED;
12513
12514 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12515 unsigned ProgramAS = CB->getDataLayout().getProgramAddressSpace();
12516 Value *FP = CB->getCalledOperand();
12517 if (FP->getType()->getPointerAddressSpace() != ProgramAS)
12518 FP = new AddrSpaceCastInst(
12519 FP, PointerType::get(C&: FP->getContext(), AddressSpace: ProgramAS),
12520 FP->getName() + ".as" + Twine(ProgramAS), CB->getIterator());
12521
12522 bool CBIsVoid = CB->getType()->isVoidTy();
12523 BasicBlock::iterator IP = CB->getIterator();
12524 FunctionType *CSFT = CB->getFunctionType();
12525 SmallVector<Value *> CSArgs(CB->args());
12526
12527 // If we know all callees and there are none, the call site is (effectively)
12528 // dead (or UB).
12529 if (AssumedCallees.empty()) {
12530 assert(AllCalleesKnown &&
12531 "Expected all callees to be known if there are none.");
12532 A.changeToUnreachableAfterManifest(I: CB);
12533 return ChangeStatus::CHANGED;
12534 }
12535
12536 // Special handling for the single callee case.
12537 if (AllCalleesKnown && AssumedCallees.size() == 1) {
12538 auto *NewCallee = AssumedCallees.front();
12539 if (isLegalToPromote(CB: *CB, Callee: NewCallee)) {
12540 promoteCall(CB&: *CB, Callee: NewCallee, RetBitCast: nullptr);
12541 NumIndirectCallsPromoted++;
12542 return ChangeStatus::CHANGED;
12543 }
12544 Instruction *NewCall =
12545 CallInst::Create(Func: FunctionCallee(CSFT, NewCallee), Args: CSArgs,
12546 NameStr: CB->getName(), InsertBefore: CB->getIterator());
12547 if (!CBIsVoid)
12548 A.changeAfterManifest(IRP: IRPosition::callsite_returned(CB: *CB), NV&: *NewCall);
12549 A.deleteAfterManifest(I&: *CB);
12550 return ChangeStatus::CHANGED;
12551 }
12552
12553 // For each potential value we create a conditional
12554 //
12555 // ```
12556 // if (ptr == value) value(args);
12557 // else ...
12558 // ```
12559 //
12560 bool SpecializedForAnyCallees = false;
12561 bool SpecializedForAllCallees = AllCalleesKnown;
12562 ICmpInst *LastCmp = nullptr;
12563 SmallVector<Function *, 8> SkippedAssumedCallees;
12564 SmallVector<std::pair<CallInst *, Instruction *>> NewCalls;
12565 for (Function *NewCallee : AssumedCallees) {
12566 if (!A.shouldSpecializeCallSiteForCallee(AA: *this, CB&: *CB, Callee&: *NewCallee,
12567 NumAssumedCallees: AssumedCallees.size())) {
12568 SkippedAssumedCallees.push_back(Elt: NewCallee);
12569 SpecializedForAllCallees = false;
12570 continue;
12571 }
12572 SpecializedForAnyCallees = true;
12573
12574 LastCmp = new ICmpInst(IP, llvm::CmpInst::ICMP_EQ, FP, NewCallee);
12575 Instruction *ThenTI =
12576 SplitBlockAndInsertIfThen(Cond: LastCmp, SplitBefore: IP, /* Unreachable */ false);
12577 BasicBlock *CBBB = CB->getParent();
12578 A.registerManifestAddedBasicBlock(BB&: *ThenTI->getParent());
12579 A.registerManifestAddedBasicBlock(BB&: *IP->getParent());
12580 auto *SplitTI = cast<CondBrInst>(Val: LastCmp->getNextNode());
12581 BasicBlock *ElseBB;
12582 if (&*IP == CB) {
12583 ElseBB = BasicBlock::Create(Context&: ThenTI->getContext(), Name: "",
12584 Parent: ThenTI->getFunction(), InsertBefore: CBBB);
12585 A.registerManifestAddedBasicBlock(BB&: *ElseBB);
12586 IP = UncondBrInst::Create(Target: CBBB, InsertBefore: ElseBB)->getIterator();
12587 SplitTI->replaceUsesOfWith(From: CBBB, To: ElseBB);
12588 } else {
12589 ElseBB = IP->getParent();
12590 ThenTI->replaceUsesOfWith(From: ElseBB, To: CBBB);
12591 }
12592 CastInst *RetBC = nullptr;
12593 CallInst *NewCall = nullptr;
12594 if (isLegalToPromote(CB: *CB, Callee: NewCallee)) {
12595 auto *CBClone = cast<CallBase>(Val: CB->clone());
12596 CBClone->insertBefore(InsertPos: ThenTI->getIterator());
12597 NewCall = &cast<CallInst>(Val&: promoteCall(CB&: *CBClone, Callee: NewCallee, RetBitCast: &RetBC));
12598 NumIndirectCallsPromoted++;
12599 } else {
12600 NewCall = CallInst::Create(Func: FunctionCallee(CSFT, NewCallee), Args: CSArgs,
12601 NameStr: CB->getName(), InsertBefore: ThenTI->getIterator());
12602 }
12603 NewCalls.push_back(Elt: {NewCall, RetBC});
12604 }
12605
12606 auto AttachCalleeMetadata = [&](CallBase &IndirectCB) {
12607 if (!AllCalleesKnown)
12608 return ChangeStatus::UNCHANGED;
12609 MDBuilder MDB(IndirectCB.getContext());
12610 MDNode *Callees = MDB.createCallees(Callees: SkippedAssumedCallees);
12611 IndirectCB.setMetadata(KindID: LLVMContext::MD_callees, Node: Callees);
12612 return ChangeStatus::CHANGED;
12613 };
12614
12615 if (!SpecializedForAnyCallees)
12616 return AttachCalleeMetadata(*CB);
12617
12618 // Check if we need the fallback indirect call still.
12619 if (SpecializedForAllCallees) {
12620 LastCmp->replaceAllUsesWith(V: ConstantInt::getTrue(Context&: LastCmp->getContext()));
12621 LastCmp->eraseFromParent();
12622 new UnreachableInst(IP->getContext(), IP);
12623 IP->eraseFromParent();
12624 } else {
12625 auto *CBClone = cast<CallInst>(Val: CB->clone());
12626 CBClone->setName(CB->getName());
12627 CBClone->insertBefore(BB&: *IP->getParent(), InsertPos: IP);
12628 NewCalls.push_back(Elt: {CBClone, nullptr});
12629 AttachCalleeMetadata(*CBClone);
12630 }
12631
12632 // Check if we need a PHI to merge the results.
12633 if (!CBIsVoid) {
12634 auto *PHI = PHINode::Create(Ty: CB->getType(), NumReservedValues: NewCalls.size(),
12635 NameStr: CB->getName() + ".phi",
12636 InsertBefore: CB->getParent()->getFirstInsertionPt());
12637 for (auto &It : NewCalls) {
12638 CallBase *NewCall = It.first;
12639 Instruction *CallRet = It.second ? It.second : It.first;
12640 if (CallRet->getType() == CB->getType())
12641 PHI->addIncoming(V: CallRet, BB: CallRet->getParent());
12642 else if (NewCall->getType()->isVoidTy())
12643 PHI->addIncoming(V: PoisonValue::get(T: CB->getType()),
12644 BB: NewCall->getParent());
12645 else
12646 llvm_unreachable("Call return should match or be void!");
12647 }
12648 A.changeAfterManifest(IRP: IRPosition::callsite_returned(CB: *CB), NV&: *PHI);
12649 }
12650
12651 A.deleteAfterManifest(I&: *CB);
12652 Changed = ChangeStatus::CHANGED;
12653
12654 return Changed;
12655 }
12656
12657 /// See AbstractAttribute::getAsStr().
12658 const std::string getAsStr(Attributor *A) const override {
12659 return std::string(AllCalleesKnown ? "eliminate" : "specialize") +
12660 " indirect call site with " + std::to_string(val: AssumedCallees.size()) +
12661 " functions";
12662 }
12663
12664 void trackStatistics() const override {
12665 if (AllCalleesKnown) {
12666 STATS_DECLTRACK(
12667 Eliminated, CallSites,
12668 "Number of indirect call sites eliminated via specialization")
12669 } else {
12670 STATS_DECLTRACK(Specialized, CallSites,
12671 "Number of indirect call sites specialized")
12672 }
12673 }
12674
12675 bool foreachCallee(function_ref<bool(Function *)> CB) const override {
12676 return isValidState() && AllCalleesKnown && all_of(Range: AssumedCallees, P: CB);
12677 }
12678
12679private:
12680 /// Map to remember filter results.
12681 DenseMap<Function *, std::optional<bool>> FilterResults;
12682
12683 /// If the !callee metadata was present, this set will contain all potential
12684 /// callees (superset).
12685 SmallSetVector<Function *, 4> PotentialCallees;
12686
12687 /// This set contains all currently assumed calllees, which might grow over
12688 /// time.
12689 SmallSetVector<Function *, 4> AssumedCallees;
12690
12691 /// Flag to indicate if all possible callees are in the AssumedCallees set or
12692 /// if there could be others.
12693 bool AllCalleesKnown = true;
12694};
12695} // namespace
12696
12697/// --------------------- Invariant Load Pointer -------------------------------
12698namespace {
12699
12700struct AAInvariantLoadPointerImpl
12701 : public StateWrapper<BitIntegerState<uint8_t, 15>,
12702 AAInvariantLoadPointer> {
12703
12704 enum {
12705 // pointer does not alias within the bounds of the function
12706 IS_NOALIAS = 1 << 0,
12707 // pointer is not involved in any effectful instructions within the bounds
12708 // of the function
12709 IS_NOEFFECT = 1 << 1,
12710 // loads are invariant within the bounds of the function
12711 IS_LOCALLY_INVARIANT = 1 << 2,
12712 // memory lifetime is constrained within the bounds of the function
12713 IS_LOCALLY_CONSTRAINED = 1 << 3,
12714
12715 IS_BEST_STATE = IS_NOALIAS | IS_NOEFFECT | IS_LOCALLY_INVARIANT |
12716 IS_LOCALLY_CONSTRAINED,
12717 };
12718 static_assert(getBestState() == IS_BEST_STATE, "Unexpected best state");
12719
12720 using Base =
12721 StateWrapper<BitIntegerState<uint8_t, 15>, AAInvariantLoadPointer>;
12722
12723 // the BitIntegerState is optimistic about IS_NOALIAS and IS_NOEFFECT, but
12724 // pessimistic about IS_KNOWN_INVARIANT
12725 AAInvariantLoadPointerImpl(const IRPosition &IRP, Attributor &A)
12726 : Base(IRP) {}
12727
12728 bool isKnownInvariant() const final {
12729 return isKnownLocallyInvariant() && isKnown(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12730 }
12731
12732 bool isKnownLocallyInvariant() const final {
12733 if (isKnown(BitsEncoding: IS_LOCALLY_INVARIANT))
12734 return true;
12735 return isKnown(BitsEncoding: IS_NOALIAS | IS_NOEFFECT);
12736 }
12737
12738 bool isAssumedInvariant() const final {
12739 return isAssumedLocallyInvariant() && isAssumed(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12740 }
12741
12742 bool isAssumedLocallyInvariant() const final {
12743 if (isAssumed(BitsEncoding: IS_LOCALLY_INVARIANT))
12744 return true;
12745 return isAssumed(BitsEncoding: IS_NOALIAS | IS_NOEFFECT);
12746 }
12747
12748 ChangeStatus updateImpl(Attributor &A) override {
12749 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12750
12751 Changed |= updateNoAlias(A);
12752 if (requiresNoAlias() && !isAssumed(BitsEncoding: IS_NOALIAS))
12753 return indicatePessimisticFixpoint();
12754
12755 Changed |= updateNoEffect(A);
12756
12757 Changed |= updateLocalInvariance(A);
12758
12759 return Changed;
12760 }
12761
12762 ChangeStatus manifest(Attributor &A) override {
12763 if (!isKnownInvariant())
12764 return ChangeStatus::UNCHANGED;
12765
12766 ChangeStatus Changed = ChangeStatus::UNCHANGED;
12767 const Value *Ptr = &getAssociatedValue();
12768 const auto TagInvariantLoads = [&](const Use &U, bool &) {
12769 if (U.get() != Ptr)
12770 return true;
12771 auto *I = dyn_cast<Instruction>(Val: U.getUser());
12772 if (!I)
12773 return true;
12774
12775 // Ensure that we are only changing uses from the corresponding callgraph
12776 // SSC in the case that the AA isn't run on the entire module
12777 if (!A.isRunOn(Fn: I->getFunction()))
12778 return true;
12779
12780 if (I->hasMetadata(KindID: LLVMContext::MD_invariant_load))
12781 return true;
12782
12783 if (auto *LI = dyn_cast<LoadInst>(Val: I)) {
12784 LI->setMetadata(KindID: LLVMContext::MD_invariant_load,
12785 Node: MDNode::get(Context&: LI->getContext(), MDs: {}));
12786 Changed = ChangeStatus::CHANGED;
12787 }
12788 return true;
12789 };
12790
12791 (void)A.checkForAllUses(Pred: TagInvariantLoads, QueryingAA: *this, V: *Ptr);
12792 return Changed;
12793 }
12794
12795 /// See AbstractAttribute::getAsStr().
12796 const std::string getAsStr(Attributor *) const override {
12797 if (isKnownInvariant())
12798 return "load-invariant pointer";
12799 return "non-invariant pointer";
12800 }
12801
12802 /// See AbstractAttribute::trackStatistics().
12803 void trackStatistics() const override {}
12804
12805private:
12806 /// Indicate that noalias is required for the pointer to be invariant.
12807 bool requiresNoAlias() const {
12808 switch (getPositionKind()) {
12809 default:
12810 // Conservatively default to require noalias.
12811 return true;
12812 case IRP_FLOAT:
12813 case IRP_RETURNED:
12814 case IRP_CALL_SITE:
12815 return false;
12816 case IRP_CALL_SITE_RETURNED: {
12817 const auto &CB = cast<CallBase>(Val&: getAnchorValue());
12818 return !isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
12819 Call: &CB, /*MustPreserveOffset=*/false);
12820 }
12821 case IRP_ARGUMENT: {
12822 const Function *F = getAssociatedFunction();
12823 assert(F && "no associated function for argument");
12824 return !isCallableCC(CC: F->getCallingConv());
12825 }
12826 }
12827 }
12828
12829 bool isExternal() const {
12830 const Function *F = getAssociatedFunction();
12831 if (!F)
12832 return true;
12833 return isCallableCC(CC: F->getCallingConv()) &&
12834 getPositionKind() != IRP_CALL_SITE_RETURNED;
12835 }
12836
12837 ChangeStatus updateNoAlias(Attributor &A) {
12838 if (isKnown(BitsEncoding: IS_NOALIAS) || !isAssumed(BitsEncoding: IS_NOALIAS))
12839 return ChangeStatus::UNCHANGED;
12840
12841 // Try to use AANoAlias.
12842 if (const auto *ANoAlias = A.getOrCreateAAFor<AANoAlias>(
12843 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED)) {
12844 if (ANoAlias->isKnownNoAlias()) {
12845 addKnownBits(Bits: IS_NOALIAS);
12846 return ChangeStatus::CHANGED;
12847 }
12848
12849 if (!ANoAlias->isAssumedNoAlias()) {
12850 removeAssumedBits(BitsEncoding: IS_NOALIAS);
12851 return ChangeStatus::CHANGED;
12852 }
12853
12854 return ChangeStatus::UNCHANGED;
12855 }
12856
12857 // Try to infer noalias from argument attribute, since it is applicable for
12858 // the duration of the function.
12859 if (const Argument *Arg = getAssociatedArgument()) {
12860 if (Arg->hasNoAliasAttr()) {
12861 addKnownBits(Bits: IS_NOALIAS);
12862 return ChangeStatus::UNCHANGED;
12863 }
12864
12865 // Noalias information is not provided, and cannot be inferred,
12866 // so we conservatively assume the pointer aliases.
12867 removeAssumedBits(BitsEncoding: IS_NOALIAS);
12868 return ChangeStatus::CHANGED;
12869 }
12870
12871 return ChangeStatus::UNCHANGED;
12872 }
12873
12874 ChangeStatus updateNoEffect(Attributor &A) {
12875 if (isKnown(BitsEncoding: IS_NOEFFECT) || !isAssumed(BitsEncoding: IS_NOEFFECT))
12876 return ChangeStatus::UNCHANGED;
12877
12878 if (!getAssociatedFunction())
12879 return indicatePessimisticFixpoint();
12880
12881 if (isa<AllocaInst>(Val: &getAssociatedValue()))
12882 return indicatePessimisticFixpoint();
12883
12884 const auto HasNoEffectLoads = [&](const Use &U, bool &) {
12885 const auto *LI = dyn_cast<LoadInst>(Val: U.getUser());
12886 return !LI || !LI->mayHaveSideEffects();
12887 };
12888 if (!A.checkForAllUses(Pred: HasNoEffectLoads, QueryingAA: *this, V: getAssociatedValue()))
12889 return indicatePessimisticFixpoint();
12890
12891 if (const auto *AMemoryBehavior = A.getOrCreateAAFor<AAMemoryBehavior>(
12892 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED)) {
12893 // For non-instructions, try to use AAMemoryBehavior to infer the readonly
12894 // attribute
12895 if (!AMemoryBehavior->isAssumedReadOnly())
12896 return indicatePessimisticFixpoint();
12897
12898 if (AMemoryBehavior->isKnownReadOnly()) {
12899 addKnownBits(Bits: IS_NOEFFECT);
12900 return ChangeStatus::UNCHANGED;
12901 }
12902
12903 return ChangeStatus::UNCHANGED;
12904 }
12905
12906 if (const Argument *Arg = getAssociatedArgument()) {
12907 if (Arg->onlyReadsMemory()) {
12908 addKnownBits(Bits: IS_NOEFFECT);
12909 return ChangeStatus::UNCHANGED;
12910 }
12911
12912 // Readonly information is not provided, and cannot be inferred from
12913 // AAMemoryBehavior.
12914 return indicatePessimisticFixpoint();
12915 }
12916
12917 return ChangeStatus::UNCHANGED;
12918 }
12919
12920 ChangeStatus updateLocalInvariance(Attributor &A) {
12921 if (isKnown(BitsEncoding: IS_LOCALLY_INVARIANT) || !isAssumed(BitsEncoding: IS_LOCALLY_INVARIANT))
12922 return ChangeStatus::UNCHANGED;
12923
12924 // try to infer invariance from underlying objects
12925 const auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(
12926 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED);
12927 if (!AUO)
12928 return ChangeStatus::UNCHANGED;
12929
12930 bool UsedAssumedInformation = false;
12931 const auto IsLocallyInvariantLoadIfPointer = [&](const Value &V) {
12932 if (!V.getType()->isPointerTy())
12933 return true;
12934 const auto *IsInvariantLoadPointer =
12935 A.getOrCreateAAFor<AAInvariantLoadPointer>(IRP: IRPosition::value(V), QueryingAA: this,
12936 DepClass: DepClassTy::REQUIRED);
12937 // Conservatively fail if invariance cannot be inferred.
12938 if (!IsInvariantLoadPointer)
12939 return false;
12940
12941 if (IsInvariantLoadPointer->isKnownLocallyInvariant())
12942 return true;
12943 if (!IsInvariantLoadPointer->isAssumedLocallyInvariant())
12944 return false;
12945
12946 UsedAssumedInformation = true;
12947 return true;
12948 };
12949 if (!AUO->forallUnderlyingObjects(Pred: IsLocallyInvariantLoadIfPointer))
12950 return indicatePessimisticFixpoint();
12951
12952 if (const auto *CB = dyn_cast<CallBase>(Val: &getAnchorValue())) {
12953 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
12954 Call: CB, /*MustPreserveOffset=*/false)) {
12955 for (const Value *Arg : CB->args()) {
12956 if (!IsLocallyInvariantLoadIfPointer(*Arg))
12957 return indicatePessimisticFixpoint();
12958 }
12959 }
12960 }
12961
12962 if (!UsedAssumedInformation) {
12963 // Pointer is known and not just assumed to be locally invariant.
12964 addKnownBits(Bits: IS_LOCALLY_INVARIANT);
12965 return ChangeStatus::CHANGED;
12966 }
12967
12968 return ChangeStatus::UNCHANGED;
12969 }
12970};
12971
12972struct AAInvariantLoadPointerFloating final : AAInvariantLoadPointerImpl {
12973 AAInvariantLoadPointerFloating(const IRPosition &IRP, Attributor &A)
12974 : AAInvariantLoadPointerImpl(IRP, A) {}
12975};
12976
12977struct AAInvariantLoadPointerReturned final : AAInvariantLoadPointerImpl {
12978 AAInvariantLoadPointerReturned(const IRPosition &IRP, Attributor &A)
12979 : AAInvariantLoadPointerImpl(IRP, A) {}
12980
12981 void initialize(Attributor &) override {
12982 removeAssumedBits(BitsEncoding: IS_LOCALLY_CONSTRAINED);
12983 }
12984};
12985
12986struct AAInvariantLoadPointerCallSiteReturned final
12987 : AAInvariantLoadPointerImpl {
12988 AAInvariantLoadPointerCallSiteReturned(const IRPosition &IRP, Attributor &A)
12989 : AAInvariantLoadPointerImpl(IRP, A) {}
12990
12991 void initialize(Attributor &A) override {
12992 // Nothing is known about the result of an indirect call.
12993 const Function *F = getAssociatedFunction();
12994 if (!F) {
12995 indicatePessimisticFixpoint();
12996 return;
12997 }
12998
12999 if (!F->isDeclaration() && !F->isIntrinsic())
13000 return AAInvariantLoadPointerImpl::initialize(A);
13001
13002 const auto &CB = cast<CallBase>(Val&: getAnchorValue());
13003 if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
13004 Call: &CB, /*MustPreserveOffset=*/false))
13005 return AAInvariantLoadPointerImpl::initialize(A);
13006
13007 if (F->onlyReadsMemory() && F->hasNoSync())
13008 return AAInvariantLoadPointerImpl::initialize(A);
13009
13010 // At this point, the function is opaque, so we conservatively assume
13011 // non-invariance.
13012 indicatePessimisticFixpoint();
13013 }
13014};
13015
13016struct AAInvariantLoadPointerArgument final : AAInvariantLoadPointerImpl {
13017 AAInvariantLoadPointerArgument(const IRPosition &IRP, Attributor &A)
13018 : AAInvariantLoadPointerImpl(IRP, A) {}
13019
13020 void initialize(Attributor &) override {
13021 const Function *F = getAssociatedFunction();
13022 assert(F && "no associated function for argument");
13023
13024 if (!isCallableCC(CC: F->getCallingConv())) {
13025 addKnownBits(Bits: IS_LOCALLY_CONSTRAINED);
13026 return;
13027 }
13028
13029 if (!F->hasLocalLinkage())
13030 removeAssumedBits(BitsEncoding: IS_LOCALLY_CONSTRAINED);
13031 }
13032};
13033
13034struct AAInvariantLoadPointerCallSiteArgument final
13035 : AAInvariantLoadPointerImpl {
13036 AAInvariantLoadPointerCallSiteArgument(const IRPosition &IRP, Attributor &A)
13037 : AAInvariantLoadPointerImpl(IRP, A) {}
13038};
13039} // namespace
13040
13041/// ------------------------ Address Space ------------------------------------
13042namespace {
13043
13044template <typename InstType>
13045static bool makeChange(Attributor &A, InstType *MemInst, const Use &U,
13046 Value *OriginalValue, PointerType *NewPtrTy,
13047 bool UseOriginalValue) {
13048 if (U.getOperandNo() != InstType::getPointerOperandIndex())
13049 return false;
13050
13051 if (MemInst->isVolatile()) {
13052 auto *TTI = A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(
13053 *MemInst->getFunction());
13054 unsigned NewAS = NewPtrTy->getPointerAddressSpace();
13055 if (!TTI || !TTI->hasVolatileVariant(MemInst, NewAS))
13056 return false;
13057 }
13058
13059 if (UseOriginalValue) {
13060 A.changeUseAfterManifest(U&: const_cast<Use &>(U), NV&: *OriginalValue);
13061 return true;
13062 }
13063
13064 Instruction *CastInst = new AddrSpaceCastInst(OriginalValue, NewPtrTy);
13065 CastInst->insertBefore(MemInst->getIterator());
13066 A.changeUseAfterManifest(U&: const_cast<Use &>(U), NV&: *CastInst);
13067 return true;
13068}
13069
13070struct AAAddressSpaceImpl : public AAAddressSpace {
13071 AAAddressSpaceImpl(const IRPosition &IRP, Attributor &A)
13072 : AAAddressSpace(IRP, A) {}
13073
13074 uint32_t getAddressSpace() const override {
13075 assert(isValidState() && "the AA is invalid");
13076 return AssumedAddressSpace;
13077 }
13078
13079 /// See AbstractAttribute::initialize(...).
13080 void initialize(Attributor &A) override {
13081 assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
13082 "Associated value is not a pointer");
13083
13084 if (!A.getInfoCache().getFlatAddressSpace().has_value()) {
13085 indicatePessimisticFixpoint();
13086 return;
13087 }
13088
13089 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13090 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13091 if (AS != FlatAS) {
13092 [[maybe_unused]] bool R = takeAddressSpace(AS);
13093 assert(R && "The take should happen");
13094 indicateOptimisticFixpoint();
13095 }
13096 }
13097
13098 ChangeStatus updateImpl(Attributor &A) override {
13099 uint32_t OldAddressSpace = AssumedAddressSpace;
13100 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13101
13102 auto CheckAddressSpace = [&](Value &Obj) {
13103 // Ignore undef.
13104 if (isa<UndefValue>(Val: &Obj))
13105 return true;
13106
13107 // If the object already has a non-flat address space, we simply take it.
13108 unsigned ObjAS = Obj.getType()->getPointerAddressSpace();
13109 if (ObjAS != FlatAS)
13110 return takeAddressSpace(AS: ObjAS);
13111
13112 // At this point, we know Obj is in the flat address space. For a final
13113 // attempt, we want to use getAssumedAddrSpace, but first we must get the
13114 // associated function, if possible.
13115 Function *F = nullptr;
13116 if (auto *Arg = dyn_cast<Argument>(Val: &Obj))
13117 F = Arg->getParent();
13118 else if (auto *I = dyn_cast<Instruction>(Val: &Obj))
13119 F = I->getFunction();
13120
13121 // Use getAssumedAddrSpace if the associated function exists.
13122 if (F) {
13123 auto *TTI =
13124 A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(F: *F);
13125 unsigned AssumedAS = TTI->getAssumedAddrSpace(V: &Obj);
13126 if (AssumedAS != ~0U)
13127 return takeAddressSpace(AS: AssumedAS);
13128 }
13129
13130 // Now we can't do anything else but to take the flat AS.
13131 return takeAddressSpace(AS: FlatAS);
13132 };
13133
13134 auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(IRP: getIRPosition(), QueryingAA: this,
13135 DepClass: DepClassTy::REQUIRED);
13136 if (!AUO->forallUnderlyingObjects(Pred: CheckAddressSpace))
13137 return indicatePessimisticFixpoint();
13138
13139 return OldAddressSpace == AssumedAddressSpace ? ChangeStatus::UNCHANGED
13140 : ChangeStatus::CHANGED;
13141 }
13142
13143 /// See AbstractAttribute::manifest(...).
13144 ChangeStatus manifest(Attributor &A) override {
13145 unsigned NewAS = getAddressSpace();
13146
13147 if (NewAS == InvalidAddressSpace ||
13148 NewAS == getAssociatedType()->getPointerAddressSpace())
13149 return ChangeStatus::UNCHANGED;
13150
13151 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13152
13153 Value *AssociatedValue = &getAssociatedValue();
13154 Value *OriginalValue = peelAddrspacecast(V: AssociatedValue, FlatAS);
13155
13156 PointerType *NewPtrTy =
13157 PointerType::get(C&: getAssociatedType()->getContext(), AddressSpace: NewAS);
13158 bool UseOriginalValue =
13159 OriginalValue->getType()->getPointerAddressSpace() == NewAS;
13160
13161 bool Changed = false;
13162
13163 auto Pred = [&](const Use &U, bool &) {
13164 if (U.get() != AssociatedValue)
13165 return true;
13166 auto *Inst = dyn_cast<Instruction>(Val: U.getUser());
13167 if (!Inst)
13168 return true;
13169 // This is a WA to make sure we only change uses from the corresponding
13170 // CGSCC if the AA is run on CGSCC instead of the entire module.
13171 if (!A.isRunOn(Fn: Inst->getFunction()))
13172 return true;
13173 if (auto *LI = dyn_cast<LoadInst>(Val: Inst)) {
13174 Changed |=
13175 makeChange(A, MemInst: LI, U, OriginalValue, NewPtrTy, UseOriginalValue);
13176 } else if (auto *SI = dyn_cast<StoreInst>(Val: Inst)) {
13177 Changed |=
13178 makeChange(A, MemInst: SI, U, OriginalValue, NewPtrTy, UseOriginalValue);
13179 } else if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: Inst)) {
13180 Changed |=
13181 makeChange(A, MemInst: RMW, U, OriginalValue, NewPtrTy, UseOriginalValue);
13182 } else if (auto *CmpX = dyn_cast<AtomicCmpXchgInst>(Val: Inst)) {
13183 Changed |=
13184 makeChange(A, MemInst: CmpX, U, OriginalValue, NewPtrTy, UseOriginalValue);
13185 }
13186 return true;
13187 };
13188
13189 // It doesn't matter if we can't check all uses as we can simply
13190 // conservatively ignore those that can not be visited.
13191 (void)A.checkForAllUses(Pred, QueryingAA: *this, V: getAssociatedValue(),
13192 /* CheckBBLivenessOnly */ true);
13193
13194 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
13195 }
13196
13197 /// See AbstractAttribute::getAsStr().
13198 const std::string getAsStr(Attributor *A) const override {
13199 if (!isValidState())
13200 return "addrspace(<invalid>)";
13201 return "addrspace(" +
13202 (AssumedAddressSpace == InvalidAddressSpace
13203 ? "none"
13204 : std::to_string(val: AssumedAddressSpace)) +
13205 ")";
13206 }
13207
13208private:
13209 uint32_t AssumedAddressSpace = InvalidAddressSpace;
13210
13211 bool takeAddressSpace(uint32_t AS) {
13212 if (AssumedAddressSpace == InvalidAddressSpace) {
13213 AssumedAddressSpace = AS;
13214 return true;
13215 }
13216 return AssumedAddressSpace == AS;
13217 }
13218
13219 static Value *peelAddrspacecast(Value *V, unsigned FlatAS) {
13220 if (auto *I = dyn_cast<AddrSpaceCastInst>(Val: V)) {
13221 assert(I->getSrcAddressSpace() != FlatAS &&
13222 "there should not be flat AS -> non-flat AS");
13223 return I->getPointerOperand();
13224 }
13225 if (auto *C = dyn_cast<ConstantExpr>(Val: V))
13226 if (C->getOpcode() == Instruction::AddrSpaceCast) {
13227 assert(C->getOperand(0)->getType()->getPointerAddressSpace() !=
13228 FlatAS &&
13229 "there should not be flat AS -> non-flat AS X");
13230 return C->getOperand(i_nocapture: 0);
13231 }
13232 return V;
13233 }
13234};
13235
13236struct AAAddressSpaceFloating final : AAAddressSpaceImpl {
13237 AAAddressSpaceFloating(const IRPosition &IRP, Attributor &A)
13238 : AAAddressSpaceImpl(IRP, A) {}
13239
13240 void trackStatistics() const override {
13241 STATS_DECLTRACK_FLOATING_ATTR(addrspace);
13242 }
13243};
13244
13245struct AAAddressSpaceReturned final : AAAddressSpaceImpl {
13246 AAAddressSpaceReturned(const IRPosition &IRP, Attributor &A)
13247 : AAAddressSpaceImpl(IRP, A) {}
13248
13249 /// See AbstractAttribute::initialize(...).
13250 void initialize(Attributor &A) override {
13251 // TODO: we don't rewrite function argument for now because it will need to
13252 // rewrite the function signature and all call sites.
13253 (void)indicatePessimisticFixpoint();
13254 }
13255
13256 void trackStatistics() const override {
13257 STATS_DECLTRACK_FNRET_ATTR(addrspace);
13258 }
13259};
13260
13261struct AAAddressSpaceCallSiteReturned final : AAAddressSpaceImpl {
13262 AAAddressSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
13263 : AAAddressSpaceImpl(IRP, A) {}
13264
13265 void trackStatistics() const override {
13266 STATS_DECLTRACK_CSRET_ATTR(addrspace);
13267 }
13268};
13269
13270struct AAAddressSpaceArgument final : AAAddressSpaceImpl {
13271 AAAddressSpaceArgument(const IRPosition &IRP, Attributor &A)
13272 : AAAddressSpaceImpl(IRP, A) {}
13273
13274 void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(addrspace); }
13275};
13276
13277struct AAAddressSpaceCallSiteArgument final : AAAddressSpaceImpl {
13278 AAAddressSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
13279 : AAAddressSpaceImpl(IRP, A) {}
13280
13281 /// See AbstractAttribute::initialize(...).
13282 void initialize(Attributor &A) override {
13283 // TODO: we don't rewrite call site argument for now because it will need to
13284 // rewrite the function signature of the callee.
13285 (void)indicatePessimisticFixpoint();
13286 }
13287
13288 void trackStatistics() const override {
13289 STATS_DECLTRACK_CSARG_ATTR(addrspace);
13290 }
13291};
13292} // namespace
13293
13294/// ------------------------ No Alias Address Space ---------------------------
13295// This attribute assumes flat address space can alias all other address space
13296
13297// TODO: this is similar to AAAddressSpace, most of the code should be merged.
13298// But merging it created failing cased on gateway test that cannot be
13299// reproduced locally. So should open a separated PR to handle the merge of
13300// AANoAliasAddrSpace and AAAddressSpace attribute
13301
13302namespace {
13303struct AANoAliasAddrSpaceImpl : public AANoAliasAddrSpace {
13304 AANoAliasAddrSpaceImpl(const IRPosition &IRP, Attributor &A)
13305 : AANoAliasAddrSpace(IRP, A) {}
13306
13307 void initialize(Attributor &A) override {
13308 assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
13309 "Associated value is not a pointer");
13310
13311 resetASRanges(A);
13312
13313 std::optional<unsigned> FlatAS = A.getInfoCache().getFlatAddressSpace();
13314 if (!FlatAS.has_value()) {
13315 indicatePessimisticFixpoint();
13316 return;
13317 }
13318
13319 removeAS(AS: *FlatAS);
13320
13321 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13322 if (AS != *FlatAS) {
13323 removeAS(AS);
13324 indicateOptimisticFixpoint();
13325 }
13326 }
13327
13328 ChangeStatus updateImpl(Attributor &A) override {
13329 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13330 uint32_t OldAssumed = getAssumed();
13331
13332 auto CheckAddressSpace = [&](Value &Obj) {
13333 if (isa<PoisonValue>(Val: &Obj))
13334 return true;
13335
13336 unsigned AS = Obj.getType()->getPointerAddressSpace();
13337 if (AS == FlatAS)
13338 return false;
13339
13340 removeAS(AS: Obj.getType()->getPointerAddressSpace());
13341 return true;
13342 };
13343
13344 const AAUnderlyingObjects *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(
13345 IRP: getIRPosition(), QueryingAA: this, DepClass: DepClassTy::REQUIRED);
13346 if (!AUO->forallUnderlyingObjects(Pred: CheckAddressSpace))
13347 return indicatePessimisticFixpoint();
13348
13349 return OldAssumed == getAssumed() ? ChangeStatus::UNCHANGED
13350 : ChangeStatus::CHANGED;
13351 }
13352
13353 /// See AbstractAttribute::manifest(...).
13354 ChangeStatus manifest(Attributor &A) override {
13355 unsigned FlatAS = A.getInfoCache().getFlatAddressSpace().value();
13356
13357 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13358 if (AS != FlatAS || Map.empty())
13359 return ChangeStatus::UNCHANGED;
13360
13361 LLVMContext &Ctx = getAssociatedValue().getContext();
13362 MDNode *NoAliasASNode = nullptr;
13363 MDBuilder MDB(Ctx);
13364 // Has to use iterator to get the range info.
13365 for (RangeMap::const_iterator I = Map.begin(); I != Map.end(); I++) {
13366 if (!I.value())
13367 continue;
13368 unsigned Upper = I.stop();
13369 unsigned Lower = I.start();
13370 if (!NoAliasASNode) {
13371 NoAliasASNode = MDB.createRange(Lo: APInt(32, Lower), Hi: APInt(32, Upper + 1));
13372 continue;
13373 }
13374 MDNode *ASRange = MDB.createRange(Lo: APInt(32, Lower), Hi: APInt(32, Upper + 1));
13375 NoAliasASNode = MDNode::getMostGenericRange(A: NoAliasASNode, B: ASRange);
13376 }
13377
13378 Value *AssociatedValue = &getAssociatedValue();
13379 bool Changed = false;
13380
13381 auto AddNoAliasAttr = [&](const Use &U, bool &) {
13382 if (U.get() != AssociatedValue)
13383 return true;
13384 Instruction *Inst = dyn_cast<Instruction>(Val: U.getUser());
13385 if (!Inst || Inst->hasMetadata(KindID: LLVMContext::MD_noalias_addrspace))
13386 return true;
13387 if (!isa<LoadInst>(Val: Inst) && !isa<StoreInst>(Val: Inst) &&
13388 !isa<AtomicCmpXchgInst>(Val: Inst) && !isa<AtomicRMWInst>(Val: Inst))
13389 return true;
13390 if (!A.isRunOn(Fn: Inst->getFunction()))
13391 return true;
13392 Inst->setMetadata(KindID: LLVMContext::MD_noalias_addrspace, Node: NoAliasASNode);
13393 Changed = true;
13394 return true;
13395 };
13396 (void)A.checkForAllUses(Pred: AddNoAliasAttr, QueryingAA: *this, V: *AssociatedValue,
13397 /*CheckBBLivenessOnly=*/true);
13398 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
13399 }
13400
13401 /// See AbstractAttribute::getAsStr().
13402 const std::string getAsStr(Attributor *A) const override {
13403 if (!isValidState())
13404 return "<invalid>";
13405 std::string Str;
13406 raw_string_ostream OS(Str);
13407 OS << "CanNotBeAddrSpace(";
13408 for (RangeMap::const_iterator I = Map.begin(); I != Map.end(); I++) {
13409 unsigned Upper = I.stop();
13410 unsigned Lower = I.start();
13411 OS << ' ' << '[' << Upper << ',' << Lower + 1 << ')';
13412 }
13413 OS << " )";
13414 return OS.str();
13415 }
13416
13417private:
13418 void removeAS(unsigned AS) {
13419 RangeMap::iterator I = Map.find(x: AS);
13420
13421 if (I != Map.end()) {
13422 unsigned Upper = I.stop();
13423 unsigned Lower = I.start();
13424 I.erase();
13425 if (Upper == Lower)
13426 return;
13427 if (AS != ~((unsigned)0) && AS + 1 <= Upper)
13428 Map.insert(a: AS + 1, b: Upper, /*what ever this variable name is=*/y: true);
13429 if (AS != 0 && Lower <= AS - 1)
13430 Map.insert(a: Lower, b: AS - 1, y: true);
13431 }
13432 }
13433
13434 void resetASRanges(Attributor &A) {
13435 Map.clear();
13436 Map.insert(a: 0, b: A.getInfoCache().getMaxAddrSpace(), y: true);
13437 }
13438};
13439
13440struct AANoAliasAddrSpaceFloating final : AANoAliasAddrSpaceImpl {
13441 AANoAliasAddrSpaceFloating(const IRPosition &IRP, Attributor &A)
13442 : AANoAliasAddrSpaceImpl(IRP, A) {}
13443
13444 void trackStatistics() const override {
13445 STATS_DECLTRACK_FLOATING_ATTR(noaliasaddrspace);
13446 }
13447};
13448
13449struct AANoAliasAddrSpaceReturned final : AANoAliasAddrSpaceImpl {
13450 AANoAliasAddrSpaceReturned(const IRPosition &IRP, Attributor &A)
13451 : AANoAliasAddrSpaceImpl(IRP, A) {}
13452
13453 void trackStatistics() const override {
13454 STATS_DECLTRACK_FNRET_ATTR(noaliasaddrspace);
13455 }
13456};
13457
13458struct AANoAliasAddrSpaceCallSiteReturned final : AANoAliasAddrSpaceImpl {
13459 AANoAliasAddrSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
13460 : AANoAliasAddrSpaceImpl(IRP, A) {}
13461
13462 void trackStatistics() const override {
13463 STATS_DECLTRACK_CSRET_ATTR(noaliasaddrspace);
13464 }
13465};
13466
13467struct AANoAliasAddrSpaceArgument final : AANoAliasAddrSpaceImpl {
13468 AANoAliasAddrSpaceArgument(const IRPosition &IRP, Attributor &A)
13469 : AANoAliasAddrSpaceImpl(IRP, A) {}
13470
13471 void trackStatistics() const override {
13472 STATS_DECLTRACK_ARG_ATTR(noaliasaddrspace);
13473 }
13474};
13475
13476struct AANoAliasAddrSpaceCallSiteArgument final : AANoAliasAddrSpaceImpl {
13477 AANoAliasAddrSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
13478 : AANoAliasAddrSpaceImpl(IRP, A) {}
13479
13480 void trackStatistics() const override {
13481 STATS_DECLTRACK_CSARG_ATTR(noaliasaddrspace);
13482 }
13483};
13484} // namespace
13485/// ----------- Allocation Info ----------
13486namespace {
13487struct AAAllocationInfoImpl : public AAAllocationInfo {
13488 AAAllocationInfoImpl(const IRPosition &IRP, Attributor &A)
13489 : AAAllocationInfo(IRP, A) {}
13490
13491 std::optional<TypeSize> getAllocatedSize() const override {
13492 assert(isValidState() && "the AA is invalid");
13493 return AssumedAllocatedSize;
13494 }
13495
13496 std::optional<TypeSize> findInitialAllocationSize(Instruction *I,
13497 const DataLayout &DL) {
13498
13499 // TODO: implement case for malloc like instructions
13500 switch (I->getOpcode()) {
13501 case Instruction::Alloca: {
13502 AllocaInst *AI = cast<AllocaInst>(Val: I);
13503 return AI->getAllocationSize(DL);
13504 }
13505 default:
13506 return std::nullopt;
13507 }
13508 }
13509
13510 ChangeStatus updateImpl(Attributor &A) override {
13511
13512 const IRPosition &IRP = getIRPosition();
13513 Instruction *I = IRP.getCtxI();
13514
13515 // TODO: update check for malloc like calls
13516 if (!isa<AllocaInst>(Val: I))
13517 return indicatePessimisticFixpoint();
13518
13519 bool IsKnownNoCapture;
13520 if (!AA::hasAssumedIRAttr<Attribute::Captures>(
13521 A, QueryingAA: this, IRP, DepClass: DepClassTy::OPTIONAL, IsKnown&: IsKnownNoCapture))
13522 return indicatePessimisticFixpoint();
13523
13524 const AAPointerInfo *PI =
13525 A.getOrCreateAAFor<AAPointerInfo>(IRP, QueryingAA: *this, DepClass: DepClassTy::REQUIRED);
13526
13527 if (!PI)
13528 return indicatePessimisticFixpoint();
13529
13530 if (!PI->getState().isValidState() || PI->reachesReturn())
13531 return indicatePessimisticFixpoint();
13532
13533 const DataLayout &DL = A.getDataLayout();
13534 const auto AllocationSize = findInitialAllocationSize(I, DL);
13535
13536 // If allocation size is nullopt, we give up.
13537 if (!AllocationSize)
13538 return indicatePessimisticFixpoint();
13539
13540 // For zero sized allocations, we give up.
13541 // Since we can't reduce further
13542 if (*AllocationSize == 0)
13543 return indicatePessimisticFixpoint();
13544
13545 int64_t BinSize = PI->numOffsetBins();
13546
13547 // TODO: implement for multiple bins
13548 if (BinSize > 1)
13549 return indicatePessimisticFixpoint();
13550
13551 if (BinSize == 0) {
13552 auto NewAllocationSize = std::make_optional<TypeSize>(args: 0, args: false);
13553 if (!changeAllocationSize(Size: NewAllocationSize))
13554 return ChangeStatus::UNCHANGED;
13555 return ChangeStatus::CHANGED;
13556 }
13557
13558 // TODO: refactor this to be part of multiple bin case
13559 const auto &It = PI->begin();
13560
13561 // TODO: handle if Offset is not zero
13562 if (It->first.Offset != 0)
13563 return indicatePessimisticFixpoint();
13564
13565 uint64_t SizeOfBin = It->first.Offset + It->first.Size;
13566
13567 if (SizeOfBin >= *AllocationSize)
13568 return indicatePessimisticFixpoint();
13569
13570 auto NewAllocationSize = std::make_optional<TypeSize>(args: SizeOfBin * 8, args: false);
13571
13572 if (!changeAllocationSize(Size: NewAllocationSize))
13573 return ChangeStatus::UNCHANGED;
13574
13575 return ChangeStatus::CHANGED;
13576 }
13577
13578 /// See AbstractAttribute::manifest(...).
13579 ChangeStatus manifest(Attributor &A) override {
13580
13581 assert(isValidState() &&
13582 "Manifest should only be called if the state is valid.");
13583
13584 Instruction *I = getIRPosition().getCtxI();
13585
13586 auto FixedAllocatedSizeInBits = getAllocatedSize()->getFixedValue();
13587
13588 unsigned long NumBytesToAllocate = (FixedAllocatedSizeInBits + 7) / 8;
13589
13590 switch (I->getOpcode()) {
13591 // TODO: add case for malloc like calls
13592 case Instruction::Alloca: {
13593
13594 AllocaInst *AI = cast<AllocaInst>(Val: I);
13595
13596 Type *CharType = Type::getInt8Ty(C&: I->getContext());
13597
13598 auto *NumBytesToValue =
13599 ConstantInt::get(Context&: I->getContext(), V: APInt(32, NumBytesToAllocate));
13600
13601 BasicBlock::iterator insertPt = AI->getIterator();
13602 insertPt = std::next(x: insertPt);
13603 AllocaInst *NewAllocaInst =
13604 new AllocaInst(CharType, AI->getAddressSpace(), NumBytesToValue,
13605 AI->getAlign(), AI->getName(), insertPt);
13606
13607 if (A.changeAfterManifest(IRP: IRPosition::inst(I: *AI), NV&: *NewAllocaInst))
13608 return ChangeStatus::CHANGED;
13609
13610 break;
13611 }
13612 default:
13613 break;
13614 }
13615
13616 return ChangeStatus::UNCHANGED;
13617 }
13618
13619 /// See AbstractAttribute::getAsStr().
13620 const std::string getAsStr(Attributor *A) const override {
13621 if (!isValidState())
13622 return "allocationinfo(<invalid>)";
13623 return "allocationinfo(" +
13624 (AssumedAllocatedSize == HasNoAllocationSize
13625 ? "none"
13626 : std::to_string(val: AssumedAllocatedSize->getFixedValue())) +
13627 ")";
13628 }
13629
13630private:
13631 std::optional<TypeSize> AssumedAllocatedSize = HasNoAllocationSize;
13632
13633 // Maintain the computed allocation size of the object.
13634 // Returns (bool) weather the size of the allocation was modified or not.
13635 bool changeAllocationSize(std::optional<TypeSize> Size) {
13636 if (AssumedAllocatedSize == HasNoAllocationSize ||
13637 AssumedAllocatedSize != Size) {
13638 AssumedAllocatedSize = Size;
13639 return true;
13640 }
13641 return false;
13642 }
13643};
13644
13645struct AAAllocationInfoFloating : AAAllocationInfoImpl {
13646 AAAllocationInfoFloating(const IRPosition &IRP, Attributor &A)
13647 : AAAllocationInfoImpl(IRP, A) {}
13648
13649 void trackStatistics() const override {
13650 STATS_DECLTRACK_FLOATING_ATTR(allocationinfo);
13651 }
13652};
13653
13654struct AAAllocationInfoReturned : AAAllocationInfoImpl {
13655 AAAllocationInfoReturned(const IRPosition &IRP, Attributor &A)
13656 : AAAllocationInfoImpl(IRP, A) {}
13657
13658 /// See AbstractAttribute::initialize(...).
13659 void initialize(Attributor &A) override {
13660 // TODO: we don't rewrite function argument for now because it will need to
13661 // rewrite the function signature and all call sites
13662 (void)indicatePessimisticFixpoint();
13663 }
13664
13665 void trackStatistics() const override {
13666 STATS_DECLTRACK_FNRET_ATTR(allocationinfo);
13667 }
13668};
13669
13670struct AAAllocationInfoCallSiteReturned : AAAllocationInfoImpl {
13671 AAAllocationInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
13672 : AAAllocationInfoImpl(IRP, A) {}
13673
13674 void trackStatistics() const override {
13675 STATS_DECLTRACK_CSRET_ATTR(allocationinfo);
13676 }
13677};
13678
13679struct AAAllocationInfoArgument : AAAllocationInfoImpl {
13680 AAAllocationInfoArgument(const IRPosition &IRP, Attributor &A)
13681 : AAAllocationInfoImpl(IRP, A) {}
13682
13683 void trackStatistics() const override {
13684 STATS_DECLTRACK_ARG_ATTR(allocationinfo);
13685 }
13686};
13687
13688struct AAAllocationInfoCallSiteArgument : AAAllocationInfoImpl {
13689 AAAllocationInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
13690 : AAAllocationInfoImpl(IRP, A) {}
13691
13692 /// See AbstractAttribute::initialize(...).
13693 void initialize(Attributor &A) override {
13694
13695 (void)indicatePessimisticFixpoint();
13696 }
13697
13698 void trackStatistics() const override {
13699 STATS_DECLTRACK_CSARG_ATTR(allocationinfo);
13700 }
13701};
13702} // namespace
13703
13704const char AANoUnwind::ID = 0;
13705const char AANoSync::ID = 0;
13706const char AANoFree::ID = 0;
13707const char AANonNull::ID = 0;
13708const char AAMustProgress::ID = 0;
13709const char AANoRecurse::ID = 0;
13710const char AANonConvergent::ID = 0;
13711const char AAWillReturn::ID = 0;
13712const char AAUndefinedBehavior::ID = 0;
13713const char AANoAlias::ID = 0;
13714const char AAIntraFnReachability::ID = 0;
13715const char AANoReturn::ID = 0;
13716const char AAIsDead::ID = 0;
13717const char AADereferenceable::ID = 0;
13718const char AAAlign::ID = 0;
13719const char AAInstanceInfo::ID = 0;
13720const char AANoCapture::ID = 0;
13721const char AAValueSimplify::ID = 0;
13722const char AAHeapToStack::ID = 0;
13723const char AAPrivatizablePtr::ID = 0;
13724const char AAMemoryBehavior::ID = 0;
13725const char AAMemoryLocation::ID = 0;
13726const char AAValueConstantRange::ID = 0;
13727const char AAPotentialConstantValues::ID = 0;
13728const char AAPotentialValues::ID = 0;
13729const char AANoUndef::ID = 0;
13730const char AANoFPClass::ID = 0;
13731const char AACallEdges::ID = 0;
13732const char AAInterFnReachability::ID = 0;
13733const char AAPointerInfo::ID = 0;
13734const char AAAssumptionInfo::ID = 0;
13735const char AAUnderlyingObjects::ID = 0;
13736const char AAInvariantLoadPointer::ID = 0;
13737const char AAAddressSpace::ID = 0;
13738const char AANoAliasAddrSpace::ID = 0;
13739const char AAAllocationInfo::ID = 0;
13740const char AAIndirectCallInfo::ID = 0;
13741const char AAGlobalValueInfo::ID = 0;
13742const char AADenormalFPMath::ID = 0;
13743
13744// Macro magic to create the static generator function for attributes that
13745// follow the naming scheme.
13746
13747#define SWITCH_PK_INV(CLASS, PK, POS_NAME) \
13748 case IRPosition::PK: \
13749 llvm_unreachable("Cannot create " #CLASS " for a " POS_NAME " position!");
13750
13751#define SWITCH_PK_CREATE(CLASS, IRP, PK, SUFFIX) \
13752 case IRPosition::PK: \
13753 AA = new (A.Allocator) CLASS##SUFFIX(IRP, A); \
13754 ++NumAAs; \
13755 break;
13756
13757#define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13758 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13759 CLASS *AA = nullptr; \
13760 switch (IRP.getPositionKind()) { \
13761 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13762 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
13763 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
13764 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13765 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
13766 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
13767 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13768 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13769 } \
13770 return *AA; \
13771 }
13772
13773#define CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13774 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13775 CLASS *AA = nullptr; \
13776 switch (IRP.getPositionKind()) { \
13777 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13778 SWITCH_PK_INV(CLASS, IRP_FUNCTION, "function") \
13779 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
13780 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13781 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13782 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
13783 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13784 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13785 } \
13786 return *AA; \
13787 }
13788
13789#define CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(POS, SUFFIX, CLASS) \
13790 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13791 CLASS *AA = nullptr; \
13792 switch (IRP.getPositionKind()) { \
13793 SWITCH_PK_CREATE(CLASS, IRP, POS, SUFFIX) \
13794 default: \
13795 llvm_unreachable("Cannot create " #CLASS " for position otherthan " #POS \
13796 " position!"); \
13797 } \
13798 return *AA; \
13799 }
13800
13801#define CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13802 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13803 CLASS *AA = nullptr; \
13804 switch (IRP.getPositionKind()) { \
13805 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13806 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13807 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13808 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13809 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13810 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
13811 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13812 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13813 } \
13814 return *AA; \
13815 }
13816
13817#define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13818 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13819 CLASS *AA = nullptr; \
13820 switch (IRP.getPositionKind()) { \
13821 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13822 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
13823 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
13824 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13825 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
13826 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
13827 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
13828 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13829 } \
13830 return *AA; \
13831 }
13832
13833#define CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13834 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13835 CLASS *AA = nullptr; \
13836 switch (IRP.getPositionKind()) { \
13837 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13838 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13839 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13840 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13841 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13842 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13843 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13844 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13845 } \
13846 return *AA; \
13847 }
13848
13849CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUnwind)
13850CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoSync)
13851CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoRecurse)
13852CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAWillReturn)
13853CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoReturn)
13854CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryLocation)
13855CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AACallEdges)
13856CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAssumptionInfo)
13857CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMustProgress)
13858
13859CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonNull)
13860CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAlias)
13861CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPrivatizablePtr)
13862CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADereferenceable)
13863CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAlign)
13864CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInstanceInfo)
13865CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoCapture)
13866CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueConstantRange)
13867CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialConstantValues)
13868CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialValues)
13869CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUndef)
13870CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFPClass)
13871CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPointerInfo)
13872CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInvariantLoadPointer)
13873CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAddressSpace)
13874CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAliasAddrSpace)
13875CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAllocationInfo)
13876
13877CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueSimplify)
13878CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIsDead)
13879CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFree)
13880CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUnderlyingObjects)
13881
13882CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_CALL_SITE, CallSite,
13883 AAIndirectCallInfo)
13884CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_FLOAT, Floating,
13885 AAGlobalValueInfo)
13886
13887CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAHeapToStack)
13888CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUndefinedBehavior)
13889CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonConvergent)
13890CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIntraFnReachability)
13891CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInterFnReachability)
13892CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADenormalFPMath)
13893
13894CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryBehavior)
13895
13896#undef CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION
13897#undef CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION
13898#undef CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION
13899#undef CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION
13900#undef CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION
13901#undef CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION
13902#undef SWITCH_PK_CREATE
13903#undef SWITCH_PK_INV
13904